def test_layer(self): self.append_layer(Layers.ZeroPaddingLayer(3)) self.append_layer(Layers.ConvLayer(64, 7, 2, use_bias_in=False)) self.append_layer(Layers.BatchNormalizationLayer()) self.append_layer(Layers.ActivationLayer('relu')) self.append_layer(Layers.ZeroPaddingLayer(1)) self.append_layer( Layers.PoolLayer(3, 2, type_in='max', concatenate=True)) self.dense_block(6) self.transition_block(128) self.dense_block(12) self.transition_block(256) self.dense_block(32) self.transition_block(640) self.dense_block(32) self.append_layer(Layers.BatchNormalizationLayer()) self.append_layer(Layers.PoolLayer(2, 2, pool_type_in='globalaverage')) self.append_layer(Layers.ActivationLayer('sigmoid'))
def ConstructTreeConvolution(nodes, numFea, numCon, numDis, numOut,\ Wleft, Wright, Bconstruct,\ Wcomb_ae, Wcomb_orig, \ Wconv_root, Wconv_left, Wconv_right, Wconv_sib, Bconv,\ Wdis, Woutput, Bdis, Boutput,\ poolCutoff ): # nodes # numFea: # of the word/symbol feature size # numCon: # of the convolution size # Wleft: left weights of continous binary tree autoencoder # Wright: right weights of continous binary tree autoencoder # Bconstruct: the biase for the autoencoder # Wcomb_ae, Wcomb_orig: the weights for the combination of # autoencoder and the original vector # (no biase for this sate) # Wconv_root, Wconv_left, Wconv_right, Bconv: the weights for covolution # Bconv: Biases for covolution numNodes = len(nodes) layers = [None] * numNodes # construct layers for each node # layers = |---leaf---|---non_leaf---| numLeaf = 0 for idx in xrange(numNodes): node = nodes[idx] if len(node.children) == 0: numLeaf += 1 layers[idx] = Lay.layer('vec_'+str(idx)+'_' + node.word,\ range( node.bidx, node.bidx + numFea),\ numFea ) layers[idx].act = 'embedding' # auto encoding # layers = |---leaf---|---non_leaf(autoencoded)---| (numNodes) # |---non_leaf(original)---| ( numNonLeaf) numNonLeaf = numNodes - numLeaf layers.extend([None] * (2 * numNonLeaf)) for idx in xrange(numLeaf, numNodes): node = nodes[idx] layers[idx + numNonLeaf] = layers[idx] tmplayer = Lay.layer('ae_'+str(idx)+'_'+node.word,\ Bconstruct, numFea) tmplayer.act = 'autoencoding' layers[idx] = tmplayer # add reconstruction connections for idx in xrange(0, numNodes): node = nodes[idx] if node.parent == None: continue tmplayer = layers[idx] parent = layers[node.parent] if node.leftRate != 0: leftcon = Con.connection(tmplayer, parent,\ numFea, numFea, Wleft, Wcoef = node.leftRate * node.leafNum/nodes[node.parent].leafNum) if node.rightRate != 0: rightcon = Con.connection(tmplayer, parent,\ numFea, numFea, Wright, Wcoef = node.rightRate * node.leafNum/nodes[node.parent].leafNum) # combinition of the constructed and original value # layers = |---leaf---|---non_leaf(combinition)---| (numNodes) # |---non_leaf(original)---|---non_leaf(ae)---| (2 * numNonLeaf) for idx in xrange(numLeaf, numNodes): aelayer = layers[idx] origlayer = layers[idx + numNonLeaf] layers[idx + numNonLeaf * 2] = aelayer comlayer = Lay.layer('comb_' + str(idx) + '_' + nodes[idx].word, None, numFea) comlayer.act = 'combination' layers[idx] = comlayer # connecton auto encoded vector and original vector con_ae = Con.connection(aelayer, comlayer, numFea, numFea, Wcomb_ae) con_orig = Con.connection(origlayer, comlayer, numFea, numFea, Wcomb_orig) # CONVOLVE!!! and POOOOL!!! # layers = |---leaf---|---non_leaf(combition)---| => (numNodes) # |---non_leaf(original)---|---non_leaf(ae)---| => (2 * numNonLeaf) # |------------convolution----------| queue = [(numNodes - 1, None)] poolTop = Lay.PoolLayer('poolTop', numCon) poolLeft = Lay.PoolLayer('poolLeft', numCon) poolRight = Lay.PoolLayer('poolRight', numCon) layerCnt = 0 rootChildrenNum = len(nodes[-1].children) - 1 while True: curLen = len(queue) #layerCnt.append( curLen ) if curLen == 0: break nextQueue = [] for (nodeidx, info) in queue: curLayer = layers[nodeidx] curNode = nodes[nodeidx] conLayer = Lay.layer('Convolve_' + curLayer.name, \ Bconv, numCon) conLayer.act = 'convolution' layers.append(conLayer) # add root connection rootCon = Con.connection(curLayer, conLayer, numFea, numCon, Wconv_root) # add sibling connections for sib in curNode.siblings: sibNode = nodes[sib] sibLayer = layers[sib] sib_childrenNum = len(sibNode.children) if sib_childrenNum == 0: sib_childrenNum = 1 sib_Weight = 1.0 * sib_childrenNum / len(curNode.siblings) sibCon = Con.connection(sibLayer, conLayer, \ numFea, numCon, Wconv_sib, sib_Weight) childNum = len(curNode.children) #print curLayer.name, info # pooling if layerCnt < poolCutoff: poolCon = Con.PoolConnection(conLayer, poolTop) else: # TODO if layerCnt >= poolCutoff if info == 'l' or info == 'lr': poolCon = Con.PoolConnection(conLayer, poolLeft) if info == 'r' or info == 'lr': poolCon = Con.PoolConnection(conLayer, poolRight) # for each child of the current node for child in curNode.children: childNode = nodes[child] childLayer = layers[child] if layerCnt != 0 and info != 'u': childinfo = info else: rootChildrenNum = len(curNode.children) - 1 if rootChildrenNum == 0: childinfo = 'u' elif childNode.pos <= rootChildrenNum / 2.0: childinfo = 'l' else: # childNode.pos > rootChildrenNum/2.0: childinfo = 'r' #else: # childinfo = 'lr' nextQueue.append((child, childinfo)) # add to child if childNum == 1: leftWeight = .5 rightWeight = .5 else: rightWeight = childNode.pos / (childNum - 1.0) leftWeight = 1 - rightWeight if leftWeight != 0: leftCon = Con.connection(childLayer, conLayer,\ numFea, numCon, Wconv_left, leftWeight) if rightWeight != 0: rightCon = Con.connection(childLayer, conLayer,\ numFea, numCon, Wconv_right, rightWeight) # end of each child of the current node queue = nextQueue layerCnt += 1 # end of current layer layers.append(poolTop) layers.append(poolLeft) layers.append(poolRight) # reorder # layers = |---leaf---|---non_leaf(ae)---| => (numNodes) # |---non_leaf(original)---|---non_leaf(comb)---| => (2 * numNonLeaf) # |------------convolution----------| for idx in xrange(numLeaf, numLeaf + numNonLeaf): tmp = layers[idx] layers[idx] = layers[idx + 2 * numNonLeaf] layers[idx + 2 * numNonLeaf] = tmp # discrimitive layer # layers = |---leaf---|---non_leaf(ae)---| => (numNodes) # |---non_leaf(original)---|---non_leaf(comb)---| => (2 * numNonLeaf) # |------------convolution----------| # |---3 pools---| ## POOOOOOOOOOOL # # pool all # poolLayer = Lay.PoolLayer('pool', numCon) # for idx in xrange( numNodes + 2* numNonLeaf, len(layers) ): # poolCon = Con.PoolConnection(layers[idx], poolLayer) # poolLayer.connectDown.append(poolCon) # layers[idx].connectUp.append(poolCon) # layers.append(poolLayer) # discriminative layer # layers = |---leaf---|---non_leaf(ae)---| => (numNodes) # |---non_leaf(original)---|---non_leaf(comb)---| => (2 * numNonLeaf) # |------------convolution----------| # |---3 pools---| # |---discriminative layer-----| # |--output--| numPool = 3 lenlayer = len(layers) conbegin = lenlayer - numPool discriminative = Lay.layer('discriminative', Bdis, numDis) discriminative.act = 'hidden' output = Lay.layer('outputlayer', Boutput, numOut) output.act = 'softmax' #One Weight Size ows = numDis * numCon for idx in xrange(numPool): poollayer = layers[idx + conbegin] con = Con.connection(poollayer, discriminative, numCon, numDis, Wdis[idx * ows:(idx * ows + ows)]) outcon = Con.connection(discriminative, output, numDis, numOut, Woutput) if numOut > 1: output._activate = Activation.softmax output._activatePrime = None layers.append(discriminative) layers.append(output) # add successive connections numlayers = len(layers) for idx in xrange(numlayers): if idx > 0: layers[idx].successiveLower = layers[idx - 1] if idx < numlayers - 1: layers[idx].successiveUpper = layers[idx + 1] return layers
def transition_block(self, conv_in): self.append_layer(Layers.BatchNormalizationLayer()) self.append_layer(Layers.ActivationLayer('relu')) self.append_layer(Layers.ConvLayer(conv_in, 1, 1, use_bias_in=False)) self.append_layer( Layers.PoolLayer(2, 2, type_in='average', concatenate=True))
def __init__(self, input, input_shape, rng=None, Ws=None, bs=None): assert( (rng is None and Ws is None and bs is None) or (rng is not None and Ws is None and bs is None) or (rng is None and Ws is not None and bs is not None) ) if Ws is None and bs is None: Ws = [None] * 5 bs = [None] * 5 self.trainable_layers = [] ############################## conv1 ############################## self.layer_conv1 = Layers.ConvLayer( input=input, input_shape=input_shape, filter_shape=(96, 3, 11, 11), stride=(4, 4), border_mode='valid', activation=T.nnet.relu, rng=rng, W=Ws[0], b=bs[0] ) conv1_output_shape = (None, 96, 55, 55) # (227 - 11) / 4 + 1 = 55 self.trainable_layers.append(self.layer_conv1) self.layer_pool1 = Layers.PoolLayer( input=self.layer_conv1.output, kernel_size=(3, 3), stride=(2, 2) ) pool1_output_shape = (None, 96, 27, 27) # (55 - 3) / 2 + 1 = 27 self.layer_norm1 = normalization.LocalResponseNormalization2DLayer( incoming=pool1_output_shape, # Actual input is param to .get_output_for() # alpha=0.0001, alpha=0.0001/5, # Caffe documentation uses alpha/n as coeff of summation k=1, # Not specified; Caffe says default is 1, but AlexNet paper says 2 beta=0.75, n=5 ) self.layer_norm1.output = self.layer_norm1.get_output_for(self.layer_pool1.output) self.layer_norm1.output_group1 = self.layer_norm1.output[:, :48, :, :] self.layer_norm1.output_group2 = self.layer_norm1.output[:, 48:, :, :] norm1_group1_output_shape = (None, 48, 27, 27) norm1_group2_output_shape = (None, 48, 27, 27) ############################## conv2 ############################## self.layer_conv2_group1 = Layers.ConvLayer( input=self.layer_norm1.output_group1, input_shape=norm1_group1_output_shape, filter_shape=(128, 48, 5, 5), stride=(1, 1), border_mode=(2, 2), # pad by 2 pixels to each edge, so height and width += 4 activation=T.nnet.relu, rng=rng, W=Ws[1][:128] if Ws[1] is not None else None, b=bs[1][:128] if bs[1] is not None else None ) conv2_group1_output_shape = (None, 128, 27, 27) # (27 + 4 - 5) / 1 + 1 = 27 self.trainable_layers.append(self.layer_conv2_group1) self.layer_conv2_group2 = Layers.ConvLayer( input=self.layer_norm1.output_group2, input_shape=norm1_group2_output_shape, filter_shape=(128, 48, 5, 5), stride=(1, 1), border_mode=(2, 2), # pad by 2 pixels to each edge, so height and width += 4 activation=T.nnet.relu, rng=rng, W=Ws[1][128:] if Ws[1] is not None else None, b=bs[1][128:] if bs[1] is not None else None ) conv2_group2_output_shape = (None, 128, 27, 27) # (27 + 4 - 5) / 1 + 1 = 27 self.trainable_layers.append(self.layer_conv2_group2) self.layer_pool2_group1 = Layers.PoolLayer( input=self.layer_conv2_group1.output, kernel_size=(3, 3), stride=(2, 2) ) pool2_group1_output_shape = (None, 128, 13, 13) # (27 - 3) / 2 + 1 = 13 self.layer_pool2_group2 = Layers.PoolLayer( input=self.layer_conv2_group2.output, kernel_size=(3, 3), stride=(2, 2) ) pool2_group2_output_shape = (None, 128, 13, 13) # (27 - 3) / 2 + 1 = 13 self.layer_norm2_group1 = normalization.LocalResponseNormalization2DLayer( incoming=pool2_group1_output_shape, # Actual input is param to .get_output_for() # alpha=0.0001, alpha=0.0001/5, # Caffe documentation uses alpha/n as coeff of summation k=1, # Not specified; Caffe says default is 1 beta=0.75, n=5 ) self.layer_norm2_group1.output = self.layer_norm2_group1.get_output_for(self.layer_pool2_group1.output) norm2_group1_output_shape = pool2_group1_output_shape self.layer_norm2_group2 = normalization.LocalResponseNormalization2DLayer( incoming=pool2_group2_output_shape, # Actual input is param to .get_output_for() # alpha=0.0001, alpha=0.0001/5, # Caffe documentation uses alpha/n as coeff of summation k=1, # Not specified; Caffe says default is 1 beta=0.75, n=5 ) self.layer_norm2_group2.output = self.layer_norm2_group2.get_output_for(self.layer_pool2_group2.output) norm2_group2_output_shape = pool2_group2_output_shape ############################## conv3 ############################## norm2_grouped_output = T.concatenate((self.layer_norm2_group1.output, self.layer_norm2_group2.output), axis=1) norm2_grouped_output_shape = (None, 256, 13, 13) self.layer_conv3 = Layers.ConvLayer( input=norm2_grouped_output, input_shape=norm2_grouped_output_shape, filter_shape=(384, 256, 3, 3), stride=(1, 1), border_mode=(1, 1), activation=T.nnet.relu, rng=rng, W=Ws[2], b=bs[2] ) conv3_output_shape = (None, 384, 13, 13) # (13 + 2 - 3) / 1 + 1 = 13 self.trainable_layers.append(self.layer_conv3) self.layer_conv3.output_group1 = self.layer_conv3.output[:, :192, :, :] self.layer_conv3.output_group2 = self.layer_conv3.output[:, 192:, :, :] conv3_group1_output_shape = (None, 192, 13, 13) conv3_group2_output_shape = (None, 192, 13, 13) ############################## conv4 ############################## self.layer_conv4_group1 = Layers.ConvLayer( input=self.layer_conv3.output_group1, input_shape=conv3_group1_output_shape, filter_shape=(192, 192, 3, 3), stride=(1, 1), border_mode=(1, 1), activation=T.nnet.relu, rng=rng, W=Ws[3][:192] if Ws[3] is not None else None, b=bs[3][:192] if bs[3] is not None else None ) conv4_group1_output_shape = (None, 192, 13, 13) # (13 + 2 - 3) / 1 + 1 = 13 self.trainable_layers.append(self.layer_conv4_group1) self.layer_conv4_group2 = Layers.ConvLayer( input=self.layer_conv3.output_group2, input_shape=conv3_group2_output_shape, filter_shape=(192, 192, 3, 3), stride=(1, 1), border_mode=(1, 1), activation=T.nnet.relu, rng=rng, W=Ws[3][192:] if Ws[3] is not None else None, b=bs[3][192:] if bs[3] is not None else None ) conv4_group2_output_shape = (None, 192, 13, 13) # (13 + 2 - 3) / 1 + 1 = 13 self.trainable_layers.append(self.layer_conv4_group2) ############################## conv5 ############################## self.layer_conv5_group1 = Layers.ConvLayer( input=self.layer_conv4_group1.output, input_shape=conv4_group1_output_shape, filter_shape=(128, 192, 3, 3), stride=(1, 1), border_mode=(1, 1), activation=T.nnet.relu, rng=rng, W=Ws[4][:128] if Ws[4] is not None else None, b=bs[4][:128] if bs[4] is not None else None ) conv5_group1_output_shape = (None, 128, 13, 13) # (13 + 2 - 3) / 1 + 1 = 13 self.trainable_layers.append(self.layer_conv5_group1) self.layer_pool5_group1 = Layers.PoolLayer( input=self.layer_conv5_group1.output, kernel_size=(3, 3), stride=(2, 2) ) pool5_group1_output_shape = (None, 128, 6, 6) # (13 - 3) / 2 + 1 = 6 self.layer_conv5_group2 = Layers.ConvLayer( input=self.layer_conv4_group2.output, input_shape=conv4_group2_output_shape, filter_shape=(128, 192, 3, 3), stride=(1, 1), border_mode=(1, 1), activation=T.nnet.relu, rng=rng, W=Ws[4][128:] if Ws[4] is not None else None, b=bs[4][128:] if bs[4] is not None else None ) conv5_group2_output_shape = (None, 128, 13, 13) # (13 + 2 - 3) / 1 + 1 = 13 self.trainable_layers.append(self.layer_conv5_group2) self.layer_pool5_group2 = Layers.PoolLayer( input=self.layer_conv5_group2.output, kernel_size=(3, 3), stride=(2, 2) ) pool5_group2_output_shape = (None, 128, 6, 6) # (13 - 3) / 2 + 1 = 6 pool5_grouped_output = T.concatenate((self.layer_pool5_group1.output, self.layer_pool5_group2.output), axis=1) self.output = pool5_grouped_output self.trainable_params = [param for layer in self.trainable_layers for param in layer.params]