def __init__(self, key, prev_keys, hidden_units, choose_from_k, cnn_keep_prob, lstm_keep_prob, att_keep_prob, att_mask): super(Layer, self).__init__(key) def conv_shortcut(kernel_size): return ConvBN(kernel_size, hidden_units, hidden_units, cnn_keep_prob, False, True) self.n_candidates = len(prev_keys) if self.n_candidates: self.prec = mutables.InputChoice( choose_from=prev_keys[-choose_from_k:], n_chosen=1) else: # first layer, skip input choice self.prec = None self.op = mutables.LayerChoice([ conv_shortcut(1), conv_shortcut(3), conv_shortcut(5), conv_shortcut(7), AvgPool(3, False, True), MaxPool(3, False, True), RNN(hidden_units, lstm_keep_prob), Attention(hidden_units, 4, att_keep_prob, att_mask) ]) if self.n_candidates: self.skipconnect = mutables.InputChoice(choose_from=prev_keys) else: self.skipconnect = None self.bn = BatchNorm(hidden_units, False, True)
def apply_shortcut(self, prev_inp, ch_in, ch_out, phase_train=None, w=None, bn=None, stride=None): if self.shortcut == 'projection': if self.dilation: prev_inp = DilatedConv2D(w, rate=stride)(prev_inp) else: prev_inp = Conv2D(w, stride=stride)(prev_inp) prev_inp = bn({'input': prev_inp, 'phase_train': phase_train}) elif self.shortcut == 'identity': pad_ch = ch_out - ch_in if pad_ch < 0: raise Exception('Must use projection when ch_in > ch_out.') prev_inp = tf.pad(prev_inp, [[0, 0], [0, 0], [0, 0], [0, pad_ch]]) if stride > 1: prev_inp = AvgPool(stride)(prev_inp) raise Exception('DEBUG Unknown') self.log.info('After proj shape: {}'.format( prev_inp.get_shape())) return prev_inp
def build(self, inp): self.lazy_init_var() x = inp['input'] phase_train = inp['phase_train'] prev_inp = x with tf.variable_scope(self.scope): for ii in xrange(self.num_stage): self.log.info( 'Stage count {:d} of {:d}'.format(ii, self.num_stage)) ch_in = self.channels[ii] ch_out = self.channels[ii + 1] s = self.strides[ii] with tf.variable_scope('stage_{}'.format(ii)): for jj in xrange(self.layers[ii]): self.log.info('Layer count {:d} of {:d}'.format( jj, self.layers[ii])) h = prev_inp if jj > 0: ch_in = ch_out else: # First unit of the layer. self.log.info( 'In {:d} Out {:d}'.format(ch_in, ch_out)) if self.compatible: # In compatible mode, always project. with tf.variable_scope('shortcut'): prev_inp = self.apply_shortcut( prev_inp, ch_in, ch_out, phase_train=phase_train, w=self.shortcut_w[ii], bn=self.shortcut_bn[ii], stride=s) else: if ch_in != ch_out: with tf.variable_scope('shortcut'): prev_inp = self.apply_shortcut( prev_inp, ch_in, ch_out, phase_train=phase_train, w=self.shortcut_w[ii], bn=self.shortcut_bn[ii], stride=s) elif s != 1: if not self.dilation: prev_inp = AvgPool(s)(prev_inp) self.log.info('After pool shape: {}'.format( prev_inp.get_shape())) with tf.variable_scope('layer_{}'.format(jj)): if self.compatible: # A compatible graph for building weights # older version of ResNet if self.bottleneck: with tf.variable_scope('unit_0'): f_, ch_in_, ch_out_ = \ self.compute_in_out( 0, ch_in, ch_out) h = Conv2D(self.w[ii][jj][0], stride=s)(h) h = self.bn[ii][jj][0]( {'input': h, 'phase_train': phase_train}) h = tf.nn.relu(h) with tf.variable_scope('unit_1'): f_, ch_in_, ch_out_ = \ self.compute_in_out( 1, ch_in, ch_out) h = Conv2D(self.w[ii][jj][1])(h) h = self.bn[ii][jj][1]( {'input': h, 'phase_train': phase_train}) h = tf.nn.relu(h) with tf.variable_scope('unit_2'): f_, ch_in_, ch_out_ = \ self.compute_in_out( 2, ch_in, ch_out) h = Conv2D(self.w[ii][jj][2])(h) h = self.bn[ii][jj][2]( {'input': h, 'phase_train': phase_train}) else: with tf.variable_scope('unit_0'): f_, ch_in_, ch_out_ = \ self.compute_in_out( 0, ch_in, ch_out) h = Conv2D(self.w[ii][jj][0], stride=s)(h) h = self.bn[ii][jj][0]( {'input': h, 'phase_train': phase_train}) h = tf.nn.relu(h) with tf.variable_scope('unit_1'): f_, ch_in_, ch_out_ = \ self.compute_in_out( 1, ch_in, ch_out) h = Conv2D(self.w[ii][jj][1])(h) h = self.bn[ii][jj][1]( {'input': h, 'phase_train': phase_train}) s = 1 else: # New version of ResNet # Full pre-activation for kk in xrange(self.unit_depth): with tf.variable_scope('unit_{}'.format(kk)): f_, ch_in_, ch_out_ = self.compute_in_out( kk, ch_in, ch_out) h = self.bn[ii][jj][kk]( {'input': h, 'phase_train': phase_train}) h = tf.nn.relu(h) if self.dilation: h = DilatedConv2D( self.w[ii][jj][kk], rate=s)(h) else: h = Conv2D(self.w[ii][jj][kk], stride=s)(h) self.log.info('Unit {} shape: {}'.format( kk, h.get_shape())) pass if not self.dilation and kk == 0: s = 1 pass if self.compatible: # Old version # Relu after add prev_inp = tf.nn.relu(prev_inp + h) # self.register_var( # 'stage_{}/layer_{}/relu'.format(ii, jj), prev_inp) else: # New version # Pure linear prev_inp = prev_inp + h self.log.info('After add shape: {}'.format( prev_inp.get_shape())) pass pass pass pass return prev_inp
print w1 b1 = tf.Variable(tf.truncated_normal_initializer( stddev=0.01)([64]), name='b1') h1 = Conv2D(w1, stride=2)(x) + b1 bn1 = BatchNorm(64) h1 = bn1({'input': h1, 'phase_train': phase_train}) h1 = MaxPool(2)(h1) hn = ResNet(layers=[3, 4, 6, 3], bottleneck=True, shortcut='projection', channels=[64, 64, 128, 256, 512], strides=[1, 2, 2, 2])( {'input': h1, 'phase_train': phase_train}) print hn.get_shape() hn = AvgPool(7)(hn) print hn.get_shape() hn = ResNet(layers=[3, 4, 6, 3], bottleneck=False, shortcut='projection', channels=[64, 64, 128, 256, 512], strides=[1, 2, 2, 2])( {'input': h1, 'phase_train': phase_train}) print hn.get_shape() hn = AvgPool(7)(hn) print hn.get_shape() hn = ResNet(layers=[3, 4, 6, 3], bottleneck=False, dilation=True,