def build_model(data_tensor, reuse, training, output_shape): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[0] with tf.variable_scope('cnn', reuse=reuse): # Add input so_filters = np.load( os.path.join('weights', 'so_filters.npy')).squeeze().reshape( 11, 11, 3, 8 * 4)[..., 0:32:4] so_filter_tensor = tf.get_variable( name='so_filters', initializer=so_filters, trainable=training) so_bias = tf.get_variable( name='so_bias', initializer=tf.zeros(so_filters.shape[-1]), trainable=training) in_emb = tf.nn.conv2d( input=data_tensor, filter=so_filter_tensor, strides=[1, 1, 1, 1], padding='SAME', name='so') in_emb = tf.nn.bias_add(in_emb, so_bias) in_emb = in_emb ** 2 layer_hgru = hgru.hGRU( 'hgru_1', x_shape=in_emb.get_shape().as_list(), timesteps=12, h_ext=1, crf=15, strides=[1, 1, 1, 1], padding='SAME', aux={ 'reuse': False, 'constrain': False, 'nonnegative': True, 'while_loop': False, 'hidden_states': 'gru', 'horizontal_dilations': [1, 1, 1, 1] }, train=training) h2 = layer_hgru.build(in_emb) h2 = normalization.batch( bottom=h2, renorm=False, name='hgru_bn', training=training) activity = conv.readout_layer( activity=h2, reuse=reuse, training=training, pool_type='max', # 'select', output_shape=output_shape, features=so_filters.shape[-1]) extra_activities = { 'activity': h2 } return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[0] with tf.variable_scope('cnn', reuse=reuse): # Add input in_emb = conv.input_layer( X=data_tensor, reuse=reuse, training=training, features=20, conv_activation=[tf.nn.relu, None], conv_kernel_size=7, pool_kernel_size=[1, 2, 2, 1], pool_kernel_strides=[1, 2, 2, 1], pool=True, name='l0') layer_hgru = hgru.hGRU( 'fgru', x_shape=in_emb.get_shape().as_list(), timesteps=1, h_ext=[{'h1': [15, 15]}, {'h2': [1, 1]}, {'fb1': [1, 1]}], strides=[1, 1, 1, 1], hgru_ids=[{'h1': 20}, {'h2': 128}, {'fb1': 20}], hgru_idx=[{'h1': 0}, {'h2': 1}, {'fb1': 2}], padding='SAME', aux={ 'readout': 'fb', 'intermediate_ff': [32, 128], 'intermediate_ks': [[3, 3], [3, 3]], 'intermediate_repeats': [3, 3], 'while_loop': False, 'skip': False, 'force_horizontal': True, 'symmetric_weights': True, 'include_pooling': True }, pool_strides=[2, 2], pooling_kernel=[2, 2], train=training) h2 = layer_hgru.build(in_emb) h2 = normalization.batch( bottom=h2, renorm=False, name='hgru_bn', training=training) activity = conv.readout_layer( activity=h2, reuse=reuse, training=training, output_shape=output_shape) extra_activities = { 'activity': h2 } return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[0] with tf.variable_scope('cnn', reuse=reuse): # Add input so_filters = np.load( '/media/data_cifs/clicktionary/pretrained_weights/alexnet.npy' ).item() so_filter_tensor = tf.get_variable(name='so_filters', initializer=so_filters['conv1'][0], trainable=training) so_bias = tf.get_variable(name='so_bias', initializer=so_filters['conv1'][1], trainable=training) in_emb = tf.nn.conv2d(input=data_tensor, filter=so_filter_tensor, strides=[1, 1, 1, 1], padding='SAME', name='so') in_emb = tf.nn.bias_add(in_emb, so_bias) # in_emb = in_emb ** 2 layer_hgru = hgru.hGRU('hgru_1', x_shape=in_emb.get_shape().as_list(), timesteps=8, h_ext=11, strides=[1, 1, 1, 1], padding='SAME', aux={ 'reuse': False, 'constrain': False, 'nonnegative': True, 'while_loop': False, 'horizontal_dilations': [1, 2, 2, 1] }, train=training) h2 = layer_hgru.build(in_emb) h2 = normalization.batch(bottom=h2, renorm=False, name='hgru_bn', training=training) activity = conv.readout_layer( activity=h2, reuse=reuse, training=training, pool_type='max', # 'select', output_shape=output_shape, features=output_shape) extra_activities = {'activity': h2} return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[0] alpha = 5. beta = 1. bs = data_tensor.get_shape().as_list()[0] dist = tf.distributions.Beta(alpha, beta) rand = dist.sample([]) sel = tf.cast(tf.round(tf.random_uniform([], minval=1, maxval=bs - 1)), tf.int32) with tf.variable_scope('cnn', reuse=reuse): with tf.variable_scope('input', reuse=reuse): o_x = tf.layers.conv2d(inputs=data_tensor, filters=24, kernel_size=11, name='l0', strides=(1, 1), padding='same', activation=tf.nn.relu, trainable=training, use_bias=True) x = rand * o_x + ((1 - rand) * tf.roll(o_x, sel, 0)) layer_hgru = hgru.hGRU('hgru_1', x_shape=x.get_shape().as_list(), timesteps=8, h_ext=15, strides=[1, 1, 1, 1], padding='SAME', aux={ 'reuse': False, 'constrain': False }, train=training) h2, inh = layer_hgru.build(x) with tf.variable_scope('ro', reuse=reuse): h2 = normalization.batch(bottom=h2, renorm=False, name='hgru_bn', training=training) activity = conv.readout_layer(activity=h2, reuse=reuse, training=training, output_shape=output_shape) extra_activities = {'activity': h2} return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[0] reduction_filters = 16 with tf.variable_scope('cnn', reuse=reuse): # Add input so_filters = np.load(os.path.join('weights', 'so_filters.npy')).squeeze().reshape( 11, 11, 3, 8 * 4) in_emb = tf.nn.conv2d(input=data_tensor, filter=so_filters, strides=[1, 1, 1, 1], padding='SAME', name='so') in_emb = tf.layers.conv2d(inputs=in_emb, filters=reduction_filters, kernel_size=(1, 1)) in_emb = in_emb**2 layer_hgru = hgru.hGRU('hgru_1', x_shape=in_emb.get_shape().as_list(), timesteps=8, h_ext=11, strides=[1, 1, 1, 1], padding='SAME', aux={ 'reuse': False, 'constrain': False, 'nonnegative': True, 'while_loop': False }, train=training) h2 = layer_hgru.build(in_emb) h2 = normalization.batch(bottom=h2, renorm=False, name='hgru_bn', training=training) activity = conv.readout_layer( activity=h2, reuse=reuse, training=training, pool_type='max', # 'select', output_shape=output_shape, features=reduction_filters) extra_activities = {'activity': h2} return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[0] with tf.variable_scope('cnn', reuse=reuse): # Add input in_emb = conv.input_layer( X=data_tensor, reuse=reuse, training=training, features=20, conv_activation=[tf.nn.relu, None], conv_kernel_size=7, pool_kernel_size=[1, 2, 2, 1], pool_kernel_strides=[1, 2, 2, 1], pool=True, name='l0') layer_hgru = hgru.hGRU( 'hgru_1', x_shape=in_emb.get_shape().as_list(), timesteps=8, h_ext=15, strides=[1, 1, 1, 1], padding='SAME', aux={'reuse': False, 'constrain': False}, train=training) h2 = layer_hgru.build(in_emb) h2 = normalization.batch( bottom=h2, renorm=False, name='hgru_bn', training=training) activity = conv.readout_layer( activity=h2, reuse=reuse, training=training, output_shape=output_shape) extra_activities = { 'activity': h2 } return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[0] with tf.variable_scope('cnn', reuse=reuse): # Add input conv_aux = { 'pretrained': os.path.join('weights', 'gabors_for_contours_11.npy'), 'pretrained_key': 's1', 'nonlinearity': 'square' } activity = conv.conv_layer(bottom=data_tensor, name='gabor_input', stride=[1, 1, 1, 1], padding='SAME', trainable=training, use_bias=True, aux=conv_aux) layer_hgru = hgru.hGRU('hgru_1', x_shape=activity.get_shape().as_list(), timesteps=8, h_ext=15, strides=[1, 1, 1, 1], padding='SAME', aux={ 'reuse': False, 'constrain': False }, train=training) h2 = layer_hgru.build(activity) h2 = normalization.batch(bottom=h2, renorm=False, name='hgru_bn', training=training) activity = conv.readout_layer(activity=h2, reuse=reuse, training=training, output_shape=output_shape) extra_activities = {'activity': h2} return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[0] elif isinstance(output_shape, dict): output_shape = output_shape['output'] with tf.variable_scope('cnn', reuse=reuse): # # Concatenate standard deviation # _, var = tf.nn.moments(data_tensor, axes=[3]) # std = tf.expand_dims(tf.sqrt(var), axis=-1) # data_tensor = tf.concat([data_tensor, std], axis=-1) # Add input in_emb = conv.input_layer(X=data_tensor, reuse=reuse, training=training, features=[12, 12], conv_activation=[tf.nn.relu, tf.nn.relu], conv_kernel_size=7, pool_kernel_size=[1, 4, 4, 1], pool_kernel_strides=[1, 2, 2, 1], pool=True, name='l0') # Run fGRU hgru_kernels = OrderedDict() hgru_kernels['h1'] = [15, 15] # height/width hgru_kernels['h2'] = [1, 1] hgru_kernels['fb1'] = [1, 1] hgru_features = OrderedDict() hgru_features['h1'] = [12, 12] # Fan-in/fan-out, I and E (match fb1) hgru_features['h2'] = [48, 48] hgru_features['fb1'] = [12, 12] # (match h1) # hgru_features['fb1'] = [24, 12] # (match h1 unless squeeze_fb) intermediate_ff = [24, 48] # Last feature must match h2 intermediate_ks = [[3, 3], [3, 3]] intermediate_repeats = [1, 1] # Repeat each interm this many times layer_hgru = hgru.hGRU( 'fgru', x_shape=in_emb.get_shape().as_list(), timesteps=8, strides=[1, 1, 1, 1], hgru_features=hgru_features, hgru_kernels=hgru_kernels, intermediate_ff=intermediate_ff, intermediate_ks=intermediate_ks, intermediate_repeats=intermediate_repeats, padding='SAME', aux={ 'readout': 'fb', 'squeeze_fb': False, # Compress Inh-hat with a 1x1 conv 'td_gate': True, # Add top-down activity to the in-gate 'attention': 'gala', 'attention_layers': 2, 'td_cell_state': False, 'excite_se': False, # Add S/E in the excitation stage 'residual': True, # intermediate resid connections 'while_loop': False, 'skip': True, 'time_skips': False, 'symmetric_weights': False, 'timestep_output': False, 'include_pooling': True }, pool_strides=[2, 2], pooling_kernel=[2, 2], train=training) h2 = layer_hgru.build(in_emb) if isinstance(h2, list): all_activities = [] for idx, it_h2 in enumerate(h2): it_h2 = normalization.batch(bottom=it_h2, renorm=False, name='hgru_bn_%s' % idx, training=training) activity = conv.readout_layer(activity=it_h2, reuse=reuse, training=training, var_scope='readout_%s' % idx, output_shape=output_shape) all_activities += [activity] h2 = all_activities[-1] activity = all_activities else: h2 = normalization.batch(bottom=h2, renorm=False, name='hgru_bn', training=training) activity = conv.readout_layer(activity=h2, reuse=reuse, training=training, output_shape=output_shape) extra_activities = {'activity': h2} return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape, data_format='NHWC'): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[-1] elif isinstance(output_shape, dict): output_shape = output_shape['output'] output_shape = 1 # norm_moments_training = training # Force instance norm # normalization_type = 'no_param_batch_norm_original' normalization_type = 'no_param_instance_norm' # output_normalization_type = 'batch_norm_original_renorm' output_normalization_type = 'instance_norm' # data_tensor = tf.concat((data_tensor, data_tensor, data_tensor), -1) # Expand to 3D data_tensor, long_data_format = tf_fun.interpret_data_format( data_tensor=data_tensor, data_format=data_format) # Prepare gammanet structure (compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts) = v2_big_working() gammanet_constructor = tf_fun.get_gammanet_constructor( compression=compression, ff_kernels=ff_kernels, ff_repeats=ff_repeats, features=features, fgru_kernels=fgru_kernels) aux = get_aux() # Build model with tf.variable_scope('vgg', reuse=reuse): aux = get_aux() vgg = vgg16.Vgg16( vgg16_npy_path= '/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy', reuse=reuse, aux=aux, train=False, # training, timesteps=8, fgru_normalization_type=normalization_type, ff_normalization_type=normalization_type) vgg(rgb=data_tensor, constructor=gammanet_constructor) # activity = vgg.fgru_0 with tf.variable_scope('fgru', reuse=reuse): # Get side weights h2_rem = [vgg.fgru_0, vgg.fgru_1, vgg.fgru_2] activities = [] for idx, h in enumerate(h2_rem): # res = normalization.apply_normalization( # activity=h, # name='output_norm1_%s' % idx, # normalization_type=output_normalization_type, # data_format=data_format, # training=False, # trainable=False, # reuse=reuse) activities.append(aux['image_resize']( h, vgg.fgru_0.get_shape().as_list()[1:3], align_corners=True)) activity = tf.concat(activities, -1) # Then read out # activity = normalization.apply_normalization( # activity=res, # name='output_norm1', # normalization_type=output_normalization_type, # data_format=data_format, # training=training, # trainable=training, # reuse=reuse) with tf.variable_scope('readout', reuse=reuse): activity = normalization.batch(bottom=activity, renorm=False, name='hgru_bn', training=training) activity = conv.readout_layer(activity=activity, reuse=reuse, training=training, output_shape=output_shape) if long_data_format is 'channels_first': activity = tf.transpose(activity, (0, 2, 3, 1)) # extra_activities = {'orientations': orientations, 'model_output_y': vgg.fgru_0, 'model_output_x': vgg.fgru_0_init} extra_activities = {} if activity.dtype != tf.float32: activity = tf.cast(activity, tf.float32) # return [activity, h_deep], extra_activities return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[0] with tf.variable_scope('cnn', reuse=reuse): # Unclear if we should include l0 in the down/upsample cascade with tf.variable_scope('g1', reuse=reuse): # Downsample act11 = conv_block(x=data_tensor, name='l1_1', filters=64, training=training, reuse=reuse, pool=False) act12 = conv_block(x=act11, name='l1_2', filters=64, training=training, reuse=reuse, pool=False) poolact12 = pooling.max_pool(bottom=act12, name='l1_2_pool') with tf.variable_scope('g2', reuse=reuse): # Downsample act21 = conv_block(x=poolact12, name='l2_1', filters=128, training=training, reuse=reuse, pool=False) act22 = conv_block(x=act21, filters=128, name='l2_2', training=training, reuse=reuse, pool=False) poolact22 = pooling.max_pool(bottom=act22, name='l2_2_pool') with tf.variable_scope('g3', reuse=reuse): # Downsample act31 = conv_block(x=poolact22, name='l3_1', filters=256, training=training, reuse=reuse, pool=False) act32 = conv_block(x=act31, filters=256, name='l3_2', training=training, reuse=reuse, pool=False) act33 = conv_block(x=act32, filters=256, name='l3_3', training=training, reuse=reuse, pool=False) poolact33 = pooling.max_pool(bottom=act33, name='l3_3_pool') with tf.variable_scope('g4', reuse=reuse): # Downsample act41 = conv_block(x=poolact33, name='l4_1', filters=512, training=training, reuse=reuse, pool=False) act42 = conv_block(x=act41, filters=512, name='l4_2', training=training, reuse=reuse, pool=False) act43 = conv_block(x=act42, filters=512, name='l4_3', training=training, reuse=reuse, pool=False) poolact43 = pooling.max_pool(bottom=act43, name='l4_3_pool') with tf.variable_scope('g5', reuse=reuse): # Downsample act51 = conv_block(x=poolact43, name='l5_1', filters=512, training=training, reuse=reuse, pool=False) act52 = conv_block(x=act51, filters=512, name='l5_2', training=training, reuse=reuse, pool=False) act53 = conv_block(x=act52, filters=512, name='l5_3', training=training, reuse=reuse, pool=False) poolact53 = pooling.max_pool(bottom=act53, name='l5_3_pool') with tf.variable_scope('g5_skip', reuse=reuse): upact5 = up_block(inputs=poolact53, skip=act53, up_filters=512, name='ul5', training=training, reuse=reuse) with tf.variable_scope('g4_skip', reuse=reuse): upact4 = up_block(inputs=upact5, skip=act43, up_filters=512, name='ul4', training=training, reuse=reuse) with tf.variable_scope('g3_skip', reuse=reuse): upact3 = up_block(inputs=upact4, skip=act33, up_filters=256, name='ul3', training=training, reuse=reuse) with tf.variable_scope('g2_skip', reuse=reuse): upact2 = up_block(inputs=upact3, skip=act22, up_filters=128, name='ul2', training=training, reuse=reuse) with tf.variable_scope('g1_skip', reuse=reuse): upact1 = up_block(inputs=upact2, skip=act12, up_filters=64, name='ul1', training=training, reuse=reuse) activity = conv.readout_layer(activity=upact1, reuse=reuse, training=training, output_shape=output_shape) extra_activities = {'activity': activity} return activity, extra_activities