def build_model(data_tensor, reuse, training, output_shape): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[0] with tf.variable_scope('cnn', reuse=reuse): # Add input in_emb = conv.input_layer( X=data_tensor, reuse=reuse, training=training, features=20, conv_activation=[tf.nn.relu, None], conv_kernel_size=7, pool_kernel_size=[1, 2, 2, 1], pool_kernel_strides=[1, 2, 2, 1], pool=True, name='l0') layer_hgru = hgru.hGRU( 'fgru', x_shape=in_emb.get_shape().as_list(), timesteps=1, h_ext=[{'h1': [15, 15]}, {'h2': [1, 1]}, {'fb1': [1, 1]}], strides=[1, 1, 1, 1], hgru_ids=[{'h1': 20}, {'h2': 128}, {'fb1': 20}], hgru_idx=[{'h1': 0}, {'h2': 1}, {'fb1': 2}], padding='SAME', aux={ 'readout': 'fb', 'intermediate_ff': [32, 128], 'intermediate_ks': [[3, 3], [3, 3]], 'intermediate_repeats': [3, 3], 'while_loop': False, 'skip': False, 'force_horizontal': True, 'symmetric_weights': True, 'include_pooling': True }, pool_strides=[2, 2], pooling_kernel=[2, 2], train=training) h2 = layer_hgru.build(in_emb) h2 = normalization.batch( bottom=h2, renorm=False, name='hgru_bn', training=training) activity = conv.readout_layer( activity=h2, reuse=reuse, training=training, output_shape=output_shape) extra_activities = { 'activity': h2 } return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[0] with tf.variable_scope('cnn', reuse=reuse): # Add input in_emb = conv.input_layer( X=data_tensor, reuse=reuse, training=training, features=20, conv_activation=[tf.nn.relu, None], conv_kernel_size=7, pool_kernel_size=[1, 2, 2, 1], pool_kernel_strides=[1, 2, 2, 1], pool=True, name='l0') layer_hgru = hgru.hGRU( 'hgru_1', x_shape=in_emb.get_shape().as_list(), timesteps=8, h_ext=15, strides=[1, 1, 1, 1], padding='SAME', aux={'reuse': False, 'constrain': False}, train=training) h2 = layer_hgru.build(in_emb) h2 = normalization.batch( bottom=h2, renorm=False, name='hgru_bn', training=training) activity = conv.readout_layer( activity=h2, reuse=reuse, training=training, output_shape=output_shape) extra_activities = { 'activity': h2 } return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[0] elif isinstance(output_shape, dict): output_shape = output_shape['output'] with tf.variable_scope('cnn', reuse=reuse): # # Concatenate standard deviation # _, var = tf.nn.moments(data_tensor, axes=[3]) # std = tf.expand_dims(tf.sqrt(var), axis=-1) # data_tensor = tf.concat([data_tensor, std], axis=-1) # Add input in_emb = conv.input_layer(X=data_tensor, reuse=reuse, training=training, features=[12, 12], conv_activation=[tf.nn.relu, tf.nn.relu], conv_kernel_size=7, pool_kernel_size=[1, 4, 4, 1], pool_kernel_strides=[1, 2, 2, 1], pool=True, name='l0') # Run fGRU hgru_kernels = OrderedDict() hgru_kernels['h1'] = [15, 15] # height/width hgru_kernels['h2'] = [1, 1] hgru_kernels['fb1'] = [1, 1] hgru_features = OrderedDict() hgru_features['h1'] = [12, 12] # Fan-in/fan-out, I and E (match fb1) hgru_features['h2'] = [48, 48] hgru_features['fb1'] = [12, 12] # (match h1) # hgru_features['fb1'] = [24, 12] # (match h1 unless squeeze_fb) intermediate_ff = [24, 48] # Last feature must match h2 intermediate_ks = [[3, 3], [3, 3]] intermediate_repeats = [1, 1] # Repeat each interm this many times layer_hgru = hgru.hGRU( 'fgru', x_shape=in_emb.get_shape().as_list(), timesteps=8, strides=[1, 1, 1, 1], hgru_features=hgru_features, hgru_kernels=hgru_kernels, intermediate_ff=intermediate_ff, intermediate_ks=intermediate_ks, intermediate_repeats=intermediate_repeats, padding='SAME', aux={ 'readout': 'fb', 'squeeze_fb': False, # Compress Inh-hat with a 1x1 conv 'td_gate': True, # Add top-down activity to the in-gate 'attention': 'gala', 'attention_layers': 2, 'td_cell_state': False, 'excite_se': False, # Add S/E in the excitation stage 'residual': True, # intermediate resid connections 'while_loop': False, 'skip': True, 'time_skips': False, 'symmetric_weights': False, 'timestep_output': False, 'include_pooling': True }, pool_strides=[2, 2], pooling_kernel=[2, 2], train=training) h2 = layer_hgru.build(in_emb) if isinstance(h2, list): all_activities = [] for idx, it_h2 in enumerate(h2): it_h2 = normalization.batch(bottom=it_h2, renorm=False, name='hgru_bn_%s' % idx, training=training) activity = conv.readout_layer(activity=it_h2, reuse=reuse, training=training, var_scope='readout_%s' % idx, output_shape=output_shape) all_activities += [activity] h2 = all_activities[-1] activity = all_activities else: h2 = normalization.batch(bottom=h2, renorm=False, name='hgru_bn', training=training) activity = conv.readout_layer(activity=h2, reuse=reuse, training=training, output_shape=output_shape) extra_activities = {'activity': h2} return activity, extra_activities