def build_model(data_tensor, reuse, training, output_shape, data_format='NHWC'): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[-1] elif isinstance(output_shape, dict): output_shape = output_shape['output'] output_shape = 2 # norm_moments_training = training # Force instance norm # normalization_type = 'no_param_batch_norm_original' normalization_type = 'no_param_instance_norm' # output_normalization_type = 'batch_norm_original_renorm' output_normalization_type = 'instance_norm' data_tensor, long_data_format = tf_fun.interpret_data_format( data_tensor=data_tensor, data_format=data_format) # Prepare gammanet structure (compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts) = v2_big_working() gammanet_constructor = tf_fun.get_gammanet_constructor( compression=compression, ff_kernels=ff_kernels, ff_repeats=ff_repeats, features=features, fgru_kernels=fgru_kernels) aux = get_aux() # Build model with tf.variable_scope('vgg', reuse=reuse): aux = get_aux() vgg = vgg16.Vgg16( vgg16_npy_path= '/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy', reuse=reuse, aux=aux, train=training, timesteps=8, fgru_normalization_type=normalization_type, ff_normalization_type=normalization_type) vgg(rgb=data_tensor, constructor=gammanet_constructor) activity = vgg.fgru_0 if long_data_format is 'channels_first': activity = tf.transpose(activity, (0, 2, 3, 1)) extra_activities = { "fgru": vgg.fgru_0 } # idx: v for idx, v in enumerate(hs_0)} if activity.dtype != tf.float32: activity = tf.cast(activity, tf.float32) # return [activity, h_deep], extra_activities return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape, data_format='NHWC'): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[-1] elif isinstance(output_shape, dict): output_shape = output_shape['output'] normalization_type = 'no_param_instance_norm' output_normalization_type = 'instance_norm' ff_kernel_size = (3, 3) # ff_nl = tf.nn.relu ff_nl = tf.nn.elu data_tensor, long_data_format = tf_fun.interpret_data_format( data_tensor=data_tensor, data_format=data_format) # Prepare gammanet structure (compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts) = spec() gammanet_constructor = tf_fun.get_gammanet_constructor( compression=compression, ff_kernels=ff_kernels, ff_repeats=ff_repeats, features=features, fgru_kernels=fgru_kernels) aux = get_aux() # Build model with tf.variable_scope('gammanet', reuse=reuse): activity = tf.layers.conv2d( inputs=data_tensor, filters=gammanet_constructor[0]['features'], kernel_size=ff_kernel_size, padding='same', data_format=long_data_format, name='l0_0', activation=ff_nl, use_bias=True, reuse=reuse) activity = tf.layers.conv2d( inputs=activity, filters=gammanet_constructor[0]['features'], kernel_size=ff_kernel_size, padding='same', data_format=long_data_format, name='l0_1', activation=ff_nl, use_bias=True, reuse=reuse) activity = normalization.apply_normalization( activity=activity, name='input_norm', normalization_type=output_normalization_type, data_format=data_format, training=training, trainable=training, reuse=reuse) # Shift conv-layers to the front # Run fGRU gn = gammanet.GN( layer_name='fgru', gammanet_constructor=gammanet_constructor, data_format=data_format, reuse=reuse, timesteps=8, fgru_connectivity='', # 'all_to_all', additional_readouts=additional_readouts, fgru_normalization_type=normalization_type, ff_normalization_type=normalization_type, horizontal_padding='SAME', ff_padding='SAME', ff_nl=ff_nl, recurrent_ff=True, horizontal_kernel_initializer=tf.initializers.orthogonal(), kernel_initializer=tf.initializers.orthogonal(), gate_initializer=tf.initializers.orthogonal(), aux=aux, strides=[1, 1, 1, 1], pool_strides=[2, 2], pool_kernel=[2, 2], up_kernel=[4, 4], train=training) h2, h_deep = gn(X=activity) with tf.variable_scope('cv_readout', reuse=reuse): # Apply one more normalization activity = normalization.apply_normalization( activity=h2, name='output_norm', normalization_type=output_normalization_type, data_format=data_format, training=training, trainable=training, reuse=reuse) activity = tf.layers.conv2d( inputs=activity, filters=output_shape, kernel_size=(5, 5), # (1, 1), padding='same', data_format=long_data_format, name='readout_conv_2', activation=None, use_bias=True, reuse=reuse) if long_data_format is 'channels_first': activity = tf.transpose(activity, (0, 2, 3, 1)) extra_activities = { 'activity_low': h2, 'activity_high': h_deep, } if activity.dtype != tf.float32: activity = tf.cast(activity, tf.float32) return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape, data_format='NHWC'): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[-1] elif isinstance(output_shape, dict): output_shape = output_shape['output'] # norm_moments_training = training # Force instance norm # normalization_type = 'no_param_batch_norm_original' normalization_type = 'no_param_instance_norm' # output_normalization_type = 'batch_norm_original_renorm' output_normalization_type = 'instance_norm' data_tensor, long_data_format = tf_fun.interpret_data_format( data_tensor=data_tensor, data_format=data_format) data_tensor = tf.concat([data_tensor, data_tensor, data_tensor], -1) # Prepare gammanet structure (compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts) = v2_big_working() gammanet_constructor = tf_fun.get_gammanet_constructor( compression=compression, ff_kernels=ff_kernels, ff_repeats=ff_repeats, features=features, fgru_kernels=fgru_kernels) aux = get_aux() # Build model with tf.variable_scope('vgg', reuse=reuse): aux = get_aux() vgg = vgg16.Vgg16( vgg16_npy_path= '/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy', reuse=reuse, aux=aux, train=training, timesteps=3, fgru_normalization_type=normalization_type, ff_normalization_type=normalization_type) vgg(rgb=data_tensor, constructor=gammanet_constructor) # activity = vgg.fgru_0 layers = 4 ds_list = [] bs = data_tensor.get_shape().as_list()[0] for idx in range(layers): inhs = getattr(vgg, 'inh_%s' % idx) xs = getattr(vgg, 'x_%s' % idx) xs = tf.reshape(xs, [bs, -1]) inhs = tf.reshape(inhs, [bs, -1]) # Augmentation # roll_choices = tf.range( # bs - 1, # dtype=tf.int32) # samples = tf.multinomial( # tf.log([tf.ones_like(tf.cast(roll_choices, tf.float32))]), 1) # rand_roll = roll_choices[tf.cast(samples[0][0], tf.int32)] # rolled_xs = tf.roll(xs, rand_roll, 0) # xs = tf.concat([xs, rolled_xs], 0) # inhs = tf.concat([inhs, inhs], 0) rolled_xs = [] for r in range(1, bs - 1): rolled_xs += [tf.roll(xs, r, 0)] xs = tf.concat([xs] + rolled_xs, 0) inhs = tf.tile(inhs, [bs - 1, 1]) # CPC distances # denom_xs = tf.sqrt( # tf.reduce_sum(tf.matmul(xs, xs, transpose_b=True), axis=-1)) # denom_inhs = tf.sqrt( # tf.reduce_sum(tf.matmul(inhs, inhs, transpose_b=True), axis=-1)) # num = tf.reduce_sum(xs * inhs, axis=-1) # ds = num / (denom_xs * denom_inhs) xs = tf.nn.l2_normalize(xs, -1) inhs = tf.nn.l2_normalize(inhs, -1) ds = 1 - tf.losses.cosine_distance( xs, inhs, axis=-1, reduction=tf.losses.Reduction.NONE) ds = tf.reshape(ds, [bs - 1, -1]) # ds = tf.nn.softmax(ds, 0) ds_list += [ds] ds_list = tf.transpose(tf.concat(ds_list, -1)) return ds_list, {}
def build_model(data_tensor, reuse, training, output_shape, data_format='NHWC'): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[-1] elif isinstance(output_shape, dict): output_shape = output_shape['output'] normalization_type = 'no_param_instance_norm' output_normalization_type = 'instance_norm' # normalization_type = 'no_param_batch_norm' # output_normalization_type = 'batch_norm' ff_kernel_size = (3, 3) ff_nl = tf.nn.relu data_tensor, long_data_format = tf_fun.interpret_data_format( data_tensor=data_tensor, data_format=data_format) # Prepare gammanet structure (compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts) = v2_big_working() gammanet_constructor = tf_fun.get_gammanet_constructor( compression=compression, ff_kernels=ff_kernels, ff_repeats=ff_repeats, features=features, fgru_kernels=fgru_kernels) aux = get_aux() # Build model fix_training = False with tf.variable_scope('gammanet', reuse=reuse): activity = tf.layers.conv2d( inputs=data_tensor, filters=gammanet_constructor[0]['features'], kernel_size=ff_kernel_size, padding='same', data_format=long_data_format, name='l0_0', activation=ff_nl, use_bias=True, # trainable=training, trainable=fix_training, reuse=reuse) activity = tf.layers.conv2d( inputs=activity, filters=gammanet_constructor[0]['features'], kernel_size=ff_kernel_size, padding='same', data_format=long_data_format, name='l0_1', activation=None, use_bias=True, # trainable=training, trainable=fix_training, reuse=reuse) activity = normalization.apply_normalization( activity=activity, name='input_norm', normalization_type=output_normalization_type, data_format=data_format, # training=training, training=fix_training, reuse=reuse) # + pre_activity activity = ff_nl(activity) # # Downsample # activity = conv.down_block( # layer_name='l1', # bottom=activity, # kernel_size=3 * [[3, 3]], # num_filters=gammanet_constructor[0]['features'], # normalization_type=output_normalization_type, # training=training, # activation=ff_nl, # include_pool=False, # reuse=reuse) # Shift conv-layers to the front # Run fGRU gn = gammanet.GN( layer_name='fgru', gammanet_constructor=gammanet_constructor, data_format=data_format, reuse=reuse, timesteps=8, fgru_connectivity='', # 'all_to_all', additional_readouts=additional_readouts, fgru_normalization_type=normalization_type, ff_normalization_type=normalization_type, horizontal_padding='SAME', ff_padding='SAME', ff_nl=ff_nl, recurrent_ff=True, horizontal_kernel_initializer=tf.initializers.orthogonal(), kernel_initializer=tf.initializers.orthogonal(), gate_initializer=tf.initializers.orthogonal(), # horizontal_kernel_initializer=tf.contrib.layers.xavier_initializer(), # gate_initializer=tf.contrib.layers.xavier_initializer(), # gate_initializer=tf.contrib.layers.xavier_initializer(), aux=aux, strides=[1, 1, 1, 1], pool_strides=[2, 2], pool_kernel=[2, 2], up_kernel=[4, 4], train=fix_training) h2, h_deep = gn(X=activity) with tf.variable_scope('gratings_readout', reuse=reuse): h2 = normalization.apply_normalization( activity=h2, name='output_norm', normalization_type=output_normalization_type, data_format=data_format, training=training, reuse=reuse) # + pre_activity # Then read out activity = tf.layers.conv2d( inputs=h2, filters=gammanet_constructor[0]['features'], kernel_size=(1, 1), padding='same', data_format=long_data_format, name='readout_l0_1', activation=ff_nl, use_bias=True, trainable=training, reuse=reuse) activity = tf.layers.conv2d(inputs=activity, filters=output_shape, kernel_size=(1, 1), padding='same', data_format=long_data_format, name='readout_l0_2', activation=None, use_bias=True, trainable=training, reuse=reuse) _, h, w, _ = activity.get_shape().as_list() activity = activity[:, h // 2, w // 2, :] if long_data_format is 'channels_first': activity = tf.transpose(activity, (0, 2, 3, 1)) extra_activities = { 'activity_low': h2, 'activity_high': h_deep, } if activity.dtype != tf.float32: activity = tf.cast(activity, tf.float32) # return [activity, h_deep], extra_activities return activity, extra_activities
def build_model( data_tensor, reuse, training, output_shape, data_format='NHWC'): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[-1] elif isinstance(output_shape, dict): output_shape = output_shape['output'] output_shape = 2 # norm_moments_training = training # Force instance norm # normalization_type = 'no_param_batch_norm_original' normalization_type = 'no_param_instance_norm' # output_normalization_type = 'batch_norm_original_renorm' output_normalization_type = 'instance_norm' data_tensor, long_data_format = tf_fun.interpret_data_format( data_tensor=data_tensor, data_format=data_format) # Prepare gammanet structure ( compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts) = v2_big_working() gammanet_constructor = tf_fun.get_gammanet_constructor( compression=compression, ff_kernels=ff_kernels, ff_repeats=ff_repeats, features=features, fgru_kernels=fgru_kernels) aux = get_aux() # Build model with tf.variable_scope('vgg', reuse=reuse): vgg = vgg16.Vgg16( vgg16_npy_path='/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy') vgg(rgb=data_tensor, train=False, ff_reuse=reuse) # activity = vgg.fgru_0 with tf.variable_scope('fgru', reuse=reuse): # Get side weights h2_rem = [ vgg.fgru_0, vgg.fgru_1, vgg.fgru_2, vgg.fgru_3] h2s = [] for idx, h in enumerate(h2_rem): res = aux['image_resize']( h, data_tensor.get_shape().as_list()[1:3], align_corners=True) h2s += [res] res = normalization.apply_normalization( activity=tf.concat(h2s, -1), name='output_norm1', normalization_type=output_normalization_type, data_format=data_format, training=False, trainable=False, reuse=reuse) activity = tf.layers.conv2d( inputs=res, filters=gammanet_constructor[0]['features'], kernel_size=(1, 1), padding='same', data_format=long_data_format, name='readout_l0_1', activation=tf.nn.relu, use_bias=True, trainable=training, reuse=reuse) orientations = tf.layers.conv2d( inputs=activity, filters=output_shape, kernel_size=(1, 1), padding='same', data_format=long_data_format, name='readout_l0_2', activation=None, use_bias=True, trainable=training, reuse=reuse) _, h, w, _ = orientations.get_shape().as_list() activity = orientations[:, h // 2, w // 2, :] if long_data_format is 'channels_first': activity = tf.transpose(activity, (0, 2, 3, 1)) extra_activities = {'orientations': orientations} if activity.dtype != tf.float32: activity = tf.cast(activity, tf.float32) # return [activity, h_deep], extra_activities return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape, data_format='NHWC'): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[-1] elif isinstance(output_shape, dict): output_shape = output_shape['output'] # norm_moments_training = training # Force instance norm # normalization_type = 'no_param_batch_norm_original' normalization_type = 'no_param_instance_norm' # output_normalization_type = 'batch_norm_original_renorm' output_normalization_type = 'instance_norm' data_tensor, long_data_format = tf_fun.interpret_data_format( data_tensor=data_tensor, data_format=data_format) # Prepare gammanet structure (compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts) = v2_big_working() gammanet_constructor = tf_fun.get_gammanet_constructor( compression=compression, ff_kernels=ff_kernels, ff_repeats=ff_repeats, features=features, fgru_kernels=fgru_kernels) aux = get_aux() # Build model with tf.variable_scope('vgg', reuse=reuse): aux = get_aux() vgg = vgg16.Vgg16( vgg16_npy_path= '/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy', reuse=reuse, aux=aux, train=training, timesteps=8, fgru_normalization_type=normalization_type, ff_normalization_type=normalization_type) vgg(rgb=data_tensor, constructor=gammanet_constructor) # activity = vgg.fgru_0 with tf.variable_scope('fgru', reuse=reuse): # Get side weights h2_rem = [vgg.fgru_0] for idx, h in enumerate(h2_rem): res = normalization.apply_normalization( activity=h, name='output_norm1_%s' % idx, normalization_type=output_normalization_type, data_format=data_format, training=training, trainable=training, reuse=reuse) res = aux['image_resize'](res, data_tensor.get_shape().as_list()[1:3], align_corners=True) activity = tf.layers.conv2d(res, filters=res.get_shape().as_list()[-1], kernel_size=(3, 3), padding='same', data_format=long_data_format, name='out', activation=tf.nn.relu, trainable=training, use_bias=True, reuse=reuse) activity = tf.layers.conv2d( activity, filters=output_shape, kernel_size=(1, 1), padding='same', data_format=long_data_format, name='out2', # activation=tf.nn.relu, activation=None, trainable=training, use_bias=True, reuse=reuse) if long_data_format is 'channels_first': activity = tf.transpose(activity, (0, 2, 3, 1)) extra_activities = {} # idx: v for idx, v in enumerate(hs_0)} if activity.dtype != tf.float32: activity = tf.cast(activity, tf.float32) # return [activity, h_deep], extra_activities return activity, extra_activities
def build_model( data_tensor, reuse, training, output_shape, data_format='NHWC'): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[-1] elif isinstance(output_shape, dict): output_shape = output_shape['output'] # output_shape = 2 # norm_moments_training = training # Force instance norm normalization_type = 'no_param_batch_norm_original' # normalization_type = 'ada_batch_norm' # normalization_type = 'no_param_instance_norm' output_normalization_type = 'batch_norm_original_renorm' # output_normalization_type = 'instance_norm' data_tensor, long_data_format = tf_fun.interpret_data_format( data_tensor=data_tensor, data_format=data_format) # Prepare gammanet structure ( compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts) = v2_big_working() gammanet_constructor = tf_fun.get_gammanet_constructor( compression=compression, ff_kernels=ff_kernels, ff_repeats=ff_repeats, features=features, fgru_kernels=fgru_kernels) aux = get_aux() # Build model with tf.variable_scope('vgg', reuse=reuse): aux = get_aux() vgg = vgg16.Vgg16( vgg16_npy_path='/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy', reuse=reuse, aux=aux, train=training, timesteps=8, fgru_normalization_type=normalization_type, ff_normalization_type=normalization_type) vgg(rgb=data_tensor, constructor=gammanet_constructor) # activity = vgg.fgru_0 with tf.variable_scope('fgru', reuse=reuse): # Get side weights activity = normalization.batch( bottom=vgg.fgru_0, # activity, renorm=True, name='readout_0_bn', training=training) # activity = normalization.group( # bottom=vgg.fgru_0) # , # activity, # # name='readout_0_bn') prepool_activity = tf.layers.conv2d( inputs=activity, filters=output_shape, # int(activity.get_shape()[-1]), kernel_size=1, name='pre_readout_conv0', strides=(1, 1), padding='same', activation=tf.nn.elu, trainable=training, use_bias=True) """ prepool_activity = normalization.batch( bottom=prepool_activity, renorm=True, name='readout_1_bn', training=False) # training) """ """ prepool_activity = tf.layers.conv2d( inputs=prepool_activity, filters=output_shape, kernel_size=1, name='pre_readout_conv1', strides=(1, 1), padding='same', activation=None, trainable=training, use_bias=True) """ out_activity = tf.reduce_max(prepool_activity, reduction_indices=[1, 2]) activity = tf.layers.dense( inputs=out_activity, units=output_shape) if long_data_format is 'channels_first': activity = tf.transpose(activity, (0, 2, 3, 1)) # extra_activities = {'orientations': orientations, 'model_output_y': vgg.fgru_0, 'model_output_x': vgg.fgru_0_init} extra_activities = {'x': vgg.fgru_0} if activity.dtype != tf.float32: activity = tf.cast(activity, tf.float32) # return [activity, h_deep], extra_activities return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape, data_format='NHWC'): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[-1] elif isinstance(output_shape, dict): output_shape = output_shape['output'] data_tensor, long_data_format = tf_fun.interpret_data_format( data_tensor=data_tensor, data_format=data_format) normalization_type = 'instance_norm' ff_nl = tf.nn.relu # Prepare gammanet structure compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts = v1( ) # compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts = v3() gammanet_constructor = tf_fun.get_gammanet_constructor( compression=compression, ff_kernels=ff_kernels, ff_repeats=ff_repeats, features=features, fgru_kernels=fgru_kernels) aux = get_aux() # Build model with tf.variable_scope('gammanet', reuse=reuse): in_emb = tf.layers.conv2d(inputs=data_tensor, filters=16, kernel_size=7, strides=(1, 1), padding='same', data_format=long_data_format, activation=ff_nl, trainable=training, use_bias=True, name='l0') in_emb = tf.layers.conv2d( inputs=in_emb, filters=16, kernel_size=7, strides=(1, 1), padding='same', data_format=long_data_format, activation=None, # ff_nl, trainable=training, use_bias=True, name='l01') in_emb = normalization.apply_normalization( activity=in_emb, name='in_norm', normalization_type=normalization_type, data_format=data_format, training=training, reuse=reuse) in_emb = ff_nl(in_emb) # Run fGRU gn = gammanet.GN( layer_name='fgru', gammanet_constructor=gammanet_constructor, data_format=data_format, reuse=reuse, timesteps=8, fgru_connectivity='', # 'all_to_all', additional_readouts=additional_readouts, fgru_normalization_type=normalization_type, ff_normalization_type=normalization_type, horizontal_padding='SAME', ff_padding='SAME', ff_nl=ff_nl, recurrent_ff=False, horizontal_kernel_initializer=tf.initializers.orthogonal(), # horizontal_kernel_initializer=tf_fun.Identity(), kernel_initializer=tf.initializers.orthogonal(), aux=aux, strides=[1, 1, 1, 1], pool_strides=[2, 2], pooling_kernel=[2, 2], up_kernel=[4, 4], train=training) h2, h_deep = gn(X=in_emb) with tf.variable_scope('cv_readout', reuse=reuse): # Apply one more normalization h2 = normalization.apply_normalization( activity=h2, name='output_norm', normalization_type=normalization_type, data_format=data_format, training=training, reuse=reuse) # h_deep = apply_normalization( # activity=h_deep, # name='output_normh_deep', # normalization_type=normalization_type, # data_format=data_format, # training=training, # reuse=reuse) # Then read out activity = tf.layers.conv2d(inputs=h2, filters=output_shape, kernel_size=(1, 1), padding='same', data_format=long_data_format, name='pre_readout_conv', use_bias=True, reuse=reuse) # h_deep = tf.layers.conv2d( # inputs=h_deep, # filters=output_shape, # kernel_size=(1, 1), # padding='same', # data_format=long_data_format, # name='pre_readout_conv1', # use_bias=True, # reuse=reuse) if long_data_format is 'channels_first': activity = tf.transpose(activity, (0, 2, 3, 1)) extra_activities = { 'activity_low': h2, 'activity_high': h_deep, } if activity.dtype != tf.float32: activity = tf.cast(activity, tf.float32) # return [activity, h_deep], extra_activities return activity, extra_activities
def build_model( data_tensor, reuse, training, output_shape, data_format='NHWC'): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[-1] elif isinstance(output_shape, dict): output_shape = output_shape['output'] # norm_moments_training = training # Force instance norm # normalization_type = 'no_param_batch_norm_original' normalization_type = 'no_param_instance_norm' # output_normalization_type = 'batch_norm_original_renorm' output_normalization_type = 'instance_norm' data_tensor, long_data_format = tf_fun.interpret_data_format( data_tensor=data_tensor, data_format=data_format) # Prepare gammanet structure ( compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts) = v2_big_working() gammanet_constructor = tf_fun.get_gammanet_constructor( compression=compression, ff_kernels=ff_kernels, ff_repeats=ff_repeats, features=features, fgru_kernels=fgru_kernels) aux = get_aux() # Build model with tf.variable_scope('vgg', reuse=reuse): aux = get_aux() vgg = vgg16.Vgg16( vgg16_npy_path='/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy', reuse=reuse, aux=aux, train=training, timesteps=8, fgru_normalization_type=normalization_type, ff_normalization_type=normalization_type) vgg(rgb=data_tensor, constructor=gammanet_constructor) # activity = vgg.fgru_0 with tf.variable_scope('fgru', reuse=reuse): # Get side weights hs_0, hs_1 = [], [] h2_rem = [ vgg.fgru_0, vgg.fgru_1, vgg.fgru_2, vgg.conv5_1, vgg.fgru_3] for idx, h in enumerate(h2_rem): res = normalization.apply_normalization( activity=h, name='output_norm1_%s' % idx, normalization_type=output_normalization_type, data_format=data_format, training=training, trainable=training, reuse=reuse) res = aux['image_resize']( res, data_tensor.get_shape().as_list()[1:3], align_corners=True) cnv_0 = tf.layers.conv2d( inputs=res, filters=output_shape, kernel_size=(1, 1), padding='same', data_format=long_data_format, name='readout_aux_00_%s' % idx, # activation=tf.nn.relu, activation=None, trainable=training, use_bias=True, reuse=reuse) cnv_1 = tf.layers.conv2d( inputs=res, filters=output_shape, kernel_size=(1, 1), padding='same', data_format=long_data_format, name='readout_aux_10_%s' % idx, # activation=tf.nn.relu, activation=None, trainable=training, use_bias=True, reuse=reuse) hs_0 += [cnv_0] hs_1 += [cnv_1] s1, s2, s3, s4, s5 = hs_0 s11, s21, s31, s41, s51 = hs_1 o1 = tf.stop_gradient(tf.identity(s1)) o2 = tf.stop_gradient(tf.identity(s2)) o3 = tf.stop_gradient(tf.identity(s3)) o4 = tf.stop_gradient(tf.identity(s4)) o21 = tf.stop_gradient(tf.identity(s21)) o31 = tf.stop_gradient(tf.identity(s31)) o41 = tf.stop_gradient(tf.identity(s41)) o51 = tf.stop_gradient(tf.identity(s51)) p1_1 = s1 p2_1 = s2 + o1 p3_1 = s3 + o2 + o1 p4_1 = s4 + o3 + o2 + o1 p5_1 = s5 + o4 + o3 + o2 + o1 p1_2 = s11 + o21 + o31 + o41 + o51 p2_2 = s21 + o31 + o41 + o51 p3_2 = s31 + o41 + o51 p4_2 = s41 + o51 p5_2 = s51 hs = [p1_1, p2_1, p3_1, p4_1, p5_1, p1_2, p2_2, p3_2, p4_2, p5_2] activity = tf.layers.conv2d( tf.concat(hs, -1), filters=output_shape, kernel_size=(1, 1), padding='same', data_format=long_data_format, name='out', # activation=tf.nn.relu, activation=None, trainable=training, use_bias=False, reuse=reuse) if long_data_format is 'channels_first': activity = tf.transpose(activity, (0, 2, 3, 1)) extra_activities = {idx: v for idx, v in enumerate(hs_0)} if activity.dtype != tf.float32: activity = tf.cast(activity, tf.float32) # return [activity, h_deep], extra_activities return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape, data_format='NHWC'): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[-1] elif isinstance(output_shape, dict): output_shape = output_shape['output'] output_shape = 1 # norm_moments_training = training # Force instance norm # normalization_type = 'no_param_batch_norm_original' normalization_type = 'no_param_instance_norm' # output_normalization_type = 'batch_norm_original_renorm' output_normalization_type = 'instance_norm' # data_tensor = tf.concat((data_tensor, data_tensor, data_tensor), -1) # Expand to 3D data_tensor, long_data_format = tf_fun.interpret_data_format( data_tensor=data_tensor, data_format=data_format) # Prepare gammanet structure (compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts) = v2_big_working() gammanet_constructor = tf_fun.get_gammanet_constructor( compression=compression, ff_kernels=ff_kernels, ff_repeats=ff_repeats, features=features, fgru_kernels=fgru_kernels) aux = get_aux() # Build model with tf.variable_scope('vgg', reuse=reuse): aux = get_aux() vgg = vgg16.Vgg16( vgg16_npy_path= '/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy', reuse=reuse, aux=aux, train=False, # training, timesteps=8, fgru_normalization_type=normalization_type, ff_normalization_type=normalization_type) vgg(rgb=data_tensor, constructor=gammanet_constructor) # activity = vgg.fgru_0 with tf.variable_scope('fgru', reuse=reuse): # Get side weights h2_rem = [vgg.fgru_0, vgg.fgru_1, vgg.fgru_2] activities = [] for idx, h in enumerate(h2_rem): # res = normalization.apply_normalization( # activity=h, # name='output_norm1_%s' % idx, # normalization_type=output_normalization_type, # data_format=data_format, # training=False, # trainable=False, # reuse=reuse) activities.append(aux['image_resize']( h, vgg.fgru_0.get_shape().as_list()[1:3], align_corners=True)) activity = tf.concat(activities, -1) # Then read out # activity = normalization.apply_normalization( # activity=res, # name='output_norm1', # normalization_type=output_normalization_type, # data_format=data_format, # training=training, # trainable=training, # reuse=reuse) with tf.variable_scope('readout', reuse=reuse): activity = normalization.batch(bottom=activity, renorm=False, name='hgru_bn', training=training) activity = conv.readout_layer(activity=activity, reuse=reuse, training=training, output_shape=output_shape) if long_data_format is 'channels_first': activity = tf.transpose(activity, (0, 2, 3, 1)) # extra_activities = {'orientations': orientations, 'model_output_y': vgg.fgru_0, 'model_output_x': vgg.fgru_0_init} extra_activities = {} if activity.dtype != tf.float32: activity = tf.cast(activity, tf.float32) # return [activity, h_deep], extra_activities return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape, data_format='NHWC'): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[-1] elif isinstance(output_shape, dict): output_shape = output_shape['output'] # normalization_type = 'no_param_instance_norm' # output_normalization_type = 'instance_norm' fgru_normalization_type = 'no_param_batch_norm_original' ff_normalization_type = 'no_param_batch_norm_original' output_normalization_type = 'batch_norm_original' ff_kernel_size = (5, 5) ff_nl = tf.nn.relu data_tensor, long_data_format = tf_fun.interpret_data_format( data_tensor=data_tensor, data_format=data_format) # Prepare gammanet structure (compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts) = v2_big_working() gammanet_constructor = tf_fun.get_gammanet_constructor( compression=compression, ff_kernels=ff_kernels, ff_repeats=ff_repeats, features=features, fgru_kernels=fgru_kernels) aux = get_aux() # Build model with tf.variable_scope('gammanet', reuse=reuse): activity = tf.layers.conv2d( inputs=data_tensor, filters=gammanet_constructor[0]['features'], kernel_size=ff_kernel_size, padding='same', # strides=(2, 2), data_format=long_data_format, name='l0_0', activation=ff_nl, use_bias=True, reuse=reuse) activity = normalization.apply_normalization( activity=activity, name='input_norm', normalization_type=output_normalization_type, data_format=data_format, training=training, trainable=training, reuse=reuse) # + pre_activity activity = tf.layers.conv2d( inputs=activity, filters=gammanet_constructor[0]['features'], kernel_size=ff_kernel_size, padding='same', # strides=(2, 2), data_format=long_data_format, name='l0_1', activation=ff_nl, use_bias=True, reuse=reuse) # Shift conv-layers to the front # Run fGRU gn = gammanet.GN( layer_name='fgru', gammanet_constructor=gammanet_constructor, data_format=data_format, reuse=reuse, timesteps=3, fgru_connectivity='', # 'all_to_all', additional_readouts=additional_readouts, fgru_normalization_type=fgru_normalization_type, ff_normalization_type=ff_normalization_type, horizontal_padding='SAME', ff_padding='SAME', ff_nl=ff_nl, stop_loop='fgru_3', recurrent_ff=True, horizontal_kernel_initializer=tf.initializers.orthogonal(), kernel_initializer=tf.initializers.orthogonal(), gate_initializer=tf.initializers.orthogonal(), # horizontal_kernel_initializer=tf.contrib.layers.xavier_initializer(), # gate_initializer=tf.contrib.layers.xavier_initializer(), # gate_initializer=tf.contrib.layers.xavier_initializer(), aux=aux, strides=[1, 1, 1, 1], pool_strides=[2, 2], pool_kernel=[2, 2], up_kernel=[4, 4], train=training) h2, h_deep = gn(X=activity) with tf.variable_scope('cv_readout', reuse=reuse): # Apply one more normalization # activity = tf.layers.max_pooling2d( # inputs=h2, # pool_size=(2, 2), # strides=(2, 2), # padding='same', # data_format=long_data_format, # name='output_pool_3') activity = normalization.apply_normalization( activity=h2, name='output_norm_activity', normalization_type=output_normalization_type, data_format=data_format, training=training, trainable=training, reuse=reuse) # + pre_activity # Then read out activity = tf.layers.conv2d( inputs=activity, filters=2048, kernel_size=ff_kernel_size, padding='same', # strides=(2, 2), data_format=long_data_format, name='readout_fc_0', activation=ff_nl, use_bias=True, reuse=reuse) activity = normalization.apply_normalization( activity=activity, name='readout_norm_activity', normalization_type=output_normalization_type, data_format=data_format, training=training, trainable=training, reuse=reuse) # + pre_activity activity = tf.reduce_mean(activity, reduction_indices=[1, 2]) activity = tf.layers.dense(inputs=activity, units=output_shape, activation=None, use_bias=True, bias_initializer=tf.constant_initializer( 1. / 1000.), name='readout_fc_2', reuse=reuse) if long_data_format is 'channels_first': activity = tf.transpose(activity, (0, 2, 3, 1)) extra_activities = { # 'activity_low': h2, # 'activity_high': h_deep, } if activity.dtype != tf.float32: activity = tf.cast(activity, tf.float32) # return [activity, h_deep], extra_activities return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape, data_format='NHWC'): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[-1] elif isinstance(output_shape, dict): output_shape = output_shape['output'] input_training = training # training readout_training = training # False norm_moments_training = training # Force instance norm norm_params_training = training fgru_kernel_training = training fgru_param_training = training ff_gate_training = training fgru_gate_training = training remaining_param_training = training normalization_type = 'no_param_batch_norm_original' output_normalization_type = 'batch_norm_original' # normalization_type = 'no_param_instance_norm' # output_normalization_type = 'instance_norm' ff_kernel_size = (3, 3) ff_nl = tf.nn.elu data_tensor, long_data_format = tf_fun.interpret_data_format( data_tensor=data_tensor, data_format=data_format) # Prepare gammanet structure (compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts) = v2_big_working() gammanet_constructor = tf_fun.get_gammanet_constructor( compression=compression, ff_kernels=ff_kernels, ff_repeats=ff_repeats, features=features, fgru_kernels=fgru_kernels) aux = get_aux() # Build model with tf.variable_scope('gammanet', reuse=reuse): aux = get_aux() vgg = vgg16.Vgg16( vgg16_npy_path= '/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy', reuse=reuse, aux=aux, train=training, timesteps=4, fgru_normalization_type=normalization_type, ff_normalization_type=normalization_type) vgg(rgb=data_tensor, constructor=gammanet_constructor) # h2_rem = [vgg.conv5_3, vgg.conv4_3, vgg.conv3_3, vgg.conv2_2, vgg.conv1_1] h2 = vgg.fgru_0 with tf.variable_scope('contour_readout', reuse=reuse): # Apply one more normalization # h_deep = apply_normalization( # activity=h_deep, # name='output_normh_deep', # normalization_type=normalization_type, # data_format=data_format, # training=training, # reuse=reuse) if 0: hs = [] for h in h2_rem: hs += [ tf.image.resize_nearest_neighbor( h, data_tensor.get_shape().as_list()[1:3]) ] activity = normalization.apply_normalization( activity=tf.concat(hs, axis=-1), name='output_norm', normalization_type=output_normalization_type, data_format=data_format, training=norm_moments_training, trainable=training, reuse=reuse) else: activity = normalization.apply_normalization( activity=h2, name='output_norm', normalization_type=output_normalization_type, data_format=data_format, training=norm_moments_training, trainable=training, reuse=reuse) # Then read out activity = tf.layers.conv2d(inputs=activity, filters=output_shape, kernel_size=(1, 1), padding='same', data_format=long_data_format, name='readout_conv', activation=None, trainable=training, use_bias=True, reuse=reuse) if long_data_format is 'channels_first': activity = tf.transpose(activity, (0, 2, 3, 1)) extra_activities = {} if activity.dtype != tf.float32: activity = tf.cast(activity, tf.float32) # return [activity, h_deep], extra_activities return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape, data_format='NHWC'): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[-1] elif isinstance(output_shape, dict): output_shape = output_shape['output'] # norm_moments_training = training # Force instance norm # normalization_type = 'no_param_batch_norm_original' normalization_type = 'no_param_instance_norm' output_normalization_type = 'batch_norm_original_renorm' # output_normalization_type = 'instance_norm' data_tensor, long_data_format = tf_fun.interpret_data_format( data_tensor=data_tensor, data_format=data_format) # Prepare gammanet structure (compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts) = v2_big_working() gammanet_constructor = tf_fun.get_gammanet_constructor( compression=compression, ff_kernels=ff_kernels, ff_repeats=ff_repeats, features=features, fgru_kernels=fgru_kernels) aux = get_aux() # Build model with tf.variable_scope('vgg', reuse=reuse): aux = get_aux() vgg = vgg16.Vgg16( vgg16_npy_path= '/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy', reuse=reuse, aux=aux, train=False, # training, timesteps=8, # train_norm_moments=training, # train_norm_params=training, fgru_normalization_type=normalization_type, downsampled=False, ff_normalization_type=normalization_type) activity = vgg(rgb=data_tensor, constructor=gammanet_constructor, store_timesteps=True) activity = tf.concat(activity, -1) # activity = vgg.fgru_0 """ with tf.variable_scope('fgru', reuse=reuse): # Get side weights res = normalization.apply_normalization( activity=activity, name='output_norm1_%s' % 0, normalization_type=output_normalization_type, data_format=data_format, training=training, trainable=training, reuse=reuse) # res = aux['image_resize']( # res, # data_tensor.get_shape().as_list()[1:3], # align_corners=True) """ with tf.variable_scope('readout', reuse=reuse): activity = tf.layers.conv2d( inputs=activity, filters=output_shape, # gammanet_constructor[0]['features'], kernel_size=(1, 1), padding='same', data_format=long_data_format, name='readout_l0_1', activation=None, use_bias=True, trainable=training, reuse=reuse) # activity = tf.nn.elu(activity) """ activity = normalization.apply_normalization( activity=activity, name='output_norm_final_%s' % 1, normalization_type=output_normalization_type, data_format=data_format, training=training, trainable=training, reuse=reuse) activity = tf.layers.conv2d( inputs=activity, filters=gammanet_constructor[0]['features'], kernel_size=(1, 1), padding='same', data_format=long_data_format, name='readout_l0_2', activation=tf.nn.elu, use_bias=True, trainable=training, reuse=reuse) bs, H, W, channels = activity.get_shape().as_list() """ activity = tf.reduce_max(activity, reduction_indices=[1, 2]) activity = tf.layers.dense(activity, output_shape, activation=None, use_bias=True, trainable=training, reuse=reuse) print("OUTPUT DIMS: {}".format(output_shape)) if long_data_format is 'channels_first': activity = tf.transpose(activity, (0, 2, 3, 1)) extra_activities = {} # 'orientations': orientations} if activity.dtype != tf.float32: activity = tf.cast(activity, tf.float32) # return [activity, h_deep], extra_activities return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape, data_format='NHWC'): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[-1] elif isinstance(output_shape, dict): output_shape = output_shape['output'] input_training = training readout_training = training # False norm_moments_training = training # Force instance norm norm_params_training = training fgru_kernel_training = training fgru_param_training = training ff_gate_training = training fgru_gate_training = training remaining_param_training = training # normalization_type = 'no_param_batch_norm_original_renorm' # output_normalization_type = 'batch_norm_original_renorm' normalization_type = 'no_param_batch_norm_original' output_normalization_type = 'batch_norm_original' # normalization_type = 'no_param_instance_norm' # output_normalization_type = 'instance_norm' ff_kernel_size = (3, 3) ff_nl = tf.nn.elu data_tensor, long_data_format = tf_fun.interpret_data_format( data_tensor=data_tensor, data_format=data_format) # Prepare gammanet structure (compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts) = v2_big_working() gammanet_constructor = tf_fun.get_gammanet_constructor( compression=compression, ff_kernels=ff_kernels, ff_repeats=ff_repeats, features=features, fgru_kernels=fgru_kernels) aux = get_aux() # Build model with tf.variable_scope('gammanet', reuse=reuse): activity_1 = tf.layers.conv2d( inputs=data_tensor, filters=gammanet_constructor[0]['features'], kernel_size=ff_kernel_size, padding='same', data_format=long_data_format, name='l0_0', activation=ff_nl, use_bias=True, trainable=input_training, reuse=reuse) activity_2 = tf.layers.conv2d( inputs=activity_1, filters=gammanet_constructor[0]['features'], kernel_size=ff_kernel_size, padding='same', data_format=long_data_format, name='l0_1', activation=ff_nl, use_bias=True, trainable=input_training, reuse=reuse) # activity = normalization.apply_normalization( # activity=activity_2, # name='input_norm_instance', # normalization_type=normalization_type, # output_normalization_type, # data_format=data_format, # training=norm_moments_training, # trainable=input_training, # reuse=reuse) # with tf.variable_scope('contour_readout', reuse=reuse): # beta = tf.get_variable( # name='hello_instance_beta', # shape=[1, 1, 1, gammanet_constructor[0]['features']], # initializer=tf.initializers.zeros, # trainable=input_training) # gamma = tf.get_variable( # name='helo_instance_gamma', # shape=[1, 1, 1, gammanet_constructor[0]['features']], # initializer=tf.initializers.ones, # trainable=input_training) # activity = beta + gamma * activity_2 activity = normalization.apply_normalization( activity=activity_2, name='input_norm', normalization_type=output_normalization_type, data_format=data_format, training=training, trainable=training, reuse=reuse) with tf.variable_scope('gammanet', reuse=reuse): # Shift conv-layers to the front # Run fGRU gn = gammanet.GN( layer_name='fgru', gammanet_constructor=gammanet_constructor, data_format=data_format, reuse=reuse, timesteps=8, train=remaining_param_training, train_fgru_gate=fgru_gate_training, train_ff_gate=ff_gate_training, train_norm_moments=norm_moments_training, train_norm_params=norm_params_training, train_fgru_kernels=fgru_kernel_training, train_fgru_params=fgru_param_training, fgru_connectivity='', # 'all_to_all', additional_readouts=additional_readouts, fgru_normalization_type=normalization_type, ff_normalization_type=normalization_type, horizontal_padding='SAME', ff_padding='SAME', ff_nl=ff_nl, recurrent_ff=True, horizontal_kernel_initializer=tf.initializers.orthogonal(), kernel_initializer=tf.initializers.orthogonal(), gate_initializer=tf.initializers.orthogonal(), # horizontal_kernel_initializer=tf.contrib.layers.xavier_initializer(), # kernel_initializer=tf.contrib.layers.xavier_initializer(), # gate_initializer=tf.contrib.layers.xavier_initializer(), aux=aux, strides=[1, 1, 1, 1], pool_strides=[2, 2], pool_kernel=[2, 2], up_kernel=[4, 4]) layers = gn(X=activity) h2 = layers[0] layers = layers[1:] # h2 = h2[0] # h2 = gn(X=activity) with tf.variable_scope('contour_readout', reuse=reuse): activity = normalization.apply_normalization( activity=h2, # activity, name='output_norm', normalization_type=normalization_type, data_format=data_format, training=training, trainable=training, reuse=reuse) beta = tf.get_variable(name='hello_instance_beta', shape=[1, 1, 1, h2.get_shape().as_list()[-1]], initializer=tf.initializers.zeros, trainable=input_training) gamma = tf.get_variable(name='helo_instance_gamma', shape=[1, 1, 1, h2.get_shape().as_list()[-1]], initializer=tf.initializers.constant(0.1), trainable=input_training) activity = beta + gamma * activity activity = tf.layers.conv2d( inputs=activity, filters=512, # gammanet_constructor[0]['features'], kernel_size=ff_kernel_size, padding='same', data_format=long_data_format, name='readout_conv', activation=ff_nl, use_bias=True, reuse=reuse) activity = normalization.apply_normalization( activity=activity, # activity, name='output_norm_1', normalization_type=output_normalization_type, data_format=data_format, training=training, trainable=training, reuse=reuse) activity = tf.layers.conv2d( inputs=activity, filters=128, # gammanet_constructor[0]['features'], kernel_size=ff_kernel_size, padding='same', data_format=long_data_format, name='readout_conv_1', activation=ff_nl, use_bias=True, reuse=reuse) # activity = normalization.apply_normalization( # activity=activity, # name='output_norm_2', # normalization_type=output_normalization_type, # data_format=data_format, # training=training, # trainable=training, # reuse=reuse) activity = tf.layers.conv2d(inputs=activity, filters=output_shape, kernel_size=ff_kernel_size, padding='same', data_format=long_data_format, name='readout_conv_2', activation=None, use_bias=True, reuse=reuse) if long_data_format is 'channels_first': activity = tf.transpose(activity, (0, 2, 3, 1)) extra_activities = {} if activity.dtype != tf.float32: activity = tf.cast(activity, tf.float32) # return [activity, h_deep], extra_activities return activity, extra_activities
def build_model(data_tensor, labels, reuse, training, output_shape, perturb_norm=False, data_format='NHWC'): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[-1] elif isinstance(output_shape, dict): output_shape = output_shape['output'] # norm_moments_training = training # Force instance norm # normalization_type = 'no_param_batch_norm_original' normalization_type = 'no_param_instance_norm' # output_normalization_type = 'batch_norm_original_renorm' output_normalization_type = 'instance_norm' data_tensor, long_data_format = tf_fun.interpret_data_format( data_tensor=data_tensor, data_format=data_format) # Prepare gammanet structure (compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts) = v2_big_working() gammanet_constructor = tf_fun.get_gammanet_constructor( compression=compression, ff_kernels=ff_kernels, ff_repeats=ff_repeats, features=features, fgru_kernels=fgru_kernels) aux = get_aux() # Build model with tf.variable_scope('vgg', reuse=reuse): hh, hw = 73, 180 aux = get_aux() fb_moments_file = "../undo_bias/neural_models/linear_moments/tb_feature_matrix.npz" fb_model_file = "../undo_bias/neural_models/linear_models/tb_model.joblib.npy" ff_moments_file = "../undo_bias/neural_models/linear_moments/conv2_2_tb_feature_matrix.npz" ff_model_file = "../undo_bias/neural_models/linear_models/conv2_2_tb_model.joblib.npy" vgg = vgg16.Vgg16( vgg16_npy_path= '/media/data_cifs_lrs/clicktionary/pretrained_weights/vgg16.npy', reuse=reuse, aux=aux, hh=hh, hw=hw, moments_file=ff_moments_file, model_file=ff_model_file, train=False, timesteps=8, perturb=60.0, # 70 might work # 2., # 1.001, # 17.1, perturb_norm=perturb_norm, # perturb=1.5, # 2., # 1.001, # 17.1, # perturb=2., # 2., # 1.001, # 17.1, fgru_normalization_type=normalization_type, ff_normalization_type=normalization_type) # gn = tf.get_default_graph() # with gn.gradient_override_map({'Conv2D': 'PerturbVizGrad'}): # Scope the entire GN (with train=False). # The lowest-level recurrent tensor is the only # trainable tensor. This grad op will freeze the # center unit, forcing other units to overcompensate # to recreate a model's prediction. # TODO: need to get the original model output... could precompute this. # noqa vgg(rgb=data_tensor, label=labels, constructor=gammanet_constructor) activity = vgg.fgru_0 # Load tuning curve transform moments = np.load(fb_moments_file) means = moments["means"] stds = moments["stds"] clf = np.load(fb_model_file).astype(np.float32) # Transform activity to outputs bs, h, w, _ = vgg.fgru_0.get_shape().as_list() sel_units = tf.reshape(vgg.fgru_0[:, hh - 2:hh + 2, hw - 2:hw + 2, :], [bs, -1]) # grad0 = tf.gradients(sel_units, vgg.conv2_2)[0] if perturb_norm: # Normalize activities -- Not normalized!! sel_units = (sel_units - means) / stds # Map responses # inv_clf = np.linalg.inv(clf.T.dot(clf)).astype(np.float32) # Precompute inversion # inv_clf = tf.linalg.inv(tf.matmul(clf, clf, transpose_a=True)) inv_clf = np.linalg.inv(clf.T.dot(clf)).astype( np.float32) # Precompute inversion activity = tf.matmul(tf.matmul(inv_clf, clf, transpose_b=True), sel_units, transpose_b=True) impatch = data_tensor[:, hh * 2 - 14:hh * 2 + 18, hw * 2 - 14:hw * 2 + 18] mx = max(h, w) x, y = np.meshgrid(np.arange(mx), np.arange(mx)) x = x[:h, :w] y = y[:h, :w] xy = np.stack((x, y), -1) coord = np.asarray([hh, hw])[None] dist = np.sqrt(((xy - coord)**2).mean(-1)) dist = dist / dist.max() # penalty = tf.reduce_mean(tf.tanh(tf.abs(tf.squeeze(vgg.fgru_0))), -1) * dist # dist = dist * tf.squeeze(vgg.mask) # zero out units in the RF # penalty = tf.cast(tf.sigmoid(tf.reduce_mean(tf.abs(tf.squeeze(vgg.fgru_0)), -1)) - 0.5, tf.float32) * dist.astype(np.float32) penalty = tf.cast( tf.sigmoid(tf.reduce_mean(tf.abs(tf.squeeze(vgg.mult)), -1)) - 0.5, tf.float32) * dist.astype(np.float32) penalty = tf.cast(penalty, tf.float32) * tf.cast( 1. - tf.squeeze(vgg.mask), tf.float32) # 0 values in the H-diameter penalty = penalty * 0.0 extra_activities = { "fgru": vgg.fgru_0, "mask": vgg.mask, "penalty": penalty, "impatch": impatch } # tf.get_variable(name="perturb_viz")} # idx: v for idx, v in enumerate(hs_0)} if activity.dtype != tf.float32: activity = tf.cast(activity, tf.float32) # return [activity, h_deep], extra_activities return activity, extra_activities
def build_model(data_tensor, labels, reuse, training, output_shape, perturb_norm=False, data_format='NHWC'): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[-1] elif isinstance(output_shape, dict): output_shape = output_shape['output'] # norm_moments_training = training # Force instance norm # normalization_type = 'no_param_batch_norm_original' normalization_type = 'no_param_instance_norm' # output_normalization_type = 'batch_norm_original_renorm' output_normalization_type = 'instance_norm' data_tensor, long_data_format = tf_fun.interpret_data_format( data_tensor=data_tensor, data_format=data_format) # Prepare gammanet structure (compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts) = v2_big_working() gammanet_constructor = tf_fun.get_gammanet_constructor( compression=compression, ff_kernels=ff_kernels, ff_repeats=ff_repeats, features=features, fgru_kernels=fgru_kernels) aux = get_aux() # Build model with tf.variable_scope('vgg', reuse=reuse): aux = get_aux() # moments_file = "../undo_bias/neural_models/linear_moments/INSILICO_BSDS_vgg_gratings_simple_tb_feature_matrix.npz" # model_file = "../undo_bias/neural_models/linear_models/INSILICO_BSDS_vgg_gratings_simple_tb_model.joblib.npy" fb_moments_file = "../undo_bias/neural_models/linear_moments/tb_feature_matrix.npz" fb_model_file = "../undo_bias/neural_models/linear_models/tb_model.joblib.npy" ff_moments_file = "../undo_bias/neural_models/linear_moments/conv2_2_tb_feature_matrix.npz" ff_model_file = "../undo_bias/neural_models/linear_models/conv2_2_tb_model.joblib.npy" vgg = vgg16.Vgg16( vgg16_npy_path= '/media/data_cifs_lrs/clicktionary/pretrained_weights/vgg16.npy', reuse=reuse, aux=aux, moments_file=ff_moments_file, # Perturb FF drive model_file=ff_model_file, train=False, timesteps=8, perturb=0.95, # 2., # 1.001, # 17.1, # perturb=.0000001, # 2., # 1.001, # 17.1, # perturb=.05, # 2., # 1.001, # 17.1 perturb_norm=perturb_norm, # perturb=1.5, # 2., # 1.001, # 17.1, # perturb=2., # 2., # 1.001, # 17.1, fgru_normalization_type=normalization_type, ff_normalization_type=normalization_type) # gn = tf.get_default_graph() # with gn.gradient_override_map({'Conv2D': 'PerturbVizGrad'}): # Scope the entire GN (with train=False). # The lowest-level recurrent tensor is the only # trainable tensor. This grad op will freeze the # center unit, forcing other units to overcompensate # to recreate a model's prediction. # TODO: need to get the original model output... could precompute this. # noqa vgg(rgb=data_tensor, label=labels, constructor=gammanet_constructor) activity = vgg.fgru_0 # Load tuning curve transform for fb output moments = np.load(fb_moments_file) means = moments["means"] stds = moments["stds"] clf = np.load(fb_model_file).astype(np.float32) # Transform activity to outputs bs, h, w, _ = vgg.fgru_0.get_shape().as_list() hh, hw = h // 2, w // 2 sel_units = tf.reshape(vgg.fgru_0[:, hh - 2:hh + 2, hw - 2:hw + 2, :], [bs, -1]) if perturb_norm: # Normalize activities -- Not normalized!! sel_units = (sel_units - means) / stds # Map responses # inv_clf = np.linalg.inv(clf.T.dot(clf)).astype(np.float32) # Precompute inversion # inv_clf = tf.linalg.inv(tf.matmul(clf, clf, transpose_a=True)) inv_clf = np.linalg.inv(clf.T.dot(clf)).astype( np.float32) # Precompute inversion activity = tf.matmul(tf.matmul(inv_clf, clf, transpose_b=True), sel_units, transpose_b=True) # bg = tf.reduce_mean(vgg.conv2_2 ** 2, reduction_indices=[-1], keep_dims=True) # bg = tf.cast(tf.greater(bg, tf.reduce_mean(bg)), tf.float32) # bg_dil = dilation2d(img=bg, extent=5) # extra_activities = {"mask": bg, "mask_dil": bg_dil} # {"mask": tf.reduce_mean(vgg.conv2_2 ** 2, reduction_indices=[-1])} # tf.get_variable(name="perturb_viz")} # idx: v for idx, v in enumerate(hs_0)} # extra_activities = {"fgru": vgg.fgru_0, "penalty": tf.constant(0.), "conv": vgg.error_1} # tf.get_variable(name="perturb_viz")} # idx: v for idx, v in enumerate(hs_0)} # extra_activities = {"fgru": vgg.fgru_0, "penalty": tf.constant(0.), "conv": vgg.error_1} # tf.get_variable(name="perturb_viz")} # idx: v for idx, v in enumerate(hs_0)} extra_activities = {} if activity.dtype != tf.float32: activity = tf.cast(activity, tf.float32) # return [activity, h_deep], extra_activities return activity, extra_activities
def build_model(data_tensor, reuse, training, output_shape, data_format='NHWC'): """Create the hgru from Learning long-range...""" if isinstance(output_shape, list): output_shape = output_shape[-1] elif isinstance(output_shape, dict): output_shape = output_shape['output'] # norm_moments_training = training # Force instance norm # normalization_type = 'no_param_batch_norm_original' normalization_type = 'no_param_instance_norm' # output_normalization_type = 'batch_norm_original_renorm' output_normalization_type = 'instance_norm' data_tensor, long_data_format = tf_fun.interpret_data_format( data_tensor=data_tensor, data_format=data_format) # Prepare gammanet structure (compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts) = v2_big_working() gammanet_constructor = tf_fun.get_gammanet_constructor( compression=compression, ff_kernels=ff_kernels, ff_repeats=ff_repeats, features=features, fgru_kernels=fgru_kernels) aux = get_aux() # Build model with tf.variable_scope('vgg', reuse=reuse): aux = get_aux() vgg = vgg16.Vgg16( vgg16_npy_path= '/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy', reuse=reuse, aux=aux, train=False, timesteps=8, fgru_normalization_type=normalization_type, ff_normalization_type=normalization_type) vgg(rgb=data_tensor, constructor=gammanet_constructor) activity = vgg.fgru_0 with tf.variable_scope('fgru_gilbert', reuse=reuse): # Then read out # activity = activity ** 2 activity = normalization.apply_normalization( activity=activity, name='output_norm1', normalization_type=output_normalization_type, data_format=data_format, training=training, trainable=training, reuse=reuse) activity = tf.layers.conv2d( inputs=activity, filters=gammanet_constructor[0]['features'], kernel_size=(1, 1), padding='same', data_format=long_data_format, name='readout_l0_1', activation=tf.nn.relu, use_bias=True, trainable=training, reuse=reuse) activity = tf.layers.conv2d(inputs=activity, filters=1, kernel_size=(1, 1), padding='same', data_format=long_data_format, name='readout_l0_2', activation=None, use_bias=True, trainable=training, reuse=reuse) activity = tf.reduce_max(activity, reduction_indices=[1, 2]) if long_data_format is 'channels_first': activity = tf.transpose(activity, (0, 2, 3, 1)) extra_activities = {} if activity.dtype != tf.float32: activity = tf.cast(activity, tf.float32) # return [activity, h_deep], extra_activities return activity, extra_activities