def build_model(data_tensor,
reuse,
training,
output_shape,
data_format='NHWC'):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[-1]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
output_normalization_type = 'batch_norm_original'
ff_kernel_size = (5, 5)
ff_nl = tf.nn.elu
data_tensor, long_data_format = tf_fun.interpret_data_format(
data_tensor=data_tensor, data_format=data_format)
# Build model
with tf.variable_scope('gammanet', reuse=reuse):
conv_aux = {
'pretrained': os.path.join('weights',
'gabors_for_contours_11.npy'),
'pretrained_key': 's1',
'nonlinearity': 'square'
}
activity = conv.conv_layer(bottom=data_tensor,
name='gabor_input',
stride=[1, 1, 1, 1],
padding='SAME',
trainable=training,
use_bias=True,
aux=conv_aux)
layer_hgru = hgru.hGRU('hgru_1',
x_shape=activity.get_shape().as_list(),
timesteps=8,
h_ext=15,
strides=[1, 1, 1, 1],
padding='SAME',
aux={
'reuse': False,
'constrain': False
},
train=training)
h2 = layer_hgru.build(activity)
h2 = normalization.batch_contrib(bottom=h2,
name='hgru_bn',
training=training)
mask = np.load('weights/cardena_mask.npy')[None, :, :, None]
activity = h2 * mask
with tf.variable_scope('cv_readout', reuse=reuse):
activity = tf.reduce_mean(activity, reduction_indices=[1, 2])
activity = tf.layers.dense(activity, output_shape)
if long_data_format is 'channels_first':
activity = tf.transpose(activity, (0, 2, 3, 1))
extra_activities = {}
if activity.dtype != tf.float32:
activity = tf.cast(activity, tf.float32)
# return [activity, h_deep], extra_activities
return activity, extra_activities
def build_model(data_tensor,
reuse,
training,
output_shape,
data_format='NHWC'):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[-1]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
output_shape = 2
# norm_moments_training = training # Force instance norm
# normalization_type = 'no_param_batch_norm_original'
normalization_type = 'no_param_instance_norm'
# output_normalization_type = 'batch_norm_original_renorm'
output_normalization_type = 'instance_norm'
data_tensor, long_data_format = tf_fun.interpret_data_format(
data_tensor=data_tensor, data_format=data_format)
# Prepare gammanet structure
(compression, ff_kernels, ff_repeats, features, fgru_kernels,
additional_readouts) = v2_big_working()
gammanet_constructor = tf_fun.get_gammanet_constructor(
compression=compression,
ff_kernels=ff_kernels,
ff_repeats=ff_repeats,
features=features,
fgru_kernels=fgru_kernels)
aux = get_aux()
# Build model
with tf.variable_scope('vgg', reuse=reuse):
aux = get_aux()
vgg = vgg16.Vgg16(
vgg16_npy_path=
'/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy',
reuse=reuse,
aux=aux,
train=training,
timesteps=8,
fgru_normalization_type=normalization_type,
ff_normalization_type=normalization_type)
vgg(rgb=data_tensor, constructor=gammanet_constructor)
activity = vgg.fgru_0
if long_data_format is 'channels_first':
activity = tf.transpose(activity, (0, 2, 3, 1))
extra_activities = {
"fgru": vgg.fgru_0
} # idx: v for idx, v in enumerate(hs_0)}
if activity.dtype != tf.float32:
activity = tf.cast(activity, tf.float32)
# return [activity, h_deep], extra_activities
return activity, extra_activities
def build_model(data_tensor,
reuse,
training,
output_shape,
data_format='NHWC'):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[-1]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
normalization_type = 'no_param_instance_norm'
output_normalization_type = 'instance_norm'
# normalization_type = 'no_param_batch_norm'
# output_normalization_type = 'batch_norm'
ff_kernel_size = (3, 3)
ff_nl = tf.nn.relu
data_tensor, long_data_format = tf_fun.interpret_data_format(
data_tensor=data_tensor, data_format=data_format)
# Prepare gammanet structure
(compression, ff_kernels, ff_repeats, features, fgru_kernels,
additional_readouts) = v2_big_working()
gammanet_constructor = tf_fun.get_gammanet_constructor(
compression=compression,
ff_kernels=ff_kernels,
ff_repeats=ff_repeats,
features=features,
fgru_kernels=fgru_kernels)
aux = get_aux()
# Build model
fix_training = False
with tf.variable_scope('gammanet', reuse=reuse):
activity = tf.layers.conv2d(
inputs=data_tensor,
filters=gammanet_constructor[0]['features'],
kernel_size=ff_kernel_size,
padding='same',
data_format=long_data_format,
name='l0_0',
activation=ff_nl,
use_bias=True,
# trainable=training,
trainable=fix_training,
reuse=reuse)
activity = tf.layers.conv2d(
inputs=activity,
filters=gammanet_constructor[0]['features'],
kernel_size=ff_kernel_size,
padding='same',
data_format=long_data_format,
name='l0_1',
activation=None,
use_bias=True,
# trainable=training,
trainable=fix_training,
reuse=reuse)
activity = normalization.apply_normalization(
activity=activity,
name='input_norm',
normalization_type=output_normalization_type,
data_format=data_format,
# training=training,
training=fix_training,
reuse=reuse) # + pre_activity
activity = ff_nl(activity)
# # Downsample
# activity = conv.down_block(
# layer_name='l1',
# bottom=activity,
# kernel_size=3 * [[3, 3]],
# num_filters=gammanet_constructor[0]['features'],
# normalization_type=output_normalization_type,
# training=training,
# activation=ff_nl,
# include_pool=False,
# reuse=reuse)
# Shift conv-layers to the front
# Run fGRU
gn = gammanet.GN(
layer_name='fgru',
gammanet_constructor=gammanet_constructor,
data_format=data_format,
reuse=reuse,
timesteps=8,
fgru_connectivity='', # 'all_to_all',
additional_readouts=additional_readouts,
fgru_normalization_type=normalization_type,
ff_normalization_type=normalization_type,
horizontal_padding='SAME',
ff_padding='SAME',
ff_nl=ff_nl,
recurrent_ff=True,
horizontal_kernel_initializer=tf.initializers.orthogonal(),
kernel_initializer=tf.initializers.orthogonal(),
gate_initializer=tf.initializers.orthogonal(),
# horizontal_kernel_initializer=tf.contrib.layers.xavier_initializer(),
# gate_initializer=tf.contrib.layers.xavier_initializer(),
# gate_initializer=tf.contrib.layers.xavier_initializer(),
aux=aux,
strides=[1, 1, 1, 1],
pool_strides=[2, 2],
pool_kernel=[2, 2],
up_kernel=[4, 4],
train=fix_training)
h2, h_deep = gn(X=activity)
with tf.variable_scope('gratings_readout', reuse=reuse):
h2 = normalization.apply_normalization(
activity=h2,
name='output_norm',
normalization_type=output_normalization_type,
data_format=data_format,
training=training,
reuse=reuse) # + pre_activity
# Then read out
activity = tf.layers.conv2d(
inputs=h2,
filters=gammanet_constructor[0]['features'],
kernel_size=(1, 1),
padding='same',
data_format=long_data_format,
name='readout_l0_1',
activation=ff_nl,
use_bias=True,
trainable=training,
reuse=reuse)
activity = tf.layers.conv2d(inputs=activity,
filters=output_shape,
kernel_size=(1, 1),
padding='same',
data_format=long_data_format,
name='readout_l0_2',
activation=None,
use_bias=True,
trainable=training,
reuse=reuse)
_, h, w, _ = activity.get_shape().as_list()
activity = activity[:, h // 2, w // 2, :]
if long_data_format is 'channels_first':
activity = tf.transpose(activity, (0, 2, 3, 1))
extra_activities = {
'activity_low': h2,
'activity_high': h_deep,
}
if activity.dtype != tf.float32:
activity = tf.cast(activity, tf.float32)
# return [activity, h_deep], extra_activities
return activity, extra_activities
def build_model(
data_tensor,
reuse,
training,
output_shape,
data_format='NHWC'):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[-1]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
output_shape = 2
# norm_moments_training = training # Force instance norm
# normalization_type = 'no_param_batch_norm_original'
normalization_type = 'no_param_instance_norm'
# output_normalization_type = 'batch_norm_original_renorm'
output_normalization_type = 'instance_norm'
data_tensor, long_data_format = tf_fun.interpret_data_format(
data_tensor=data_tensor,
data_format=data_format)
# Prepare gammanet structure
(
compression,
ff_kernels,
ff_repeats,
features,
fgru_kernels,
additional_readouts) = v2_big_working()
gammanet_constructor = tf_fun.get_gammanet_constructor(
compression=compression,
ff_kernels=ff_kernels,
ff_repeats=ff_repeats,
features=features,
fgru_kernels=fgru_kernels)
aux = get_aux()
# Build model
with tf.variable_scope('vgg', reuse=reuse):
vgg = vgg16.Vgg16(
vgg16_npy_path='/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy')
vgg(rgb=data_tensor, train=False, ff_reuse=reuse)
# activity = vgg.fgru_0
with tf.variable_scope('fgru', reuse=reuse):
# Get side weights
h2_rem = [
vgg.fgru_0, vgg.fgru_1, vgg.fgru_2, vgg.fgru_3]
h2s = []
for idx, h in enumerate(h2_rem):
res = aux['image_resize'](
h,
data_tensor.get_shape().as_list()[1:3],
align_corners=True)
h2s += [res]
res = normalization.apply_normalization(
activity=tf.concat(h2s, -1),
name='output_norm1',
normalization_type=output_normalization_type,
data_format=data_format,
training=False,
trainable=False,
reuse=reuse)
activity = tf.layers.conv2d(
inputs=res,
filters=gammanet_constructor[0]['features'],
kernel_size=(1, 1),
padding='same',
data_format=long_data_format,
name='readout_l0_1',
activation=tf.nn.relu,
use_bias=True,
trainable=training,
reuse=reuse)
orientations = tf.layers.conv2d(
inputs=activity,
filters=output_shape,
kernel_size=(1, 1),
padding='same',
data_format=long_data_format,
name='readout_l0_2',
activation=None,
use_bias=True,
trainable=training,
reuse=reuse)
_, h, w, _ = orientations.get_shape().as_list()
activity = orientations[:, h // 2, w // 2, :]
if long_data_format is 'channels_first':
activity = tf.transpose(activity, (0, 2, 3, 1))
extra_activities = {'orientations': orientations}
if activity.dtype != tf.float32:
activity = tf.cast(activity, tf.float32)
# return [activity, h_deep], extra_activities
return activity, extra_activities
def build_model(data_tensor,
reuse,
training,
output_shape,
data_format='NHWC'):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[-1]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
# norm_moments_training = training # Force instance norm
# normalization_type = 'no_param_batch_norm_original'
normalization_type = 'no_param_instance_norm'
# output_normalization_type = 'batch_norm_original_renorm'
output_normalization_type = 'instance_norm'
data_tensor, long_data_format = tf_fun.interpret_data_format(
data_tensor=data_tensor, data_format=data_format)
# Prepare gammanet structure
(compression, ff_kernels, ff_repeats, features, fgru_kernels,
additional_readouts) = v2_big_working()
gammanet_constructor = tf_fun.get_gammanet_constructor(
compression=compression,
ff_kernels=ff_kernels,
ff_repeats=ff_repeats,
features=features,
fgru_kernels=fgru_kernels)
aux = get_aux()
# Build model
with tf.variable_scope('vgg', reuse=reuse):
aux = get_aux()
vgg = vgg16.Vgg16(
vgg16_npy_path=
'/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy',
reuse=reuse,
aux=aux,
train=training,
timesteps=8,
fgru_normalization_type=normalization_type,
ff_normalization_type=normalization_type)
vgg(rgb=data_tensor, constructor=gammanet_constructor)
# activity = vgg.fgru_0
with tf.variable_scope('fgru', reuse=reuse):
# Get side weights
h2_rem = [vgg.fgru_0]
for idx, h in enumerate(h2_rem):
res = normalization.apply_normalization(
activity=h,
name='output_norm1_%s' % idx,
normalization_type=output_normalization_type,
data_format=data_format,
training=training,
trainable=training,
reuse=reuse)
res = aux['image_resize'](res,
data_tensor.get_shape().as_list()[1:3],
align_corners=True)
activity = tf.layers.conv2d(res,
filters=res.get_shape().as_list()[-1],
kernel_size=(3, 3),
padding='same',
data_format=long_data_format,
name='out',
activation=tf.nn.relu,
trainable=training,
use_bias=True,
reuse=reuse)
activity = tf.layers.conv2d(
activity,
filters=output_shape,
kernel_size=(1, 1),
padding='same',
data_format=long_data_format,
name='out2',
# activation=tf.nn.relu,
activation=None,
trainable=training,
use_bias=True,
reuse=reuse)
if long_data_format is 'channels_first':
activity = tf.transpose(activity, (0, 2, 3, 1))
extra_activities = {} # idx: v for idx, v in enumerate(hs_0)}
if activity.dtype != tf.float32:
activity = tf.cast(activity, tf.float32)
# return [activity, h_deep], extra_activities
return activity, extra_activities
def build_model(
data_tensor,
reuse,
training,
output_shape,
data_format='NHWC'):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[-1]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
# output_shape = 2
# norm_moments_training = training # Force instance norm
normalization_type = 'no_param_batch_norm_original'
# normalization_type = 'ada_batch_norm'
# normalization_type = 'no_param_instance_norm'
output_normalization_type = 'batch_norm_original_renorm'
# output_normalization_type = 'instance_norm'
data_tensor, long_data_format = tf_fun.interpret_data_format(
data_tensor=data_tensor,
data_format=data_format)
# Prepare gammanet structure
(
compression,
ff_kernels,
ff_repeats,
features,
fgru_kernels,
additional_readouts) = v2_big_working()
gammanet_constructor = tf_fun.get_gammanet_constructor(
compression=compression,
ff_kernels=ff_kernels,
ff_repeats=ff_repeats,
features=features,
fgru_kernels=fgru_kernels)
aux = get_aux()
# Build model
with tf.variable_scope('vgg', reuse=reuse):
aux = get_aux()
vgg = vgg16.Vgg16(
vgg16_npy_path='/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy',
reuse=reuse,
aux=aux,
train=training,
timesteps=8,
fgru_normalization_type=normalization_type,
ff_normalization_type=normalization_type)
vgg(rgb=data_tensor, constructor=gammanet_constructor)
# activity = vgg.fgru_0
with tf.variable_scope('fgru', reuse=reuse):
# Get side weights
activity = normalization.batch(
bottom=vgg.fgru_0, # activity,
renorm=True,
name='readout_0_bn',
training=training)
# activity = normalization.group(
# bottom=vgg.fgru_0) # , # activity,
# # name='readout_0_bn')
prepool_activity = tf.layers.conv2d(
inputs=activity,
filters=output_shape, # int(activity.get_shape()[-1]),
kernel_size=1,
name='pre_readout_conv0',
strides=(1, 1),
padding='same',
activation=tf.nn.elu,
trainable=training,
use_bias=True)
"""
prepool_activity = normalization.batch(
bottom=prepool_activity,
renorm=True,
name='readout_1_bn',
training=False) # training)
"""
"""
prepool_activity = tf.layers.conv2d(
inputs=prepool_activity,
filters=output_shape,
kernel_size=1,
name='pre_readout_conv1',
strides=(1, 1),
padding='same',
activation=None,
trainable=training,
use_bias=True)
"""
out_activity = tf.reduce_max(prepool_activity, reduction_indices=[1, 2])
activity = tf.layers.dense(
inputs=out_activity,
units=output_shape)
if long_data_format is 'channels_first':
activity = tf.transpose(activity, (0, 2, 3, 1))
# extra_activities = {'orientations': orientations, 'model_output_y': vgg.fgru_0, 'model_output_x': vgg.fgru_0_init}
extra_activities = {'x': vgg.fgru_0}
if activity.dtype != tf.float32:
activity = tf.cast(activity, tf.float32)
# return [activity, h_deep], extra_activities
return activity, extra_activities
def build_model(data_tensor,
labels,
reuse,
training,
output_shape,
perturb_norm=False,
data_format='NHWC'):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[-1]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
# norm_moments_training = training # Force instance norm
# normalization_type = 'no_param_batch_norm_original'
normalization_type = 'no_param_instance_norm'
# output_normalization_type = 'batch_norm_original_renorm'
output_normalization_type = 'instance_norm'
data_tensor, long_data_format = tf_fun.interpret_data_format(
data_tensor=data_tensor, data_format=data_format)
# Prepare gammanet structure
(compression, ff_kernels, ff_repeats, features, fgru_kernels,
additional_readouts) = v2_big_working()
gammanet_constructor = tf_fun.get_gammanet_constructor(
compression=compression,
ff_kernels=ff_kernels,
ff_repeats=ff_repeats,
features=features,
fgru_kernels=fgru_kernels)
aux = get_aux()
# Build model
with tf.variable_scope('vgg', reuse=reuse):
hh, hw = 73, 180
aux = get_aux()
fb_moments_file = "../undo_bias/neural_models/linear_moments/tb_feature_matrix.npz"
fb_model_file = "../undo_bias/neural_models/linear_models/tb_model.joblib.npy"
ff_moments_file = "../undo_bias/neural_models/linear_moments/conv2_2_tb_feature_matrix.npz"
ff_model_file = "../undo_bias/neural_models/linear_models/conv2_2_tb_model.joblib.npy"
vgg = vgg16.Vgg16(
vgg16_npy_path=
'/media/data_cifs_lrs/clicktionary/pretrained_weights/vgg16.npy',
reuse=reuse,
aux=aux,
hh=hh,
hw=hw,
moments_file=ff_moments_file,
model_file=ff_model_file,
train=False,
timesteps=8,
perturb=60.0, # 70 might work # 2., # 1.001, # 17.1,
perturb_norm=perturb_norm,
# perturb=1.5, # 2., # 1.001, # 17.1,
# perturb=2., # 2., # 1.001, # 17.1,
fgru_normalization_type=normalization_type,
ff_normalization_type=normalization_type)
# gn = tf.get_default_graph()
# with gn.gradient_override_map({'Conv2D': 'PerturbVizGrad'}):
# Scope the entire GN (with train=False).
# The lowest-level recurrent tensor is the only
# trainable tensor. This grad op will freeze the
# center unit, forcing other units to overcompensate
# to recreate a model's prediction.
# TODO: need to get the original model output... could precompute this. # noqa
vgg(rgb=data_tensor, label=labels, constructor=gammanet_constructor)
activity = vgg.fgru_0
# Load tuning curve transform
moments = np.load(fb_moments_file)
means = moments["means"]
stds = moments["stds"]
clf = np.load(fb_model_file).astype(np.float32)
# Transform activity to outputs
bs, h, w, _ = vgg.fgru_0.get_shape().as_list()
sel_units = tf.reshape(vgg.fgru_0[:, hh - 2:hh + 2, hw - 2:hw + 2, :],
[bs, -1])
# grad0 = tf.gradients(sel_units, vgg.conv2_2)[0]
if perturb_norm:
# Normalize activities -- Not normalized!!
sel_units = (sel_units - means) / stds
# Map responses
# inv_clf = np.linalg.inv(clf.T.dot(clf)).astype(np.float32) # Precompute inversion
# inv_clf = tf.linalg.inv(tf.matmul(clf, clf, transpose_a=True))
inv_clf = np.linalg.inv(clf.T.dot(clf)).astype(
np.float32) # Precompute inversion
activity = tf.matmul(tf.matmul(inv_clf, clf, transpose_b=True),
sel_units,
transpose_b=True)
impatch = data_tensor[:, hh * 2 - 14:hh * 2 + 18, hw * 2 - 14:hw * 2 + 18]
mx = max(h, w)
x, y = np.meshgrid(np.arange(mx), np.arange(mx))
x = x[:h, :w]
y = y[:h, :w]
xy = np.stack((x, y), -1)
coord = np.asarray([hh, hw])[None]
dist = np.sqrt(((xy - coord)**2).mean(-1))
dist = dist / dist.max()
# penalty = tf.reduce_mean(tf.tanh(tf.abs(tf.squeeze(vgg.fgru_0))), -1) * dist
# dist = dist * tf.squeeze(vgg.mask) # zero out units in the RF
# penalty = tf.cast(tf.sigmoid(tf.reduce_mean(tf.abs(tf.squeeze(vgg.fgru_0)), -1)) - 0.5, tf.float32) * dist.astype(np.float32)
penalty = tf.cast(
tf.sigmoid(tf.reduce_mean(tf.abs(tf.squeeze(vgg.mult)), -1)) - 0.5,
tf.float32) * dist.astype(np.float32)
penalty = tf.cast(penalty, tf.float32) * tf.cast(
1. - tf.squeeze(vgg.mask), tf.float32) # 0 values in the H-diameter
penalty = penalty * 0.0
extra_activities = {
"fgru": vgg.fgru_0,
"mask": vgg.mask,
"penalty": penalty,
"impatch": impatch
} # tf.get_variable(name="perturb_viz")} # idx: v for idx, v in enumerate(hs_0)}
if activity.dtype != tf.float32:
activity = tf.cast(activity, tf.float32)
# return [activity, h_deep], extra_activities
return activity, extra_activities
def build_model(data_tensor,
reuse,
training,
output_shape,
data_format='NHWC'):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[-1]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
data_tensor, long_data_format = tf_fun.interpret_data_format(
data_tensor=data_tensor, data_format=data_format)
normalization_type = 'instance_norm'
ff_nl = tf.nn.relu
# Prepare gammanet structure
compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts = v1(
)
# compression, ff_kernels, ff_repeats, features, fgru_kernels, additional_readouts = v3()
gammanet_constructor = tf_fun.get_gammanet_constructor(
compression=compression,
ff_kernels=ff_kernels,
ff_repeats=ff_repeats,
features=features,
fgru_kernels=fgru_kernels)
aux = get_aux()
# Build model
with tf.variable_scope('gammanet', reuse=reuse):
in_emb = tf.layers.conv2d(inputs=data_tensor,
filters=16,
kernel_size=7,
strides=(1, 1),
padding='same',
data_format=long_data_format,
activation=ff_nl,
trainable=training,
use_bias=True,
name='l0')
in_emb = tf.layers.conv2d(
inputs=in_emb,
filters=16,
kernel_size=7,
strides=(1, 1),
padding='same',
data_format=long_data_format,
activation=None, # ff_nl,
trainable=training,
use_bias=True,
name='l01')
in_emb = normalization.apply_normalization(
activity=in_emb,
name='in_norm',
normalization_type=normalization_type,
data_format=data_format,
training=training,
reuse=reuse)
in_emb = ff_nl(in_emb)
# Run fGRU
gn = gammanet.GN(
layer_name='fgru',
gammanet_constructor=gammanet_constructor,
data_format=data_format,
reuse=reuse,
timesteps=8,
fgru_connectivity='', # 'all_to_all',
additional_readouts=additional_readouts,
fgru_normalization_type=normalization_type,
ff_normalization_type=normalization_type,
horizontal_padding='SAME',
ff_padding='SAME',
ff_nl=ff_nl,
recurrent_ff=False,
horizontal_kernel_initializer=tf.initializers.orthogonal(),
# horizontal_kernel_initializer=tf_fun.Identity(),
kernel_initializer=tf.initializers.orthogonal(),
aux=aux,
strides=[1, 1, 1, 1],
pool_strides=[2, 2],
pooling_kernel=[2, 2],
up_kernel=[4, 4],
train=training)
h2, h_deep = gn(X=in_emb)
with tf.variable_scope('cv_readout', reuse=reuse):
# Apply one more normalization
h2 = normalization.apply_normalization(
activity=h2,
name='output_norm',
normalization_type=normalization_type,
data_format=data_format,
training=training,
reuse=reuse)
# h_deep = apply_normalization(
# activity=h_deep,
# name='output_normh_deep',
# normalization_type=normalization_type,
# data_format=data_format,
# training=training,
# reuse=reuse)
# Then read out
activity = tf.layers.conv2d(inputs=h2,
filters=output_shape,
kernel_size=(1, 1),
padding='same',
data_format=long_data_format,
name='pre_readout_conv',
use_bias=True,
reuse=reuse)
# h_deep = tf.layers.conv2d(
# inputs=h_deep,
# filters=output_shape,
# kernel_size=(1, 1),
# padding='same',
# data_format=long_data_format,
# name='pre_readout_conv1',
# use_bias=True,
# reuse=reuse)
if long_data_format is 'channels_first':
activity = tf.transpose(activity, (0, 2, 3, 1))
extra_activities = {
'activity_low': h2,
'activity_high': h_deep,
}
if activity.dtype != tf.float32:
activity = tf.cast(activity, tf.float32)
# return [activity, h_deep], extra_activities
return activity, extra_activities
def build_model(data_tensor,
reuse,
training,
output_shape,
data_format='NHWC'):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[-1]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
input_training = training
readout_training = training # False
norm_moments_training = training # Force instance norm
norm_params_training = training
fgru_kernel_training = training
fgru_param_training = training
ff_gate_training = training
fgru_gate_training = training
remaining_param_training = training
# normalization_type = 'no_param_batch_norm_original_renorm'
# output_normalization_type = 'batch_norm_original_renorm'
normalization_type = 'no_param_batch_norm_original'
output_normalization_type = 'batch_norm_original'
# normalization_type = 'no_param_instance_norm'
# output_normalization_type = 'instance_norm'
ff_kernel_size = (3, 3)
ff_nl = tf.nn.elu
data_tensor, long_data_format = tf_fun.interpret_data_format(
data_tensor=data_tensor, data_format=data_format)
# Prepare gammanet structure
(compression, ff_kernels, ff_repeats, features, fgru_kernels,
additional_readouts) = v2_big_working()
gammanet_constructor = tf_fun.get_gammanet_constructor(
compression=compression,
ff_kernels=ff_kernels,
ff_repeats=ff_repeats,
features=features,
fgru_kernels=fgru_kernels)
aux = get_aux()
# Build model
with tf.variable_scope('gammanet', reuse=reuse):
activity_1 = tf.layers.conv2d(
inputs=data_tensor,
filters=gammanet_constructor[0]['features'],
kernel_size=ff_kernel_size,
padding='same',
data_format=long_data_format,
name='l0_0',
activation=ff_nl,
use_bias=True,
trainable=input_training,
reuse=reuse)
activity_2 = tf.layers.conv2d(
inputs=activity_1,
filters=gammanet_constructor[0]['features'],
kernel_size=ff_kernel_size,
padding='same',
data_format=long_data_format,
name='l0_1',
activation=ff_nl,
use_bias=True,
trainable=input_training,
reuse=reuse)
# activity = normalization.apply_normalization(
# activity=activity_2,
# name='input_norm_instance',
# normalization_type=normalization_type, # output_normalization_type,
# data_format=data_format,
# training=norm_moments_training,
# trainable=input_training,
# reuse=reuse)
# with tf.variable_scope('contour_readout', reuse=reuse):
# beta = tf.get_variable(
# name='hello_instance_beta',
# shape=[1, 1, 1, gammanet_constructor[0]['features']],
# initializer=tf.initializers.zeros,
# trainable=input_training)
# gamma = tf.get_variable(
# name='helo_instance_gamma',
# shape=[1, 1, 1, gammanet_constructor[0]['features']],
# initializer=tf.initializers.ones,
# trainable=input_training)
# activity = beta + gamma * activity_2
activity = normalization.apply_normalization(
activity=activity_2,
name='input_norm',
normalization_type=output_normalization_type,
data_format=data_format,
training=training,
trainable=training,
reuse=reuse)
with tf.variable_scope('gammanet', reuse=reuse):
# Shift conv-layers to the front
# Run fGRU
gn = gammanet.GN(
layer_name='fgru',
gammanet_constructor=gammanet_constructor,
data_format=data_format,
reuse=reuse,
timesteps=8,
train=remaining_param_training,
train_fgru_gate=fgru_gate_training,
train_ff_gate=ff_gate_training,
train_norm_moments=norm_moments_training,
train_norm_params=norm_params_training,
train_fgru_kernels=fgru_kernel_training,
train_fgru_params=fgru_param_training,
fgru_connectivity='', # 'all_to_all',
additional_readouts=additional_readouts,
fgru_normalization_type=normalization_type,
ff_normalization_type=normalization_type,
horizontal_padding='SAME',
ff_padding='SAME',
ff_nl=ff_nl,
recurrent_ff=True,
horizontal_kernel_initializer=tf.initializers.orthogonal(),
kernel_initializer=tf.initializers.orthogonal(),
gate_initializer=tf.initializers.orthogonal(),
# horizontal_kernel_initializer=tf.contrib.layers.xavier_initializer(),
# kernel_initializer=tf.contrib.layers.xavier_initializer(),
# gate_initializer=tf.contrib.layers.xavier_initializer(),
aux=aux,
strides=[1, 1, 1, 1],
pool_strides=[2, 2],
pool_kernel=[2, 2],
up_kernel=[4, 4])
layers = gn(X=activity)
h2 = layers[0]
layers = layers[1:]
# h2 = h2[0]
# h2 = gn(X=activity)
with tf.variable_scope('contour_readout', reuse=reuse):
activity = normalization.apply_normalization(
activity=h2, # activity,
name='output_norm',
normalization_type=normalization_type,
data_format=data_format,
training=training,
trainable=training,
reuse=reuse)
beta = tf.get_variable(name='hello_instance_beta',
shape=[1, 1, 1,
h2.get_shape().as_list()[-1]],
initializer=tf.initializers.zeros,
trainable=input_training)
gamma = tf.get_variable(name='helo_instance_gamma',
shape=[1, 1, 1,
h2.get_shape().as_list()[-1]],
initializer=tf.initializers.constant(0.1),
trainable=input_training)
activity = beta + gamma * activity
activity = tf.layers.conv2d(
inputs=activity,
filters=512, # gammanet_constructor[0]['features'],
kernel_size=ff_kernel_size,
padding='same',
data_format=long_data_format,
name='readout_conv',
activation=ff_nl,
use_bias=True,
reuse=reuse)
activity = normalization.apply_normalization(
activity=activity, # activity,
name='output_norm_1',
normalization_type=output_normalization_type,
data_format=data_format,
training=training,
trainable=training,
reuse=reuse)
activity = tf.layers.conv2d(
inputs=activity,
filters=128, # gammanet_constructor[0]['features'],
kernel_size=ff_kernel_size,
padding='same',
data_format=long_data_format,
name='readout_conv_1',
activation=ff_nl,
use_bias=True,
reuse=reuse)
# activity = normalization.apply_normalization(
# activity=activity,
# name='output_norm_2',
# normalization_type=output_normalization_type,
# data_format=data_format,
# training=training,
# trainable=training,
# reuse=reuse)
activity = tf.layers.conv2d(inputs=activity,
filters=output_shape,
kernel_size=ff_kernel_size,
padding='same',
data_format=long_data_format,
name='readout_conv_2',
activation=None,
use_bias=True,
reuse=reuse)
if long_data_format is 'channels_first':
activity = tf.transpose(activity, (0, 2, 3, 1))
extra_activities = {}
if activity.dtype != tf.float32:
activity = tf.cast(activity, tf.float32)
# return [activity, h_deep], extra_activities
return activity, extra_activities
def build_model(data_tensor,
reuse,
training,
output_shape,
data_format='NHWC'):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[-1]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
# normalization_type = 'no_param_instance_norm'
# output_normalization_type = 'instance_norm'
fgru_normalization_type = 'no_param_batch_norm_original'
ff_normalization_type = 'no_param_batch_norm_original'
output_normalization_type = 'batch_norm_original'
ff_kernel_size = (5, 5)
ff_nl = tf.nn.relu
data_tensor, long_data_format = tf_fun.interpret_data_format(
data_tensor=data_tensor, data_format=data_format)
# Prepare gammanet structure
(compression, ff_kernels, ff_repeats, features, fgru_kernels,
additional_readouts) = v2_big_working()
gammanet_constructor = tf_fun.get_gammanet_constructor(
compression=compression,
ff_kernels=ff_kernels,
ff_repeats=ff_repeats,
features=features,
fgru_kernels=fgru_kernels)
aux = get_aux()
# Build model
with tf.variable_scope('gammanet', reuse=reuse):
activity = tf.layers.conv2d(
inputs=data_tensor,
filters=gammanet_constructor[0]['features'],
kernel_size=ff_kernel_size,
padding='same',
# strides=(2, 2),
data_format=long_data_format,
name='l0_0',
activation=ff_nl,
use_bias=True,
reuse=reuse)
activity = normalization.apply_normalization(
activity=activity,
name='input_norm',
normalization_type=output_normalization_type,
data_format=data_format,
training=training,
trainable=training,
reuse=reuse) # + pre_activity
activity = tf.layers.conv2d(
inputs=activity,
filters=gammanet_constructor[0]['features'],
kernel_size=ff_kernel_size,
padding='same',
# strides=(2, 2),
data_format=long_data_format,
name='l0_1',
activation=ff_nl,
use_bias=True,
reuse=reuse)
# Shift conv-layers to the front
# Run fGRU
gn = gammanet.GN(
layer_name='fgru',
gammanet_constructor=gammanet_constructor,
data_format=data_format,
reuse=reuse,
timesteps=3,
fgru_connectivity='', # 'all_to_all',
additional_readouts=additional_readouts,
fgru_normalization_type=fgru_normalization_type,
ff_normalization_type=ff_normalization_type,
horizontal_padding='SAME',
ff_padding='SAME',
ff_nl=ff_nl,
stop_loop='fgru_3',
recurrent_ff=True,
horizontal_kernel_initializer=tf.initializers.orthogonal(),
kernel_initializer=tf.initializers.orthogonal(),
gate_initializer=tf.initializers.orthogonal(),
# horizontal_kernel_initializer=tf.contrib.layers.xavier_initializer(),
# gate_initializer=tf.contrib.layers.xavier_initializer(),
# gate_initializer=tf.contrib.layers.xavier_initializer(),
aux=aux,
strides=[1, 1, 1, 1],
pool_strides=[2, 2],
pool_kernel=[2, 2],
up_kernel=[4, 4],
train=training)
h2, h_deep = gn(X=activity)
with tf.variable_scope('cv_readout', reuse=reuse):
# Apply one more normalization
# activity = tf.layers.max_pooling2d(
# inputs=h2,
# pool_size=(2, 2),
# strides=(2, 2),
# padding='same',
# data_format=long_data_format,
# name='output_pool_3')
activity = normalization.apply_normalization(
activity=h2,
name='output_norm_activity',
normalization_type=output_normalization_type,
data_format=data_format,
training=training,
trainable=training,
reuse=reuse) # + pre_activity
# Then read out
activity = tf.layers.conv2d(
inputs=activity,
filters=2048,
kernel_size=ff_kernel_size,
padding='same',
# strides=(2, 2),
data_format=long_data_format,
name='readout_fc_0',
activation=ff_nl,
use_bias=True,
reuse=reuse)
activity = normalization.apply_normalization(
activity=activity,
name='readout_norm_activity',
normalization_type=output_normalization_type,
data_format=data_format,
training=training,
trainable=training,
reuse=reuse) # + pre_activity
activity = tf.reduce_mean(activity, reduction_indices=[1, 2])
activity = tf.layers.dense(inputs=activity,
units=output_shape,
activation=None,
use_bias=True,
bias_initializer=tf.constant_initializer(
1. / 1000.),
name='readout_fc_2',
reuse=reuse)
if long_data_format is 'channels_first':
activity = tf.transpose(activity, (0, 2, 3, 1))
extra_activities = {
# 'activity_low': h2,
# 'activity_high': h_deep,
}
if activity.dtype != tf.float32:
activity = tf.cast(activity, tf.float32)
# return [activity, h_deep], extra_activities
return activity, extra_activities
def build_model(data_tensor,
reuse,
training,
output_shape,
data_format='NHWC'):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[-1]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
# norm_moments_training = training # Force instance norm
# normalization_type = 'no_param_batch_norm_original'
# output_normalization_type = 'batch_norm_original_renorm'
output_normalization_type = 'instance_norm'
data_tensor, long_data_format = tf_fun.interpret_data_format(
data_tensor=data_tensor, data_format=data_format)
# Build model
with tf.variable_scope('vgg', reuse=reuse):
vgg = vgg16.Vgg16(
vgg16_npy_path=
'/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy')
vgg(rgb=data_tensor, train=training, ff_reuse=reuse)
with tf.variable_scope('fgru', reuse=reuse):
# Get side weights
h2_rem = [
vgg.conv1_2, vgg.conv2_2, vgg.conv3_3, vgg.conv4_3, vgg.conv5_3
]
res_act = []
for idx, h in enumerate(h2_rem):
res = normalization.apply_normalization(
activity=h,
name='output_norm1_%s' % idx,
normalization_type=output_normalization_type,
data_format=data_format,
training=training,
trainable=training,
reuse=reuse)
res_act += [
tf.image.resize_bilinear(
res,
data_tensor.get_shape().as_list()[1:3],
align_corners=True)
]
activity = tf.layers.conv2d(tf.concat(res_act, -1),
filters=output_shape,
kernel_size=(1, 1),
padding='same',
data_format=long_data_format,
name='out',
activation=None,
trainable=training,
use_bias=True,
reuse=reuse)
if long_data_format is 'channels_first':
activity = tf.transpose(activity, (0, 2, 3, 1))
extra_activities = {} # idx: v for idx, v in enumerate(hs_0)}
if activity.dtype != tf.float32:
activity = tf.cast(activity, tf.float32)
return activity, extra_activities
def build_model(data_tensor,
reuse,
training,
output_shape,
learned_temp=False,
data_format='NHWC'):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[-1]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
# norm_moments_training = training # Force instance norm
# normalization_type = 'no_param_batch_norm_original'
# output_normalization_type = 'batch_norm_original_renorm'
output_normalization_type = 'instance_norm'
data_tensor, long_data_format = tf_fun.interpret_data_format(
data_tensor=data_tensor, data_format=data_format)
# Build model
with tf.variable_scope('vgg', reuse=reuse):
vgg = vgg16.Vgg16(
vgg16_npy_path=
'/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy')
vgg(rgb=data_tensor, train=training, ff_reuse=reuse)
with tf.variable_scope('fgru', reuse=reuse):
# Get side weights
h2_rem = [
vgg.conv1_2, vgg.conv2_2, vgg.conv3_3, vgg.conv4_3, vgg.conv5_3
]
res_act = []
for idx, h in enumerate(h2_rem):
res = normalization.apply_normalization(
activity=h,
name='output_norm1_%s' % idx,
normalization_type=output_normalization_type,
data_format=data_format,
training=training,
trainable=training,
reuse=reuse)
res_act += [
tf.image.resize_bilinear(
res,
data_tensor.get_shape().as_list()[1:3],
align_corners=True)
]
# # 3-step readout
# (1) Per-pixel competition
activity = tf.layers.conv2d(tf.concat(res_act, -1),
filters=1,
kernel_size=(3, 3),
padding='same',
data_format=long_data_format,
name='out_0',
activation=tf.nn.sigmoid,
trainable=training,
use_bias=True,
reuse=reuse)
# (2) Softmax across locations
activity = tf.layers.conv2d(tf.concat(activity, -1),
filters=1,
kernel_size=(3, 3),
padding='same',
data_format=long_data_format,
name='out_1',
activation=None,
trainable=training,
use_bias=True,
reuse=reuse)
act_shape = activity.get_shape().as_list()
activity = tf.reshape(activity, [act_shape[0], -1])
if learned_temp:
# Potentially add another non-linearity pre-GAP for VAF:q":
temperature = tf.layers.dense(tf.reduce_mean(activity,
reduction_indices=[1
]),
1,
name='temperature',
activation=None,
use_bias=True)
sigmoid_attention = tf.nn.sigmoid(temperature)
map_activity = tf.nn.softmax(activity / sigmoid_attention)
else:
map_activity = tf.nn.softmax(activity)
map_activity = tf.reshape(map_activity, act_shape)
# (3) GAP & readout
activity = tf.reduce_mean(map_activity, reduction_indices=[1, 2])
activity = tf.layers.dense(activity,
output_shape,
name='out_2',
reuse=reuse,
activation=None,
use_bias=True)
if long_data_format is 'channels_first':
activity = tf.transpose(activity, (0, 2, 3, 1))
if learned_temp:
extra_activities = {
'temperature': temperature,
'map_activity': map_activity
}
else:
extra_activities = {'map_activity': map_activity}
if activity.dtype != tf.float32:
activity = tf.cast(activity, tf.float32)
return activity, extra_activities
def build_model(data_tensor,
reuse,
training,
output_shape,
data_format='NHWC'):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[-1]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
input_training = training # training
readout_training = training # False
norm_moments_training = training # Force instance norm
norm_params_training = training
fgru_kernel_training = training
fgru_param_training = training
ff_gate_training = training
fgru_gate_training = training
remaining_param_training = training
normalization_type = 'no_param_batch_norm_original'
output_normalization_type = 'batch_norm_original'
# normalization_type = 'no_param_instance_norm'
# output_normalization_type = 'instance_norm'
ff_kernel_size = (3, 3)
ff_nl = tf.nn.elu
data_tensor, long_data_format = tf_fun.interpret_data_format(
data_tensor=data_tensor, data_format=data_format)
# Prepare gammanet structure
(compression, ff_kernels, ff_repeats, features, fgru_kernels,
additional_readouts) = v2_big_working()
gammanet_constructor = tf_fun.get_gammanet_constructor(
compression=compression,
ff_kernels=ff_kernels,
ff_repeats=ff_repeats,
features=features,
fgru_kernels=fgru_kernels)
aux = get_aux()
# Build model
with tf.variable_scope('gammanet', reuse=reuse):
aux = get_aux()
vgg = vgg16.Vgg16(
vgg16_npy_path=
'/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy',
reuse=reuse,
aux=aux,
train=training,
timesteps=4,
fgru_normalization_type=normalization_type,
ff_normalization_type=normalization_type)
vgg(rgb=data_tensor, constructor=gammanet_constructor)
# h2_rem = [vgg.conv5_3, vgg.conv4_3, vgg.conv3_3, vgg.conv2_2, vgg.conv1_1]
h2 = vgg.fgru_0
with tf.variable_scope('contour_readout', reuse=reuse):
# Apply one more normalization
# h_deep = apply_normalization(
# activity=h_deep,
# name='output_normh_deep',
# normalization_type=normalization_type,
# data_format=data_format,
# training=training,
# reuse=reuse)
if 0:
hs = []
for h in h2_rem:
hs += [
tf.image.resize_nearest_neighbor(
h,
data_tensor.get_shape().as_list()[1:3])
]
activity = normalization.apply_normalization(
activity=tf.concat(hs, axis=-1),
name='output_norm',
normalization_type=output_normalization_type,
data_format=data_format,
training=norm_moments_training,
trainable=training,
reuse=reuse)
else:
activity = normalization.apply_normalization(
activity=h2,
name='output_norm',
normalization_type=output_normalization_type,
data_format=data_format,
training=norm_moments_training,
trainable=training,
reuse=reuse)
# Then read out
activity = tf.layers.conv2d(inputs=activity,
filters=output_shape,
kernel_size=(1, 1),
padding='same',
data_format=long_data_format,
name='readout_conv',
activation=None,
trainable=training,
use_bias=True,
reuse=reuse)
if long_data_format is 'channels_first':
activity = tf.transpose(activity, (0, 2, 3, 1))
extra_activities = {}
if activity.dtype != tf.float32:
activity = tf.cast(activity, tf.float32)
# return [activity, h_deep], extra_activities
return activity, extra_activities
def build_model(data_tensor,
reuse,
training,
output_shape,
data_format='NHWC'):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[-1]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
# norm_moments_training = training # Force instance norm
# normalization_type = 'no_param_batch_norm_original'
normalization_type = 'no_param_instance_norm'
output_normalization_type = 'batch_norm_original_renorm'
# output_normalization_type = 'instance_norm'
data_tensor, long_data_format = tf_fun.interpret_data_format(
data_tensor=data_tensor, data_format=data_format)
# Prepare gammanet structure
(compression, ff_kernels, ff_repeats, features, fgru_kernels,
additional_readouts) = v2_big_working()
gammanet_constructor = tf_fun.get_gammanet_constructor(
compression=compression,
ff_kernels=ff_kernels,
ff_repeats=ff_repeats,
features=features,
fgru_kernels=fgru_kernels)
aux = get_aux()
# Build model
with tf.variable_scope('vgg', reuse=reuse):
aux = get_aux()
vgg = vgg16.Vgg16(
vgg16_npy_path=
'/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy',
reuse=reuse,
aux=aux,
train=False, # training,
timesteps=8,
# train_norm_moments=training,
# train_norm_params=training,
fgru_normalization_type=normalization_type,
downsampled=False,
ff_normalization_type=normalization_type)
activity = vgg(rgb=data_tensor,
constructor=gammanet_constructor,
store_timesteps=True)
activity = tf.concat(activity, -1)
# activity = vgg.fgru_0
"""
with tf.variable_scope('fgru', reuse=reuse):
# Get side weights
res = normalization.apply_normalization(
activity=activity,
name='output_norm1_%s' % 0,
normalization_type=output_normalization_type,
data_format=data_format,
training=training,
trainable=training,
reuse=reuse)
# res = aux['image_resize'](
# res,
# data_tensor.get_shape().as_list()[1:3],
# align_corners=True)
"""
with tf.variable_scope('readout', reuse=reuse):
activity = tf.layers.conv2d(
inputs=activity,
filters=output_shape, # gammanet_constructor[0]['features'],
kernel_size=(1, 1),
padding='same',
data_format=long_data_format,
name='readout_l0_1',
activation=None,
use_bias=True,
trainable=training,
reuse=reuse)
# activity = tf.nn.elu(activity)
"""
activity = normalization.apply_normalization(
activity=activity,
name='output_norm_final_%s' % 1,
normalization_type=output_normalization_type,
data_format=data_format,
training=training,
trainable=training,
reuse=reuse)
activity = tf.layers.conv2d(
inputs=activity,
filters=gammanet_constructor[0]['features'],
kernel_size=(1, 1),
padding='same',
data_format=long_data_format,
name='readout_l0_2',
activation=tf.nn.elu,
use_bias=True,
trainable=training,
reuse=reuse)
bs, H, W, channels = activity.get_shape().as_list()
"""
activity = tf.reduce_max(activity, reduction_indices=[1, 2])
activity = tf.layers.dense(activity,
output_shape,
activation=None,
use_bias=True,
trainable=training,
reuse=reuse)
print("OUTPUT DIMS: {}".format(output_shape))
if long_data_format is 'channels_first':
activity = tf.transpose(activity, (0, 2, 3, 1))
extra_activities = {} # 'orientations': orientations}
if activity.dtype != tf.float32:
activity = tf.cast(activity, tf.float32)
# return [activity, h_deep], extra_activities
return activity, extra_activities
def build_model(data_tensor,
reuse,
training,
output_shape,
data_format='NHWC'):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[-1]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
# norm_moments_training = training # Force instance norm
# normalization_type = 'no_param_batch_norm_original'
normalization_type = 'no_param_instance_norm'
# output_normalization_type = 'batch_norm_original_renorm'
output_normalization_type = 'instance_norm'
data_tensor, long_data_format = tf_fun.interpret_data_format(
data_tensor=data_tensor, data_format=data_format)
data_tensor = tf.concat([data_tensor, data_tensor, data_tensor], -1)
# Prepare gammanet structure
(compression, ff_kernels, ff_repeats, features, fgru_kernels,
additional_readouts) = v2_big_working()
gammanet_constructor = tf_fun.get_gammanet_constructor(
compression=compression,
ff_kernels=ff_kernels,
ff_repeats=ff_repeats,
features=features,
fgru_kernels=fgru_kernels)
aux = get_aux()
# Build model
with tf.variable_scope('vgg', reuse=reuse):
aux = get_aux()
vgg = vgg16.Vgg16(
vgg16_npy_path=
'/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy',
reuse=reuse,
aux=aux,
train=training,
timesteps=3,
fgru_normalization_type=normalization_type,
ff_normalization_type=normalization_type)
vgg(rgb=data_tensor, constructor=gammanet_constructor)
# activity = vgg.fgru_0
layers = 4
ds_list = []
bs = data_tensor.get_shape().as_list()[0]
for idx in range(layers):
inhs = getattr(vgg, 'inh_%s' % idx)
xs = getattr(vgg, 'x_%s' % idx)
xs = tf.reshape(xs, [bs, -1])
inhs = tf.reshape(inhs, [bs, -1])
# Augmentation
# roll_choices = tf.range(
# bs - 1,
# dtype=tf.int32)
# samples = tf.multinomial(
# tf.log([tf.ones_like(tf.cast(roll_choices, tf.float32))]), 1)
# rand_roll = roll_choices[tf.cast(samples[0][0], tf.int32)]
# rolled_xs = tf.roll(xs, rand_roll, 0)
# xs = tf.concat([xs, rolled_xs], 0)
# inhs = tf.concat([inhs, inhs], 0)
rolled_xs = []
for r in range(1, bs - 1):
rolled_xs += [tf.roll(xs, r, 0)]
xs = tf.concat([xs] + rolled_xs, 0)
inhs = tf.tile(inhs, [bs - 1, 1])
# CPC distances
# denom_xs = tf.sqrt(
# tf.reduce_sum(tf.matmul(xs, xs, transpose_b=True), axis=-1))
# denom_inhs = tf.sqrt(
# tf.reduce_sum(tf.matmul(inhs, inhs, transpose_b=True), axis=-1))
# num = tf.reduce_sum(xs * inhs, axis=-1)
# ds = num / (denom_xs * denom_inhs)
xs = tf.nn.l2_normalize(xs, -1)
inhs = tf.nn.l2_normalize(inhs, -1)
ds = 1 - tf.losses.cosine_distance(
xs, inhs, axis=-1, reduction=tf.losses.Reduction.NONE)
ds = tf.reshape(ds, [bs - 1, -1])
# ds = tf.nn.softmax(ds, 0)
ds_list += [ds]
ds_list = tf.transpose(tf.concat(ds_list, -1))
return ds_list, {}
def build_model(data_tensor,
reuse,
training,
output_shape,
data_format='NHWC'):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[-1]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
output_shape = 1
# norm_moments_training = training # Force instance norm
# normalization_type = 'no_param_batch_norm_original'
normalization_type = 'no_param_instance_norm'
# output_normalization_type = 'batch_norm_original_renorm'
output_normalization_type = 'instance_norm'
# data_tensor = tf.concat((data_tensor, data_tensor, data_tensor), -1) # Expand to 3D
data_tensor, long_data_format = tf_fun.interpret_data_format(
data_tensor=data_tensor, data_format=data_format)
# Prepare gammanet structure
(compression, ff_kernels, ff_repeats, features, fgru_kernels,
additional_readouts) = v2_big_working()
gammanet_constructor = tf_fun.get_gammanet_constructor(
compression=compression,
ff_kernels=ff_kernels,
ff_repeats=ff_repeats,
features=features,
fgru_kernels=fgru_kernels)
aux = get_aux()
# Build model
with tf.variable_scope('vgg', reuse=reuse):
aux = get_aux()
vgg = vgg16.Vgg16(
vgg16_npy_path=
'/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy',
reuse=reuse,
aux=aux,
train=False, # training,
timesteps=8,
fgru_normalization_type=normalization_type,
ff_normalization_type=normalization_type)
vgg(rgb=data_tensor, constructor=gammanet_constructor)
# activity = vgg.fgru_0
with tf.variable_scope('fgru', reuse=reuse):
# Get side weights
h2_rem = [vgg.fgru_0, vgg.fgru_1, vgg.fgru_2]
activities = []
for idx, h in enumerate(h2_rem):
# res = normalization.apply_normalization(
# activity=h,
# name='output_norm1_%s' % idx,
# normalization_type=output_normalization_type,
# data_format=data_format,
# training=False,
# trainable=False,
# reuse=reuse)
activities.append(aux['image_resize'](
h, vgg.fgru_0.get_shape().as_list()[1:3], align_corners=True))
activity = tf.concat(activities, -1)
# Then read out
# activity = normalization.apply_normalization(
# activity=res,
# name='output_norm1',
# normalization_type=output_normalization_type,
# data_format=data_format,
# training=training,
# trainable=training,
# reuse=reuse)
with tf.variable_scope('readout', reuse=reuse):
activity = normalization.batch(bottom=activity,
renorm=False,
name='hgru_bn',
training=training)
activity = conv.readout_layer(activity=activity,
reuse=reuse,
training=training,
output_shape=output_shape)
if long_data_format is 'channels_first':
activity = tf.transpose(activity, (0, 2, 3, 1))
# extra_activities = {'orientations': orientations, 'model_output_y': vgg.fgru_0, 'model_output_x': vgg.fgru_0_init}
extra_activities = {}
if activity.dtype != tf.float32:
activity = tf.cast(activity, tf.float32)
# return [activity, h_deep], extra_activities
return activity, extra_activities
def build_model(data_tensor,
reuse,
training,
output_shape,
data_format='NHWC'):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[-1]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
normalization_type = 'no_param_instance_norm'
output_normalization_type = 'instance_norm'
ff_kernel_size = (3, 3)
# ff_nl = tf.nn.relu
ff_nl = tf.nn.elu
data_tensor, long_data_format = tf_fun.interpret_data_format(
data_tensor=data_tensor, data_format=data_format)
# Prepare gammanet structure
(compression, ff_kernels, ff_repeats, features, fgru_kernels,
additional_readouts) = spec()
gammanet_constructor = tf_fun.get_gammanet_constructor(
compression=compression,
ff_kernels=ff_kernels,
ff_repeats=ff_repeats,
features=features,
fgru_kernels=fgru_kernels)
aux = get_aux()
# Build model
with tf.variable_scope('gammanet', reuse=reuse):
activity = tf.layers.conv2d(
inputs=data_tensor,
filters=gammanet_constructor[0]['features'],
kernel_size=ff_kernel_size,
padding='same',
data_format=long_data_format,
name='l0_0',
activation=ff_nl,
use_bias=True,
reuse=reuse)
activity = tf.layers.conv2d(
inputs=activity,
filters=gammanet_constructor[0]['features'],
kernel_size=ff_kernel_size,
padding='same',
data_format=long_data_format,
name='l0_1',
activation=ff_nl,
use_bias=True,
reuse=reuse)
activity = normalization.apply_normalization(
activity=activity,
name='input_norm',
normalization_type=output_normalization_type,
data_format=data_format,
training=training,
trainable=training,
reuse=reuse)
# Shift conv-layers to the front
# Run fGRU
gn = gammanet.GN(
layer_name='fgru',
gammanet_constructor=gammanet_constructor,
data_format=data_format,
reuse=reuse,
timesteps=8,
fgru_connectivity='', # 'all_to_all',
additional_readouts=additional_readouts,
fgru_normalization_type=normalization_type,
ff_normalization_type=normalization_type,
horizontal_padding='SAME',
ff_padding='SAME',
ff_nl=ff_nl,
recurrent_ff=True,
horizontal_kernel_initializer=tf.initializers.orthogonal(),
kernel_initializer=tf.initializers.orthogonal(),
gate_initializer=tf.initializers.orthogonal(),
aux=aux,
strides=[1, 1, 1, 1],
pool_strides=[2, 2],
pool_kernel=[2, 2],
up_kernel=[4, 4],
train=training)
h2, h_deep = gn(X=activity)
with tf.variable_scope('cv_readout', reuse=reuse):
# Apply one more normalization
activity = normalization.apply_normalization(
activity=h2,
name='output_norm',
normalization_type=output_normalization_type,
data_format=data_format,
training=training,
trainable=training,
reuse=reuse)
activity = tf.layers.conv2d(
inputs=activity,
filters=output_shape,
kernel_size=(5, 5), # (1, 1),
padding='same',
data_format=long_data_format,
name='readout_conv_2',
activation=None,
use_bias=True,
reuse=reuse)
if long_data_format is 'channels_first':
activity = tf.transpose(activity, (0, 2, 3, 1))
extra_activities = {
'activity_low': h2,
'activity_high': h_deep,
}
if activity.dtype != tf.float32:
activity = tf.cast(activity, tf.float32)
return activity, extra_activities
def build_model(
data_tensor,
reuse,
training,
output_shape,
data_format='NHWC'):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[-1]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
# norm_moments_training = training # Force instance norm
# normalization_type = 'no_param_batch_norm_original'
normalization_type = 'no_param_instance_norm'
# output_normalization_type = 'batch_norm_original_renorm'
output_normalization_type = 'instance_norm'
data_tensor, long_data_format = tf_fun.interpret_data_format(
data_tensor=data_tensor,
data_format=data_format)
# Prepare gammanet structure
(
compression,
ff_kernels,
ff_repeats,
features,
fgru_kernels,
additional_readouts) = v2_big_working()
gammanet_constructor = tf_fun.get_gammanet_constructor(
compression=compression,
ff_kernels=ff_kernels,
ff_repeats=ff_repeats,
features=features,
fgru_kernels=fgru_kernels)
aux = get_aux()
# Build model
with tf.variable_scope('vgg', reuse=reuse):
aux = get_aux()
vgg = vgg16.Vgg16(
vgg16_npy_path='/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy',
reuse=reuse,
aux=aux,
train=training,
timesteps=8,
fgru_normalization_type=normalization_type,
ff_normalization_type=normalization_type)
vgg(rgb=data_tensor, constructor=gammanet_constructor)
# activity = vgg.fgru_0
with tf.variable_scope('fgru', reuse=reuse):
# Get side weights
hs_0, hs_1 = [], []
h2_rem = [
vgg.fgru_0,
vgg.fgru_1,
vgg.fgru_2,
vgg.conv5_1,
vgg.fgru_3]
for idx, h in enumerate(h2_rem):
res = normalization.apply_normalization(
activity=h,
name='output_norm1_%s' % idx,
normalization_type=output_normalization_type,
data_format=data_format,
training=training,
trainable=training,
reuse=reuse)
res = aux['image_resize'](
res,
data_tensor.get_shape().as_list()[1:3],
align_corners=True)
cnv_0 = tf.layers.conv2d(
inputs=res,
filters=output_shape,
kernel_size=(1, 1),
padding='same',
data_format=long_data_format,
name='readout_aux_00_%s' % idx,
# activation=tf.nn.relu,
activation=None,
trainable=training,
use_bias=True,
reuse=reuse)
cnv_1 = tf.layers.conv2d(
inputs=res,
filters=output_shape,
kernel_size=(1, 1),
padding='same',
data_format=long_data_format,
name='readout_aux_10_%s' % idx,
# activation=tf.nn.relu,
activation=None,
trainable=training,
use_bias=True,
reuse=reuse)
hs_0 += [cnv_0]
hs_1 += [cnv_1]
s1, s2, s3, s4, s5 = hs_0
s11, s21, s31, s41, s51 = hs_1
o1 = tf.stop_gradient(tf.identity(s1))
o2 = tf.stop_gradient(tf.identity(s2))
o3 = tf.stop_gradient(tf.identity(s3))
o4 = tf.stop_gradient(tf.identity(s4))
o21 = tf.stop_gradient(tf.identity(s21))
o31 = tf.stop_gradient(tf.identity(s31))
o41 = tf.stop_gradient(tf.identity(s41))
o51 = tf.stop_gradient(tf.identity(s51))
p1_1 = s1
p2_1 = s2 + o1
p3_1 = s3 + o2 + o1
p4_1 = s4 + o3 + o2 + o1
p5_1 = s5 + o4 + o3 + o2 + o1
p1_2 = s11 + o21 + o31 + o41 + o51
p2_2 = s21 + o31 + o41 + o51
p3_2 = s31 + o41 + o51
p4_2 = s41 + o51
p5_2 = s51
hs = [p1_1, p2_1, p3_1, p4_1, p5_1, p1_2, p2_2, p3_2, p4_2, p5_2]
activity = tf.layers.conv2d(
tf.concat(hs, -1),
filters=output_shape,
kernel_size=(1, 1),
padding='same',
data_format=long_data_format,
name='out',
# activation=tf.nn.relu,
activation=None,
trainable=training,
use_bias=False,
reuse=reuse)
if long_data_format is 'channels_first':
activity = tf.transpose(activity, (0, 2, 3, 1))
extra_activities = {idx: v for idx, v in enumerate(hs_0)}
if activity.dtype != tf.float32:
activity = tf.cast(activity, tf.float32)
# return [activity, h_deep], extra_activities
return activity, extra_activities
def build_model(data_tensor,
labels,
reuse,
training,
output_shape,
perturb_norm=False,
data_format='NHWC'):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[-1]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
# norm_moments_training = training # Force instance norm
# normalization_type = 'no_param_batch_norm_original'
normalization_type = 'no_param_instance_norm'
# output_normalization_type = 'batch_norm_original_renorm'
output_normalization_type = 'instance_norm'
data_tensor, long_data_format = tf_fun.interpret_data_format(
data_tensor=data_tensor, data_format=data_format)
# Prepare gammanet structure
(compression, ff_kernels, ff_repeats, features, fgru_kernels,
additional_readouts) = v2_big_working()
gammanet_constructor = tf_fun.get_gammanet_constructor(
compression=compression,
ff_kernels=ff_kernels,
ff_repeats=ff_repeats,
features=features,
fgru_kernels=fgru_kernels)
aux = get_aux()
# Build model
with tf.variable_scope('vgg', reuse=reuse):
aux = get_aux()
# moments_file = "../undo_bias/neural_models/linear_moments/INSILICO_BSDS_vgg_gratings_simple_tb_feature_matrix.npz"
# model_file = "../undo_bias/neural_models/linear_models/INSILICO_BSDS_vgg_gratings_simple_tb_model.joblib.npy"
fb_moments_file = "../undo_bias/neural_models/linear_moments/tb_feature_matrix.npz"
fb_model_file = "../undo_bias/neural_models/linear_models/tb_model.joblib.npy"
ff_moments_file = "../undo_bias/neural_models/linear_moments/conv2_2_tb_feature_matrix.npz"
ff_model_file = "../undo_bias/neural_models/linear_models/conv2_2_tb_model.joblib.npy"
vgg = vgg16.Vgg16(
vgg16_npy_path=
'/media/data_cifs_lrs/clicktionary/pretrained_weights/vgg16.npy',
reuse=reuse,
aux=aux,
moments_file=ff_moments_file, # Perturb FF drive
model_file=ff_model_file,
train=False,
timesteps=8,
perturb=0.95, # 2., # 1.001, # 17.1,
# perturb=.0000001, # 2., # 1.001, # 17.1,
# perturb=.05, # 2., # 1.001, # 17.1
perturb_norm=perturb_norm,
# perturb=1.5, # 2., # 1.001, # 17.1,
# perturb=2., # 2., # 1.001, # 17.1,
fgru_normalization_type=normalization_type,
ff_normalization_type=normalization_type)
# gn = tf.get_default_graph()
# with gn.gradient_override_map({'Conv2D': 'PerturbVizGrad'}):
# Scope the entire GN (with train=False).
# The lowest-level recurrent tensor is the only
# trainable tensor. This grad op will freeze the
# center unit, forcing other units to overcompensate
# to recreate a model's prediction.
# TODO: need to get the original model output... could precompute this. # noqa
vgg(rgb=data_tensor, label=labels, constructor=gammanet_constructor)
activity = vgg.fgru_0
# Load tuning curve transform for fb output
moments = np.load(fb_moments_file)
means = moments["means"]
stds = moments["stds"]
clf = np.load(fb_model_file).astype(np.float32)
# Transform activity to outputs
bs, h, w, _ = vgg.fgru_0.get_shape().as_list()
hh, hw = h // 2, w // 2
sel_units = tf.reshape(vgg.fgru_0[:, hh - 2:hh + 2, hw - 2:hw + 2, :],
[bs, -1])
if perturb_norm:
# Normalize activities -- Not normalized!!
sel_units = (sel_units - means) / stds
# Map responses
# inv_clf = np.linalg.inv(clf.T.dot(clf)).astype(np.float32) # Precompute inversion
# inv_clf = tf.linalg.inv(tf.matmul(clf, clf, transpose_a=True))
inv_clf = np.linalg.inv(clf.T.dot(clf)).astype(
np.float32) # Precompute inversion
activity = tf.matmul(tf.matmul(inv_clf, clf, transpose_b=True),
sel_units,
transpose_b=True)
# bg = tf.reduce_mean(vgg.conv2_2 ** 2, reduction_indices=[-1], keep_dims=True)
# bg = tf.cast(tf.greater(bg, tf.reduce_mean(bg)), tf.float32)
# bg_dil = dilation2d(img=bg, extent=5)
# extra_activities = {"mask": bg, "mask_dil": bg_dil} # {"mask": tf.reduce_mean(vgg.conv2_2 ** 2, reduction_indices=[-1])} # tf.get_variable(name="perturb_viz")} # idx: v for idx, v in enumerate(hs_0)}
# extra_activities = {"fgru": vgg.fgru_0, "penalty": tf.constant(0.), "conv": vgg.error_1} # tf.get_variable(name="perturb_viz")} # idx: v for idx, v in enumerate(hs_0)}
# extra_activities = {"fgru": vgg.fgru_0, "penalty": tf.constant(0.), "conv": vgg.error_1} # tf.get_variable(name="perturb_viz")} # idx: v for idx, v in enumerate(hs_0)}
extra_activities = {}
if activity.dtype != tf.float32:
activity = tf.cast(activity, tf.float32)
# return [activity, h_deep], extra_activities
return activity, extra_activities
def build_model(data_tensor,
reuse,
training,
output_shape,
data_format='NHWC'):
"""Create the hgru from Learning long-range..."""
if isinstance(output_shape, list):
output_shape = output_shape[-1]
elif isinstance(output_shape, dict):
output_shape = output_shape['output']
# norm_moments_training = training # Force instance norm
# normalization_type = 'no_param_batch_norm_original'
normalization_type = 'no_param_instance_norm'
# output_normalization_type = 'batch_norm_original_renorm'
output_normalization_type = 'instance_norm'
data_tensor, long_data_format = tf_fun.interpret_data_format(
data_tensor=data_tensor, data_format=data_format)
# Prepare gammanet structure
(compression, ff_kernels, ff_repeats, features, fgru_kernels,
additional_readouts) = v2_big_working()
gammanet_constructor = tf_fun.get_gammanet_constructor(
compression=compression,
ff_kernels=ff_kernels,
ff_repeats=ff_repeats,
features=features,
fgru_kernels=fgru_kernels)
aux = get_aux()
# Build model
with tf.variable_scope('vgg', reuse=reuse):
aux = get_aux()
vgg = vgg16.Vgg16(
vgg16_npy_path=
'/media/data_cifs/clicktionary/pretrained_weights/vgg16.npy',
reuse=reuse,
aux=aux,
train=False,
timesteps=8,
fgru_normalization_type=normalization_type,
ff_normalization_type=normalization_type)
vgg(rgb=data_tensor, constructor=gammanet_constructor)
activity = vgg.fgru_0
with tf.variable_scope('fgru_gilbert', reuse=reuse):
# Then read out
# activity = activity ** 2
activity = normalization.apply_normalization(
activity=activity,
name='output_norm1',
normalization_type=output_normalization_type,
data_format=data_format,
training=training,
trainable=training,
reuse=reuse)
activity = tf.layers.conv2d(
inputs=activity,
filters=gammanet_constructor[0]['features'],
kernel_size=(1, 1),
padding='same',
data_format=long_data_format,
name='readout_l0_1',
activation=tf.nn.relu,
use_bias=True,
trainable=training,
reuse=reuse)
activity = tf.layers.conv2d(inputs=activity,
filters=1,
kernel_size=(1, 1),
padding='same',
data_format=long_data_format,
name='readout_l0_2',
activation=None,
use_bias=True,
trainable=training,
reuse=reuse)
activity = tf.reduce_max(activity, reduction_indices=[1, 2])
if long_data_format is 'channels_first':
activity = tf.transpose(activity, (0, 2, 3, 1))
extra_activities = {}
if activity.dtype != tf.float32:
activity = tf.cast(activity, tf.float32)
# return [activity, h_deep], extra_activities
return activity, extra_activities
