def _gan28_discr(experiment=None, X=None, reuse=False, **kwargs):
lrelu = partial(leaky_relu, leakiness=experiment.leak)
h_dim = experiment.h_dim
logger = experiment.logger
msg = "D_SHAPE {} {} [reuse={}]"
logger.debug(msg.format("in", X.shape, reuse))
with tf.variable_scope("discriminator", reuse=reuse):
with tf.variable_scope("hidden1"):
d_hidden1 = conv2d(
X, filters=h_dim, kernel_size=5, strides=2, padding="same",
activation=lrelu, kernel_initializer=xavier_init())
logger.debug(msg.format("dh1", d_hidden1.shape, reuse))
with tf.variable_scope("hidden2"):
d_hidden2 = conv2d(
d_hidden1, filters=h_dim * 2, kernel_size=5, strides=2,
padding="same", activation=lrelu,
kernel_initializer=xavier_init())
logger.debug(msg.format("dh2", d_hidden2.shape, reuse))
with tf.variable_scope("hidden3"):
d_hidden2 = tf.reshape(
d_hidden2, [-1, np.prod(d_hidden2.shape[1:], dtype=int)])
d_hidden3 = dense(d_hidden2, 1024, activation=lrelu,
kernel_initializer=xavier_init())
logger.debug(msg.format("dh3", d_hidden3.shape, reuse))
with tf.variable_scope("output"):
d_out = dense(d_hidden3, 1, kernel_initializer=xavier_init())
logger.debug(msg.format("out", d_out.shape, reuse))
# define summaries on the last layer
d_summaries = stats_summaries(d_out)
return d_out, d_summaries
def apply_constraints():
logger.debug("Using constraints: {}".format(
str(experiment.constraints)))
with tf.variable_scope("constrained_ll_out"):
constraints_out = dense(
constraints_features, 1, activation=lrelu,
kernel_initializer=xavier_init())
d_out_kernel = tf.get_variable(
"d_out_kernel", shape=[2, 1], initializer=xavier_init())
logger.debug(
msg.format("d_out_kernel", d_out_kernel.shape, reuse))
d_out_bias = tf.get_variable(
"d_out_bias", shape=[1, 1], initializer=xavier_init())
logger.debug(msg.format("d_out_bias", d_out_bias.shape, reuse))
input_concat = tf.concat(
[d_out, constraints_out],
axis=1, name="input_concat_{}".format(reuse))
logger.debug(msg.format(
"input_concat", input_concat.shape, reuse))
d_constrained_out = tf.add(
tf.matmul(input_concat, d_out_kernel), d_out_bias,
name="d_constrained_out_{}".format(reuse))
logger.debug(msg.format(
"constrained_out", d_constrained_out.shape, reuse))
# define summaries on the last layer
d_summaries = tf.summary.merge(
[stats_summaries(d_constrained_out), d_out_summaries])
return d_constrained_out, d_summaries
def build_ae(incoming, num, cp, SEED, reuse=False):
act_dict = get_act_dictionary()
enclabel = 'Encoder' + str(num)
declabel = 'Decoder' + str(num)
# Enable/Disable sharing weights
if reuse:
tf.get_variable_scope().reuse_variables()
with tf.name_scope(enclabel):
# Input
encoder = incoming
# Dropout
encoder = tf.layers.dropout(encoder,
float(cp.get('Dropout', 'rate')),
seed=SEED)
# Hidden Layer(s)
for sect in [i for i in cp.sections() if enclabel in i]:
if cp.get('Experiment', 'PREFIX') == 'reu100k':
encoder = tf.layers.dense(
encoder,
cp.getint(sect, 'Width'),
activation=act_dict[cp.get(sect, 'Activation')],
name='Pre_' + sect,
kernel_initializer=xavier_init(uniform=False, seed=SEED))
else:
encoder = tf.layers.dense(
encoder,
cp.getint(sect, 'Width'),
activation=act_dict[cp.get(sect, 'Activation')],
name='Pre_' + sect,
kernel_initializer=normal_init(mean=0.0,
stddev=0.01,
seed=SEED))
with tf.name_scope(declabel):
decoder = encoder
# Decoder Layer(s)
for sect in [i for i in cp.sections() if declabel in i]:
if cp.get('Experiment', 'PREFIX') == 'reu100k':
decoder = tf.layers.dense(
decoder,
cp.getint(sect, 'Width'),
activation=act_dict[cp.get(sect, 'Activation')],
name='Pre_' + sect,
kernel_initializer=xavier_init(uniform=False, seed=SEED))
else:
decoder = tf.layers.dense(
decoder,
cp.getint(sect, 'Width'),
activation=act_dict[cp.get(sect, 'Activation')],
name='Pre_' + sect,
kernel_initializer=normal_init(mean=0.0,
stddev=0.01,
seed=SEED))
return decoder, encoder
def discriminator(self, x, reuse=False):
with tf.variable_scope("discriminator", reuse=reuse):
d_w1 = tf.get_variable('d_w1',
[self.config.out_dim, self.config.h_dim],
initializer=xavier_init())
d_b1 = tf.get_variable('d_b1', [self.config.h_dim],
initializer=tf.zeros_initializer())
d_w2 = tf.get_variable('d_w2', [self.config.h_dim, 1],
initializer=xavier_init())
d_b2 = tf.get_variable('d_b2', [1],
initializer=tf.zeros_initializer())
h = tf.nn.relu(tf.matmul(x, d_w1) + d_b1)
logits = tf.matmul(h, d_w2) + d_b2
return tf.nn.sigmoid(logits), logits
def generator(self, z):
with tf.variable_scope("generator") as scope:
g_w1 = tf.get_variable("g_w1",
[self.config.z_dim, self.config.h_dim],
initializer=xavier_init())
g_b1 = tf.get_variable("g_b1", [self.config.h_dim],
initializer=tf.zeros_initializer())
g_w2 = tf.get_variable("g_w2",
[self.config.h_dim, self.config.out_dim],
initializer=xavier_init())
g_b2 = tf.get_variable("g_b2", [self.config.out_dim],
initializer=tf.zeros_initializer())
h = tf.nn.relu(tf.matmul(z, g_w1) + g_b1)
return tf.nn.sigmoid(tf.matmul(h, g_w2) + g_b2)
def make_wbkernels(
self, name, shp=None, initializer=xavier_init(uniform=False),
):
""" make weights, biases, kernels """
return tf.get_variable(
name, shp, initializer=initializer, regularizer=self.reg
)
def apply_constraints():
logger.debug("Using constraints: {}".format(
str(experiment.constraints)))
with tf.variable_scope("constrained_out"):
d_constraints_kernel = tf.get_variable(
"d_constraints_kernel",
shape=[constraints_features.shape[1], 1],
initializer=xavier_init())
logger.debug(msg.format(
"d_constraints_kernel", d_constraints_kernel.shape, reuse))
input_concat = tf.concat(
[d_hidden4, constraints_features],
axis=1, name="input_concat_{}".format(reuse))
logger.debug(msg.format(
"input_concat", input_concat.shape, reuse))
weight_concat = tf.concat(
[d_out_kernel, d_constraints_kernel],
axis=0, name="weight_concat_{}".format(reuse))
logger.debug(msg.format(
"weight_concat", weight_concat.shape, reuse))
d_constrained_out = tf.add(
tf.matmul(input_concat, weight_concat), d_out_bias,
name="d_constrained_out_{}".format(reuse))
logger.debug(msg.format(
"constrained_out", d_constrained_out.shape, reuse))
# define summaries on the last layer
d_summaries = tf.summary.merge(
[stats_summaries(d_constrained_out),
d_summaries_dh4])
return d_constrained_out, d_summaries
def make_fc_layer(self,
inp_lyr,
name_fc_lyr,
name_w,
shp_w,
name_b=None,
shp_b=None,
initializer=xavier_init(uniform=False)):
""" TODO - regularize batch norm params? """
W = self.make_wbkernels(name_w, shp_w, initializer=initializer)
b = self.make_wbkernels(name_b,
shp_b,
initializer=tf.zeros_initializer())
fc_lyr = tf.nn.bias_add(
tf.matmul(inp_lyr, W, name=name_fc_lyr + '_matmul'),
b,
data_format=self.data_format,
name=name_fc_lyr,
)
if self.use_batch_norm:
fc_lyr = tf.contrib.layers.batch_norm(fc_lyr,
decay=self.batch_norm_decay,
center=True,
scale=True,
data_format=self.data_format,
is_training=self.is_training)
return fc_lyr
def make_wbkernels(
self, name, shp=None, initializer=xavier_init(uniform=False),
):
""" make weights, biases, kernels """
return tf.get_variable(
name, shp, initializer=initializer, regularizer=self.reg
)
def fully_connected(input, num_output, name):
# does input*weight, expects 2D input
with tf.variable_scope(name) as scope:
weight = tf.get_variable("weight",
shape=[input.shape.as_list()[1], num_output],
initializer=xavier_init())
return tf.matmul(input, weight)
def make_active_fc_layer(
self, inp_lyr, name_fc_lyr,
name_w, shp_w, name_b=None, shp_b=None,
act=tf.nn.relu, initializer=xavier_init(uniform=False)
):
return act(self.make_fc_layer(
inp_lyr, name_fc_lyr, name_w, shp_w, name_b, shp_b,
initializer=initializer
), name=name_fc_lyr+'_act')
def build_from_inputs(self, inputs, output_space, trainable=True):
net = inputs
for i, units in enumerate(self.layer_sizes):
net = tf.layers.dense(net,
units=units,
activation=tf.nn.relu,
trainable=trainable,
name='fc%d' % i)
v = tf.layers.dense(net,
units=1,
activation=None,
kernel_initializer=xavier_init(),
bias_initializer=xavier_init(),
name='out_value',
trainable=trainable)
v = tf.squeeze(v, axis=1)
out_pi = policy_head(net, output_space, trainable)
return {'input': inputs, 'out': out_pi, 'policy': out_pi, 'value': v}
def add_conv(X, shape, strides, padding = 'SAME'):
'''Create convolution layer with relu activation. Kernel weights and biases are
initalized with Xavier initialization and zero respectivly.'''
kernel = tf.get_variable('kernel',shape = shape, initializer = xavier_init())
conv = tf.nn.conv2d(input = X, filter = kernel, strides = strides, padding = padding)
bias = tf.get_variable('bias',shape = [shape[-1]],
initializer = tf.constant_initializer(value = 0.0))
relu = tf.nn.relu(features = conv + bias)
return relu
def policy_head(input_layer, output_space, trainable=True):
if not isinstance(output_space, Continuous):
pi = tf.layers.dense(input_layer,
units=output_space.shape[0],
activation=tf.nn.softmax,
kernel_initializer=xavier_init(),
bias_initializer=xavier_init(),
name='out_policy',
trainable=trainable)
return pi
mean = tf.layers.dense(input_layer,
units=output_space.shape[0],
activation=tf.nn.tanh,
kernel_initializer=xavier_init(),
bias_initializer=xavier_init(),
name='mean',
trainable=trainable)
std = tf.layers.dense(input_layer,
units=output_space.shape[0],
activation=tf.nn.softplus,
kernel_initializer=xavier_init(),
bias_initializer=xavier_init(),
name='std',
trainable=trainable)
mean = tf.squeeze(mean)
std = tf.squeeze(std)
dist = tf.contrib.distributions.Normal(mean, std)
pi = tf.clip_by_value(dist.sample(1), output_space.low, output_space.high)
return pi
def make_active_fc_layer(
self, inp_lyr, name_fc_lyr,
name_w, shp_w, name_b=None, shp_b=None,
act=tf.nn.relu, initializer=xavier_init(uniform=False)
):
with tf.variable_scope(name_fc_lyr):
return act(
self.make_fc_layer(
inp_lyr, name_fc_lyr, name_w, shp_w, name_b, shp_b,
initializer=initializer
),
name=name_fc_lyr+'_act'
)
def add_fc(X, n_in, n_out, act_func = True):
'''Create feed forward layer. Weights and biases are initalized with
Xavier initialization and zero respectivly. If act_func = True then
relu is used and non-linear layer is returned. If act_func = False then logits are
returned for softmax classification.'''
weights = tf.get_variable('weights', shape = [n_in, n_out], initializer = xavier_init())
biases = tf.get_variable('bias', shape = [n_out],
initializer = tf.constant_initializer(value = 0.0))
z = tf.matmul(X, weights) + biases
if act_func:
return tf.nn.relu(features = z)
else:
return z
def build_EviTRAM(cp, SEED):
# Initialize I/O tensors
conv_In = tf.placeholder(tf.float32,
shape=[None, cp.getint('Input', 'Width')],
name='conv_IN')
# Initiliaze placeholders for each source of evidence
sect = 'Experiment'
ev_paths = [cp.get(sect, i) for i in cp.options(sect) if 'evidence' in i]
ks_IN = []
for ev_path_id, ev_path in enumerate(ev_paths):
ks_IN.append(tf.placeholder(tf.float32, shape=[
None, cp.getint('Q' + str(ev_path_id), 'Width')],
name='k_IN' + str(ev_path_id)))
# Building network
conv_Z, last_shape = build_encoder(conv_In, cp, SEED)
conv_Xrec, conv_Prev = build_decoder(conv_Z, cp, SEED, last_shape)
# Initialize additional prediction layer to minimize cross entropy,
# for each source of evidence
Qs = []
for ev_path_id, ev_path in enumerate(ev_paths):
with tf.name_scope('COND' + str(ev_path_id)):
# Get activation ditionary
act_dict = get_act_dictionary()
sect = 'Q' + str(ev_path_id)
Q = tf.layers.dense(conv_Xrec,
cp.getint(sect, 'Width'),
activation=act_dict[cp.get(
sect, 'Activation')],
name='Pre_' + sect,
kernel_initializer=xavier_init(uniform=False,
seed=SEED),
reuse=tf.AUTO_REUSE)
Qs.append(Q)
TV = [v for v in tf.trainable_variables() if 'Pre_' in v.name or
'beta' in v.name]
# Building loss of EviTRAM
cond_loss, cond_lr, cond_t_op, px_mse = EviTRAM_loss(conv_In, conv_Xrec,
conv_Z, Qs, ks_IN, TV)
ret_dict = {'conv_in': conv_In, 'conv_z': conv_Z, 'conv_out': conv_Xrec,
'conv_prev': conv_Prev, 'Qs': Qs, 'ks_IN': ks_IN, 'TV': TV,
'evitram_t_op': cond_t_op, 'evitram_loss': cond_loss,
'px_mse': px_mse, 'evitram_lr': cond_lr}
return ret_dict
def build_encoder(incoming, cp, SEED, scope='px_encoder', reuse=False):
# Get activation ditionary
act_dict = get_act_dictionary()
# Enable/Disable sharing weights
if reuse:
tf.get_variable_scope().reuse_variables()
with tf.name_scope(scope):
# Reshape input to [samples, height, width, channels]
try:
channels = cp.getint('Input', 'Channels')
height = width = int(
np.sqrt(cp.getint('Input', 'Width') / channels))
cae = tf.reshape(
incoming, shape=[-1, height, width, channels])
except:
raise ValueError('width x height matrix must be square')
# Add convolutional layers
for sect in [i for i in cp.sections() if 'Conv' in i]:
cae = tf.layers.conv2d(cae,
filters=cp.getint(sect, 'Filters'),
kernel_size=cp.getint(
sect, 'Fsize'),
strides=cp.getint(sect, 'Stride'),
padding=cp.get(sect, 'Pad'),
activation=act_dict['ReLU'],
name='Pre_' + sect,
data_format='channels_last')
cae = tf.contrib.layers.batch_norm(cae,
scope=sect.split('Conv')[1])
# Store shape for later reshaping
last_shape = cae.shape[1:]
cae = tf.layers.flatten(cae)
keep_prob = float(cp.get('Dropout', 'Rate'))
cae = tf.layers.dropout(cae, rate=keep_prob)
# Most inner layer of AE / Last layer of encoder
sect = 'Encoder0'
cae = tf.layers.dense(cae,
cp.getint(sect, 'Width'),
activation=act_dict[cp.get(
sect, 'Activation')],
name='Pre_' + sect,
kernel_initializer=xavier_init
(uniform=False, seed=SEED)
)
return cae, last_shape
def build_decoder(incoming,
cp,
SEED,
last_shape,
scope='px_decoder',
reuse=False):
# Get activation ditionary
act_dict = get_act_dictionary()
# Enable/Disable sharing weights
if reuse:
tf.get_variable_scope().reuse_variables()
with tf.name_scope(scope):
cae = incoming
for sect in [i for i in cp.sections() if 'Encoder' in i]:
if sect != 'Encoder0':
cae = tf.layers.dense(
cae,
cp.getint(sect, 'Width'),
activation=act_dict[cp.get(sect, 'Activation')],
name='Pre_' + sect,
kernel_initializer=xavier_init(uniform=False, seed=SEED))
# Reshape to [sample, width, height, channels]
cae = tf.reshape(
cae, shape=[-1, last_shape[0], last_shape[1], last_shape[2]])
decon_lst = [i for i in cp.sections() if 'DeCon' in i]
decon_num = len(decon_lst)
# Add deconvolutional layers
for sect in decon_lst:
cae = tf.layers.conv2d_transpose(
cae,
filters=cp.getint(sect, 'Filters'),
kernel_size=cp.getint(sect, 'Fsize'),
strides=cp.getint(sect, 'Stride'),
padding=cp.get(sect, 'Pad'),
activation=act_dict['ReLU'],
name='Pre_De_' + sect,
data_format='channels_last')
# Return last layer before reconstruction
if sect == decon_lst[decon_num - 2]:
prev = cae
prev = tf.contrib.layers.max_pool2d(cae, (2, 2))
prev = tf.layers.flatten(prev)
# Flatten output
cae = tf.layers.flatten(cae)
return cae, prev
def build_from_inputs(self, inputs, output_space, trainable=True):
net = tf.layers.conv2d(inputs=inputs,
filters=16,
kernel_size=[8, 8],
strides=[4, 4],
activation=tf.nn.relu,
padding="same",
name="conv1",
trainable=trainable,
kernel_initializer=xavier_init(),
bias_initializer=xavier_init())
net = tf.layers.conv2d(inputs=net,
filters=32,
kernel_size=[4, 4],
strides=[2, 2],
activation=tf.nn.relu,
padding="same",
name="conv2",
trainable=trainable,
kernel_initializer=xavier_init(),
bias_initializer=xavier_init())
net = tf.layers.flatten(net)
fc1 = tf.layers.dense(net,
units=256,
activation=tf.nn.relu,
name='fc1',
trainable=trainable,
kernel_initializer=xavier_init(),
bias_initializer=xavier_init())
v = tf.layers.dense(fc1,
units=1,
activation=None,
kernel_initializer=xavier_init(),
bias_initializer=xavier_init(),
name='out_value',
trainable=trainable)
v = tf.squeeze(v, axis=1)
out_pi = policy_head(net, output_space, trainable)
return {'input': inputs, 'out': out_pi, 'policy': out_pi, 'value': v}
def make_fc_layer(
self, inp_lyr, name_fc_lyr,
name_w, shp_w, name_b=None, shp_b=None,
initializer=xavier_init(uniform=False)
):
""" Note: `name_fc_lyr` is a _basename_ for the ops """
# TODO - regularize batch norm params?
W = self.make_wbkernels(name_w, shp_w, initializer=initializer)
b = self.make_wbkernels(
name_b, shp_b, initializer=tf.zeros_initializer()
)
fc_lyr = tf.nn.bias_add(
tf.matmul(inp_lyr, W, name=name_fc_lyr+'_matmul'), b,
data_format=self.data_format, name=name_fc_lyr+'_plus_biases',
)
if self.use_batch_norm:
fc_lyr = tf.contrib.layers.batch_norm(
fc_lyr, decay=self.batch_norm_decay, center=True, scale=True,
data_format=self.data_format, is_training=self.is_training
)
return fc_lyr
def _can60_discr_32layer_auto(
experiment=None, X=None, reuse=False, use_constraints=None,
constraints_features=None, **kwargs):
lrelu = partial(leaky_relu, leakiness=experiment.leak)
h_dim = experiment.h_dim
logger = experiment.logger
msg = "D_SHAPE {} {} [reuse={}]"
logger.debug(msg.format("in", X.shape, reuse))
with tf.variable_scope("discriminator", reuse=reuse):
with tf.variable_scope("hidden1"):
d_hidden1 = conv2d(
X, filters=h_dim, kernel_size=5, strides=2, padding="same",
activation=lrelu, kernel_initializer=xavier_init())
logger.debug(msg.format("dh1", d_hidden1.shape, reuse))
with tf.variable_scope("hidden15"):
d_hidden15 = conv2d(
d_hidden1, filters=h_dim, kernel_size=5, strides=2,
padding="same", activation=lrelu,
kernel_initializer=xavier_init())
logger.debug(msg.format("dh15", d_hidden15.shape, reuse))
with tf.variable_scope("hidden2"):
d_hidden2 = conv2d(
d_hidden15, filters=h_dim * 2, kernel_size=5, strides=2,
padding="same", activation=lrelu,
kernel_initializer=xavier_init())
logger.debug(msg.format("dh2", d_hidden2.shape, reuse))
d_hidden2 = tf.reshape(
d_hidden2, [-1, np.prod(d_hidden2.shape[1:], dtype=int)])
logger.debug(msg.format("dh2", d_hidden2.shape, reuse))
with tf.variable_scope("hidden3"):
d_hidden3 = dense(d_hidden2, 1024, activation=lrelu,
kernel_initializer=xavier_init())
logger.debug(msg.format("dh3", d_hidden3.shape, reuse))
with tf.variable_scope("hidden4"):
d_hidden4 = dense(d_hidden3, 32, activation=lrelu,
kernel_initializer=xavier_init())
logger.debug(msg.format("dh4", d_hidden4.shape, reuse))
d_summaries_dh4 = stats_summaries(d_hidden4, "dh4_pre_cond")
with tf.variable_scope("shared_weights"):
d_out_kernel = tf.get_variable(
"d_out_kernel", shape=[32, 1], initializer=xavier_init())
logger.debug(msg.format("d_out_kernel", d_out_kernel.shape, reuse))
d_out_bias = tf.get_variable(
"d_out_bias", shape=[1, 1], initializer=xavier_init())
logger.debug(msg.format("d_out_bias", d_out_bias.shape, reuse))
def skip_constraints():
with tf.variable_scope("output"):
d_out = tf.add(tf.matmul(d_hidden4, d_out_kernel),
d_out_bias, name="d_out_{}".format(reuse))
logger.debug(msg.format("out", d_out.shape, reuse))
# define summaries on the last layer
d_summaries = tf.summary.merge(
[stats_summaries(d_out), d_summaries_dh4])
return d_out, d_summaries
def apply_constraints():
logger.debug("Using constraints: {}".format(
str(experiment.constraints)))
with tf.variable_scope("constrained_out"):
d_constraints_kernel = tf.get_variable(
"d_constraints_kernel",
shape=[constraints_features.shape[1], 1],
initializer=xavier_init())
logger.debug(msg.format(
"d_constraints_kernel", d_constraints_kernel.shape, reuse))
input_concat = tf.concat(
[d_hidden4, constraints_features],
axis=1, name="input_concat_{}".format(reuse))
logger.debug(msg.format(
"input_concat", input_concat.shape, reuse))
weight_concat = tf.concat(
[d_out_kernel, d_constraints_kernel],
axis=0, name="weight_concat_{}".format(reuse))
logger.debug(msg.format(
"weight_concat", weight_concat.shape, reuse))
d_constrained_out = tf.add(
tf.matmul(input_concat, weight_concat), d_out_bias,
name="d_constrained_out_{}".format(reuse))
logger.debug(msg.format(
"constrained_out", d_constrained_out.shape, reuse))
# define summaries on the last layer
d_summaries = tf.summary.merge(
[stats_summaries(d_constrained_out),
d_summaries_dh4])
return d_constrained_out, d_summaries
return tf.cond(tf.cast(use_constraints, tf.bool),
skip_constraints, apply_constraints)
def build_EviTRAM(cp, ae_ids, SEED):
sae_dict = build_px(cp, ae_ids, SEED)
# Initiliaze placeholders for each source of evidence
sect = 'Experiment'
ev_paths = [cp.get(sect, i) for i in cp.options(sect) if 'evidence' in i]
ks_IN = []
for ev_path_id, ev_path in enumerate(ev_paths):
ks_IN.append(
tf.placeholder(
tf.float32,
shape=[None, cp.getint('Q' + str(ev_path_id), 'Width')],
name='k_IN' + str(ev_path_id)))
# Initialize additional prediction layer to minimize cross entropy,
# for each source of evidence
Qs = []
for ev_path_id, ev_path in enumerate(ev_paths):
with tf.name_scope('COND' + str(ev_path_id)):
# Get activation ditionary
act_dict = get_act_dictionary()
sect = 'Q' + str(ev_path_id)
if cp.get('Experiment', 'PREFIX') == 'CIFAR':
Q = tf.layers.dense(sae_dict['sda_prev'],
sae_dict['sda_prev'].shape.as_list()[1] *
0.2,
activation=act_dict['ReLU'],
name='Pre_Comp_' + sect,
kernel_initializer=normal_init(mean=0.0,
stddev=0.01,
seed=SEED))
Q = tf.layers.dense(
Q,
cp.getint(sect, 'Width'),
activation=act_dict[cp.get(sect, 'Activation')],
name='Pre_' + sect,
kernel_initializer=xavier_init(uniform=False, seed=SEED),
reuse=tf.AUTO_REUSE)
else:
Q = tf.layers.dense(
sae_dict['sda_prev'],
cp.getint(sect, 'Width'),
activation=act_dict[cp.get(sect, 'Activation')],
name='Pre_' + sect,
kernel_initializer=xavier_init(uniform=False, seed=SEED),
reuse=tf.AUTO_REUSE)
Qs.append(Q)
TV = [
v for v in tf.trainable_variables()
if 'Pre_' in v.name or 'beta' in v.name
]
if cp.get('Experiment', 'PREFIX') == 'CIFAR':
cmse = 0.5
ccond = 1.0
else:
cmse = 1.0
ccond = 1.0
if cp.get('Experiment', 'PREFIX') == 'reu100k':
TV1 = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope='Pre_Encoder|Pre_Decoder')
for v in TV:
if v.name == 'Pre_Decoder0/kernel:0':
TV.remove(v)
for v in TV:
if v.name == 'Pre_Decoder0/bias:0':
TV.remove(v)
# Building loss of EviTRAM
cond_loss, cond_lr, cond_t_op, px_mse, px_lr, px_t_op = disj_EviTRAM_loss(
sae_dict['sda_in'], sae_dict['sda_out'], sae_dict['sda_hidden'],
Qs, ks_IN, TV1, TV)
else:
# Building loss of EviTRAM
cond_loss, cond_lr, cond_t_op, px_mse = EviTRAM_loss(
sae_dict['sda_in'], sae_dict['sda_out'], sae_dict['sda_hidden'],
Qs, ks_IN, TV, cmse, ccond)
if cp.get('Experiment', 'PREFIX') == 'reu100k':
ret_dict = {
'sda_in': sae_dict['sda_in'],
'sda_hidden': sae_dict['sda_hidden'],
'sda_out': sae_dict['sda_out'],
'sda_prev': sae_dict['sda_prev'],
'Qs': Qs,
'ks_IN': ks_IN,
'TV': TV,
'evitram_t_op': cond_t_op,
'evitram_loss': cond_loss,
'px_mse': px_mse,
'px_t_op': px_t_op,
'px_lr': px_lr,
'evitram_lr': cond_lr
}
else:
ret_dict = {
'sda_in': sae_dict['sda_in'],
'sda_hidden': sae_dict['sda_hidden'],
'sda_out': sae_dict['sda_out'],
'sda_prev': sae_dict['sda_prev'],
'Qs': Qs,
'ks_IN': ks_IN,
'TV': TV,
'evitram_t_op': cond_t_op,
'evitram_loss': cond_loss,
'px_mse': px_mse,
'evitram_lr': cond_lr
}
return ret_dict
