def build_stacked(incoming, cp, reuse=False):
# Enable/Disable sharing weights
if reuse:
tf.get_variable_scope().reuse_variables()
# Loop through encoders
for enc in [i for i in cp.sections() if 'Encoder' in i]:
num = enc.split('Encoder')[1]
act_dict = get_act_dictionary()
enclabel = 'Encoder' + str(num)
with tf.name_scope(enclabel):
# Input
if num == '0':
sda = incoming
# Hidden layers
for sect in [i for i in cp.sections() if enclabel in i]:
sda = tf.layers.dense(sda,
cp.getint(sect, 'Width'),
activation=act_dict[cp.get(
sect, 'Activation')],
name='SAE_' + sect)
encoder = sda
# Loop through Decoders
for dec in reversed([i for i in cp.sections() if 'Decoder' in i]):
num = dec.split('Decoder')[1]
declabel = 'Decoder' + str(num)
maxaenum = cp.getint('Experiment', 'AENUM')
with tf.name_scope(declabel):
# End to enc autoencoder
for sect in [i for i in cp.sections() if declabel in i]:
sda = tf.layers.dense(sda,
cp.getint(sect, 'Width'),
activation=act_dict[cp.get(
sect, 'Activation')],
name='SAE_' + sect)
return sda, encoder
def build_ae(incoming, num, cp, SEED, reuse=False):
act_dict = get_act_dictionary()
enclabel = 'Encoder' + str(num)
declabel = 'Decoder' + str(num)
# Enable/Disable sharing weights
if reuse:
tf.get_variable_scope().reuse_variables()
with tf.name_scope(enclabel):
# Input
encoder = incoming
# Hidden Layer(s)
for sect in [i for i in cp.sections() if enclabel in i]:
encoder = tf.layers.dense(encoder,
cp.getint(sect, 'Width'),
activation=act_dict[cp.get(
sect, 'Activation')],
name='SAE_' + sect,
kernel_initializer=normal_init(mean=0.0,
stddev=0.01, seed=SEED)
)
with tf.name_scope(declabel):
decoder = encoder
# Decoder Layer(s)
for sect in [i for i in cp.sections() if declabel in i]:
decoder = tf.layers.dense(decoder,
cp.getint(sect, 'Width'),
activation=act_dict[cp.get(
sect, 'Activation')],
name='SAE_' + sect,
kernel_initializer=normal_init(mean=0.0,
stddev=0.01, seed=SEED)
)
return decoder, encoder
def build_encoder(incoming, cp, SEED, scope='px_encoder', reuse=False):
# Get activation ditionary
act_dict = get_act_dictionary()
# Enable/Disable sharing weights
if reuse:
tf.get_variable_scope().reuse_variables()
with tf.name_scope(scope):
cae = incoming
cae = tf.layers.dropout(cae,
float(cp.get('Dropout', 'rate')),
seed=SEED)
# Reshape input to [samples, height, width, channels]
try:
channels = cp.getint('Input', 'Channels')
height = width = int(
np.sqrt(cp.getint('Input', 'Width') / channels))
frames = cp.getint('Input', 'Frames')
cae = tf.reshape(incoming,
shape=[-1, height * width * channels, frames])
print cae.shape
except:
raise ValueError('width x height matrix must be square')
# Add convolutional layers
for sect in [i for i in cp.sections() if 'Conv' in i]:
cae = tf.layers.conv1d(cae,
filters=cp.getint(sect, 'Filters'),
kernel_size=cp.getint(sect, 'Fsize'),
strides=cp.getint(sect, 'Stride'),
padding=cp.get(sect, 'Pad'),
activation=act_dict['ReLU'],
name='Pre_' + sect,
data_format='channels_last')
print cae.shape
height = width = int(np.sqrt(cae.get_shape().as_list()[1]))
cae = tf.reshape(cae, shape=[-1, height, width, cae.shape[2]])
print cae.shape
# cae = tf.contrib.layers.batch_norm(cae,
# scope=sect.split('Conv')[1])
cae = tf.contrib.layers.max_pool2d(cae, (2, 2))
print cae.shape
cae = tf.contrib.layers.max_pool2d(cae, (2, 2))
print cae.shape
last_shape = cae.shape[1:]
cae = tf.layers.flatten(cae)
for sect in [i for i in cp.sections() if 'Encoder' in i]:
cae = tf.layers.dense(cae,
cp.getint(sect, 'Width'),
activation=act_dict[cp.get(
sect, 'Activation')],
name='Pre_' + sect)
print cae.shape
# print cae.shape
return cae, last_shape
def build_ae(incoming, num, cp, SEED, reuse=False):
act_dict = get_act_dictionary()
enclabel = 'Encoder' + str(num)
declabel = 'Decoder' + str(num)
# Enable/Disable sharing weights
if reuse:
tf.get_variable_scope().reuse_variables()
with tf.name_scope(enclabel):
# Input
encoder = incoming
# Dropout
encoder = tf.layers.dropout(encoder,
float(cp.get('Dropout', 'rate')),
seed=SEED)
# Hidden Layer(s)
for sect in [i for i in cp.sections() if enclabel in i]:
if cp.get('Experiment', 'PREFIX') == 'reu100k':
encoder = tf.layers.dense(
encoder,
cp.getint(sect, 'Width'),
activation=act_dict[cp.get(sect, 'Activation')],
name='Pre_' + sect,
kernel_initializer=xavier_init(uniform=False, seed=SEED))
else:
encoder = tf.layers.dense(
encoder,
cp.getint(sect, 'Width'),
activation=act_dict[cp.get(sect, 'Activation')],
name='Pre_' + sect,
kernel_initializer=normal_init(mean=0.0,
stddev=0.01,
seed=SEED))
with tf.name_scope(declabel):
decoder = encoder
# Decoder Layer(s)
for sect in [i for i in cp.sections() if declabel in i]:
if cp.get('Experiment', 'PREFIX') == 'reu100k':
decoder = tf.layers.dense(
decoder,
cp.getint(sect, 'Width'),
activation=act_dict[cp.get(sect, 'Activation')],
name='Pre_' + sect,
kernel_initializer=xavier_init(uniform=False, seed=SEED))
else:
decoder = tf.layers.dense(
decoder,
cp.getint(sect, 'Width'),
activation=act_dict[cp.get(sect, 'Activation')],
name='Pre_' + sect,
kernel_initializer=normal_init(mean=0.0,
stddev=0.01,
seed=SEED))
return decoder, encoder
def build_EviTRAM(cp, SEED):
# Initialize I/O tensors
conv_In = tf.placeholder(tf.float32,
shape=[None, cp.getint('Input', 'Width')],
name='conv_IN')
# Initiliaze placeholders for each source of evidence
sect = 'Experiment'
ev_paths = [cp.get(sect, i) for i in cp.options(sect) if 'evidence' in i]
ks_IN = []
for ev_path_id, ev_path in enumerate(ev_paths):
ks_IN.append(tf.placeholder(tf.float32, shape=[
None, cp.getint('Q' + str(ev_path_id), 'Width')],
name='k_IN' + str(ev_path_id)))
# Building network
conv_Z, last_shape = build_encoder(conv_In, cp, SEED)
conv_Xrec, conv_Prev = build_decoder(conv_Z, cp, SEED, last_shape)
# Initialize additional prediction layer to minimize cross entropy,
# for each source of evidence
Qs = []
for ev_path_id, ev_path in enumerate(ev_paths):
with tf.name_scope('COND' + str(ev_path_id)):
# Get activation ditionary
act_dict = get_act_dictionary()
sect = 'Q' + str(ev_path_id)
Q = tf.layers.dense(conv_Xrec,
cp.getint(sect, 'Width'),
activation=act_dict[cp.get(
sect, 'Activation')],
name='Pre_' + sect,
kernel_initializer=xavier_init(uniform=False,
seed=SEED),
reuse=tf.AUTO_REUSE)
Qs.append(Q)
TV = [v for v in tf.trainable_variables() if 'Pre_' in v.name or
'beta' in v.name]
# Building loss of EviTRAM
cond_loss, cond_lr, cond_t_op, px_mse = EviTRAM_loss(conv_In, conv_Xrec,
conv_Z, Qs, ks_IN, TV)
ret_dict = {'conv_in': conv_In, 'conv_z': conv_Z, 'conv_out': conv_Xrec,
'conv_prev': conv_Prev, 'Qs': Qs, 'ks_IN': ks_IN, 'TV': TV,
'evitram_t_op': cond_t_op, 'evitram_loss': cond_loss,
'px_mse': px_mse, 'evitram_lr': cond_lr}
return ret_dict
def build_decoder(incoming,
cp,
SEED,
last_shape,
scope='px_decoder',
reuse=False):
# Get activation ditionary
act_dict = get_act_dictionary()
# Enable/Disable sharing weights
if reuse:
tf.get_variable_scope().reuse_variables()
with tf.name_scope(scope):
cae = incoming
for sect in [i for i in cp.sections() if 'Encoder' in i]:
if sect != 'Encoder0':
cae = tf.layers.dense(
cae,
cp.getint(sect, 'Width'),
activation=act_dict[cp.get(sect, 'Activation')],
name='Pre_' + sect,
kernel_initializer=xavier_init(uniform=False, seed=SEED))
# Reshape to [sample, width, height, channels]
cae = tf.reshape(
cae, shape=[-1, last_shape[0], last_shape[1], last_shape[2]])
decon_lst = [i for i in cp.sections() if 'DeCon' in i]
decon_num = len(decon_lst)
# Add deconvolutional layers
for sect in decon_lst:
cae = tf.layers.conv2d_transpose(
cae,
filters=cp.getint(sect, 'Filters'),
kernel_size=cp.getint(sect, 'Fsize'),
strides=cp.getint(sect, 'Stride'),
padding=cp.get(sect, 'Pad'),
activation=act_dict['ReLU'],
name='Pre_De_' + sect,
data_format='channels_last')
# Return last layer before reconstruction
if sect == decon_lst[decon_num - 2]:
prev = cae
prev = tf.contrib.layers.max_pool2d(cae, (2, 2))
prev = tf.layers.flatten(prev)
# Flatten output
cae = tf.layers.flatten(cae)
return cae, prev
def build_encoder(incoming, cp, SEED, scope='px_encoder', reuse=False):
# Get activation ditionary
act_dict = get_act_dictionary()
# Enable/Disable sharing weights
if reuse:
tf.get_variable_scope().reuse_variables()
with tf.name_scope(scope):
# Reshape input to [samples, height, width, channels]
try:
channels = cp.getint('Input', 'Channels')
height = width = int(
np.sqrt(cp.getint('Input', 'Width') / channels))
cae = tf.reshape(
incoming, shape=[-1, height, width, channels])
except:
raise ValueError('width x height matrix must be square')
# Add convolutional layers
for sect in [i for i in cp.sections() if 'Conv' in i]:
cae = tf.layers.conv2d(cae,
filters=cp.getint(sect, 'Filters'),
kernel_size=cp.getint(
sect, 'Fsize'),
strides=cp.getint(sect, 'Stride'),
padding=cp.get(sect, 'Pad'),
activation=act_dict['ReLU'],
name='Pre_' + sect,
data_format='channels_last')
cae = tf.contrib.layers.batch_norm(cae,
scope=sect.split('Conv')[1])
# Store shape for later reshaping
last_shape = cae.shape[1:]
cae = tf.layers.flatten(cae)
keep_prob = float(cp.get('Dropout', 'Rate'))
cae = tf.layers.dropout(cae, rate=keep_prob)
# Most inner layer of AE / Last layer of encoder
sect = 'Encoder0'
cae = tf.layers.dense(cae,
cp.getint(sect, 'Width'),
activation=act_dict[cp.get(
sect, 'Activation')],
name='Pre_' + sect,
kernel_initializer=xavier_init
(uniform=False, seed=SEED)
)
return cae, last_shape
def build_stacked(incoming, cp, SEED, reuse=False):
# Enable/Disable sharing weights
if reuse:
tf.get_variable_scope().reuse_variables()
# Loop through encoders
for enc in [i for i in cp.sections() if 'Encoder' in i]:
num = enc.split('Encoder')[1]
act_dict = get_act_dictionary()
enclabel = 'Encoder' + str(num)
with tf.name_scope(enclabel):
# Input
if num == '0':
sda = incoming
if cp.get('Experiment', 'PREFIX') == '20ng':
sda = tf.contrib.layers.batch_norm(sda,
scope='Pre_INBN',
reuse=tf.AUTO_REUSE)
sda = tf.layers.dropout(sda,
float(cp.get('Dropout', 'rate')),
seed=SEED)
# Hidden layers
for sect in [i for i in cp.sections() if enclabel in i]:
sda = tf.layers.dense(sda,
cp.getint(sect, 'Width'),
activation=act_dict[cp.get(
sect, 'Activation')],
name='Pre_' + sect)
encoder = sda
# Loop through Decoders
for dec in reversed([i for i in cp.sections() if 'Decoder' in i]):
num = dec.split('Decoder')[1]
declabel = 'Decoder' + str(num)
maxaenum = cp.getint('Experiment', 'AENUM')
with tf.name_scope(declabel):
# End to enc autoencoder
for sect in [i for i in cp.sections() if declabel in i]:
sda = tf.layers.dense(sda,
cp.getint(sect, 'Width'),
activation=act_dict[cp.get(
sect, 'Activation')],
name='Pre_' + sect)
if sect == 'Decoder1':
prev = sda
return sda, encoder, prev
def build_decoder(incoming,
cp,
SEED,
last_shape,
scope='px_decoder',
reuse=False):
# Get activation ditionary
act_dict = get_act_dictionary()
# Enable/Disable sharing weights
if reuse:
tf.get_variable_scope().reuse_variables()
with tf.name_scope(scope):
batch_size = cp.getint('Hyperparameters', 'BatchSize')
cae = incoming
for sect in [i for i in cp.sections() if 'Decoder' in i]:
cae = tf.layers.dense(cae,
cp.getint(sect, 'Width'),
activation=act_dict[cp.get(
sect, 'Activation')],
name='Pre_' + sect)
print cae.shape
# cae = tf.tile(cae, [1, last_shape[2]])
# print cae.shape
cae = tf.reshape(
cae, shape=[-1, last_shape[0], last_shape[1], last_shape[2]])
print cae.shape
cae = tf.image.resize_images(cae, (cae.shape[1] * 2, cae.shape[2] * 2))
print cae.shape
cae = tf.image.resize_images(cae, (cae.shape[1] * 2, cae.shape[2] * 2))
print cae.shape
cae = tf.reshape(cae,
shape=[-1, cae.shape[1] * cae.shape[2], cae.shape[3]])
print cae.shape
cae_last = cae
decon_lst = [i for i in cp.sections() if 'DeCon' in i]
decon_num = len(decon_lst)
# Add deconvolutional layers
for sect in decon_lst:
if sect == decon_lst[decon_num - 1]:
fsize = cp.getint(sect, 'Fsize')
filters = cp.getint(sect, 'Filters')
cae = tf.contrib.nn.conv1d_transpose(
cae,
tf.Variable(
tf.random_normal(
[fsize, filters,
cae.get_shape().as_list()[2]])),
[batch_size,
cae.get_shape().as_list()[1] * 2, filters],
cp.getint(sect, 'Stride'),
name='Pre_De_' + sect)
else:
fsize = cp.getint(sect, 'Fsize')
filters = cp.getint(sect, 'Filters')
cae = tf.contrib.nn.conv1d_transpose(
cae,
tf.Variable(
tf.random_normal(
[fsize, filters,
cae.get_shape().as_list()[2]])),
[batch_size,
cae.get_shape().as_list()[1] * 2, filters],
cp.getint(sect, 'Stride'),
name='Pre_De_' + sect)
cae = tf.nn.relu(cae)
print cae.shape
print '--------'
prev = cae_last
print prev.shape
height = width = int(np.sqrt(prev.get_shape().as_list()[1]))
prev = tf.reshape(prev, shape=[-1, height, width, cae.shape[2]])
print prev.shape
prev = tf.contrib.layers.max_pool2d(prev, (2, 2))
print prev.shape
prev = tf.contrib.layers.max_pool2d(prev, (2, 2))
print prev.shape
prev = tf.layers.flatten(prev)
print prev.shape
print '--------'
print cae.shape
return cae, prev
def build_EviTRAM(cp, ae_ids, SEED):
sae_dict = build_px(cp, ae_ids, SEED)
# Initiliaze placeholders for each source of evidence
sect = 'Experiment'
ev_paths = [cp.get(sect, i) for i in cp.options(sect) if 'evidence' in i]
ks_IN = []
for ev_path_id, ev_path in enumerate(ev_paths):
ks_IN.append(
tf.placeholder(
tf.float32,
shape=[None, cp.getint('Q' + str(ev_path_id), 'Width')],
name='k_IN' + str(ev_path_id)))
# Initialize additional prediction layer to minimize cross entropy,
# for each source of evidence
Qs = []
for ev_path_id, ev_path in enumerate(ev_paths):
with tf.name_scope('COND' + str(ev_path_id)):
# Get activation ditionary
act_dict = get_act_dictionary()
sect = 'Q' + str(ev_path_id)
if cp.get('Experiment', 'PREFIX') == 'CIFAR':
Q = tf.layers.dense(sae_dict['sda_prev'],
sae_dict['sda_prev'].shape.as_list()[1] *
0.2,
activation=act_dict['ReLU'],
name='Pre_Comp_' + sect,
kernel_initializer=normal_init(mean=0.0,
stddev=0.01,
seed=SEED))
Q = tf.layers.dense(
Q,
cp.getint(sect, 'Width'),
activation=act_dict[cp.get(sect, 'Activation')],
name='Pre_' + sect,
kernel_initializer=xavier_init(uniform=False, seed=SEED),
reuse=tf.AUTO_REUSE)
else:
Q = tf.layers.dense(
sae_dict['sda_prev'],
cp.getint(sect, 'Width'),
activation=act_dict[cp.get(sect, 'Activation')],
name='Pre_' + sect,
kernel_initializer=xavier_init(uniform=False, seed=SEED),
reuse=tf.AUTO_REUSE)
Qs.append(Q)
TV = [
v for v in tf.trainable_variables()
if 'Pre_' in v.name or 'beta' in v.name
]
if cp.get('Experiment', 'PREFIX') == 'CIFAR':
cmse = 0.5
ccond = 1.0
else:
cmse = 1.0
ccond = 1.0
if cp.get('Experiment', 'PREFIX') == 'reu100k':
TV1 = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope='Pre_Encoder|Pre_Decoder')
for v in TV:
if v.name == 'Pre_Decoder0/kernel:0':
TV.remove(v)
for v in TV:
if v.name == 'Pre_Decoder0/bias:0':
TV.remove(v)
# Building loss of EviTRAM
cond_loss, cond_lr, cond_t_op, px_mse, px_lr, px_t_op = disj_EviTRAM_loss(
sae_dict['sda_in'], sae_dict['sda_out'], sae_dict['sda_hidden'],
Qs, ks_IN, TV1, TV)
else:
# Building loss of EviTRAM
cond_loss, cond_lr, cond_t_op, px_mse = EviTRAM_loss(
sae_dict['sda_in'], sae_dict['sda_out'], sae_dict['sda_hidden'],
Qs, ks_IN, TV, cmse, ccond)
if cp.get('Experiment', 'PREFIX') == 'reu100k':
ret_dict = {
'sda_in': sae_dict['sda_in'],
'sda_hidden': sae_dict['sda_hidden'],
'sda_out': sae_dict['sda_out'],
'sda_prev': sae_dict['sda_prev'],
'Qs': Qs,
'ks_IN': ks_IN,
'TV': TV,
'evitram_t_op': cond_t_op,
'evitram_loss': cond_loss,
'px_mse': px_mse,
'px_t_op': px_t_op,
'px_lr': px_lr,
'evitram_lr': cond_lr
}
else:
ret_dict = {
'sda_in': sae_dict['sda_in'],
'sda_hidden': sae_dict['sda_hidden'],
'sda_out': sae_dict['sda_out'],
'sda_prev': sae_dict['sda_prev'],
'Qs': Qs,
'ks_IN': ks_IN,
'TV': TV,
'evitram_t_op': cond_t_op,
'evitram_loss': cond_loss,
'px_mse': px_mse,
'evitram_lr': cond_lr
}
return ret_dict