def _gan28_discr(experiment=None, X=None, reuse=False, **kwargs): lrelu = partial(leaky_relu, leakiness=experiment.leak) h_dim = experiment.h_dim logger = experiment.logger msg = "D_SHAPE {} {} [reuse={}]" logger.debug(msg.format("in", X.shape, reuse)) with tf.variable_scope("discriminator", reuse=reuse): with tf.variable_scope("hidden1"): d_hidden1 = conv2d( X, filters=h_dim, kernel_size=5, strides=2, padding="same", activation=lrelu, kernel_initializer=xavier_init()) logger.debug(msg.format("dh1", d_hidden1.shape, reuse)) with tf.variable_scope("hidden2"): d_hidden2 = conv2d( d_hidden1, filters=h_dim * 2, kernel_size=5, strides=2, padding="same", activation=lrelu, kernel_initializer=xavier_init()) logger.debug(msg.format("dh2", d_hidden2.shape, reuse)) with tf.variable_scope("hidden3"): d_hidden2 = tf.reshape( d_hidden2, [-1, np.prod(d_hidden2.shape[1:], dtype=int)]) d_hidden3 = dense(d_hidden2, 1024, activation=lrelu, kernel_initializer=xavier_init()) logger.debug(msg.format("dh3", d_hidden3.shape, reuse)) with tf.variable_scope("output"): d_out = dense(d_hidden3, 1, kernel_initializer=xavier_init()) logger.debug(msg.format("out", d_out.shape, reuse)) # define summaries on the last layer d_summaries = stats_summaries(d_out) return d_out, d_summaries
def apply_constraints(): logger.debug("Using constraints: {}".format( str(experiment.constraints))) with tf.variable_scope("constrained_ll_out"): constraints_out = dense( constraints_features, 1, activation=lrelu, kernel_initializer=xavier_init()) d_out_kernel = tf.get_variable( "d_out_kernel", shape=[2, 1], initializer=xavier_init()) logger.debug( msg.format("d_out_kernel", d_out_kernel.shape, reuse)) d_out_bias = tf.get_variable( "d_out_bias", shape=[1, 1], initializer=xavier_init()) logger.debug(msg.format("d_out_bias", d_out_bias.shape, reuse)) input_concat = tf.concat( [d_out, constraints_out], axis=1, name="input_concat_{}".format(reuse)) logger.debug(msg.format( "input_concat", input_concat.shape, reuse)) d_constrained_out = tf.add( tf.matmul(input_concat, d_out_kernel), d_out_bias, name="d_constrained_out_{}".format(reuse)) logger.debug(msg.format( "constrained_out", d_constrained_out.shape, reuse)) # define summaries on the last layer d_summaries = tf.summary.merge( [stats_summaries(d_constrained_out), d_out_summaries]) return d_constrained_out, d_summaries
def build_ae(incoming, num, cp, SEED, reuse=False): act_dict = get_act_dictionary() enclabel = 'Encoder' + str(num) declabel = 'Decoder' + str(num) # Enable/Disable sharing weights if reuse: tf.get_variable_scope().reuse_variables() with tf.name_scope(enclabel): # Input encoder = incoming # Dropout encoder = tf.layers.dropout(encoder, float(cp.get('Dropout', 'rate')), seed=SEED) # Hidden Layer(s) for sect in [i for i in cp.sections() if enclabel in i]: if cp.get('Experiment', 'PREFIX') == 'reu100k': encoder = tf.layers.dense( encoder, cp.getint(sect, 'Width'), activation=act_dict[cp.get(sect, 'Activation')], name='Pre_' + sect, kernel_initializer=xavier_init(uniform=False, seed=SEED)) else: encoder = tf.layers.dense( encoder, cp.getint(sect, 'Width'), activation=act_dict[cp.get(sect, 'Activation')], name='Pre_' + sect, kernel_initializer=normal_init(mean=0.0, stddev=0.01, seed=SEED)) with tf.name_scope(declabel): decoder = encoder # Decoder Layer(s) for sect in [i for i in cp.sections() if declabel in i]: if cp.get('Experiment', 'PREFIX') == 'reu100k': decoder = tf.layers.dense( decoder, cp.getint(sect, 'Width'), activation=act_dict[cp.get(sect, 'Activation')], name='Pre_' + sect, kernel_initializer=xavier_init(uniform=False, seed=SEED)) else: decoder = tf.layers.dense( decoder, cp.getint(sect, 'Width'), activation=act_dict[cp.get(sect, 'Activation')], name='Pre_' + sect, kernel_initializer=normal_init(mean=0.0, stddev=0.01, seed=SEED)) return decoder, encoder
def discriminator(self, x, reuse=False): with tf.variable_scope("discriminator", reuse=reuse): d_w1 = tf.get_variable('d_w1', [self.config.out_dim, self.config.h_dim], initializer=xavier_init()) d_b1 = tf.get_variable('d_b1', [self.config.h_dim], initializer=tf.zeros_initializer()) d_w2 = tf.get_variable('d_w2', [self.config.h_dim, 1], initializer=xavier_init()) d_b2 = tf.get_variable('d_b2', [1], initializer=tf.zeros_initializer()) h = tf.nn.relu(tf.matmul(x, d_w1) + d_b1) logits = tf.matmul(h, d_w2) + d_b2 return tf.nn.sigmoid(logits), logits
def generator(self, z): with tf.variable_scope("generator") as scope: g_w1 = tf.get_variable("g_w1", [self.config.z_dim, self.config.h_dim], initializer=xavier_init()) g_b1 = tf.get_variable("g_b1", [self.config.h_dim], initializer=tf.zeros_initializer()) g_w2 = tf.get_variable("g_w2", [self.config.h_dim, self.config.out_dim], initializer=xavier_init()) g_b2 = tf.get_variable("g_b2", [self.config.out_dim], initializer=tf.zeros_initializer()) h = tf.nn.relu(tf.matmul(z, g_w1) + g_b1) return tf.nn.sigmoid(tf.matmul(h, g_w2) + g_b2)
def make_wbkernels( self, name, shp=None, initializer=xavier_init(uniform=False), ): """ make weights, biases, kernels """ return tf.get_variable( name, shp, initializer=initializer, regularizer=self.reg )
def apply_constraints(): logger.debug("Using constraints: {}".format( str(experiment.constraints))) with tf.variable_scope("constrained_out"): d_constraints_kernel = tf.get_variable( "d_constraints_kernel", shape=[constraints_features.shape[1], 1], initializer=xavier_init()) logger.debug(msg.format( "d_constraints_kernel", d_constraints_kernel.shape, reuse)) input_concat = tf.concat( [d_hidden4, constraints_features], axis=1, name="input_concat_{}".format(reuse)) logger.debug(msg.format( "input_concat", input_concat.shape, reuse)) weight_concat = tf.concat( [d_out_kernel, d_constraints_kernel], axis=0, name="weight_concat_{}".format(reuse)) logger.debug(msg.format( "weight_concat", weight_concat.shape, reuse)) d_constrained_out = tf.add( tf.matmul(input_concat, weight_concat), d_out_bias, name="d_constrained_out_{}".format(reuse)) logger.debug(msg.format( "constrained_out", d_constrained_out.shape, reuse)) # define summaries on the last layer d_summaries = tf.summary.merge( [stats_summaries(d_constrained_out), d_summaries_dh4]) return d_constrained_out, d_summaries
def make_fc_layer(self, inp_lyr, name_fc_lyr, name_w, shp_w, name_b=None, shp_b=None, initializer=xavier_init(uniform=False)): """ TODO - regularize batch norm params? """ W = self.make_wbkernels(name_w, shp_w, initializer=initializer) b = self.make_wbkernels(name_b, shp_b, initializer=tf.zeros_initializer()) fc_lyr = tf.nn.bias_add( tf.matmul(inp_lyr, W, name=name_fc_lyr + '_matmul'), b, data_format=self.data_format, name=name_fc_lyr, ) if self.use_batch_norm: fc_lyr = tf.contrib.layers.batch_norm(fc_lyr, decay=self.batch_norm_decay, center=True, scale=True, data_format=self.data_format, is_training=self.is_training) return fc_lyr
def make_wbkernels( self, name, shp=None, initializer=xavier_init(uniform=False), ): """ make weights, biases, kernels """ return tf.get_variable( name, shp, initializer=initializer, regularizer=self.reg )
def fully_connected(input, num_output, name): # does input*weight, expects 2D input with tf.variable_scope(name) as scope: weight = tf.get_variable("weight", shape=[input.shape.as_list()[1], num_output], initializer=xavier_init()) return tf.matmul(input, weight)
def make_active_fc_layer( self, inp_lyr, name_fc_lyr, name_w, shp_w, name_b=None, shp_b=None, act=tf.nn.relu, initializer=xavier_init(uniform=False) ): return act(self.make_fc_layer( inp_lyr, name_fc_lyr, name_w, shp_w, name_b, shp_b, initializer=initializer ), name=name_fc_lyr+'_act')
def build_from_inputs(self, inputs, output_space, trainable=True): net = inputs for i, units in enumerate(self.layer_sizes): net = tf.layers.dense(net, units=units, activation=tf.nn.relu, trainable=trainable, name='fc%d' % i) v = tf.layers.dense(net, units=1, activation=None, kernel_initializer=xavier_init(), bias_initializer=xavier_init(), name='out_value', trainable=trainable) v = tf.squeeze(v, axis=1) out_pi = policy_head(net, output_space, trainable) return {'input': inputs, 'out': out_pi, 'policy': out_pi, 'value': v}
def add_conv(X, shape, strides, padding = 'SAME'): '''Create convolution layer with relu activation. Kernel weights and biases are initalized with Xavier initialization and zero respectivly.''' kernel = tf.get_variable('kernel',shape = shape, initializer = xavier_init()) conv = tf.nn.conv2d(input = X, filter = kernel, strides = strides, padding = padding) bias = tf.get_variable('bias',shape = [shape[-1]], initializer = tf.constant_initializer(value = 0.0)) relu = tf.nn.relu(features = conv + bias) return relu
def policy_head(input_layer, output_space, trainable=True): if not isinstance(output_space, Continuous): pi = tf.layers.dense(input_layer, units=output_space.shape[0], activation=tf.nn.softmax, kernel_initializer=xavier_init(), bias_initializer=xavier_init(), name='out_policy', trainable=trainable) return pi mean = tf.layers.dense(input_layer, units=output_space.shape[0], activation=tf.nn.tanh, kernel_initializer=xavier_init(), bias_initializer=xavier_init(), name='mean', trainable=trainable) std = tf.layers.dense(input_layer, units=output_space.shape[0], activation=tf.nn.softplus, kernel_initializer=xavier_init(), bias_initializer=xavier_init(), name='std', trainable=trainable) mean = tf.squeeze(mean) std = tf.squeeze(std) dist = tf.contrib.distributions.Normal(mean, std) pi = tf.clip_by_value(dist.sample(1), output_space.low, output_space.high) return pi
def make_active_fc_layer( self, inp_lyr, name_fc_lyr, name_w, shp_w, name_b=None, shp_b=None, act=tf.nn.relu, initializer=xavier_init(uniform=False) ): with tf.variable_scope(name_fc_lyr): return act( self.make_fc_layer( inp_lyr, name_fc_lyr, name_w, shp_w, name_b, shp_b, initializer=initializer ), name=name_fc_lyr+'_act' )
def add_fc(X, n_in, n_out, act_func = True): '''Create feed forward layer. Weights and biases are initalized with Xavier initialization and zero respectivly. If act_func = True then relu is used and non-linear layer is returned. If act_func = False then logits are returned for softmax classification.''' weights = tf.get_variable('weights', shape = [n_in, n_out], initializer = xavier_init()) biases = tf.get_variable('bias', shape = [n_out], initializer = tf.constant_initializer(value = 0.0)) z = tf.matmul(X, weights) + biases if act_func: return tf.nn.relu(features = z) else: return z
def build_EviTRAM(cp, SEED): # Initialize I/O tensors conv_In = tf.placeholder(tf.float32, shape=[None, cp.getint('Input', 'Width')], name='conv_IN') # Initiliaze placeholders for each source of evidence sect = 'Experiment' ev_paths = [cp.get(sect, i) for i in cp.options(sect) if 'evidence' in i] ks_IN = [] for ev_path_id, ev_path in enumerate(ev_paths): ks_IN.append(tf.placeholder(tf.float32, shape=[ None, cp.getint('Q' + str(ev_path_id), 'Width')], name='k_IN' + str(ev_path_id))) # Building network conv_Z, last_shape = build_encoder(conv_In, cp, SEED) conv_Xrec, conv_Prev = build_decoder(conv_Z, cp, SEED, last_shape) # Initialize additional prediction layer to minimize cross entropy, # for each source of evidence Qs = [] for ev_path_id, ev_path in enumerate(ev_paths): with tf.name_scope('COND' + str(ev_path_id)): # Get activation ditionary act_dict = get_act_dictionary() sect = 'Q' + str(ev_path_id) Q = tf.layers.dense(conv_Xrec, cp.getint(sect, 'Width'), activation=act_dict[cp.get( sect, 'Activation')], name='Pre_' + sect, kernel_initializer=xavier_init(uniform=False, seed=SEED), reuse=tf.AUTO_REUSE) Qs.append(Q) TV = [v for v in tf.trainable_variables() if 'Pre_' in v.name or 'beta' in v.name] # Building loss of EviTRAM cond_loss, cond_lr, cond_t_op, px_mse = EviTRAM_loss(conv_In, conv_Xrec, conv_Z, Qs, ks_IN, TV) ret_dict = {'conv_in': conv_In, 'conv_z': conv_Z, 'conv_out': conv_Xrec, 'conv_prev': conv_Prev, 'Qs': Qs, 'ks_IN': ks_IN, 'TV': TV, 'evitram_t_op': cond_t_op, 'evitram_loss': cond_loss, 'px_mse': px_mse, 'evitram_lr': cond_lr} return ret_dict
def build_encoder(incoming, cp, SEED, scope='px_encoder', reuse=False): # Get activation ditionary act_dict = get_act_dictionary() # Enable/Disable sharing weights if reuse: tf.get_variable_scope().reuse_variables() with tf.name_scope(scope): # Reshape input to [samples, height, width, channels] try: channels = cp.getint('Input', 'Channels') height = width = int( np.sqrt(cp.getint('Input', 'Width') / channels)) cae = tf.reshape( incoming, shape=[-1, height, width, channels]) except: raise ValueError('width x height matrix must be square') # Add convolutional layers for sect in [i for i in cp.sections() if 'Conv' in i]: cae = tf.layers.conv2d(cae, filters=cp.getint(sect, 'Filters'), kernel_size=cp.getint( sect, 'Fsize'), strides=cp.getint(sect, 'Stride'), padding=cp.get(sect, 'Pad'), activation=act_dict['ReLU'], name='Pre_' + sect, data_format='channels_last') cae = tf.contrib.layers.batch_norm(cae, scope=sect.split('Conv')[1]) # Store shape for later reshaping last_shape = cae.shape[1:] cae = tf.layers.flatten(cae) keep_prob = float(cp.get('Dropout', 'Rate')) cae = tf.layers.dropout(cae, rate=keep_prob) # Most inner layer of AE / Last layer of encoder sect = 'Encoder0' cae = tf.layers.dense(cae, cp.getint(sect, 'Width'), activation=act_dict[cp.get( sect, 'Activation')], name='Pre_' + sect, kernel_initializer=xavier_init (uniform=False, seed=SEED) ) return cae, last_shape
def build_decoder(incoming, cp, SEED, last_shape, scope='px_decoder', reuse=False): # Get activation ditionary act_dict = get_act_dictionary() # Enable/Disable sharing weights if reuse: tf.get_variable_scope().reuse_variables() with tf.name_scope(scope): cae = incoming for sect in [i for i in cp.sections() if 'Encoder' in i]: if sect != 'Encoder0': cae = tf.layers.dense( cae, cp.getint(sect, 'Width'), activation=act_dict[cp.get(sect, 'Activation')], name='Pre_' + sect, kernel_initializer=xavier_init(uniform=False, seed=SEED)) # Reshape to [sample, width, height, channels] cae = tf.reshape( cae, shape=[-1, last_shape[0], last_shape[1], last_shape[2]]) decon_lst = [i for i in cp.sections() if 'DeCon' in i] decon_num = len(decon_lst) # Add deconvolutional layers for sect in decon_lst: cae = tf.layers.conv2d_transpose( cae, filters=cp.getint(sect, 'Filters'), kernel_size=cp.getint(sect, 'Fsize'), strides=cp.getint(sect, 'Stride'), padding=cp.get(sect, 'Pad'), activation=act_dict['ReLU'], name='Pre_De_' + sect, data_format='channels_last') # Return last layer before reconstruction if sect == decon_lst[decon_num - 2]: prev = cae prev = tf.contrib.layers.max_pool2d(cae, (2, 2)) prev = tf.layers.flatten(prev) # Flatten output cae = tf.layers.flatten(cae) return cae, prev
def build_from_inputs(self, inputs, output_space, trainable=True): net = tf.layers.conv2d(inputs=inputs, filters=16, kernel_size=[8, 8], strides=[4, 4], activation=tf.nn.relu, padding="same", name="conv1", trainable=trainable, kernel_initializer=xavier_init(), bias_initializer=xavier_init()) net = tf.layers.conv2d(inputs=net, filters=32, kernel_size=[4, 4], strides=[2, 2], activation=tf.nn.relu, padding="same", name="conv2", trainable=trainable, kernel_initializer=xavier_init(), bias_initializer=xavier_init()) net = tf.layers.flatten(net) fc1 = tf.layers.dense(net, units=256, activation=tf.nn.relu, name='fc1', trainable=trainable, kernel_initializer=xavier_init(), bias_initializer=xavier_init()) v = tf.layers.dense(fc1, units=1, activation=None, kernel_initializer=xavier_init(), bias_initializer=xavier_init(), name='out_value', trainable=trainable) v = tf.squeeze(v, axis=1) out_pi = policy_head(net, output_space, trainable) return {'input': inputs, 'out': out_pi, 'policy': out_pi, 'value': v}
def make_fc_layer( self, inp_lyr, name_fc_lyr, name_w, shp_w, name_b=None, shp_b=None, initializer=xavier_init(uniform=False) ): """ Note: `name_fc_lyr` is a _basename_ for the ops """ # TODO - regularize batch norm params? W = self.make_wbkernels(name_w, shp_w, initializer=initializer) b = self.make_wbkernels( name_b, shp_b, initializer=tf.zeros_initializer() ) fc_lyr = tf.nn.bias_add( tf.matmul(inp_lyr, W, name=name_fc_lyr+'_matmul'), b, data_format=self.data_format, name=name_fc_lyr+'_plus_biases', ) if self.use_batch_norm: fc_lyr = tf.contrib.layers.batch_norm( fc_lyr, decay=self.batch_norm_decay, center=True, scale=True, data_format=self.data_format, is_training=self.is_training ) return fc_lyr
def _can60_discr_32layer_auto( experiment=None, X=None, reuse=False, use_constraints=None, constraints_features=None, **kwargs): lrelu = partial(leaky_relu, leakiness=experiment.leak) h_dim = experiment.h_dim logger = experiment.logger msg = "D_SHAPE {} {} [reuse={}]" logger.debug(msg.format("in", X.shape, reuse)) with tf.variable_scope("discriminator", reuse=reuse): with tf.variable_scope("hidden1"): d_hidden1 = conv2d( X, filters=h_dim, kernel_size=5, strides=2, padding="same", activation=lrelu, kernel_initializer=xavier_init()) logger.debug(msg.format("dh1", d_hidden1.shape, reuse)) with tf.variable_scope("hidden15"): d_hidden15 = conv2d( d_hidden1, filters=h_dim, kernel_size=5, strides=2, padding="same", activation=lrelu, kernel_initializer=xavier_init()) logger.debug(msg.format("dh15", d_hidden15.shape, reuse)) with tf.variable_scope("hidden2"): d_hidden2 = conv2d( d_hidden15, filters=h_dim * 2, kernel_size=5, strides=2, padding="same", activation=lrelu, kernel_initializer=xavier_init()) logger.debug(msg.format("dh2", d_hidden2.shape, reuse)) d_hidden2 = tf.reshape( d_hidden2, [-1, np.prod(d_hidden2.shape[1:], dtype=int)]) logger.debug(msg.format("dh2", d_hidden2.shape, reuse)) with tf.variable_scope("hidden3"): d_hidden3 = dense(d_hidden2, 1024, activation=lrelu, kernel_initializer=xavier_init()) logger.debug(msg.format("dh3", d_hidden3.shape, reuse)) with tf.variable_scope("hidden4"): d_hidden4 = dense(d_hidden3, 32, activation=lrelu, kernel_initializer=xavier_init()) logger.debug(msg.format("dh4", d_hidden4.shape, reuse)) d_summaries_dh4 = stats_summaries(d_hidden4, "dh4_pre_cond") with tf.variable_scope("shared_weights"): d_out_kernel = tf.get_variable( "d_out_kernel", shape=[32, 1], initializer=xavier_init()) logger.debug(msg.format("d_out_kernel", d_out_kernel.shape, reuse)) d_out_bias = tf.get_variable( "d_out_bias", shape=[1, 1], initializer=xavier_init()) logger.debug(msg.format("d_out_bias", d_out_bias.shape, reuse)) def skip_constraints(): with tf.variable_scope("output"): d_out = tf.add(tf.matmul(d_hidden4, d_out_kernel), d_out_bias, name="d_out_{}".format(reuse)) logger.debug(msg.format("out", d_out.shape, reuse)) # define summaries on the last layer d_summaries = tf.summary.merge( [stats_summaries(d_out), d_summaries_dh4]) return d_out, d_summaries def apply_constraints(): logger.debug("Using constraints: {}".format( str(experiment.constraints))) with tf.variable_scope("constrained_out"): d_constraints_kernel = tf.get_variable( "d_constraints_kernel", shape=[constraints_features.shape[1], 1], initializer=xavier_init()) logger.debug(msg.format( "d_constraints_kernel", d_constraints_kernel.shape, reuse)) input_concat = tf.concat( [d_hidden4, constraints_features], axis=1, name="input_concat_{}".format(reuse)) logger.debug(msg.format( "input_concat", input_concat.shape, reuse)) weight_concat = tf.concat( [d_out_kernel, d_constraints_kernel], axis=0, name="weight_concat_{}".format(reuse)) logger.debug(msg.format( "weight_concat", weight_concat.shape, reuse)) d_constrained_out = tf.add( tf.matmul(input_concat, weight_concat), d_out_bias, name="d_constrained_out_{}".format(reuse)) logger.debug(msg.format( "constrained_out", d_constrained_out.shape, reuse)) # define summaries on the last layer d_summaries = tf.summary.merge( [stats_summaries(d_constrained_out), d_summaries_dh4]) return d_constrained_out, d_summaries return tf.cond(tf.cast(use_constraints, tf.bool), skip_constraints, apply_constraints)
def build_EviTRAM(cp, ae_ids, SEED): sae_dict = build_px(cp, ae_ids, SEED) # Initiliaze placeholders for each source of evidence sect = 'Experiment' ev_paths = [cp.get(sect, i) for i in cp.options(sect) if 'evidence' in i] ks_IN = [] for ev_path_id, ev_path in enumerate(ev_paths): ks_IN.append( tf.placeholder( tf.float32, shape=[None, cp.getint('Q' + str(ev_path_id), 'Width')], name='k_IN' + str(ev_path_id))) # Initialize additional prediction layer to minimize cross entropy, # for each source of evidence Qs = [] for ev_path_id, ev_path in enumerate(ev_paths): with tf.name_scope('COND' + str(ev_path_id)): # Get activation ditionary act_dict = get_act_dictionary() sect = 'Q' + str(ev_path_id) if cp.get('Experiment', 'PREFIX') == 'CIFAR': Q = tf.layers.dense(sae_dict['sda_prev'], sae_dict['sda_prev'].shape.as_list()[1] * 0.2, activation=act_dict['ReLU'], name='Pre_Comp_' + sect, kernel_initializer=normal_init(mean=0.0, stddev=0.01, seed=SEED)) Q = tf.layers.dense( Q, cp.getint(sect, 'Width'), activation=act_dict[cp.get(sect, 'Activation')], name='Pre_' + sect, kernel_initializer=xavier_init(uniform=False, seed=SEED), reuse=tf.AUTO_REUSE) else: Q = tf.layers.dense( sae_dict['sda_prev'], cp.getint(sect, 'Width'), activation=act_dict[cp.get(sect, 'Activation')], name='Pre_' + sect, kernel_initializer=xavier_init(uniform=False, seed=SEED), reuse=tf.AUTO_REUSE) Qs.append(Q) TV = [ v for v in tf.trainable_variables() if 'Pre_' in v.name or 'beta' in v.name ] if cp.get('Experiment', 'PREFIX') == 'CIFAR': cmse = 0.5 ccond = 1.0 else: cmse = 1.0 ccond = 1.0 if cp.get('Experiment', 'PREFIX') == 'reu100k': TV1 = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='Pre_Encoder|Pre_Decoder') for v in TV: if v.name == 'Pre_Decoder0/kernel:0': TV.remove(v) for v in TV: if v.name == 'Pre_Decoder0/bias:0': TV.remove(v) # Building loss of EviTRAM cond_loss, cond_lr, cond_t_op, px_mse, px_lr, px_t_op = disj_EviTRAM_loss( sae_dict['sda_in'], sae_dict['sda_out'], sae_dict['sda_hidden'], Qs, ks_IN, TV1, TV) else: # Building loss of EviTRAM cond_loss, cond_lr, cond_t_op, px_mse = EviTRAM_loss( sae_dict['sda_in'], sae_dict['sda_out'], sae_dict['sda_hidden'], Qs, ks_IN, TV, cmse, ccond) if cp.get('Experiment', 'PREFIX') == 'reu100k': ret_dict = { 'sda_in': sae_dict['sda_in'], 'sda_hidden': sae_dict['sda_hidden'], 'sda_out': sae_dict['sda_out'], 'sda_prev': sae_dict['sda_prev'], 'Qs': Qs, 'ks_IN': ks_IN, 'TV': TV, 'evitram_t_op': cond_t_op, 'evitram_loss': cond_loss, 'px_mse': px_mse, 'px_t_op': px_t_op, 'px_lr': px_lr, 'evitram_lr': cond_lr } else: ret_dict = { 'sda_in': sae_dict['sda_in'], 'sda_hidden': sae_dict['sda_hidden'], 'sda_out': sae_dict['sda_out'], 'sda_prev': sae_dict['sda_prev'], 'Qs': Qs, 'ks_IN': ks_IN, 'TV': TV, 'evitram_t_op': cond_t_op, 'evitram_loss': cond_loss, 'px_mse': px_mse, 'evitram_lr': cond_lr } return ret_dict