def encoder(x, y, phase, scope='enc', reuse=None, internal_update=False): with tf.variable_scope(scope, reuse=reuse): with arg_scope([conv2d, dense], bn=True, phase=phase, activation=leaky_relu), \ arg_scope([batch_norm], internal_update=internal_update): # Ignore y x = conv2d(x, 64, 3, 2) x = conv2d(x, 128, 3, 2) x = conv2d(x, 256, 3, 2) x = dense(x, 1024) # Autoregression (4 steps) ms = [] vs = [] zs = [x] for i in xrange(5): h = tf.concat(zs, axis=-1) h = dense(h, 100) m = dense(h, 20, activation=None) v = dense(h, 20, activation=tf.nn.softplus) + 1e-5 z = gaussian_sample(m, v) ms += [m] vs += [v] zs += [z] m = tf.concat(ms, 1) v = tf.concat(vs, 1) z = tf.concat(zs[1:], 1) return z, (m, v)
def fit(self, x_input, epochs = 1000, learning_rate = 0.001, batch_size = 100, print_size = 50, train=True): # training setting self.DO_SHARE = False self.epochs = epochs self.learning_rate = learning_rate self.batch_size = batch_size self.print_size = print_size self.g = tf.Graph() # inference process x_ = placeholder((None, self.input_dim)) x = x_ depth_inf = len(self.encoding_dims) for i in range(depth_inf): x = dense(x, self.encoding_dims[i], scope="enc_layer"+"%s" %i, activation=tf.nn.sigmoid) h_encode = x z_mu = dense(h_encode, self.z_dim, scope="mu_layer") z_log_sigma_sq = dense(h_encode, self.z_dim, scope = "sigma_layer") e = tf.random_normal(tf.shape(z_mu)) z = z_mu + tf.sqrt(tf.maximum(tf.exp(z_log_sigma_sq), self.eps)) * e # generative process if self.useTranse == False: depth_gen = len(self.decoding_dims) for i in range(depth_gen): y = dense(z, self.decoding_dims[i], scope="dec_layer"+"%s" %i, activation=tf.nn.sigmoid) # if last_layer_nonelinear: depth_gen -1 else: depth_gen = depth_inf ## haven't finnished yet... x_recons = y if self.loss == "cross_entropy": loss_recons = tf.reduce_mean(tf.reduce_sum(binary_crossentropy(x_recons, x_, self.eps), axis=1)) loss_kl = 0.5 * tf.reduce_mean(tf.reduce_sum(tf.square(z_mu) + tf.exp(z_log_sigma_sq) - z_log_sigma_sq - 1, 1)) # loss_kl = 0.5 * tf.reduce_mean(tf.reduce_sum(tf.square(z_mu) + tf.exp(z_log_sigma_sq) - z_log_sigma_sq - 1, 1)) loss = loss_recons + loss_kl # other cases not finished yet train_op = tf.train.AdamOptimizer(self.learning_rate).minimize(loss) sess = tf.Session() sess.run(tf.global_variables_initializer()) saver = tf.train.Saver() ckpt_dir = "pre_model/" + "vae.ckpt" if train == True: # num_turn = x_input.shape[0] / self.batch_size for i in range(epochs): idx = np.random.choice(x_input.shape[0], batch_size, replace=False) x_batch = x_input[idx] _, l = sess.run((train_op, loss), feed_dict={x_:x_batch}) if i % self.print_size == 0: print "{:>10s}{:>10s}".format("epoces","loss") print "{:10.2e}{:10.2e}".format(i, l) saver.save(sess, ckpt_dir) else: saver.restore(sess, ckpt_dir)
def feature_discriminator(x, phase, C=1, reuse=None): with tf.variable_scope('disc/feat', reuse=reuse): with arg_scope([dense], activation=tf.nn.relu): # Switch to leaky? x = dense(x, 100) x = dense(x, C, activation=None) return x
def feature_discriminator(x, phase, C=1, reuse=None): with tf.variable_scope('disc/feat', reuse=reuse): with arg_scope([dense], activation=tf.nn.relu): # Switch to leaky? x = dense(x, 100) x = dense(x, C, activation=None) return x
def classifier(x, phase, reuse=None): with tf.variable_scope('class', reuse=reuse): with arg_scope([conv2d, dense], bn=True, phase=phase, activation=tf.nn.relu): for i in range(4): x = conv2d(x, 64 + 64 * i, 3, 2) x = conv2d(x, 64 + 64 * i, 3, 1) x = dense(x, 500) x = dense(x, 10, activation=None) return x
def encoder(x, y, phase, reuse=None): with tf.variable_scope('enc', reuse=reuse): with arg_scope([conv2d, dense], bn=True, phase=phase, activation=leaky_relu), \ arg_scope([noise], phase=phase): # Ignore y x = conv2d(x, 64, 3, 2, bn=False) x = conv2d(x, 128, 3, 2) x = conv2d(x, 256, 3, 2) x = dense(x, 1024) m = dense(x, 100, activation=None) v = dense(x, 100, activation=tf.nn.softplus) + 1e-5 return (m, v)
def discriminator(x, phase, reuse=None, depth=1): with tf.variable_scope('disc', reuse=reuse): with arg_scope([conv2d, dense], bn=True, phase=phase, activation=lrelu), \ arg_scope([noise], phase=phase): x = dropout(x, rate=0.2, training=phase) x = conv2d(x, 64, 3, 2, bn=False) x = dropout(x, training=phase) x = conv2d(x, 128, 3, 2) x = dropout(x, training=phase) x = conv2d(x, 256, 3, 2) x = dropout(x, training=phase) x = dense(x, 1024) x = dense(x, depth, activation=None, bn=False) return x
def generator(x, phase, reuse=None): with tf.variable_scope('gen', reuse=reuse): with arg_scope([dense], bn=True, phase=phase, activation=tf.nn.relu), \ arg_scope([conv2d_transpose], bn=True, phase=phase, activation=tf.nn.relu): if args.dense: raise NotImplementedError('yep') # Level 0: 1 l0 = tf.reshape(x, [-1, 1, 1, 128]) # Level 1: 1 -> 4, 8, 16 a1 = conv2d_transpose(l0, 64, 1, 1) a2 = conv2d_transpose(l0, 64, 1, 1) a3 = conv2d_transpose(l0, 64, 1, 1) l1 = a1 # Level 2: 4 -> 8, 16 b2 = conv2d_transpose(l1, 64, ) # Level 2: 8 -> 16, 32 b2 = conv2d_transpose(l1, 64, 3, 2) l2 = tf.concat([a2, b2], -1) # Level 3: 16 -> 32 c3 = conv2d_transpose(l2, 64, 3, 2) l3 = tf.concat([a3, b3, c3], -1) else: x = dense(x, 4 * 4 * 512) x = tf.reshape(x, [-1, 4, 4, 512]) x = conv2d_transpose(x, 256, 5, 2) x = conv2d_transpose(x, 128, 5, 2) x = conv2d_transpose(x, 1, 5, 2, bn=False, activation=tf.nn.tanh) return x
def encoder(x, y, phase, scope='enc', reuse=None, internal_update=False): with tf.variable_scope(scope, reuse=reuse): with arg_scope([conv2d, dense], bn=True, phase=phase, activation=leaky_relu), \ arg_scope([batch_norm], internal_update=internal_update): # Ignore y x = conv2d(x, 64, 3, 2) x = conv2d(x, 128, 3, 2) x = conv2d(x, 256, 3, 2) x = dense(x, 1024) m = dense(x, 100, activation=None) v = dense(x, 100, activation=tf.nn.softplus) + 1e-5 z = gaussian_sample(m, v) return z, (m, v)
def generator(z, y, phase, scope='gen', reuse=None, internal_update=False): with tf.variable_scope(scope, reuse=reuse): with arg_scope([dense, conv2d_transpose], bn=True, phase=phase, activation=leaky_relu), \ arg_scope([batch_norm], internal_update=internal_update): x = tf.concat([z, y], 1) x = dense(x, 4 * 4 * 512) x = tf.reshape(x, [-1, 4, 4, 512]) x = conv2d_transpose(x, 256, 5, 2) x = conv2d_transpose(x, 128, 5, 2) x = conv2d_transpose(x, 3, 5, 2, bn=False, activation=tf.nn.tanh) return x
def encoder(x, phase, reuse=None): with tf.variable_scope('enc', reuse=reuse): with arg_scope([dense], bn=True, phase=phase, activation=leaky_relu), \ arg_scope([conv2d], bn=True, phase=phase, activation=leaky_relu): if x._shape_as_list()[-1] == 1: x = tf.image.grayscale_to_rgb(x) x = conv2d(x, 64, 3, 2) x = conv2d(x, 128, 3, 2) x = conv2d(x, 256, 3, 2) x = dense(x, 128) return x
def encoder(x, phase, reuse=None): with tf.variable_scope('enc', reuse=reuse): with arg_scope([dense], bn=True, phase=phase, activation=leaky_relu), \ arg_scope([conv2d], bn=True, phase=phase, activation=leaky_relu): if x._shape_as_list()[-1] == 1: x = tf.image.grayscale_to_rgb(x) if args.dense: # Level 0: 32 l0 = x # Level 1: 32 -> 16, 8, 4 a1 = conv2d(l0, 64, 3, 2) a2 = conv2d(l0, 64, 5, 4) a3 = conv2d(l0, 64, 9, 8) l1 = a1 # Level 2: 16 -> 8, 4 b2 = conv2d(l1, 64, 3, 2) b3 = conv2d(l1, 64, 5, 4) l2 = tf.concat([a2, b2], -1) # Level 3: 8 -> 4 c3 = conv2d(l2, 64, 3, 2) l3 = tf.concat([a3, b3, c3], -1) # Level 4: Dense x = dense(l3, 128) else: x = conv2d(x, 64, 3, 2) x = conv2d(x, 128, 3, 2) x = conv2d(x, 256, 3, 2) x = dense(x, 128) return x
def generator(x, y, phase, reuse=None): with tf.variable_scope('gen', reuse=reuse): with arg_scope([dense], bn=True, phase=phase, activation=tf.nn.relu), \ arg_scope([conv2d_transpose], bn=True, phase=phase, activation=tf.nn.relu): if y is not None: x = tf.concat([x, y], 1) x = dense(x, 4 * 4 * 512) x = tf.reshape(x, [-1, 4, 4, 512]) x = conv2d_transpose(x, 256, 5, 2) x = conv2d_transpose(x, 128, 5, 2) x = wnconv2d_transpose(x, 3, 5, 2, bn=False, activation=tf.nn.tanh, scale=True) return x
def discriminator(x, phase, reuse=None): with tf.variable_scope('disc/gan', reuse=reuse): with arg_scope([wnconv2d, wndense], activation=leaky_relu): x = dropout(x, rate=0.2, training=phase) x = wnconv2d(x, 64, 3, 2) x = dropout(x, training=phase) x = wnconv2d(x, 128, 3, 2) x = dropout(x, training=phase) x = wnconv2d(x, 256, 3, 2) x = dropout(x, training=phase) x = wndense(x, 1024) x = dense(x, 1, activation=None, bn=False) return x
def classify(x, phase, reuse=None): z = encoder(x, phase, reuse=reuse) with tf.variable_scope('enc/final', reuse=reuse): y = dense(z, 10, activation=None, bn=False) return y
def e_step(self, x_data): print "e_step finetuning" tf.reset_default_graph() self.x_ = placeholder( (None, self.input_dim)) # we need these global nodes self.v_ = placeholder((None, self.num_factors)) self.sess = tf.Session() self.sess.run(tf.global_variables_initializer()) # inference process x = self.x_ depth_inf = len(self.encoding_dims) for i in range(depth_inf): x = dense(x, self.encoding_dims[i], scope="enc_layer" + "%s" % i, activation=tf.nn.sigmoid) # print("enc_layer0/weights:0".graph) h_encode = x z_mu = dense(h_encode, self.z_dim, scope="mu_layer") z_log_sigma_sq = dense(h_encode, self.z_dim, scope="sigma_layer") e = tf.random_normal(tf.shape(z_mu)) z = z_mu + tf.sqrt(tf.maximum(tf.exp(z_log_sigma_sq), self.eps)) * e # generative process depth_gen = len(self.decoding_dims) for i in range(depth_gen): y = dense(z, self.decoding_dims[i], scope="dec_layer" + "%s" % i, activation=tf.nn.sigmoid) # if last_layer_nonelinear: depth_gen -1 x_recons = y if self.loss_type == "cross_entropy": loss_recons = tf.reduce_mean( tf.reduce_sum(binary_crossentropy(x_recons, self.x_, self.eps), axis=1)) loss_kl = 0.5 * tf.reduce_mean( tf.reduce_sum( tf.square(z_mu) + tf.exp(z_log_sigma_sq) - z_log_sigma_sq - 1, 1)) loss_v = 1.0 * self.params.lambda_v / self.params.lambda_r * tf.reduce_mean( tf.reduce_sum(tf.square(self.v_ - z), 1)) # reg_loss we don't use reg_loss temporailly self.loss_e_step = loss_recons + loss_kl + loss_v train_op = tf.train.AdamOptimizer(self.params.learning_rate).minimize( self.loss_e_step) ckpt_file = "pre_model/" + "vae.ckpt" self.saver = tf.train.Saver() # if init == True: self.saver.restore(self.sess, ckpt_file) for i in range(self.params.num_iter): idx = np.random.choice(self.num_items, self.params.batch_size, replace=False) x_batch = x_data[idx] v_batch = self.V[idx] _, l = self.sess.run((train_op, self.loss_e_step), feed_dict={ self.x_: x_batch, self.v_: v_batch }) if i % 50 == 0: print "{:>10s}{:>10s}".format("epochs", "loss_e_step") print "{:>10d}{:>10.2e}".format(i, l) self.z_mu = z_mu self.x_recons = x_recons self.saver.save(self.sess, ckpt_file) return None