dtype=tf.float32) cov = tf.Variable(tf.eye(DNA_SIZE), dtype=tf.float32) mvn = MultivariateNormalFullCovariance( loc=mean, covariance_matrix=abs( cov + tf.Variable(0.001 * tf.eye(DNA_SIZE), dtype=tf.float32))) make_kid = mvn.sample(N_POP) #==========================compute gradient and update mean and covariance matrix from sample and fitness tfkids_fit = tf.placeholder(tf.float32, [ N_POP, ]) tfkids = tf.placeholder(tf.float32, [N_POP, DNA_SIZE]) loss = -tf.reduce_mean( mvn.log_prob(tfkids) * tfkids_fit + 0.01 * mvn.log_prob(tfkids) * mvn.prob(tfkids)) # log prob * fitness # print(0.01 * mvn.log_prob(tfkids) * mvn.prob(tfkids)) train_op = tf.train.GradientDescentOptimizer(LR).minimize( loss) # compute and apply gradients for mean and cov sess = tf.Session() sess.run(tf.global_variables_initializer()) max = 0 dr = 0 for g in range(N_GENERATION): if N_GENERATION % 10 == 0: LR = LR * 0.9 kids = sess.run(make_kid) kids_fit = [] feature_set = []
def build_network(self, status): # get parameters from the given input(status) param_stack = self.proposal.get_latent_samples(status=status) list = [] with tf.variable_scope(name_or_scope=self.scope, reuse=tf.AUTO_REUSE): # iterate over the time step # when t == 0 time = 0 [q_mean, q_cov] = param_stack.pop() samples = q_mean + tf.matmul( tf.random_normal(shape=(1, self.num_units)), q_cov) # transition p_mean = tf.zeros(shape=(1, self.num_units)) p_cov = tf.eye(num_rows=self.num_units) # emission h = tf.layers.dense(inputs=samples, units=self.num_units, activation=self.activation, kernel_initializer=self.init, name="emission") mu = tf.layers.dense(inputs=h, units=self.num_units, kernel_initializer=self.init, name="mu") logd = tf.layers.dense(inputs=h, units=self.num_units, kernel_initializer=self.init, name="logd") cov = tf.diag(tf.exp(logd[0])) x_dist = MultivariateNormalFullCovariance(loc=mu, covariance_matrix=cov) x = tf.reshape(tensor=status[0], shape=(1, self.num_units)) x_prob = x_dist.prob(value=x) list.append([q_mean, q_cov, p_mean, p_cov, x_prob, samples]) while len(param_stack) != 0: [q_mean, q_cov] = param_stack.pop() # transition for i in range(self.num_layers): p_mean = tf.layers.dense(inputs=samples, units=self.num_units, use_bias=False, kernel_initializer=self.init, name="dense" + str(i)) samples = q_mean + tf.matmul( tf.random_normal(shape=(1, self.num_units)), q_cov) p_cov = tf.diag( tf.exp(tf.random_normal(shape=(self.num_units)))) # emission h = tf.layers.dense(inputs=samples, units=self.num_units, activation=self.activation, kernel_initializer=self.init, name="emission") mu = tf.layers.dense(inputs=h, units=self.num_units, kernel_initializer=self.init, name="mu") logd = tf.layers.dense(inputs=h, units=self.num_units, kernel_initializer=self.init, name="logd") cov = tf.diag(tf.exp(logd[0])) x_dist = MultivariateNormalFullCovariance( loc=mu, covariance_matrix=cov) x = tf.reshape(tensor=status[0], shape=(1, self.num_units)) x_prob = x_dist.prob(value=x) list.append([q_mean, q_cov, p_mean, p_cov, x_prob, samples]) return list
class Worker_pop(object): def __init__(self, name, data): with tf.variable_scope(name): self.name = name self.DNA_size = data.getDNA_size() # self.mean_params, self.cov_params, self.mean, self.cov = self._creat_net(name, data.getDNA_size()) with tf.variable_scope('mean'): self.mean = tf.Variable(tf.truncated_normal([ self.DNA_size, ], stddev=0.1, mean=0.0), dtype=tf.float32, name=name + '_mean') with tf.variable_scope('cov'): self.cov = tf.Variable(1.0 * tf.eye(self.DNA_size), dtype=tf.float32, name=name + '_cov') # self.mvn = MultivariateNormalFullCovariance(loc=self.mean, covariance_matrix=self.cov) self.mvn = MultivariateNormalFullCovariance(loc=self.mean, covariance_matrix=abs( self.cov)) # self.mvn = MultivariateNormalFullCovariance(loc=self.mean, covariance_matrix=abs(self.cov + tf.Variable(0.05 * tf.eye(self.DNA_size), dtype=tf.float32))) self.make_kid = self.mvn.sample(N_POP) self.tfkids_fit = tf.placeholder(tf.float32, [ N_POP, ]) self.tfkids = tf.placeholder(tf.float32, [N_POP, self.DNA_size]) self.loss = -tf.reduce_mean( self.mvn.log_prob(self.tfkids) * self.tfkids_fit + 0.001 * self.mvn.log_prob(self.tfkids) * self.mvn.prob(self.tfkids)) self.train_op = tf.train.GradientDescentOptimizer( LEARNING_RATE).minimize( self.loss) # compute and apply gradients for mean and cov self.mean_params = tf.get_collection( tf.GraphKeys.TRAINABLE_VARIABLES, scope=name + '/mean') self.cov_params = tf.get_collection( tf.GraphKeys.TRAINABLE_VARIABLES, scope=name + '/cov') # sess.run(tf.global_variables_initializer()) # def _creat_net(self, scope, DNA_size): # with tf.variable_scope('mean'): # mean = tf.Variable(tf.truncated_normal([DNA_size, ], stddev=0.02, mean=2.5), dtype=tf.float32, # name=scope + '_mean') # with tf.variable_scope('cov'): # cov = tf.Variable(1.0 * tf.eye(DNA_size), dtype=tf.float32, name=scope + '_cov') # # with tf.variable_scope('mvn'): # # mvn = MultivariateNormalFullCovariance(loc=mean, covariance_matrix=cov + # # tf.Variable(0.05 * tf.eye(DNA_size), dtype=tf.float32)) # mean_params = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=scope + '/mean') # cov_params = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=scope + '/cov') # # sess.run(tf.global_variables_initializer()) # return mean_params, cov_params, mean, cov # # def _make_kids(self): # lock_kids.acquire() # kids = sess.run(self.mvn.sample(N_POP)) # lock_kids.release() # return kids def _update_net(self): lock_push.acquire() self.push_mean_params_op = [ g_p.assign(l_p) for g_p, l_p in zip(global_pop.mean_params, self.mean_params) ] self.push_cov_params_op = [ g_p.assign(l_p) for g_p, l_p in zip(global_pop.cov_params, self.cov_params) ] sess.run([self.push_mean_params_op, self.push_cov_params_op]) # self.update_mean = self.train_op.apply_gradients(zip(self.mean_grads, global_pop.mean_params)) # self.update_cov = self.train_op.apply_gradients(zip(self.cov_grads, global_pop.cov_params)) # sess.run([self.update_mean, self.update_cov]) # local grads applies to global net lock_push.release() def _pull_net(self): lock_pull.acquire() self.pull_mean_params_op = [ l_p.assign(g_p) for l_p, g_p in zip(self.mean_params, global_pop.mean_params) ] self.pull_cov_params_op = [ l_p.assign(g_p) for l_p, g_p in zip(self.cov_params, global_pop.cov_params) ] sess.run([self.pull_mean_params_op, self.pull_cov_params_op]) lock_pull.release()
import numpy as np from tensorflow.contrib.distributions import MultivariateNormalFullCovariance rand_init = tf.random_normal_initializer() mu = tf.get_variable(name="mu", shape=[1, 16], dtype=tf.float32, initializer=rand_init) log_d = tf.get_variable(name="log_d", shape=[1, 16], dtype=tf.float32, initializer=rand_init) with tf.name_scope(name="sample"): cov = tf.diag(tf.exp(log_d[0])) dist = MultivariateNormalFullCovariance(loc=mu, covariance_matrix=cov) sample = dist.sample() prob = dist.prob(value=sample) with tf.name_scope("gradient"): optimizer = tf.train.AdamOptimizer() list = optimizer.compute_gradients(loss=prob, var_list=[log_d]) with tf.name_scope("miscellaneous"): init = tf.global_variables_initializer() with tf.Session() as sess: sess.run(init) res = sess.run(list) print(res)