def __init__(self, sess, ob_space, ac_space, nenv, nsteps, nstack, size, reuse=False): nbatch = nenv * nsteps nh, nw, nc = ob_space.shape ob_shape = (nbatch, nh, nw, nc * nstack) nact = size * size # ac_space.n X = tf.placeholder(tf.float32, ob_shape) # obs with tf.variable_scope("model", reuse=reuse): h = conv(tf.cast(X, tf.float32), 'c1m', nf=32, rf=2, stride=1, init_scale=np.sqrt(2)) h2 = conv(h, 'c2m', nf=64, rf=2, stride=1, init_scale=np.sqrt(2)) # h3 = conv(h2, 'c3m', nf=64, rf=3, stride=1, init_scale=np.sqrt(2)) h3 = conv_to_fc(h2) h4 = fc(h3, 'fc1m', nh=512, init_scale=np.sqrt(2)) pi = fc(h4, 'pim', nact, act=lambda x: x) vf = fc(h4, 'vm', 1, act=lambda x: x) v0 = vf[:, 0] a0 = sample_without_exploration(pi) noise = tf.random_uniform(tf.shape(pi), tf.reduce_max(pi)) / 5 pi = noise + pi p0 = pi self.initial_state = [] # not stateful def step(ob, *_args, **_kwargs): a, v, prob = sess.run([a0, v0, p0], {X: ob}) # print('sum', (prob + 1) / (np.sum(prob) + len(prob[0]))) return a, v, [], (prob + 1) / (np.sum(prob) + len(prob[0])) def value(ob, *_args, **_kwargs): return sess.run(v0, {X: ob}) self.X = X self.pi = pi self.vf = vf self.step = step self.value = value
def __init__(self, sess, ob_space, ac_space, nenv, nsteps, nstack, scope, reuse=False): nbatch = nenv * nsteps nh, nw, nc = ob_space.shape ob_shape = (nbatch, nh, nw, nc * nstack) nact = ac_space X = tf.placeholder(tf.float32, ob_shape) # obs X = X + 1.1 / 2.2 TEMP = tf.placeholder(tf.float32, 1) with tf.variable_scope(scope, reuse=reuse): conv1 = slim.conv2d( activation_fn=tf.nn.relu, inputs=X, num_outputs=64, kernel_size=[5, 5], stride=[1, 1], padding='VALID', weights_regularizer=slim.l2_regularizer(0.0005)) conv2 = slim.conv2d( activation_fn=tf.nn.relu, inputs=conv1, num_outputs=128, kernel_size=[2, 2], stride=[1, 1], padding='VALID', weights_regularizer=slim.l2_regularizer(0.0005)) conv3 = slim.conv2d( activation_fn=tf.nn.relu, inputs=conv2, num_outputs=256, kernel_size=[2, 2], stride=[1, 1], padding='VALID', weights_regularizer=slim.l2_regularizer(0.0005)) hidden = slim.fully_connected(slim.flatten(conv3), 512, activation_fn=tf.nn.relu) pi = slim.fully_connected( hidden, nact, activation_fn=None, weights_initializer=normalized_columns_initializer(0.01), biases_initializer=None, weights_regularizer=slim.l2_regularizer(0.0005)) vf = slim.fully_connected( hidden, 1, activation_fn=None, weights_initializer=normalized_columns_initializer(1.0), biases_initializer=None, weights_regularizer=slim.l2_regularizer(0.0005)) # pi = tf.nn.softmax(pi / TEMP) v0 = vf[:, 0] p0 = [pi] a0 = sample_without_exploration(pi) self.initial_state = [] # not stateful def step(ob, temp, *_args, **_kwargs): a, v, pi = sess.run([a0, v0, p0], {X: ob, TEMP: temp}) return a, v, [], pi def value(ob, temp, *_args, **_kwargs): return sess.run(v0, {X: ob, TEMP: temp}) def get_pi(ob, *_args, **_kwargs): a, v, pi = sess.run([a0, v0, p0], {X: ob}) return a, v, [], pi self.X = X self.TEMP = TEMP self.pi = pi self.vf = vf self.step = step self.value = value
def __init__(self, sess, ob_space, ac_space, nenv, nsteps, nstack, scope, reuse=False): nbatch = nenv * nsteps nh, nw, nc = ob_space.shape # ob_shape = (nbatch, nh, nw, nc * nstack) nact = ac_space self.X = tf.placeholder(tf.float32, shape=[None, 4, nh, nw]) self.X_reshaped = tf.reshape(self.X, [-1, nh, nw, 4]) self.TEMP = tf.placeholder(tf.float32, 1) with tf.variable_scope(scope, reuse=reuse): conv1 = slim.conv2d(activation_fn=tf.nn.relu, inputs=self.X_reshaped, num_outputs=32, kernel_size=[3, 3], stride=[1, 1], padding='same') conv2 = slim.conv2d(activation_fn=tf.nn.relu, inputs=conv1, num_outputs=64, kernel_size=[3, 3], stride=[1, 1], padding='same') conv3 = slim.conv2d(activation_fn=tf.nn.relu, inputs=conv2, num_outputs=128, kernel_size=[3, 3], stride=[1, 1], padding='same') action_conv = slim.conv2d(activation_fn=tf.nn.relu, inputs=conv3, num_outputs=4, kernel_size=[1, 1], stride=[1, 1], padding='same') action_conv_flat = tf.reshape(action_conv, [-1, 4 * nh * nw]) pi = tf.layers.dense(action_conv_flat, nh * nw, activation=tf.nn.softmax) eval_conv = tf.layers.conv2d(inputs=conv3, filters=2, kernel_size=[1, 1], padding="same", activation=tf.nn.relu) eval_conv_flat = tf.reshape(eval_conv, [-1, 2 * nh * nw]) conv_fc = tf.layers.dense(inputs=eval_conv_flat, units=nh * nw, activation=tf.nn.relu) vf = tf.layers.dense(inputs=conv_fc, units=1, activation=tf.nn.tanh) # pi = slim.fully_connected(pi, nact, # activation_fn=None, # weights_initializer=normalized_columns_initializer(0.01), # biases_initializer=None, # weights_regularizer=slim.l2_regularizer(0.0005)) # vf = slim.fully_connected(pi, 1, # activation_fn=None, # weights_initializer=normalized_columns_initializer(1.0), # biases_initializer=None, # weights_regularizer=slim.l2_regularizer(0.0005)) # pi = tf.nn.softmax(pi / TEMP) v0 = vf[:, 0] p0 = [pi] a0 = sample_without_exploration(pi) self.initial_state = [] # not stateful def step(ob, temp, *_args, **_kwargs): a, v, pi = sess.run([a0, v0, p0], {self.X: ob, self.TEMP: temp}) return a, v, [], pi def value(ob, temp, *_args, **_kwargs): return sess.run(v0, {self.X: ob, self.TEMP: temp}) def get_pi(ob, *_args, **_kwargs): a, v, pi = sess.run([a0, v0, p0], {self.X: ob}) return a, v, [], pi self.pi = pi self.vf = vf self.step = step self.value = value