def C(img, q, scope='Classifier'):
with tf.variable_scope(scope) as scope:
log.warn(scope.name)
conv_1 = conv2d(img,
conv_info[0],
is_train,
s_h=3,
s_w=3,
name='conv_1')
conv_2 = conv2d(conv_1,
conv_info[1],
is_train,
s_h=3,
s_w=3,
name='conv_2')
conv_3 = conv2d(conv_2, conv_info[2], is_train, name='conv_3')
conv_4 = conv2d(conv_3, conv_info[3], is_train, name='conv_4')
conv_q = tf.concat(
[tf.reshape(conv_4, [self.batch_size, -1]), q], axis=1)
fc_1 = fc(conv_q, 256, name='fc_1')
fc_1 = slim.dropout(fc_1,
keep_prob=0.5,
is_training=is_train,
scope='fc_2/')
fc_2 = fc(fc_1, n, activation_fn=None, name='fc_2')
return fc_2
def discriminator(self, image, is_training, reuse=False):
with tf.variable_scope("discriminator"):
if reuse:
tf.get_variable_scope().reuse_variables()
h0 = lrelu(conv2d(image, self.discriminator_dim,
scope="d_h0_conv"))
h1 = lrelu(
batch_norm(conv2d(h0,
self.discriminator_dim * 2,
scope="d_h1_conv"),
is_training,
scope="d_bn_1"))
h2 = lrelu(
batch_norm(conv2d(h1,
self.discriminator_dim * 4,
scope="d_h2_conv"),
is_training,
scope="d_bn_2"))
h3 = lrelu(
batch_norm(conv2d(h2,
self.discriminator_dim * 8,
sh=1,
sw=1,
scope="d_h3_conv"),
is_training,
scope="d_bn_3"))
# real or fake binary loss
fc1 = fc(tf.reshape(h3, [self.batch_size, -1]), 1, scope="d_fc1")
# category loss
fc2 = fc(tf.reshape(h3, [self.batch_size, -1]),
self.embedding_num,
scope="d_fc2")
return tf.nn.sigmoid(fc1), fc1, fc2
def network(X, dropout_keep_prob=0.8, label_cnt=1000, type_=16): ####
out_channels = 64
layers16 = [2, 2, 3, 3, 3]
layers19 = [2, 2, 4, 4, 4]
if type_ == 19:
layers = layers19
else:
layers = layers16
for x in range(5):
for y in range(layers[x]):
layer_name = 'conv{}_{}layer'.format(x + 1, y + 1)
with tf.name_scope(layer_name):
X = op.conv(X,
filter_size=3,
out_channels=out_channels,
stride_size=1,
padding='SAME',
a=tf.nn.relu)
X = tf.nn.max_pool(X,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='VALID')
out_channels = 2 * out_channels
with tf.name_scope('fc1layer'):
X = op.fc(X, output_size=4096, a=tf.nn.relu)
with tf.name_scope('fc2layer'):
X = op.fc(X, output_size=4096, a=tf.nn.relu)
with tf.name_scope('fc3layer'):
X = op.fc(X, output_size=label_cnt, a=None)
with tf.name_scope('softmaxlayer'):
out_probs = tf.nn.softmax(logits=X, axis=-1, name='softmax_op')
return X, out_probs
def __init__(
self,
action_size,
thread_index, # -1 for global
device="/cpu:0"):
GameACNetwork.__init__(self, action_size, thread_index, device)
scope_name = "net_" + str(self._thread_index)
with tf.device(self._device), tf.variable_scope(scope_name):
# weight for policy output layer
self.W_fc2, self.b_fc2 = self._fc_variable([15, 5])
# weight for value output layer
self.W_fc3, self.b_fc3 = self._fc_variable([15, 1])
# state (input)
self.s = tf.placeholder("float", [None, 10])
h_fc1 = fc(self.s, 30, name='fc1')
h_fc2 = fc(h_fc1, 20, name='fc2')
h_fc3 = fc(h_fc2, 20, name='fc3')
h_fc4 = fc(h_fc3, 15, name='fc4')
# policy (output)
self.pi = tf.nn.softmax(tf.matmul(h_fc4, self.W_fc2) + self.b_fc2)
# value (output)
v_ = tf.matmul(h_fc4, self.W_fc3) + self.b_fc3
self.v = tf.reshape(v_, [-1])
def discriminator_z(self,
z,
is_training=True,
reuse_variables=False,
num_hidden_layer_channels=(64, 32, 16),
enable_bn=True):
if reuse_variables:
tf.get_variable_scope().reuse_variables()
current = z
# fully connection layer
for i in range(len(num_hidden_layer_channels)):
name = 'D_z_fc' + str(i)
current = ops.fc(input_vector=current,
num_output_length=num_hidden_layer_channels[i],
name=name)
if enable_bn:
name = 'D_z_bn' + str(i)
current = tf.contrib.layers.batch_norm(current,
scale=False,
is_training=is_training,
scope=name,
reuse=reuse_variables)
current = ops.lrelu(current)
# output layer
name = 'D_z_fc' + str(i + 1)
current = ops.fc(input_vector=current, num_output_length=1, name=name)
return tf.nn.sigmoid(current), current
def discriminator_ae(x, reuse=False):
#x_small = resize(x, (56, 56))
with tf.variable_scope('discriminator', reuse=reuse):
net = conv2d(x, 3, 32, d=2, scope='conv_1', dropout=False)
net = conv2d(net, 32, 64, d=2, scope='conv_2', dropout=False)
net = tf.reshape(net, [-1, 56 * 56 * 64])
code = lrelu(fc(net, 1000, scope='fc_3'))
net = lrelu(fc(code, 56 * 56 * 64, scope='fc_4'))
net = tf.reshape(net, [-1, 56, 56, 64])
net = resize(net, (112, 112))
net = deconv2d(net, 64, 32, 112, scope='conv_5', dropout=False)
net = resize(net, (224, 224))
#net = deconv2d(net, 32, 16, 224, scope='conv_6')
dimension = 3
if dimension == 2:
sc_1, x = tf.split(x, [1, 2], axis=3)
net = deconv2d(net, 32, 2, 224, scope='deconv_7', bn=False)
#loss = tf.sqrt(2 * tf.nn.l2_loss(x - net)) / (224*224)
loss = tf.norm(x - net, ord=1) / (224 * 224)
net = tf.concat([sc_1, net], axis=3)
else:
net = deconv2d(net, 32, 3, 224, scope='deconv_10', bn=False)
loss = tf.sqrt(2 * tf.nn.l2_loss(x - net)) / (224 * 224)
return net, loss
def __call__(self, x, is_reuse=False, is_train=True):
with tf.variable_scope('classifier') as scope:
if is_reuse:
scope.reuse_variables()
unit_n = self.smallest_unit_n
conv_ns = [2, 2]
for layer_i, conv_n in enumerate(conv_ns):
with tf.variable_scope('layer{}'.format(layer_i)):
for conv_i in range(conv_n):
x = conv2d(x, unit_n, self.k_size, 1, 'SAME', name='conv2d_{}'.format(conv_i))
x = tf.nn.relu(x)
x = maxpool2d(x, self.k_size, 2, 'SAME')
unit_n *= 2
unit_n = 256
fc_n = 1
for layer_i in range(len(conv_ns), len(conv_ns) + fc_n):
with tf.variable_scope('layer{}'.format(layer_i)):
x = fc(x, unit_n)
x = tf.nn.relu(x)
x = batch_norm(x, is_train)
x = tf.nn.dropout(x, 0.5)
with tf.variable_scope('output'.format(layer_i)):
x = fc(x, self.class_n)
return x
def f_phi(g, scope='f_phi'):
with tf.variable_scope(scope) as scope:
log.warn(scope.name)
fc_1 = fc(g, 256, name='fc_1')
fc_1 = slim.dropout(fc_1, keep_prob=0.5, is_training=is_train, scope='fc_1/')
fc_2 = fc(fc_1, n, activation_fn=None, name='fc_2')
return fc_2
def testCreateFCWithoutWD(self):
height, width = 3, 3
with self.test_session():
inputs = tf.random_uniform((5, height * width * 3), seed=1)
ops.fc(inputs, 32, weight_decay=0)
self.assertEquals(
tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES), [])
def network_model(features,
layer_channel_num_list,
device,
is_training,
initializer,
weight_decay,
weight_decay_rate,
reuse=False):
network_name_prefix = "FullyConnected_%dLayers_Channels" % len(
layer_channel_num_list)
for ii in layer_channel_num_list:
network_name_prefix = network_name_prefix + '_%d' % ii
if is_training:
print(print_separater)
print("Training on %s" % network_name_prefix)
print(print_separater)
drop_v = 0.5
else:
drop_v = 0
with tf.variable_scope(network_name_prefix):
if reuse:
tf.get_variable_scope().reuse_variables()
for ii in range(len(layer_channel_num_list)):
if ii == 0:
input_feature = features
else:
input_feature = previous_output_feature
if not ii == len(layer_channel_num_list) - 1:
current_output = dropout(relu(
batch_norm(
x=fc(x=input_feature,
output_size=layer_channel_num_list[ii],
weight_decay_rate=weight_decay_rate,
weight_decay=weight_decay,
parameter_update_device=device,
initializer=initializer,
scope='fc%d' % (ii + 1),
name_prefix=network_name_prefix),
is_training=is_training,
scope='bn%d' % (ii + 1),
parameter_update_device=device,
)),
drop_v=drop_v)
else:
current_output = fc(x=input_feature,
output_size=1,
weight_decay_rate=weight_decay_rate,
weight_decay=weight_decay,
parameter_update_device=device,
initializer=initializer,
scope='fc%d' % (ii + 1),
name_prefix=network_name_prefix)
previous_output_feature = current_output
output = tf.nn.sigmoid(previous_output_feature)
return previous_output_feature, output, network_name_prefix
def g_theta(o_i, o_j, q, scope='g_theta', reuse=True):
with tf.variable_scope(scope, reuse=reuse) as scope:
if not reuse: log.warn(scope.name)
g_1 = fc(tf.concat([o_i, o_j, q], axis=1), 256, name='g_1')
g_2 = fc(g_1, 256, name='g_2')
g_3 = fc(g_2, 256, name='g_3')
return g_3
def __init__(self, sess, img_height, img_width, c_dim, a_dim, reuse=False):
self.sess = sess
with tf.variable_scope("policy", reuse=reuse):
#ob_ph: (mb_size, img_height, img_width, c_dim)
self.ob_ph = tf.placeholder(tf.uint8, [None, img_height, img_width, c_dim], name="observation")
ob_normalized = tf.cast(self.ob_ph, tf.float32) / 255.0
#conv1: (mb_size, img_height1, img_width1, 32)
h = ops.conv2d(ob_normalized, 32, 8, 8, 4, 4, name="conv1")
h = tf.nn.relu(h)
#conv2: (mb_size, img_height2, img_width2, 64)
h = ops.conv2d(h, 64, 4, 4, 2, 2, name="conv2")
h = tf.nn.relu(h)
#conv3: (mb_size, img_height3, img_width3, 64)
h = ops.conv2d(h, 64, 3, 3, 1, 1, name="conv3")
h = tf.nn.relu(h)
#fc: (mb_size, 512)
h = ops.fc(tf.reshape(h, [-1, h.shape[1]*h.shape[2]*h.shape[3]]), 512, name="fc1")
h = tf.nn.relu(h)
#pi: (mb_size, a_dim)
#value: (mb_size, 1)
pi = ops.fc(h, a_dim, name="fc_pi")
value = ops.fc(h, 1, name="fc_value")
#value: (mb_size)
#action: (mb_size)
self.value = value[:, 0]
self.cat_dist = tf.distributions.Categorical(pi)
self.action = self.cat_dist.sample(1)[0]
self.pi = pi
def g_theta_faster(inputs, scope='g_theta', reuse=True):
with tf.variable_scope(scope, reuse=reuse) as scope:
# if not reuse: log.warn(scope.name)
g_1 = fc(inputs, 256, name='g_1')
g_2 = fc(g_1, 256, name='g_2')
g_3 = fc(g_2, 256, name='g_3')
g_4 = fc(g_3, 256, name='g_4')
return g_4
def f_phi(g, is_train, scope='f_phi'):
with tf.variable_scope(scope) as scope:
# log.warn(scope.name)
fc_1 = fc(g, 256, name='fc_1')
fc_2 = fc(fc_1, 256, name='fc_2')
fc_2 = slim.dropout(fc_2, keep_prob=0.5, is_training=is_train, scope='fc_3/')
fc_3 = fc(fc_2, ANSWER_DICT_LENGTH, activation_fn=None, name='fc_3')
return fc_3
def testNonReuseVars(self):
height, width = 3, 3
inputs = tf.random_uniform((5, height * width * 3), seed=1)
with self.test_session():
ops.fc(inputs, 32)
self.assertEquals(len(variables.get_variables('FC')), 2)
ops.fc(inputs, 32)
self.assertEquals(len(variables.get_variables('FC')), 4)
def testReuseVars(self):
height, width = 3, 3
inputs = tf.random_uniform((5, height * width * 3), seed=1)
with self.test_session():
ops.fc(inputs, 32, scope='fc1')
self.assertEquals(len(variables.get_variables('fc1')), 2)
ops.fc(inputs, 32, scope='fc1', reuse=True)
self.assertEquals(len(variables.get_variables('fc1')), 2)
def testReuseFCWithBatchNorm(self):
height, width = 3, 3
with self.test_session():
images = tf.random_uniform((5, height * width * 3), seed=1)
with scopes.arg_scope([ops.fc], batch_norm_params={'decay': 0.9}):
net = ops.fc(images, 27, scope='fc1')
net = ops.fc(net, 27, scope='fc1', reuse=True)
self.assertEquals(len(variables.get_variables()), 4)
self.assertEquals(len(variables.get_variables('fc1/BatchNorm')), 3)
def testCreateFcCreatesWeightsAndBiasesVars(self):
height, width = 3, 3
inputs = tf.random_uniform((5, height * width * 3), seed=1)
with self.test_session():
self.assertFalse(variables.get_variables('fc1/weights'))
self.assertFalse(variables.get_variables('fc1/biases'))
ops.fc(inputs, 32, scope='fc1')
self.assertTrue(variables.get_variables('fc1/weights'))
self.assertTrue(variables.get_variables('fc1/biases'))
def Counter(img, reuse=True, scope='Counter'):
with tf.variable_scope(scope, reuse=reuse) as scope:
if not reuse: log.warn(scope.name)
_ = conv2d(img, 64, is_train, info=not reuse, name='conv1_1')
_ = conv2d(_, 64, is_train, info=not reuse, name='conv1_2')
conv1 = max_pool(_, name='conv1')
_ = conv2d(conv1,
128,
is_train,
info=not reuse,
name='conv2_1')
_ = conv2d(_, 128, is_train, info=not reuse, name='conv2_2')
conv2 = max_pool(_, name='conv2')
_ = conv2d(conv2,
256,
is_train,
info=not reuse,
name='conv3_1')
_ = conv2d(_, 256, is_train, info=not reuse, name='conv3_2')
_ = conv2d(_, 256, is_train, info=not reuse, name='conv3_3')
conv3 = max_pool(_, name='conv3')
_ = conv2d(conv3,
512,
is_train,
info=not reuse,
name='conv4_1')
_ = conv2d(_, 512, is_train, info=not reuse, name='conv4_2')
_ = conv2d(_, 512, is_train, info=not reuse, name='conv4_3')
conv4 = max_pool(_, name='conv4')
_ = conv2d(conv4,
512,
is_train,
info=not reuse,
name='conv5_1')
_ = conv2d(_, 512, is_train, info=not reuse, name='conv5_2')
_ = conv2d(_, 512, is_train, info=not reuse, name='conv5_3')
conv5 = max_pool(_, name='conv5')
fc1 = fc(tf.reshape(conv5, [self.batch_size, -1]),
4096,
is_train,
info=not reuse,
name='fc_1')
fc2 = fc(fc1, 4096, is_train, info=not reuse, name='fc_2')
fc3 = fc(fc2, 1000, is_train, info=not reuse, name='fc_3')
fc4 = fc(fc3,
1000,
is_train,
info=not reuse,
batch_norm=False,
name='fc_4')
return [conv1, conv2, conv3, conv4, conv5, fc1, fc2, fc3, fc4]
def value_net(self, state_logits_last, reuse=False):
with tf.variable_scope("value_net", reuse=reuse):
#h: (mb_size, 128)
h = ops.fc(state_logits_last, 128, name="fc1")
h = tf.nn.relu(h)
#value: (mb_size, 1)
value = ops.fc(h, 1, name="fc_out")
return value
def testCreateFCWithWD(self):
height, width = 3, 3
with self.test_session() as sess:
inputs = tf.random_uniform((5, height * width * 3), seed=1)
ops.fc(inputs, 32, weight_decay=0.01)
wd = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)[0]
self.assertEquals(wd.op.name,
'FC/weights/Regularizer/L2Regularizer/value')
sess.run(tf.global_variables_initializer())
self.assertTrue(sess.run(wd) <= 0.01)
def g_theta(o_i, o_j, q, scope='g_theta', reuse=True):
with tf.variable_scope(scope, reuse=reuse) as scope:
if not reuse: log.warn(scope.name)
g_1 = fc(tf.concat([o_i, o_j, q], axis=1), 256, name='g_1')
g_1 = tf.layers.dropout(g_1, rate=0.5, training=is_train)
g_2 = fc(g_1, 256, name='g_2')
g_2 = tf.layers.dropout(g_2, rate=0.5, training=is_train)
g_3 = fc(g_2, 256, name='g_3')
g_3 = tf.layers.dropout(g_3, rate=0.5, training=is_train)
g_4 = fc(g_3, 256, name='g_4')
return g_4
def g_theta(o_i, o_j, q, scope='g_theta', reuse=True):
with tf.variable_scope(scope, reuse=reuse) as scope:
if not reuse: log.warn(scope.name)
# check_tensor(o_i, name="o_i") [batch, 26]
# check_tensor(o_j, name="o_j") [batch, 26]
# check_tensor(q, name='q') [batch, 11]
g_1 = fc(tf.concat([o_i, o_j, q], axis=1), 256, name='g_1')
g_2 = fc(g_1, 256, name='g_2')
g_3 = fc(g_2, 256, name='g_3')
g_4 = fc(g_3, 256, name='g_4')
return g_4
def testFCWithBatchNorm(self):
height, width = 3, 3
with self.test_session():
images = tf.random_uniform((5, height * width * 3), seed=1)
with scopes.arg_scope([ops.fc], batch_norm_params={}):
net = ops.fc(images, 27)
net = ops.fc(net, 27)
self.assertEquals(len(variables.get_variables()), 8)
self.assertEquals(len(variables.get_variables('FC/BatchNorm')), 3)
self.assertEquals(len(variables.get_variables('FC_1/BatchNorm')),
3)
def final(f, scope='final'):
with tf.variable_scope(scope) as scope:
log.warn(scope.name)
fi_1 = fc(f, 256, name='final_1')
fi_2 = fc(fi_1, 256, name='final_2')
fi_3 = slim.dropout(fi_2,
keep_prob=0.5,
is_training=is_train,
scope='final_3/')
fi_4 = fc(fi_3, 10, activation=None, name='final_4')
return fi_4
def f_phi(g, scope='f_phi', reuse=True):
with tf.variable_scope(scope, reuse=reuse) as scope:
if not reuse: log.warn(scope.name)
fc_1 = fc(g, 256, name='fc_1')
fc_2 = fc(fc_1, 256, name='fc_2')
fc_3 = slim.dropout(fc_2,
keep_prob=0.5,
is_training=is_train,
scope='fc_3/')
fc_4 = fc(fc_3, 256, activation=None, name='fc_4')
return fc_4
def testReuseFCWithWD(self):
height, width = 3, 3
with self.test_session():
inputs = tf.random_uniform((5, height * width * 3), seed=1)
ops.fc(inputs, 32, weight_decay=0.01, scope='fc')
self.assertEquals(len(variables.get_variables()), 2)
self.assertEquals(
len(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)), 1)
ops.fc(inputs, 32, weight_decay=0.01, scope='fc', reuse=True)
self.assertEquals(len(variables.get_variables()), 2)
self.assertEquals(
len(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)), 1)
def discriminator(self, state_logits_last, reuse=False):
with tf.variable_scope("dis", reuse=reuse):
#h: (mb_size, 128)
h = ops.fc(state_logits_last, 128, name="fc1")
h = tf.nn.relu(h)
#logits: (mb_size, 1)
#prob: (mb_size, 1)
logits = ops.fc(h, 1, name="fc_out")
prob = tf.nn.sigmoid(logits)
return logits, prob
def state_action_discriminator(self, state_logits_last, action_embed, reuse=False):
with tf.variable_scope("sa_dis", reuse=reuse):
#h: (mb_size, 128)
h = ops.fc(tf.concat([state_logits_last, action_embed], 1), 128, name="fc1")
h = tf.nn.relu(h)
#logits: (mb_size, 1)
#prob: (mb_size, 1)
logits = ops.fc(h, 1, name="fc_out")
prob = tf.nn.sigmoid(logits)
return logits, prob
def f_phi(g, scope='f_phi'):
with tf.variable_scope(scope) as scope:
fc_1 = fc(g, 256, name='fc_1')
fc_2 = fc(fc_1, 256, name='fc_2')
fc_2 = slim.dropout(fc_2,
keep_prob=0.5,
is_training=is_train,
scope='fc_3/')
fc_3 = fc(fc_2,
self.vocab_size,
activation_fn=None,
name='fc_3')
return fc_3
def __init__(self, env_type, state_dims, num_actions):
if env_type == EnvTypes.ATARI:
state_size = [state_dims[0], state_dims[1]*FRAME_STACK, state_dims[2]]
elif env_type == EnvTypes.STANDARD:
state_size = state_dims
self.replay_memory = ReplayMemory(REPLAY_MEMORY_CAPACITY, state_size)
self.exploration = 1.0
self.train_iter = 0
self.env_type = env_type
if env_type == EnvTypes.ATARI:
buffer_size = FRAME_STACK*FRAME_SKIP
self.observation_buffer = [np.zeros((state_dims[0], state_dims[1], state_dims[2]))
for _ in range(buffer_size)]
else:
self.observation_buffer = [np.zeros((state_dims[0]))]
self.config = tf.ConfigProto()
self.config.gpu_options.per_process_gpu_memory_fraction = GPU_MEMORY_FRACTION
self.sess = tf.Session(config=self.config)
# build q network
self.dqn_vars = dict()
with tf.variable_scope(DQN_SCOPE):
if env_type == EnvTypes.ATARI:
self.x, self.initial_layers = self.add_atari_layers(state_dims, self.dqn_vars)
elif env_type == EnvTypes.STANDARD:
self.x, self.initial_layers = self.add_standard_layers(state_dims, self.dqn_vars)
# add final hidden layers
self.hid = fc(self.initial_layers, 128, HIDDEN, var_dict=self.dqn_vars)
self.q = fc(self.hid, num_actions, OUTPUT,
var_dict=self.dqn_vars, activation=False)
tf.histogram_summary('q_values', self.q)
# build target network
self.target_vars = dict()
with tf.variable_scope(TARGET_SCOPE):
if env_type == EnvTypes.ATARI:
self.t_x, self.t_initial_layers = self.add_atari_layers(state_dims,
self.target_vars)
elif env_type == EnvTypes.STANDARD:
self.t_x, self.t_initial_layers = self.add_standard_layers(state_dims,
self.target_vars)
self.t_hid = fc(self.t_initial_layers, 128, HIDDEN, var_dict=self.target_vars)
self.t_q = fc(self.t_hid, num_actions, OUTPUT,
var_dict=self.target_vars, activation=False)
tf.histogram_summary('target_q_values', self.t_q)
# add weight transfer operations from primary dqn network to target network
self.assign_ops = []
with tf.variable_scope(TRANSFER_SCOPE):
for variable in self.dqn_vars.keys():
target_variable = TARGET_SCOPE + variable[len(DQN_SCOPE):]
decay = tf.mul(1 - TAU, self.target_vars[target_variable])
update = tf.mul(TAU, self.dqn_vars[variable])
new_target_weight = tf.add(decay, update)
target_assign = self.target_vars[target_variable].assign(new_target_weight)
self.assign_ops.append(target_assign)
# build dqn evaluation
with tf.variable_scope(EVALUATION_SCOPE):
# one-hot action selection
self.action = tf.placeholder(tf.int32, shape=[None])
self.action_one_hot = tf.one_hot(self.action, num_actions)
# reward
self.reward = tf.placeholder(tf.float32, shape=[None, 1])
# terminal state
self.nonterminal = tf.placeholder(tf.float32, shape=[None, 1])
self.target = tf.add(self.reward, tf.mul(GAMMA, tf.mul(self.nonterminal,
tf.reduce_max(self.t_q, 1, True))))
self.predict = tf.reduce_sum(tf.mul(self.action_one_hot, self.q), 1, True)
self.error = tf.reduce_mean(mse(self.predict, self.target))
tf.scalar_summary('error', self.error)
val_print = tf.Print(self.error, [self.predict, self.target])
self.optimize = tf.train.RMSPropOptimizer(ALPHA, decay=RMS_DECAY, momentum=MOMENTUM,
epsilon=EPSILON).minimize(self.error, var_list=self.dqn_vars.values())
# write out the graph and summaries for tensorboard
self.summaries = tf.merge_all_summaries()
if os.path.isdir(TENSORBOARD_GRAPH_DIR):
shutil.rmtree(TENSORBOARD_GRAPH_DIR)
self.writer = tf.train.SummaryWriter(TENSORBOARD_GRAPH_DIR, self.sess.graph)
# initialize variables
self.sess.run(tf.initialize_all_variables())
# create saver
self.saver = tf.train.Saver()
def add_standard_layers(self, dims, var_dict):
x = tf.placeholder(tf.float32, shape=[None, dims[0]])
fc1 = fc(x, 256, FC, var_dict=var_dict)
return x, fc1
