def _construct_default_model(self):
with self.graph.as_default():
with tf.variable_scope('actor'):
self.actor_model = NNM.NeuralNetworkModel(graph=self.graph)
self.actor_model.td_error = tf.placeholder(dtype=self.dtype,
shape=None)
self.actor_model.action = tf.placeholder(dtype=self.dtype,
shape=None)
self.actor_model.build(NNU.NeuronLayer(hidden_dim=20),
input_dim=self.env.features_dim)
self.actor_model.build(
NNU.NeuronLayer(hidden_dim=self.env.actions_num))
self.actor_model.compile(
optimizer=tf.train.GradientDescentOptimizer,
loss_fun=NNL.NeuralNetworkLoss.exploss,
action=self.actor_model.action,
td_error=self.actor_model.td_error)
with tf.variable_scope('critic'):
self.critic_model = NNM.NeuralNetworkModel(graph=self.graph)
self.critic_model.reward = tf.placeholder(dtype=self.dtype,
shape=None)
self.critic_model.value = tf.placeholder(dtype=self.dtype,
shape=[1, 1])
self.critic_model.build(NNU.NeuronLayer(hidden_dim=10),
input_dim=self.env.features_dim)
self.critic_model.build(NNU.NeuronLayer(hidden_dim=1))
self.critic_model.compile(
optimizer=tf.train.GradientDescentOptimizer,
loss_fun=NNL.NeuralNetworkLoss.tdsquared,
reward=self.critic_model.reward,
gamma=self.gamma)
def example3():
def load_data(filename):
this_dir, _ = os.path.split(__file__)
data_path = os.path.join(this_dir, 'data', filename)
return np.load(data_path)
imgs, labels, shape = load_data('FERET.npy')
x_train, y_train, x_test, y_test = UF.split_train_test(imgs, labels, 2)
y_train = UF.OneHot(y_train)
y_test = UF.OneHot(y_test)
model = NNM.NeuralNetworkModel(dtype=tf.float64)
# shape=(5,5,3) means the kernel's height=5 width=5 num of ker=3.
# first layer should give the input dim, which in this case is (image height, image width, num channel).
model.build(NNU.ConvolutionUnit(dtype=tf.float64, shape=(4, 4, 2),
transfer_fun=tf.tanh), input_dim=x_train.shape[1:])
# model.build(NNU.BatchNormalization())
# model.build(NNU.Dropout(keep_prob=0.8))
# model.build(NNU.ConvolutionUnit(dtype=tf.float64, shape=(2, 2, 4),
# transfer_fun=tf.sigmoid))
# model.build(NNU.AvgPooling(dtype=tf.float64, shape=(1, 4, 4, 1)))
model.build(NNU.Dropout(keep_prob=0.8))
model.build(NNU.Flatten())
model.build(NNU.NeuronLayer(hidden_dim=shape[1], dtype=tf.float64))
model.build(NNU.BatchNormalization())
model.build(NNU.SoftMaxLayer())
model.fit(x_train, y_train, loss_fun=NNL.NeuralNetworkLoss.crossentropy, show_graph=False, num_epochs=501,
mini_batch=10, learning_rate=0.5)
acc, result, correctness = model.evaluate(x_test, y_test)
print(acc, result, correctness)
def example2():
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets('MNIST_data/', one_hot=True)
x_train = mnist.train.images
# To reduce computational cost, we let the training size be 500.
x_train = x_train[:500, :]
y_train = mnist.train.labels
y_train = y_train[:500, :]
x_train = UF.vectors2imgs(x_train, (None, 28, 28, 1))
model = NNM.NeuralNetworkModel(dtype=tf.float32, img_size=(28, 28))
# shape=(5,5,3) means the kernel's height=5 width=5 num of ker=3
model.build(NNU.ConvolutionUnit(dtype=tf.float32, shape=(5, 5, 3), transfer_fun=tf.tanh))
model.build(NNU.AvgPooling(dtype=tf.float32, shape=(1, 4, 4, 1)))
model.build(NNU.Dropout(keep_prob=0.5))
model.build(NNU.Flatten())
model.build(NNU.NeuronLayer(hidden_dim=10, dtype=tf.float32))
model.build(NNU.SoftMaxLayer())
model.fit(x_train, y_train, loss_fun=NNL.NeuralNetworkLoss.crossentropy, show_graph=True, num_epochs=1000)
x_test = mnist.test.images
y_test = mnist.test.labels
x_test = x_test[0:20, :]
y_test = y_test[0:20, :]
x_test = UF.vectors2imgs(x_test, (None, 28, 28, 1))
print(model.Evaluate(x_test, y_test))
def example1():
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets('MNIST_data/', one_hot=True)
x_train = mnist.train.images
# To reduce computational cost, we let the training size be 500.
x_train = x_train[:500, :]
model = NNM.NeuralNetworkModel()
model.build(NNU.NeuronLayer(hidden_dim=10), input_dim=784)
model.build(NNU.BatchNormalization())
model.build(NNU.NeuronLayer(hidden_dim=5, transfer_fun=tf.nn.sigmoid))
model.build(NNU.BatchNormalization())
model.build(NNU.NeuronLayer(hidden_dim=784))
import time
t1 = time.time()
model.Fit(x_train, x_train, show_graph=False, num_epochs=500,
mini_size=40, loss_fun=NNL.NeuralNetworkLoss.meansquared)
print(time.time()-t1)
n = 1 # how many digits we will display
x_test = mnist.test.images
x_test = x_test[:n, :]
results = model.predict(x_test)
model.print_output_detail(x_test)
def _construct_default_models(self):
with self.graph.as_default():
with tf.variable_scope('eval'):
self.eval_model = NNM.NeuralNetworkModel(graph=self.graph)
self.eval_model.build(NNU.NeuronLayer(
hidden_dim=10, transfer_fun=tf.nn.sigmoid),
input_dim=self.env.features_size)
self.eval_model.build(NNU.BatchNormalization())
# self.eval_model.build(NNU.NeuronLayer(hidden_dim=5, transfer_fun=tf.nn.sigmoid))
# self.eval_model.build(NNU.BatchNormalization())
self.eval_model.split(names=['adv', 'value'])
self.eval_model.build(NNU.NeuronLayer(hidden_dim=1),
name='value')
self.eval_model.build(
NNU.NeuronLayer(hidden_dim=self.actions_size), name='adv')
self.eval_model.build(NNU.ReduceMean(), name='adv')
self.eval_model.merge(op='add', names=['adv', 'value'])
self.eval_model.batch_size = self.batch_size
self.eval_model.mini_batch = self.eval_model.batch_size
self.eval_model.compile(
optimizer=tf.train.GradientDescentOptimizer(
learning_rate=self.learning_rate),
loss_fun=NNL.NeuralNetworkLoss.meansquared)
self.eval_model.sess.close()
with tf.variable_scope('target'):
self.targ_model = NNM.NeuralNetworkModel(graph=self.graph)
self.targ_model.build(NNU.NeuronLayer(
hidden_dim=10, transfer_fun=tf.nn.sigmoid),
input_dim=self.env.features_size)
self.targ_model.build(NNU.BatchNormalization())
# self.targ_model.build(NNU.NeuronLayer(hidden_dim=5, transfer_fun=tf.nn.sigmoid))
# self.targ_model.build(NNU.BatchNormalization())
self.targ_model.split(names=['adv', 'value'])
self.targ_model.build(NNU.NeuronLayer(hidden_dim=1),
name='value')
self.targ_model.build(
NNU.NeuronLayer(hidden_dim=self.actions_size), name='adv')
self.targ_model.build(NNU.ReduceMean(), name='adv')
self.targ_model.merge(op='add',
names=['adv', 'value'],
output_name='last')
self.targ_model.sess.close()
def _construct_default_models(self):
self.eval_model = NNM.NeuralNetworkModel()
self.eval_model.build(NNU.NeuronLayer(hidden_dim=20))
self.eval_model.build(NNU.NeuronLayer(hidden_dim=15))
self.eval_model.build(NNU.NeuronLayer(hidden_dim=self.actions_size))
# For priority reply
self.eval_model.ISWeights = tf.placeholder(tf.float32, [None, 1])
# target model and eval model share the same structure.
self.targ_model = cp.deepcopy(self.eval_model)
def _construct_default_models(self):
with self.graph.as_default():
with tf.variable_scope('eval'):
self.eval_model = NNM.NeuralNetworkModel(graph=self.graph)
self.eval_model.build(NNU.NeuronLayer(
hidden_dim=30, transfer_fun=tf.nn.sigmoid),
input_dim=self.env.features_size)
self.eval_model.build(NNU.BatchNormalization())
self.eval_model.build(
NNU.NeuronLayer(hidden_dim=20, transfer_fun=tf.nn.sigmoid))
self.eval_model.build(NNU.BatchNormalization())
self.eval_model.build(
NNU.NeuronLayer(hidden_dim=self.actions_size))
# self.eval_model.Build(NNU.BatchNormalization())
self.eval_model.batch_size = self.batch_size
self.eval_model.mini_batch = self.eval_model.batch_size
self.eval_model.compile(
optimizer=tf.train.GradientDescentOptimizer(
learning_rate=self.learning_rate),
loss_fun=NNL.NeuralNetworkLoss.meansquared)
self.eval_model.sess.close()
# target model and eval model share the same structure.
with tf.variable_scope('target'):
self.targ_model = NNM.NeuralNetworkModel(graph=self.graph)
self.targ_model.build(NNU.NeuronLayer(
hidden_dim=30, transfer_fun=tf.nn.sigmoid),
input_dim=self.env.features_size)
self.targ_model.build(NNU.BatchNormalization())
self.targ_model.build(
NNU.NeuronLayer(hidden_dim=20, transfer_fun=tf.nn.sigmoid))
self.targ_model.build(NNU.BatchNormalization())
self.targ_model.build(
NNU.NeuronLayer(hidden_dim=self.actions_size,
transfer_fun=tf.nn.sigmoid))
# self.targ_model.build(NNU.BatchNormalization())
self.targ_model.sess.close()
def _construct_default_models(self):
with self.graph.as_default():
with tf.variable_scope('Policy_Network'):
self.policy_model = NNM.NeuralNetworkModel(graph=self.graph)
self.policy_model.build(NNU.NeuronLayer(
hidden_dim=30, transfer_fun=tf.nn.sigmoid),
input_dim=self.env.features_size)
self.policy_model.build(
NNU.NeuronLayer(hidden_dim=30, transfer_fun=None))
self.policy_model.build(NNU.SoftMaxLayer())
self.policy_model.action_state_value = tf.placeholder(shape=[
None,
])
self.policy_model.mini_batch = self.batch_size
self.policy_model.compile(
optimizer=tf.train.GradientDescentOptimizer(
learning_rate=self.learning_rate),
loss_fun=NNL.NeuralNetworkLoss.crossentropy,
loss_and_optimize=False)
with self.graph.as_default():
self.policy_model.target = tf.placeholder(shape=[
None,
])
self.policy_model.loss = self.policy_model.loss_fun(
output=self.policy_model.output,
target=self.policy_model.target,
batch_size=1,
axis=1)
self.policy_model.loss = tf.reduce_mean(
self.policy_model.loss *
self.policy_model.action_state_value)
if self.policy_model.update:
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
grads_and_vars = self.policy_model.optimizer.compute_gradients(
self.policy_model.loss)
self.policy_model.train = self.policy_model.optimizer.apply_gradients(
grads_and_vars)
else:
grads_and_vars = self.policy_model.optimizer.compute_gradients(
self.policy_model.loss)
self.policy_model.train = self.policy_model.optimizer.apply_gradients(
grads_and_vars)
def _construct_default_model(self):
with self.graph.as_default():
with tf.variable_scope('actor'):
with tf.variable_scope('eval'):
self.actor_eval_model = NNM.NeuralNetworkModel(
graph=self.graph)
self.actor_eval_model.build(
NNU.NeuronLayer(hidden_dim=30, trainable=True),
input_dim=self.env.features_dim)
self.actor_eval_model.build(
NNU.NeuronLayer(hidden_dim=self.env.actions_num,
trainable=True))
with tf.variable_scope('targ'):
self.actor_targ_model = NNM.NeuralNetworkModel(
graph=self.graph)
self.actor_targ_model.build(
NNU.NeuronLayer(hidden_dim=30, trainable=False),
input_dim=self.env.features_dim)
self.actor_targ_model.build(
NNU.NeuronLayer(hidden_dim=self.env.actions_num,
trainable=False))
self.actor_e_params = tf.get_collection(
tf.GraphKeys.GLOBAL_VARIABLES, scope='actor/eval')
self.actor_t_params = tf.get_collection(
tf.GraphKeys.GLOBAL_VARIABLES, scope='actor/targ')
with tf.variable_scope('critic'):
with tf.variable_scope('eval'):
self.critic_eval_model_state = NNM.NeuralNetworkModel(
graph=self.graph)
self.critic_eval_model_state.build(
NNU.NeuronLayer(hidden_dim=10, trainable=True),
input_dim=self.env.features_dim)
self.critic_eval_model_action = NNM.NeuralNetworkModel(
graph=self.graph)
self.critic_eval_model_action.build(
NNU.NeuronLayer(hidden_dim=10, trainable=True),
input_dim=self.env.features_dim)
self.critic_eval_model_action.input = self.actor_eval_model.output
# integrate all the models into one.
self.critic_eval_model = self.critic_eval_model_action + self.critic_eval_model_state
self.critic_eval_model.build(NNU.Relu())
self.critic_eval_model.build(NNU.NeuronLayer(
hidden_dim=1, trainable=True),
input_dim=10)
with tf.variable_scope('targ'):
self.critic_targ_model_state = NNM.NeuralNetworkModel(
graph=self.graph)
self.critic_targ_model_state.build(
NNU.NeuronLayer(hidden_dim=10, trainable=False),
input_dim=self.env.features_dim)
self.critic_targ_model_action = NNM.NeuralNetworkModel(
graph=self.graph)
self.critic_targ_model_action.build(
NNU.NeuronLayer(hidden_dim=10, trainable=False),
input_dim=self.env.features_dim)
self.critic_targ_model_state.action = self.actor_targ_model.output
# integrate all the models into one.
self.critic_targ_model = self.critic_targ_model_action + self.critic_targ_model_state
self.critic_targ_model.build(NNU.Relu())
self.critic_targ_model.build(NNU.NeuronLayer(
hidden_dim=1, trainable=False),
input_dim=10)
self.critic_e_params = tf.get_collection(
tf.GraphKeys.GLOBAL_VARIABLES, scope='critic/eval')
self.critic_t_params = tf.get_collection(
tf.GraphKeys.GLOBAL_VARIABLES, scope='critic/targ')
with tf.variable_scope('target_q'):
self.critic_target_q = self.reward + self.gamma * self.critic_targ_model.output
with tf.variable_scope('loss'):
critic_loss = tf.reduce_mean(
tf.squared_difference(self.critic_target_q,
self.critic_eval_model.output))
self.critic_train_op = tf.train.AdamOptimizer(
self.learning_rate).minimize(critic_loss)
with tf.variable_scope('a_grad'):
self.critic_action_grads = tf.gradients(
self.critic_eval_model.output,
self.actor_eval_model.output)[0]
# Connect the actor to the critic.
with tf.variable_scope('policy_grads'):
# dq/da * da/dp
self.policy_grads = tf.gradients(
ys=self.actor_eval_model.output,
xs=self.actor_e_params,
grad_ys=self.critic_action_grads)
opt = tf.train.AdamOptimizer(-self.learning_rate)
self.actor_train_op = opt.apply_gradients(
zip(self.policy_grads, self.actor_e_params))