def _construct_default_model(self): with self.graph.as_default(): with tf.variable_scope('actor'): self.actor_model = NNM.NeuralNetworkModel(graph=self.graph) self.actor_model.td_error = tf.placeholder(dtype=self.dtype, shape=None) self.actor_model.action = tf.placeholder(dtype=self.dtype, shape=None) self.actor_model.build(NNU.NeuronLayer(hidden_dim=20), input_dim=self.env.features_dim) self.actor_model.build( NNU.NeuronLayer(hidden_dim=self.env.actions_num)) self.actor_model.compile( optimizer=tf.train.GradientDescentOptimizer, loss_fun=NNL.NeuralNetworkLoss.exploss, action=self.actor_model.action, td_error=self.actor_model.td_error) with tf.variable_scope('critic'): self.critic_model = NNM.NeuralNetworkModel(graph=self.graph) self.critic_model.reward = tf.placeholder(dtype=self.dtype, shape=None) self.critic_model.value = tf.placeholder(dtype=self.dtype, shape=[1, 1]) self.critic_model.build(NNU.NeuronLayer(hidden_dim=10), input_dim=self.env.features_dim) self.critic_model.build(NNU.NeuronLayer(hidden_dim=1)) self.critic_model.compile( optimizer=tf.train.GradientDescentOptimizer, loss_fun=NNL.NeuralNetworkLoss.tdsquared, reward=self.critic_model.reward, gamma=self.gamma)
def example3(): def load_data(filename): this_dir, _ = os.path.split(__file__) data_path = os.path.join(this_dir, 'data', filename) return np.load(data_path) imgs, labels, shape = load_data('FERET.npy') x_train, y_train, x_test, y_test = UF.split_train_test(imgs, labels, 2) y_train = UF.OneHot(y_train) y_test = UF.OneHot(y_test) model = NNM.NeuralNetworkModel(dtype=tf.float64) # shape=(5,5,3) means the kernel's height=5 width=5 num of ker=3. # first layer should give the input dim, which in this case is (image height, image width, num channel). model.build(NNU.ConvolutionUnit(dtype=tf.float64, shape=(4, 4, 2), transfer_fun=tf.tanh), input_dim=x_train.shape[1:]) # model.build(NNU.BatchNormalization()) # model.build(NNU.Dropout(keep_prob=0.8)) # model.build(NNU.ConvolutionUnit(dtype=tf.float64, shape=(2, 2, 4), # transfer_fun=tf.sigmoid)) # model.build(NNU.AvgPooling(dtype=tf.float64, shape=(1, 4, 4, 1))) model.build(NNU.Dropout(keep_prob=0.8)) model.build(NNU.Flatten()) model.build(NNU.NeuronLayer(hidden_dim=shape[1], dtype=tf.float64)) model.build(NNU.BatchNormalization()) model.build(NNU.SoftMaxLayer()) model.fit(x_train, y_train, loss_fun=NNL.NeuralNetworkLoss.crossentropy, show_graph=False, num_epochs=501, mini_batch=10, learning_rate=0.5) acc, result, correctness = model.evaluate(x_test, y_test) print(acc, result, correctness)
def example2(): from tensorflow.examples.tutorials.mnist import input_data mnist = input_data.read_data_sets('MNIST_data/', one_hot=True) x_train = mnist.train.images # To reduce computational cost, we let the training size be 500. x_train = x_train[:500, :] y_train = mnist.train.labels y_train = y_train[:500, :] x_train = UF.vectors2imgs(x_train, (None, 28, 28, 1)) model = NNM.NeuralNetworkModel(dtype=tf.float32, img_size=(28, 28)) # shape=(5,5,3) means the kernel's height=5 width=5 num of ker=3 model.build(NNU.ConvolutionUnit(dtype=tf.float32, shape=(5, 5, 3), transfer_fun=tf.tanh)) model.build(NNU.AvgPooling(dtype=tf.float32, shape=(1, 4, 4, 1))) model.build(NNU.Dropout(keep_prob=0.5)) model.build(NNU.Flatten()) model.build(NNU.NeuronLayer(hidden_dim=10, dtype=tf.float32)) model.build(NNU.SoftMaxLayer()) model.fit(x_train, y_train, loss_fun=NNL.NeuralNetworkLoss.crossentropy, show_graph=True, num_epochs=1000) x_test = mnist.test.images y_test = mnist.test.labels x_test = x_test[0:20, :] y_test = y_test[0:20, :] x_test = UF.vectors2imgs(x_test, (None, 28, 28, 1)) print(model.Evaluate(x_test, y_test))
def example1(): from tensorflow.examples.tutorials.mnist import input_data mnist = input_data.read_data_sets('MNIST_data/', one_hot=True) x_train = mnist.train.images # To reduce computational cost, we let the training size be 500. x_train = x_train[:500, :] model = NNM.NeuralNetworkModel() model.build(NNU.NeuronLayer(hidden_dim=10), input_dim=784) model.build(NNU.BatchNormalization()) model.build(NNU.NeuronLayer(hidden_dim=5, transfer_fun=tf.nn.sigmoid)) model.build(NNU.BatchNormalization()) model.build(NNU.NeuronLayer(hidden_dim=784)) import time t1 = time.time() model.Fit(x_train, x_train, show_graph=False, num_epochs=500, mini_size=40, loss_fun=NNL.NeuralNetworkLoss.meansquared) print(time.time()-t1) n = 1 # how many digits we will display x_test = mnist.test.images x_test = x_test[:n, :] results = model.predict(x_test) model.print_output_detail(x_test)
def _construct_default_models(self): with self.graph.as_default(): with tf.variable_scope('eval'): self.eval_model = NNM.NeuralNetworkModel(graph=self.graph) self.eval_model.build(NNU.NeuronLayer( hidden_dim=10, transfer_fun=tf.nn.sigmoid), input_dim=self.env.features_size) self.eval_model.build(NNU.BatchNormalization()) # self.eval_model.build(NNU.NeuronLayer(hidden_dim=5, transfer_fun=tf.nn.sigmoid)) # self.eval_model.build(NNU.BatchNormalization()) self.eval_model.split(names=['adv', 'value']) self.eval_model.build(NNU.NeuronLayer(hidden_dim=1), name='value') self.eval_model.build( NNU.NeuronLayer(hidden_dim=self.actions_size), name='adv') self.eval_model.build(NNU.ReduceMean(), name='adv') self.eval_model.merge(op='add', names=['adv', 'value']) self.eval_model.batch_size = self.batch_size self.eval_model.mini_batch = self.eval_model.batch_size self.eval_model.compile( optimizer=tf.train.GradientDescentOptimizer( learning_rate=self.learning_rate), loss_fun=NNL.NeuralNetworkLoss.meansquared) self.eval_model.sess.close() with tf.variable_scope('target'): self.targ_model = NNM.NeuralNetworkModel(graph=self.graph) self.targ_model.build(NNU.NeuronLayer( hidden_dim=10, transfer_fun=tf.nn.sigmoid), input_dim=self.env.features_size) self.targ_model.build(NNU.BatchNormalization()) # self.targ_model.build(NNU.NeuronLayer(hidden_dim=5, transfer_fun=tf.nn.sigmoid)) # self.targ_model.build(NNU.BatchNormalization()) self.targ_model.split(names=['adv', 'value']) self.targ_model.build(NNU.NeuronLayer(hidden_dim=1), name='value') self.targ_model.build( NNU.NeuronLayer(hidden_dim=self.actions_size), name='adv') self.targ_model.build(NNU.ReduceMean(), name='adv') self.targ_model.merge(op='add', names=['adv', 'value'], output_name='last') self.targ_model.sess.close()
def _construct_default_models(self): self.eval_model = NNM.NeuralNetworkModel() self.eval_model.build(NNU.NeuronLayer(hidden_dim=20)) self.eval_model.build(NNU.NeuronLayer(hidden_dim=15)) self.eval_model.build(NNU.NeuronLayer(hidden_dim=self.actions_size)) # For priority reply self.eval_model.ISWeights = tf.placeholder(tf.float32, [None, 1]) # target model and eval model share the same structure. self.targ_model = cp.deepcopy(self.eval_model)
def _construct_default_models(self): with self.graph.as_default(): with tf.variable_scope('eval'): self.eval_model = NNM.NeuralNetworkModel(graph=self.graph) self.eval_model.build(NNU.NeuronLayer( hidden_dim=30, transfer_fun=tf.nn.sigmoid), input_dim=self.env.features_size) self.eval_model.build(NNU.BatchNormalization()) self.eval_model.build( NNU.NeuronLayer(hidden_dim=20, transfer_fun=tf.nn.sigmoid)) self.eval_model.build(NNU.BatchNormalization()) self.eval_model.build( NNU.NeuronLayer(hidden_dim=self.actions_size)) # self.eval_model.Build(NNU.BatchNormalization()) self.eval_model.batch_size = self.batch_size self.eval_model.mini_batch = self.eval_model.batch_size self.eval_model.compile( optimizer=tf.train.GradientDescentOptimizer( learning_rate=self.learning_rate), loss_fun=NNL.NeuralNetworkLoss.meansquared) self.eval_model.sess.close() # target model and eval model share the same structure. with tf.variable_scope('target'): self.targ_model = NNM.NeuralNetworkModel(graph=self.graph) self.targ_model.build(NNU.NeuronLayer( hidden_dim=30, transfer_fun=tf.nn.sigmoid), input_dim=self.env.features_size) self.targ_model.build(NNU.BatchNormalization()) self.targ_model.build( NNU.NeuronLayer(hidden_dim=20, transfer_fun=tf.nn.sigmoid)) self.targ_model.build(NNU.BatchNormalization()) self.targ_model.build( NNU.NeuronLayer(hidden_dim=self.actions_size, transfer_fun=tf.nn.sigmoid)) # self.targ_model.build(NNU.BatchNormalization()) self.targ_model.sess.close()
def _construct_default_models(self): with self.graph.as_default(): with tf.variable_scope('Policy_Network'): self.policy_model = NNM.NeuralNetworkModel(graph=self.graph) self.policy_model.build(NNU.NeuronLayer( hidden_dim=30, transfer_fun=tf.nn.sigmoid), input_dim=self.env.features_size) self.policy_model.build( NNU.NeuronLayer(hidden_dim=30, transfer_fun=None)) self.policy_model.build(NNU.SoftMaxLayer()) self.policy_model.action_state_value = tf.placeholder(shape=[ None, ]) self.policy_model.mini_batch = self.batch_size self.policy_model.compile( optimizer=tf.train.GradientDescentOptimizer( learning_rate=self.learning_rate), loss_fun=NNL.NeuralNetworkLoss.crossentropy, loss_and_optimize=False) with self.graph.as_default(): self.policy_model.target = tf.placeholder(shape=[ None, ]) self.policy_model.loss = self.policy_model.loss_fun( output=self.policy_model.output, target=self.policy_model.target, batch_size=1, axis=1) self.policy_model.loss = tf.reduce_mean( self.policy_model.loss * self.policy_model.action_state_value) if self.policy_model.update: update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): grads_and_vars = self.policy_model.optimizer.compute_gradients( self.policy_model.loss) self.policy_model.train = self.policy_model.optimizer.apply_gradients( grads_and_vars) else: grads_and_vars = self.policy_model.optimizer.compute_gradients( self.policy_model.loss) self.policy_model.train = self.policy_model.optimizer.apply_gradients( grads_and_vars)
def _construct_default_model(self): with self.graph.as_default(): with tf.variable_scope('actor'): with tf.variable_scope('eval'): self.actor_eval_model = NNM.NeuralNetworkModel( graph=self.graph) self.actor_eval_model.build( NNU.NeuronLayer(hidden_dim=30, trainable=True), input_dim=self.env.features_dim) self.actor_eval_model.build( NNU.NeuronLayer(hidden_dim=self.env.actions_num, trainable=True)) with tf.variable_scope('targ'): self.actor_targ_model = NNM.NeuralNetworkModel( graph=self.graph) self.actor_targ_model.build( NNU.NeuronLayer(hidden_dim=30, trainable=False), input_dim=self.env.features_dim) self.actor_targ_model.build( NNU.NeuronLayer(hidden_dim=self.env.actions_num, trainable=False)) self.actor_e_params = tf.get_collection( tf.GraphKeys.GLOBAL_VARIABLES, scope='actor/eval') self.actor_t_params = tf.get_collection( tf.GraphKeys.GLOBAL_VARIABLES, scope='actor/targ') with tf.variable_scope('critic'): with tf.variable_scope('eval'): self.critic_eval_model_state = NNM.NeuralNetworkModel( graph=self.graph) self.critic_eval_model_state.build( NNU.NeuronLayer(hidden_dim=10, trainable=True), input_dim=self.env.features_dim) self.critic_eval_model_action = NNM.NeuralNetworkModel( graph=self.graph) self.critic_eval_model_action.build( NNU.NeuronLayer(hidden_dim=10, trainable=True), input_dim=self.env.features_dim) self.critic_eval_model_action.input = self.actor_eval_model.output # integrate all the models into one. self.critic_eval_model = self.critic_eval_model_action + self.critic_eval_model_state self.critic_eval_model.build(NNU.Relu()) self.critic_eval_model.build(NNU.NeuronLayer( hidden_dim=1, trainable=True), input_dim=10) with tf.variable_scope('targ'): self.critic_targ_model_state = NNM.NeuralNetworkModel( graph=self.graph) self.critic_targ_model_state.build( NNU.NeuronLayer(hidden_dim=10, trainable=False), input_dim=self.env.features_dim) self.critic_targ_model_action = NNM.NeuralNetworkModel( graph=self.graph) self.critic_targ_model_action.build( NNU.NeuronLayer(hidden_dim=10, trainable=False), input_dim=self.env.features_dim) self.critic_targ_model_state.action = self.actor_targ_model.output # integrate all the models into one. self.critic_targ_model = self.critic_targ_model_action + self.critic_targ_model_state self.critic_targ_model.build(NNU.Relu()) self.critic_targ_model.build(NNU.NeuronLayer( hidden_dim=1, trainable=False), input_dim=10) self.critic_e_params = tf.get_collection( tf.GraphKeys.GLOBAL_VARIABLES, scope='critic/eval') self.critic_t_params = tf.get_collection( tf.GraphKeys.GLOBAL_VARIABLES, scope='critic/targ') with tf.variable_scope('target_q'): self.critic_target_q = self.reward + self.gamma * self.critic_targ_model.output with tf.variable_scope('loss'): critic_loss = tf.reduce_mean( tf.squared_difference(self.critic_target_q, self.critic_eval_model.output)) self.critic_train_op = tf.train.AdamOptimizer( self.learning_rate).minimize(critic_loss) with tf.variable_scope('a_grad'): self.critic_action_grads = tf.gradients( self.critic_eval_model.output, self.actor_eval_model.output)[0] # Connect the actor to the critic. with tf.variable_scope('policy_grads'): # dq/da * da/dp self.policy_grads = tf.gradients( ys=self.actor_eval_model.output, xs=self.actor_e_params, grad_ys=self.critic_action_grads) opt = tf.train.AdamOptimizer(-self.learning_rate) self.actor_train_op = opt.apply_gradients( zip(self.policy_grads, self.actor_e_params))