def __init__(self, batch_size, n_history, image_y_size, image_x_size, n_action, gamma, name):
self.batch_size = batch_size
self.n_history = n_history
self.image_y_size = image_y_size
self.image_x_size = image_x_size
self.n_action = n_action
self.gamma = gamma
self.layers = []
# 网络结构
print('\n%-10s\t%-20s\t%-20s\t%s' % ('Name', 'Filter', 'Input', 'Output'))
self.conv_layer1 = ConvLayer(
input_shape=(None, self.image_y_size, self.image_x_size, self.n_history),
n_size=8, n_filter=32, stride=4, activation='relu',
batch_normal=False, weight_decay=None, name='%s_conv1' % (name))
self.layers.append(self.conv_layer1)
self.conv_layer2 = ConvLayer(
input_shape=(None, int(self.image_y_size/4), int(self.image_x_size/4), 32),
n_size=4, n_filter=64, stride=2, activation='relu',
batch_normal=False, weight_decay=None, name='%s_conv2' % (name))
self.layers.append(self.conv_layer2)
self.conv_layer3 = ConvLayer(
input_shape=(None, int(self.image_y_size/8), int(self.image_x_size/8), 64),
n_size=3, n_filter=64, stride=1, activation='relu',
batch_normal=False, weight_decay=None, name='%s_conv3' % (name))
self.layers.append(self.conv_layer3)
self.dense_layer1 = DenseLayer(
input_shape=(None, int(self.image_y_size/8) * int(self.image_x_size/8) * 64),
hidden_dim=512, activation='relu', dropout=False,
keep_prob=None, batch_normal=False, weight_decay=None, name='%s_dense1' % (name))
self.layers.append(self.dense_layer1)
self.dense_layer2 = DenseLayer(
input_shape=(None, 512),
hidden_dim=2, activation='none', dropout=False,
keep_prob=None, batch_normal=False, weight_decay=None, name='%s_dense2' % (name))
self.layers.append(self.dense_layer2)
print('')
sys.stdout.flush()
def __init__(self, batch_size, state_size, n_action, gamma, name):
self.batch_size = batch_size
self.state_size = state_size
self.n_action = n_action
self.gamma = gamma
self.layers = []
# 网络结构
print('\n%-10s\t%-20s\t%-20s\t%s' %
('Name', 'Filter', 'Input', 'Output'))
self.dense_layer1 = DenseLayer(input_shape=(None, self.state_size),
hidden_dim=64,
activation='tanh',
dropout=False,
keep_prob=None,
batch_normal=False,
weight_decay=None,
name='%s_dense1' % (name))
self.layers.append(self.dense_layer1)
self.dense_layer2 = DenseLayer(input_shape=(None, 64),
hidden_dim=128,
activation='tanh',
dropout=False,
keep_prob=None,
batch_normal=False,
weight_decay=None,
name='%s_dense2' % (name))
self.layers.append(self.dense_layer2)
self.dense_layer3 = DenseLayer(input_shape=(None, 128),
hidden_dim=self.n_action,
activation='none',
dropout=False,
keep_prob=None,
batch_normal=False,
weight_decay=None,
name='%s_dense3' % (name))
self.layers.append(self.dense_layer3)
print('')
sys.stdout.flush()
def inference(self, images,istraining = False):
conv_layer0_list = ConvLayer(
input_shape=(None, self.image_size+6, self.image_size+6, self.n_channel),
n_size=7, n_filter=64, stride=2, activation='relu',
batch_normal=istraining, weight_decay=1e-4, name='conv0', padding='SAME')
conv_layer1_list1 = ConvLayer(
input_shape=(None, int(self.image_size/4), int(self.image_size/4), 64),
n_size=3, n_filter=64, stride=1, activation='relu',
batch_normal=istraining, weight_decay=1e-4, name='conv1_1', padding='SAME')
conv_layer1_list2 = ConvLayer(
input_shape=(None, int(self.image_size/4), int(self.image_size/4), 64),
n_size=3, n_filter=64, stride=1, activation='none',
batch_normal=istraining, weight_decay=1e-4, name='conv1_2', padding='SAME')
conv_layer2_list1 = ConvLayer(
input_shape=(None, int(self.image_size/4)+1, int(self.image_size/4)+1, 64),
n_size=3, n_filter=128, stride=2, activation='relu',
batch_normal=istraining, weight_decay=1e-4, name='conv2_1', padding='SAME')
conv_layer2_list2 = ConvLayer(
input_shape=(None, int(self.image_size/8), int(self.image_size/8), 128),
n_size=3, n_filter=128, stride=1, activation='none',
batch_normal=istraining, weight_decay=1e-4, name='conv2_2', padding='SAME')
conv_layer2_list3 = ConvLayer(
input_shape=(None, int(self.image_size/4), int(self.image_size/4), 64),
n_size=1, n_filter=128, stride=2, activation='none',
batch_normal=istraining, weight_decay=1e-4, name='conv2_shortcut', padding='SAME')
conv_layer3_list1 = ConvLayer(
input_shape=(None, int(self.image_size/8)+1, int(self.image_size/8)+1, 128),
n_size=3, n_filter=256, stride=2, activation='relu',
batch_normal=istraining, weight_decay=1e-4, name='conv3_1', padding='SAME')
conv_layer3_list2 = ConvLayer(
input_shape=(None, int(self.image_size/16), int(self.image_size/16), 256),
n_size=3, n_filter=256, stride=1, activation='none',
batch_normal=istraining, weight_decay=1e-4, name='conv3_2', padding='SAME')
conv_layer3_list3 = ConvLayer(
input_shape=(None, int(self.image_size/8), int(self.image_size/8), 128),
n_size=1, n_filter=256, stride=2, activation='none',
batch_normal=istraining, weight_decay=1e-4, name='conv3_shortcut', padding='SAME')
conv_layer4_list1 = ConvLayer(
input_shape=(None, int(self.image_size/16), int(self.image_size/16), 256),
n_size=3, n_filter=256, stride=1, activation='relu',
batch_normal=istraining, weight_decay=1e-4, name='conv4_1', padding='SAME')
conv_layer4_list2 = ConvLayer(
input_shape=(None, int(self.image_size/16), int(self.image_size/16), 256),
n_size=3, n_filter=256, stride=1, activation='relu',
batch_normal=istraining, weight_decay=1e-4, name='conv4_2', padding='SAME')
dense_layer1 = DenseLayer(
input_shape=(None, 256),
hidden_dim=self.n_classes,
activation='none', dropout=False, keep_prob=None,
batch_normal=False, weight_decay=1e-4, name='dense1')
# images_size: 128x128
hidden_conv = conv_layer0_list.get_output(input=images)
#64xx64
hidden_pool = tf.nn.max_pool(hidden_conv, ksize=[1,3,3,1], strides=[1,2,2,1], padding='VALID')
#32x32
hidden_conv1 = conv_layer1_list1.get_output(input=hidden_pool)
hidden_conv2 = conv_layer1_list2.get_output(input=hidden_conv1)
hidden_conv = tf.nn.relu(hidden_pool + hidden_conv2)
#32x32
hidden_conv1 = conv_layer2_list1.get_output(input=hidden_conv)
hidden_conv2 = conv_layer2_list2.get_output(input=hidden_conv1)
hidden_shortcut = conv_layer2_list3.get_output(input=hidden_conv)
hidden_conv = tf.nn.relu(hidden_shortcut + hidden_conv2)
#16x16
hidden_conv1 = conv_layer3_list1.get_output(input=hidden_conv)
hidden_conv2 = conv_layer3_list2.get_output(input=hidden_conv1)
hidden_shortcut = conv_layer3_list3.get_output(input=hidden_conv)
hidden_conv = tf.nn.relu(hidden_shortcut + hidden_conv2)
#8x8
hidden_conv1 = conv_layer4_list1.get_output(input=hidden_conv)
hidden_conv2 = conv_layer4_list2.get_output(input=hidden_conv1)
hidden_conv = tf.nn.relu(hidden_conv + hidden_conv2)
#8x8
#hidden_pool = tf.nn.avg_pool(hidden_conv, ksize=[1,8,8,1], strides=[1,1,1,1], padding='VALID')
#global average pooling
input_dense1 = tf.reduce_mean(hidden_conv, reduction_indices=[1, 2])
#1x1
logits = dense_layer1.get_output(input=input_dense1)
return logits
def __init__(self, network_path, n_channel=3, n_classes=10, image_size=24):
# 输入变量
self.images = tf.placeholder(
dtype=tf.float32,
shape=[None, image_size, image_size, n_channel],
name='images')
self.labels = tf.placeholder(dtype=tf.int64,
shape=[None],
name='labels')
self.keep_prob = tf.placeholder(dtype=tf.float32, name='keep_prob')
self.global_step = tf.Variable(0, dtype=tf.int32, name='global_step')
# os.path.join（）该函数用以链接字符串
network_option_path = os.path.join(network_path)
self.network_option = yaml.load(open(network_option_path,
'r')) # 读取网络结构配置文件
# 网络结构
print()
self.conv_lists, self.dense_lists = [], []
for layer_dict in self.network_option['net']['conv_first']:
layer = ConvLayer(x_size=layer_dict['x_size'],
y_size=layer_dict['y_size'],
x_stride=layer_dict['x_stride'],
y_stride=layer_dict['y_stride'],
n_filter=layer_dict['n_filter'],
activation=layer_dict['activation'],
batch_normal=layer_dict['bn'],
weight_decay=1e-4,
data_format='channels_last',
name=layer_dict['name'],
input_shape=(image_size, image_size, n_channel))
self.conv_lists.append(layer)
for layer_dict in self.network_option['net']['conv']:
if layer_dict['type'] == 'conv':
layer = ConvLayer(x_size=layer_dict['x_size'],
y_size=layer_dict['y_size'],
x_stride=layer_dict['x_stride'],
y_stride=layer_dict['y_stride'],
n_filter=layer_dict['n_filter'],
activation=layer_dict['activation'],
batch_normal=layer_dict['bn'],
weight_decay=1e-4,
data_format='channels_last',
name=layer_dict['name'],
prev_layer=layer)
elif layer_dict['type'] == 'pool':
layer = PoolLayer(x_size=layer_dict['x_size'],
y_size=layer_dict['y_size'],
x_stride=layer_dict['x_stride'],
y_stride=layer_dict['y_stride'],
mode=layer_dict['mode'],
resp_normal=False,
data_format='channels_last',
name=layer_dict['name'],
prev_layer=layer)
self.conv_lists.append(layer)
for layer_dict in self.network_option['net']['dense_first']:
layer = DenseLayer(hidden_dim=layer_dict['hidden_dim'],
activation=layer_dict['activation'],
dropout=layer_dict['dropout'],
keep_prob=self.keep_prob,
batch_normal=layer_dict['bn'],
weight_decay=1e-4,
name=layer_dict['name'],
input_shape=(int(image_size / 8) *
int(image_size / 8) * 256, ))
self.dense_lists.append(layer)
for layer_dict in self.network_option['net']['dense']:
layer = DenseLayer(hidden_dim=layer_dict['hidden_dim'],
activation=layer_dict['activation'],
dropout=layer_dict['dropout'],
keep_prob=self.keep_prob,
batch_normal=layer_dict['bn'],
weight_decay=1e-4,
name=layer_dict['name'],
prev_layer=layer)
self.dense_lists.append(layer)
print()
# 数据流
hidden_state = self.images
for layer in self.conv_lists:
hidden_state = layer.get_output(inputs=hidden_state)
hidden_state = tf.reshape(
hidden_state,
[-1, int(image_size / 8) * int(image_size / 8) * 256])
for layer in self.dense_lists:
hidden_state = layer.get_output(inputs=hidden_state)
logits = hidden_state
# 目标函数
self.objective = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits,
labels=self.labels))
self.avg_loss = self.objective
tf.summary.scalar('loss', self.avg_loss) #记录损失函数的变化
# 优化器
lr = tf.cond(
tf.less(self.global_step, 50000), lambda: tf.constant(0.001),
lambda: tf.cond(tf.less(self.global_step, 100000), lambda: tf.
constant(0.0001), lambda: tf.constant(0.00001)))
self.optimizer = tf.train.AdamOptimizer(learning_rate=lr).minimize(
self.avg_loss, global_step=self.global_step)
#tf.summary.scalar('optimizer', self.optimizer)#记录优化器的变化
# 观察值
correct_prediction = tf.equal(self.labels, tf.argmax(logits, 1))
self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float'))
print(correct_prediction)
tf.summary.scalar('optimizer', self.accuracy) # 记录优化器的变化
class Network:
def __init__(self, batch_size, state_size, n_action, gamma, name):
self.batch_size = batch_size
self.state_size = state_size
self.n_action = n_action
self.gamma = gamma
self.layers = []
# 网络结构
print('\n%-10s\t%-20s\t%-20s\t%s' %
('Name', 'Filter', 'Input', 'Output'))
self.dense_layer1 = DenseLayer(input_shape=(None, self.state_size),
hidden_dim=64,
activation='tanh',
dropout=False,
keep_prob=None,
batch_normal=False,
weight_decay=None,
name='%s_dense1' % (name))
self.layers.append(self.dense_layer1)
self.dense_layer2 = DenseLayer(input_shape=(None, 64),
hidden_dim=128,
activation='tanh',
dropout=False,
keep_prob=None,
batch_normal=False,
weight_decay=None,
name='%s_dense2' % (name))
self.layers.append(self.dense_layer2)
self.dense_layer3 = DenseLayer(input_shape=(None, 128),
hidden_dim=self.n_action,
activation='none',
dropout=False,
keep_prob=None,
batch_normal=False,
weight_decay=None,
name='%s_dense3' % (name))
self.layers.append(self.dense_layer3)
print('')
sys.stdout.flush()
def get_inference(self, states, batch_size=1):
# 数据流
hidden_dense1 = self.dense_layer1.get_output(input=states)
hidden_dense2 = self.dense_layer2.get_output(input=hidden_dense1)
hidden_dense3 = self.dense_layer3.get_output(input=hidden_dense2)
return hidden_dense3
def cal_labels(self, next_states, rewards, is_terminals):
next_action_score = self.get_inference(next_states,
batch_size=self.batch_size)
max_action_score = tf.reduce_max(next_action_score,
axis=1,
keep_dims=True)
labels = tf.stop_gradient(rewards +
self.gamma * max_action_score * is_terminals)
return labels
def get_loss(self, states, actions, labels):
action_score = self.get_inference(states, batch_size=self.batch_size)
preds = tf.reduce_sum(action_score * actions, axis=1, keep_dims=True)
loss = tf.nn.l2_loss(labels - preds)
tf.add_to_collection('losses', loss / self.batch_size)
avg_loss = tf.add_n(tf.get_collection('losses'))
return avg_loss
class Network:
def __init__(self, batch_size, n_history, image_y_size, image_x_size, n_action, gamma, name):
self.batch_size = batch_size
self.n_history = n_history
self.image_y_size = image_y_size
self.image_x_size = image_x_size
self.n_action = n_action
self.gamma = gamma
self.layers = []
# 网络结构
print('\n%-10s\t%-20s\t%-20s\t%s' % ('Name', 'Filter', 'Input', 'Output'))
self.conv_layer1 = ConvLayer(
input_shape=(None, self.image_y_size, self.image_x_size, self.n_history),
n_size=8, n_filter=32, stride=4, activation='relu',
batch_normal=False, weight_decay=None, name='%s_conv1' % (name))
self.layers.append(self.conv_layer1)
self.conv_layer2 = ConvLayer(
input_shape=(None, int(self.image_y_size/4), int(self.image_x_size/4), 32),
n_size=4, n_filter=64, stride=2, activation='relu',
batch_normal=False, weight_decay=None, name='%s_conv2' % (name))
self.layers.append(self.conv_layer2)
self.conv_layer3 = ConvLayer(
input_shape=(None, int(self.image_y_size/8), int(self.image_x_size/8), 64),
n_size=3, n_filter=64, stride=1, activation='relu',
batch_normal=False, weight_decay=None, name='%s_conv3' % (name))
self.layers.append(self.conv_layer3)
self.dense_layer1 = DenseLayer(
input_shape=(None, int(self.image_y_size/8) * int(self.image_x_size/8) * 64),
hidden_dim=512, activation='relu', dropout=False,
keep_prob=None, batch_normal=False, weight_decay=None, name='%s_dense1' % (name))
self.layers.append(self.dense_layer1)
self.dense_layer2 = DenseLayer(
input_shape=(None, 512),
hidden_dim=2, activation='none', dropout=False,
keep_prob=None, batch_normal=False, weight_decay=None, name='%s_dense2' % (name))
self.layers.append(self.dense_layer2)
print('')
sys.stdout.flush()
def get_inference(self, images, batch_size=1):
# 数据流
hidden_conv1 = self.conv_layer1.get_output(input=images)
hidden_conv2 = self.conv_layer2.get_output(input=hidden_conv1)
hidden_conv3 = self.conv_layer3.get_output(input=hidden_conv2)
hidden_conv3 = tf.reshape(hidden_conv3, shape=(
batch_size, int(self.image_y_size/8) * int(self.image_x_size/8) * 64))
hidden_dense1 = self.dense_layer1.get_output(input=hidden_conv3)
hidden_dense2 = self.dense_layer2.get_output(input=hidden_dense1)
return hidden_dense2
def cal_labels(self,next_images, rewards, is_terminals):
next_action_score = self.get_inference(next_images, batch_size=self.batch_size)
max_action_score = tf.reduce_max(next_action_score, axis=1, keep_dims=True)
labels = tf.stop_gradient(rewards + self.gamma * max_action_score * is_terminals)
return labels
def get_loss(self, images, actions, labels):
action_score = self.get_inference(images, batch_size=self.batch_size)
preds = tf.reduce_sum(action_score * actions, axis=1, keep_dims=True)
loss = tf.nn.l2_loss(labels - preds)
tf.add_to_collection('losses', loss / self.batch_size)
avg_loss = tf.add_n(tf.get_collection('losses'))
return avg_loss