def __init__(self, num_filters, kernel_size):
# inputs
self.inputs = tf.placeholder(tf.float32, shape=[None, 28, 28, 1])
# first convolutional layer, default values: strides=1, use_bias=True
conv1 = layers.Conv2D(filters=num_filters,
kernel_size=kernel_size,
padding="same",
activation="relu")
pooling = layers.MaxPooling2D(pool_size=2, strides=2)
conv2 = layers.Conv2D(filters=num_filters,
kernel_size=kernel_size,
padding="same",
activation="relu")
# flatten layer before the dense layers
flatten = layers.Flatten()
# dense layers -> first one with relu?
linear1 = layers.Dense(units=128, activation="relu")
# second dense layer only computes logits
linear2 = layers.Dense(units=10, activation=None)
# define the graph
self.logits = conv1(self.inputs)
for layer in [pooling, conv2, pooling, flatten, linear1, linear2]:
self.logits = layer(self.logits)
self.out_soft = tf.nn.softmax(self.logits)
# intialize the variables
init = tf.global_variables_initializer()
self.sess = tf.Session()
self.sess.run(init)
def __call__(self, inp):
with tf.variable_scope("discriminator",
reuse=tf.AUTO_REUSE) as model_scope:
x = tfl.Flatten()(inp)
x = tfl.Dense(256)(x)
x = self.batchnorm()(x)
x = tf.nn.leaky_relu(x)
x = self.dropout(0.25)(x)
x = tfl.Dense(256)(x)
x = self.batchnorm()(x)
x = tf.nn.leaky_relu(x)
x = self.dropout(0.25)(x)
x = tfl.Dense(64)(x)
x = self.batchnorm()(x)
x = tf.nn.leaky_relu(x)
x = self.dropout(0.25)(x)
x = tfl.Dense(1, activation=None)(x)
self.variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope=model_scope.name)
self.post_call(model_scope)
return x
def __init__(self, num_classes):
self.num_classes = num_classes
self.denseNet_1 = layers.Dense(256,
activation=nn.sigmoid,
name="classifier_dense_kayer_1")
self.denseNet_2 = layers.Dense(self.num_classes,
activation=nn.sigmoid,
name="classifier_dense_kayer_1")
self.softmax_layer = nn.softmax
return
def build_model():
model = keras.Sequential([
layers.Dense(64,
activation=tf.nn.relu,
input_shape=[len(train_dataset.keys())]),
layers.Dense(64, activation=tf.nn.relu),
layers.Dense(1)
])
optimizer = tf.keras.optimizers.RMSprop(0.001)
model.compile(loss='mse', optimizer=optimizer, metrics=['mae', 'mse'])
return model
def cell_input_fn(inputs, attention):
'''
根据attn_input_feeding属性来判断是否在attention计算前进行一次投影的计算
使用注意力机制才会进行的运算
:param inputs:
:param attention:
:return:
'''
if not self.use_residual:
print(inputs.get_shape, 'inputs_shape')
print(attention.get_shape, 'inputs_shape')
print(array_ops.concat([inputs, attention], -1), 'inputs和attention拼接之后的形状')
return array_ops.concat([inputs, attention], -1)
attn_projection = layers.Dense(self.hidden_units,
dtype=tf.float32,
use_bias=False,
name='attention_cell_input_fn')
'''
这个attn_projection(array_ops.concat([inputs,attention],-1))我的理解就是
layers.Dense(self.hidden_units,
dtype=tf.float32,
use_bias=False,
name='attention_cell_input_fn')(array_ops.concat([inputs,attention],-1))
Dense最终继承了Layer类，Layer中定义了call方法和__call__ 方法，Dense也重写了call方法，__call__方法中调用call方法，call方法中还是起一个全连接层层的作用，__call__
方法中执行流程是：pre process，call，post process
'''
return attn_projection(array_ops.concat([inputs, attention], -1))
def cell_input_fn(inputs,attention):
'''
根据attn_input_feeding属性来判断是否在attention计算前进行一次投影的计算
:param inputs:
:param attention:
:return:
'''
if not self.use_residual:
return array_ops.concat([inputs,attention],-1)
attn_projection = layers.Dense(self.hidden_units,
dtype=tf.float32,
use_bias=False,
name='attention_cell_input_fn')
'''
这个attn_projection(array_ops.concat([inputs,attention],-1))我的理解就是
layers.Dense(self.hidden_units,
dtype=tf.float32,
use_bias=False,
name='attention_cell_input_fn')(array_ops.concat([inputs,attention],-1))
因为Dense内部实际上是定义了__call__(self): 的方法，因此可以这样使用
'''
return attn_projection(array_ops.concat([inputs,attention],-1))
def __call__(self, inp):
with tf.variable_scope("decoder", reuse=tf.AUTO_REUSE) as model_scope:
x = tfl.Dense(11*11*16)(inp)
x = self.batchnorm()(x)
x = tf.nn.leaky_relu(x)
x = tf.reshape(x, [-1, 11, 11, 16])
x = tfl.Conv2D(32, 3, padding="valid")(x)
x = self.batchnorm()(x)
x = tf.nn.leaky_relu(x)
x = UpSampling2D()(x)
x = tfl.Conv2D(128, 5, padding="valid")(x)
x = self.batchnorm()(x)
x = tf.nn.leaky_relu(x)
x = self.pad()(x)
x = tfl.Conv2D(64, 3, padding="valid")(x)
x = self.batchnorm()(x)
x = tf.nn.leaky_relu(x)
x = UpSampling2D()(x)
x = self.pad()(x)
x = tfl.Conv2D(64, 3, padding="valid")(x)
x = tf.nn.leaky_relu(x)
x = self.pad()(x)
x = tfl.Conv2D(1, 3, padding="valid")(x)
self.variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=model_scope.name)
self.post_call(model_scope)
return x
def __init__(self, observation_space, action_space, epsilon=1.0):
self.exploration_rate = EXPLORATION_MAX
self.action_space = action_space
self.env = gym.make('CartPole-v0')
self.epsilon = epsilon
self.memory = deque(maxlen=MEMORY_SIZE)
self.model = tf.keras.models.Sequential()
self.model.add(
layers.Dense(24, input_dim=observation_space, activation="relu"))
self.model.add(layers.Dense(24, activation="relu"))
self.model.add(layers.Dense(self.action_space, activation="linear"))
self.model.compile(loss='mse',
optimizer=tf.train.GradientDescentOptimizer(
learning_rate=LEARNING_RATE))
def __call__(self, inputs, batch_sz):
pr_shape = lambda var: print(var.shape)
if self.rnntype == "GRU":
print("rnntype: " + self.rnntype)
cell = nn.rnn_cell.GRUCell(self.n_hidden)
else:
print("rnntype: " + self.rnntype)
cell = nn.rnn_cell.LSTMCell(self.n_hidden)
initial_state = cell.zero_state(batch_sz, tf.float32)
# initial_state = cell.zero_state(inputs.shape[0], tf.float32)
# dynamic_rnn inputs shape = [batch_size, max_time, ...]
# outs shape = [batch_size, max_time, cell.output_size]
# states shape = [batch_size, cell.state_size]
# n_step = int(inputs.shape[1]) # n_step
outs, states = nn.dynamic_rnn(cell,
inputs,
initial_state=initial_state,
dtype=tf.float32)
print('outs shape: ')
pr_shape(outs) # (batch_sz, max_time, n_hidden)
# final_state = states[-1] #
print('states shape: ')
pr_shape(states) # (batch_sz, n_hidden)
FC = tl.Dense(self.n_classes,
use_bias=True,
kernel_initializer=tc.layers.xavier_initializer(
tf.float32))
outs = FC(states)
return outs
def cell_input_fn(inputs, attention):
mul = 2 if self.bidirection else 1
attn_projection = layers.Dense(self.hidden_size * mul,
dtype=tf.float32,
use_bias=False,
name='attention_cell_input_fn')
return attn_projection(array_ops.concat([inputs, attention], -1))
def __call__(self, inp):
with tf.variable_scope("decoder", reuse=tf.AUTO_REUSE) as model_scope:
x = tfl.Dense(9 * 9 * 8)(inp)
x = self.batchnorm()(x)
x = tf.nn.leaky_relu(x)
x = tf.reshape(x, [-1, 9, 9, 8])
x = tfl.Conv2D(16, 3, padding="valid")(x)
x = self.batchnorm()(x)
x = tf.nn.leaky_relu(x)
x = tfl.Conv2DTranspose(128, 5, 2, padding="same")(x)
x = self.batchnorm()(x)
x = tf.nn.leaky_relu(x)
print(x)
x = self.pad()(x)
x = tfl.Conv2D(64, 3, padding="valid")(x)
x = self.batchnorm()(x)
x = tf.nn.leaky_relu(x)
print(x)
x = tfl.Conv2DTranspose(64, 3, 2, padding="same")(x)
x = tf.nn.leaky_relu(x)
print(x)
x = self.pad()(x)
x = tfl.Conv2D(1, 3, padding="valid")(x)
self.variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope=model_scope.name)
self.post_call(model_scope)
return x
def build_decoder(self, encoder_outputs, encoder_final_state):
"""
构建完整解码器
:return:
"""
with tf.variable_scope("decode"):
decoder_cell, decoder_initial_state = self.build_decoder_cell(
encoder_outputs, encoder_final_state, self.hidden_size,
self.cell_type, self.layer_size)
# 输出层投影
decoder_output_projection = layers.Dense(
self.decoder_vocab_size,
dtype=tf.float32,
use_bias=False,
kernel_initializer=tf.truncated_normal_initializer(mean=0.0,
stddev=0.1),
name='decoder_output_projection')
if self.mode == 'train':
# 训练模式
decoder_inputs_embdedded = tf.nn.embedding_lookup(
self.decoder_embeddings, self.decoder_inputs_train)
training_helper = TrainingHelper(
inputs=decoder_inputs_embdedded,
sequence_length=self.decoder_inputs_length,
name='training_helper')
training_decoder = BasicDecoder(decoder_cell, training_helper,
decoder_initial_state,
decoder_output_projection)
max_decoder_length = tf.reduce_max(self.decoder_inputs_length)
training_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(
training_decoder, maximum_iterations=max_decoder_length)
self.masks = tf.sequence_mask(self.decoder_inputs_length,
maxlen=max_decoder_length,
dtype=tf.float32,
name='masks')
self.loss = tf.contrib.seq2seq.sequence_loss(
logits=training_decoder_output.rnn_output,
targets=self.decoder_inputs,
weights=self.masks,
average_across_timesteps=True,
average_across_batch=True)
else:
# 预测模式
start_token = [DataUnit.START_INDEX] * self.batch_size
end_token = DataUnit.END_INDEX
inference_decoder = BeamSearchDecoder(
cell=decoder_cell,
embedding=lambda x: tf.nn.embedding_lookup(
self.decoder_embeddings, x),
start_tokens=start_token,
end_token=end_token,
initial_state=decoder_initial_state,
beam_width=self.beam_width,
output_layer=decoder_output_projection)
inference_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(
inference_decoder, maximum_iterations=self.max_decode_step)
self.decoder_pred_decode = inference_decoder_output.predicted_ids
self.decoder_pred_decode = tf.transpose(
self.decoder_pred_decode, perm=[0, 2, 1])
def cell_input_fn(inputs, attention):
if not self.use_residual:
return array_ops.concat([inputs, attention], -1)
attn_projection = layers.Dense(self.hidden_size,
dtype=tf.float32,
use_bias=False,
name='attention_cell_input_fn')
return attn_projection(array_ops.concat([inputs, attention], -1))
def cell_input_fn(inputs, attention):
""" 根据attn_input_feeding属性来判断是否在attention计算前进行一次投影的计算"""
if not self.use_residual:
return array_ops.concat([inputs, attention], -1)
attn_projection = layers.Dense(self.hidden_units,
dtype=tf.float32,
use_bias=False,
name='attention_cell_input_fn')
return attn_projection(array_ops.concat([inputs, attention], -1))
def __init__(self, block, layers, modality='RGB',
shortcut_type='B', num_classes=400,dropout=0.5,ST_struc=('A','B','C')):
self.w_initer=tc.layers.xavier_initializer(tf.float32)
self.data_format='NCHW'
self.layer_data_format = 'channels_last' if data_format == 'NHWC' \
else 'channels_first'
self.is_training=True
self.inplanes = 64
self.input_channel = 3 if modality=='RGB' else 2 # 2 is for flow
self.ST_struc=ST_struc
self.conv1_custom = nn.Conv3d(self.input_channel, 64, kernel_size=(1,7,7), stride=(1,2,2),
padding=(0,3,3), bias=False)
self.conv1_custom = tl.Conv3D(filters=64, kernel_size=(1,7,7), strides=(1,2,2),
padding=self.layer_data_format, use_bias=False, kernel_initializer=self.w_initer)
self.depth_3d = sum(layers[:3])# C3D layers are only (res2,res3,res4), res5 is C2D
axis = 1 if self.data_format=="NCHW" else -1
self.bn1 = tl.BatchNormalization(axis=axis, scale=False fused=True)
# out = self.bn(in, training=True) False for eval
self.cnt = 0
self.relu = lambda input : nn.relu(input)
self.maxpool = tl.MaxPooling3D(pool_size=(2, 3, 3), strides=2, padding='valid',
data_format=self.layer_data_format) # pooling layer for conv1.
self.maxpool = tl.MaxPooling3D(pool_size=(2, 1, 1), strides=(2, 1, 1), padding='valid',
data_format=self.layer_data_format) # pooling layer for res2, 3, 4.
self.layer1 = self._make_layer(block, 64, layers[0], shortcut_type)
self.layer2 = self._make_layer(block, 128, layers[1], shortcut_type, stride=2)
self.layer3 = self._make_layer(block, 256, layers[2], shortcut_type, stride=2)
self.layer4 = self._make_layer(block, 512, layers[3], shortcut_type, stride=2)
self.avgpool = tl.AveragePooling2D(pool_size=5, strides=1,
data_format=self.layer_data_format) # pooling layer for res5.
self.dropout=tl.Dropout(dropout)
self.fc = tl.Dense(num_classes, use_bias=False, kernel_initializer=self.w_initer)
# self.fc = nn.Linear(512 * block.expansion, num_classes)
for m in self.modules():
if isinstance(m, nn.Conv3d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2. / n))
elif isinstance(m, nn.BatchNorm3d):
m.weight.data.fill_(1)
m.bias.data.zero_()
# some private attribute
self.input_size=(self.input_channel,16,160,160) # input of the network
self.input_mean = [0.485, 0.456, 0.406] if modality=='RGB' else [0.5]
self.input_std = [0.229, 0.224, 0.225] if modality=='RGB' else [np.mean([0.229, 0.224, 0.225])]
return
def building(self, input_layer):
model_layers = [input_layer]
for hidden_layer in self.hidden_layers_info: #关键字，层数1 2 3 4 5...
this_layer_info = self.hidden_layers_info[hidden_layer]
prev_layer = model_layers[-1]
#普通层
if this_layer_info['layer_type'] == 'Dense':
prev_layer_shape = prev_layer.get_shape().as_list()
if len(prev_layer_shape) == 3:
prev_layer = tf.reshape(prev_layer,
(-1, prev_layer_shape[-1]))
this_layer = layers.Dense( units = this_layer_info['units_filters'],
activation = this_layer_info['activation'],
name = 'hidden_'+str(hidden_layer)+'_'+\
this_layer_info['layer_type'])(prev_layer)
model_layers.append(this_layer)
#一维卷积层
elif this_layer_info['layer_type'] == 'Conv1D':
prev_layer_shape = prev_layer.get_shape().as_list()
if len(prev_layer_shape) == 2:
prev_layer = tf.reshape(prev_layer,
(-1, 1, prev_layer_shape[-1]))
this_layer = layers.Conv1D( filters = this_layer_info['units_filters'],
activation = this_layer_info['activation'],
kernel_size = this_layer_info['kernel_size'],
strides = this_layer_info['strides'],
padding = 'same',
name = 'hidden_'+str(hidden_layer)+'_'+\
this_layer_info['layer_type']
)(prev_layer)
model_layers.append(this_layer)
last_hidden_layer = model_layers[-1]
last_hidden_layer_shape = last_hidden_layer.get_shape().as_list()
if len(last_hidden_layer_shape) > 2:
last_hidden_layer_shape = tf.reshape(
last_hidden_layer, (-1, last_hidden_layer_shape[-1]))
output_layer = layers.Dense(units=self.output_dim,
name='output_layer')(last_hidden_layer)
output = tf.multiply(output_layer, -100, name='magnification')
return output
def __init__(self):
self.conv_1 = layers.Conv2D(filters=512,
kernel_size=[3, 3],
strides=[2, 2],
padding='same',
activation=nn.sigmoid,
name="gfn_conv_1")
self.conv_2 = layers.Conv2D(filters=512,
kernel_size=[3, 3],
strides=[1, 1],
padding='same',
activation=nn.sigmoid,
name="gfn_conv_2")
self.conv_3 = layers.Conv2D(filters=512,
kernel_size=[3, 3],
strides=[2, 2],
padding='same',
activation=nn.sigmoid,
name="gfn_conv_3")
self.conv_4 = layers.Conv2D(filters=512,
kernel_size=[3, 3],
strides=[1, 1],
padding='same',
activation=nn.sigmoid,
name="gfn_conv_4")
self.flatten = layers.Flatten(name="flatten")
self.dense_5 = layers.Dense(1024,
activation=nn.sigmoid,
name="gfn_dense_1")
self.dense_6 = layers.Dense(512,
activation=nn.sigmoid,
name="gfn_dense_1")
self.dense_7 = layers.Dense(256,
activation=nn.sigmoid,
name="gfn_desne_3")
return
def __call__(self, inp):
with tf.variable_scope("decoder", reuse=tf.AUTO_REUSE) as model_scope:
x = tfl.Dense(64)(inp)
x = self.batchnorm()(x)
x = tf.nn.leaky_relu(x)
x = tfl.Dense(256)(x)
x = self.batchnorm()(x)
x = tf.nn.leaky_relu(x)
x = tfl.Dense(256)(x)
x = self.batchnorm()(x)
x = tf.nn.leaky_relu(x)
x = tfl.Dense(28 * 28, activation=tf.nn.tanh)(x)
x = tf.reshape(x, [-1, 28, 28, 1])
self.variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope=model_scope.name)
self.post_call(model_scope)
return x
def cell_input_fn(inputs, attention):
'''
根据attn_input_feeding 属性来判断是否在attention计算前进行一次投影计算
:param inputs:
:param attention:
:return:
'''
if not self.use_residual:
return array_ops.concat([inputs, attention], -1)
attn_projection = layers.Dense(self.hidden_size,
dtype=tf.float32,
use_bias=False,
name='attention_cell_input_fn')
return attn_projection(array_ops.concat([inputs, attention], -1))
def attn_decoder_input_fn(inputs, attention):
"""根据attn_input_feeding属性来判断是否在attention计算前进行一次投影计算
"""
if not self.attn_input_feeding:
return inputs
# Essential when use_residual=True
hidden_units = self.hidden_units
if self.bidirectional:
hidden_units *= 2
input_layer = layers.Dense(hidden_units,
dtype=tf.float32,
use_bias=False,
name='attn_input_feeding')
return input_layer(array_ops.concat([inputs, attention], -1))
def build_decoder(self, encoder_outputs, encoder_state):
with tf.variable_scope('decoder') as decoder_scope:
(
self.decoder_cell,
self.decoder_initial_state
) = self.build_decoder_cell(encoder_outputs, encoder_state)
with tf.device(_get_embed_device(self.target_vocab_size)):
if self.share_embedding:
self.decoder_embeddings = self.encoder_embeddings
elif self.pretrained_embedding:
self.decoder_embeddings = tf.Variable(
tf.constant(0.0, shape=(self.target_vocab_size, self.embedding_size)),
trainable=True,
name='embeddings'
)
self.decoder_embeddings_placeholder =\
tf.placeholder(tf.float32, (self.target_vocab_size,
self.embedding_size))
self.decoder_embeddings_init = self.decoder_embeddings.assign(self.decoder_embeddings_placeholder)
else:
self.decoder_embeddings = tf.get_variable(
name='embedding',
shape=(self.target_vocab_size, self.embedding_size),
initializer=self.initializer,
dtype=tf.float32
)
self.decoder_output_projection = layers.Dense(self.target_vocab_size,
dtype=tf.float32,
use_bias=False,
name='decoder_output_projection')
if self.mode == 'train':
self.decoder_inputs_embdedded = tf.nn.embedding_lookup(
params=self.decoder_embeddings,
ids=self.decoder_inputs_train
)
inputs = self.decoder_inputs_embdedded
if self.time_major:
inputs = tf.transpose(inputs, (1, 0, 2))
training_helper = seq2seq.TrainingHelper(
inputs=inputs,
sequence_length=self.decoder_inputs_length,
time_major=self.time_major,
name='training_helper'
)
training_decoder = seq2seq.BasicDecoder(
cell=self.decoder_cell,
helper=training_helper,
initial_state=self.decoder_initial_state
)
max_decoder_length = tf.reduce_max(
self.decoder_inputs_length
)
(
outputs,
self.final_state,
_
) = seq2seq.dynamic_decode(
decoder=training_decoder,
output_time_major=self.time_major,
impute_finished=True,
maximum_iterations=max_decoder_length,
parallel_iterations=self.parallel_iterations,
swap_memory=True,
scope=decoder_scope
)
self.decoder_logits_train = self.decoder_output_projection(
outputs.rnn_output
)
self.masks = tf.sequence_mask(
lengths=self.decoder_inputs_length,
maxlen=max_decoder_length,
dtype=tf.float32,
name='masks'
)
decoder_logits_train = self.decoder_logits_train
if self.time_major:
decoder_logits_train = tf.transpose(decoder_logits_train, (1, 0, 2))
self.decoder_pred_train = tf.argmax(
decoder_logits_train, axis=-1, name='decoder_pred_train'
)
self.train_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=self.decoder_inputs,
logits=decoder_logits_train)
self.masks_rewards = self.masks * self.rewards
self.loss_rewards = seq2seq.sequence_loss(
logits=decoder_logits_train,
targets=self.decoder_inputs,
weights=self.masks_rewards,
average_across_timesteps=True,
average_across_batch=True
)
self.loss = seq2seq.sequence_loss(
logits=decoder_logits_train,
targets=self.decoder_inputs,
weights=self.masks,
average_across_timesteps=True,
average_across_batch=True
)
self.add_loss = self.loss + self.add_loss
elif self.mode == 'decode':
start_token = tf.tile(
[WordSequence.START],
[self.batch_size]
)
end_token = WordSequence.END
def embed_and_input_proj(inputs):
return tf.nn.embedding_lookup(
self.decoder_embeddings,
inputs
)
if not self.use_beamsearch_decode:
decoder_helper = seq2seq.GreedyEmbeddingHelper(
start_tokens=start_token,
end_token=end_token,
embedding=embed_and_input_proj
)
inference_decoder = seq2seq.BasicDecoder(
cell=self.decoder_cell,
helper=decoder_helper,
initial_state=self.decoder_initial_state,
output_layer=self.decoder_output_projection
)
else:
inference_decoder = BeamSearchDecoder(
cell=self.decoder_cell,
embedding=embed_and_input_proj,
start_tokens=start_token,
end_token=end_token,
initial_state=self.decoder_initial_state,
beam_width=self.beam_width,
output_layer=self.decoder_output_projection
)
if self.max_decode_step is not None:
max_decoder_step = self.max_decode_step
else:
max_decoder_step = tf.round(tf.reduce_max(
self.encoder_inputs_length
) * 4)
(
self.decoder_outputs_decode,
self.final_state
) = (seq2seq.dynamic_decode(
decoder=inference_decoder,
output_time_major=self.time_major,
maximum_iterations=max_decoder_step,
parallel_iterations=self.parallel_iterations,
swap_memory=True,
scope=decoder_scope
))
if not self.use_beamsearch_decode:
dod = self.decoder_outputs_decode
self.decoder_pred_train = tf.transpose(
self.decoder_pred_decode, (1, 0)
)
else:
self.decoder_pred_decode = self.decoder_outputs_decode.predicted_ids
if self.time_major:
self.decoder_pred_decode = tf.transpose(
self.decoder_pred_decode, (1, 0, 2)
)
self.decoder_pred_decode = tf.transpose(
self.decoder_pred_decode,
perm=[0, 2, 1]
)
dod = self.decoder_pred_decode
self.beam_prob = dod.beam_search_decoder_output.scores
def build_decoder(self, encoder_outputs, encoder_state):
"""构建解码器
"""
with tf.variable_scope('decoder') as decoder_scope:
#创建解码器单元
(self.decoder_cell,self.decoder_initial_state)\
= self.build_decoder_cell(encoder_outputs, encoder_state)
# 解码器embedding 根据词表大小选择CPU还是GPU上训练
with tf.device(_get_embed_device(self.target_vocab_size)):
#如果是共享的embedding 则赋值，否则加载预训练 或者初始化进行后续的训练
if self.share_embedding:
self.decoder_embeddings = self.encoder_embeddings
#如果是预训练的embedding
elif self.pretrained_embedding:
self.decoder_embeddings = tf.Variable(
tf.constant(
0.0,
shape=(self.target_vocab_size,self.embedding_size)),
trainable=True,#是否可以被训练
name='embeddings')
self.decoder_embeddings_placeholder = tf.placeholder(
tf.float32,
(self.target_vocab_size, self.embedding_size))
self.decoder_embeddings_init = self.decoder_embeddings.assign(
self.decoder_embeddings_placeholder)
else:
self.decoder_embeddings = tf.get_variable(
name='embeddings',
shape=(self.target_vocab_size, self.embedding_size),
initializer=self.initializer,
dtype=tf.float32
)
#解码器的输出
self.decoder_output_projection = layers.Dense(
self.target_vocab_size, #一共有词表大小个输出
dtype=tf.float32,
use_bias=False,
name='decoder_output_projection'
)
if self.mode == 'train':
self.decoder_inputs_embedded = tf.nn.embedding_lookup(
params=self.decoder_embeddings,
ids=self.decoder_inputs_train #placeholder初始化时设定
)
inputs = self.decoder_inputs_embedded
if self.time_major:
inputs = tf.transpose(inputs, (1, 0, 2))
#帮助feed参数 一般用于训练阶段Decoder解码，辅助Decoder解码过程
training_helper = seq2seq.TrainingHelper(
inputs=inputs,
sequence_length=self.decoder_inputs_length,
time_major=self.time_major,
name='training_helper'
)
# 训练的时候不在这里应用 output_layer
# 因为这里会每个 time_step 的进行 output_layer 的投影计算，比较慢
# 注意这个trick要成功必须设置 dynamic_decode 的 scope 参数
training_decoder = seq2seq.BasicDecoder(
cell=self.decoder_cell,
helper=training_helper,
initial_state=self.decoder_initial_state, #用之前定义的初始化单元的状态进行初始化
)
# decoder在当前batch下最大的time_steps
max_decoder_length = tf.reduce_max(self.decoder_inputs_length)
#定义动态解码的输出
(outputs,self.final_state,_)\
= seq2seq.dynamic_decode( #动态decoder
decoder=training_decoder,
output_time_major=self.time_major, #True是以time(seq_length)为第一维，False是以batch_size为第一维
impute_finished=True, #追踪finished，如果一个序列已经finished，那么后面的每一步output为0
maximum_iterations=max_decoder_length,#最大迭代次数（可以理解为decoder最多可以生成几个词）
parallel_iterations=self.parallel_iterations,##while_loop的并行次数
swap_memory=True, ##True时，当遇到OOM(out of memory)，是否把张量从显存转到内存
scope=decoder_scope)
#在训练时将所有的结果在全连接层一次性做投影运算 可以提高效率官方提倡
self.decoder_logits_train = self.decoder_output_projection(
outputs.rnn_output #上面定义的解码器的输出
)
# masks: masking for valid and padded time steps,
#tf.sequence_mask的作用是构建序列长度的mask标志
"""
tf.sequence_mask([1,2], 4)
-->
[[ True False False False]
[ True True False False]]
"""
# [batch_size, max_time_step + 1]
self.masks = tf.sequence_mask(
lengths=self.decoder_inputs_length,
maxlen=max_decoder_length,
dtype=tf.float32,
name='masks'
)
decoder_logits_train = self.decoder_logits_train
if self.time_major:
decoder_logits_train = tf.transpose(decoder_logits_train,
(1,0,2))
#解码器训练时的预测输出 decoder_logits_train一共有词表大小个输出，现仅取值最大的那个下标即为预测的对应下标
self.decoder_pred_train = tf.argmax(
decoder_logits_train,
axis=-1,
name='decoder_pred_train')
# 下面的一些变量用于特殊的学习训练
# 自定义rewards，其实我这里是修改了masks 损失之类
# train_entropy = cross entropy
self.train_entropy = \
tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=self.decoder_inputs,#标签
logits=decoder_logits_train)#预测
self.masks_rewards = self.masks * self.rewards
#seq2sqe中的损失函数 就是将各个时间步输出相加求平均 权重为mask 当句子长度短于最大长度，为0部分的权重为0
self.loss_rewards = seq2seq.sequence_loss(
logits=decoder_logits_train,
targets=self.decoder_inputs,
weights=self.masks_rewards, #这里权重跟下面的不同
average_across_timesteps=True, #损失将除以总的权重
average_across_batch=True, #损失将是总的损失处于批次大小
)
self.loss = seq2seq.sequence_loss(
logits=decoder_logits_train,
targets=self.decoder_inputs,
weights=self.masks,
average_across_timesteps=True,
average_across_batch=True,
)
self.loss_add = self.loss + self.add_loss
elif self.mode == 'decode':
# 预测模式，非训练
#对原数据进行扩展 参考
#https://blog.csdn.net/tsyccnh/article/details/82459859
start_tokens = tf.tile(
[WordSequence.START],
[self.batch_size]
)
end_token = WordSequence.END
def embed_and_input_proj(inputs):
"""输入层的投影层wrapper
将输入转换成对应词表对应下的embedding
"""
return tf.nn.embedding_lookup(
self.decoder_embeddings,
inputs
)
#如果不使用集束搜索解码 这里定义helper和decoder的结构
if not self.use_beamsearch_decode:
# Helper to feed inputs for greedy decoding:
# uses the argmax of the output
#贪婪搜索解码
decoding_helper = seq2seq.GreedyEmbeddingHelper(
start_tokens=start_tokens,#起始token
end_token=end_token, #结束token
embedding=embed_and_input_proj #已经将输入转换成对应的embedding
)
# Basic decoder performs greedy decoding at each time step
# print("building greedy decoder..")
inference_decoder = seq2seq.BasicDecoder(
cell=self.decoder_cell,
helper=decoding_helper,
initial_state=self.decoder_initial_state,
output_layer=self.decoder_output_projection
)
else:
#使用beamsearch解码
# Beamsearch is used to approximately
# find the most likely translation
# print("building beamsearch decoder..")
inference_decoder = BeamSearchDecoder(
cell=self.decoder_cell,
embedding=embed_and_input_proj,
start_tokens=start_tokens,
end_token=end_token,
initial_state=self.decoder_initial_state,
beam_width=self.beam_width,
output_layer=self.decoder_output_projection,
)
#一般使用最大值
if self.max_decode_step is not None:
max_decode_step = self.max_decode_step
else:
# 默认 4 倍输入长度的输出解码
max_decode_step = tf.round(tf.reduce_max(
self.encoder_inputs_length) * 4)
( self.decoder_outputs_decode,#输出
self.final_state, #最后的状态
_ # self.decoder_outputs_length_decode
) = seq2seq.dynamic_decode(
decoder=inference_decoder, #这里包含了使用哪种解码方式
output_time_major=self.time_major,
# impute_finished=True, # error occurs
maximum_iterations=max_decode_step,
parallel_iterations=self.parallel_iterations,
swap_memory=True,
scope=decoder_scope
)
#如果不使用beamsearch解码，使用贪婪解码
#调用dynamic_decode进行解码，decoder_outputs_decode是一个namedtuple，里面包含两项(rnn_outputs, sample_id)
# rnn_output: [batch_size, decoder_targets_length, vocab_size]，保存decode每个时刻每个单词的概率，可以用来计算loss
# sample_id: [batch_size], tf.int32，保存最终的编码结果。可以表示最后的答案
if not self.use_beamsearch_decode:
dod = self.decoder_outputs_decode
self.decoder_pred_decode = dod.sample_id #就是最终的答案
if self.time_major:
self.decoder_pred_decode = tf.transpose(
self.decoder_pred_decode, (1, 0))
#如果使用beamsearch
#参考 https://blog.csdn.net/liuchonge/article/details/79021938
# 对于使用beam_search的时候，decoder_outputs_decode它里面包含两项(predicted_ids, beam_search_decoder_output)
# predicted_ids: [batch_size, decoder_targets_length, beam_size],保存输出结果
# beam_search_decoder_output: BeamSearchDecoderOutput instance namedtuple(scores, predicted_ids, parent_ids)
# 所以对应只需要返回predicted_ids或者sample_id即可翻译成最终的结果
else:
self.decoder_pred_decode = \
self.decoder_outputs_decode.predicted_ids
if self.time_major:
self.decoder_pred_decode = tf.transpose(
self.decoder_pred_decode, (1, 0, 2))
self.decoder_pred_decode = tf.transpose(
self.decoder_pred_decode,
perm=[0, 2, 1])
dod = self.decoder_outputs_decode
self.beam_prob = dod.beam_search_decoder_output.scores
def build_decoder(self, encoder_outputs, encoder_state):
"""构建解码器
"""
with tf.variable_scope('decoder') as decoder_scope:
# Building decoder_cell and decoder_initial_state
(self.decoder_cell,
self.decoder_initial_state) = self.build_decoder_cell(
encoder_outputs, encoder_state)
# 解码器embedding
with tf.device(_get_embed_device(self.target_vocab_size)):
if self.share_embedding:
self.decoder_embeddings = self.encoder_embeddings
elif self.pretrained_embedding:
self.decoder_embeddings = tf.Variable(tf.constant(
0.0,
shape=(self.target_vocab_size, self.embedding_size)),
trainable=True,
name='embeddings')
self.decoder_embeddings_placeholder = tf.placeholder(
tf.float32,
(self.target_vocab_size, self.embedding_size))
self.decoder_embeddings_init = \
self.decoder_embeddings.assign(
self.decoder_embeddings_placeholder)
else:
self.decoder_embeddings = tf.get_variable(
name='embeddings',
shape=(self.target_vocab_size, self.embedding_size),
initializer=self.initializer,
dtype=tf.float32)
self.decoder_output_projection = layers.Dense(
self.target_vocab_size,
dtype=tf.float32,
use_bias=False,
name='decoder_output_projection')
if self.mode == 'train':
# decoder_inputs_embedded:
# [batch_size, max_time_step + 1, embedding_size]
self.decoder_inputs_embedded = tf.nn.embedding_lookup(
params=self.decoder_embeddings,
ids=self.decoder_inputs_train)
# Helper to feed inputs for training:
# read inputs from dense ground truth vectors
inputs = self.decoder_inputs_embedded
if self.time_major:
inputs = tf.transpose(inputs, (1, 0, 2))
training_helper = seq2seq.TrainingHelper(
inputs=inputs,
sequence_length=self.decoder_inputs_length,
time_major=self.time_major,
name='training_helper')
# 训练的时候不在这里应用 output_layer
# 因为这里会每个 time_step 的进行 output_layer 的投影计算，比较慢
# 注意这个trick要成功必须设置 dynamic_decode 的 scope 参数
training_decoder = seq2seq.BasicDecoder(
cell=self.decoder_cell,
helper=training_helper,
initial_state=self.decoder_initial_state,
# output_layer=self.decoder_output_projection
)
# Maximum decoder time_steps in current batch
max_decoder_length = tf.reduce_max(self.decoder_inputs_length)
# decoder_outputs_train: BasicDecoderOutput
# namedtuple(rnn_outputs, sample_id)
# decoder_outputs_train.rnn_output:
# if output_time_major=False:
# [batch_size, max_time_step + 1, num_decoder_symbols]
# if output_time_major=True:
# [max_time_step + 1, batch_size, num_decoder_symbols]
# decoder_outputs_train.sample_id: [batch_size], tf.int32
(
outputs,
self.final_state, # contain attention
_ # self.final_sequence_lengths
) = seq2seq.dynamic_decode(
decoder=training_decoder,
output_time_major=self.time_major,
impute_finished=True,
maximum_iterations=max_decoder_length,
parallel_iterations=self.parallel_iterations,
swap_memory=True,
scope=decoder_scope)
# More efficient to do the projection
# on the batch-time-concatenated tensor
# logits_train:
# [batch_size, max_time_step + 1, num_decoder_symbols]
# 训练的时候一次性对所有的结果进行 output_layer 的投影运算
# 官方NMT库说这样能提高10~20%的速度
# 实际上我提高的速度会更大
self.decoder_logits_train = self.decoder_output_projection(
outputs.rnn_output)
# masks: masking for valid and padded time steps,
# [batch_size, max_time_step + 1]
self.masks = tf.sequence_mask(
lengths=self.decoder_inputs_length,
maxlen=max_decoder_length,
dtype=tf.float32,
name='masks')
# Computes per word average cross-entropy over a batch
# Internally calls
# 'nn_ops.sparse_softmax_cross_entropy_with_logits' by default
decoder_logits_train = self.decoder_logits_train
if self.time_major:
decoder_logits_train = tf.transpose(
decoder_logits_train, (1, 0, 2))
self.decoder_pred_train = tf.argmax(decoder_logits_train,
axis=-1,
name='decoder_pred_train')
# 下面的一些变量用于特殊的学习训练
# 自定义rewards，其实我这里是修改了masks
# train_entropy = cross entropy
self.train_entropy = \
tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=self.decoder_inputs,
logits=decoder_logits_train)
self.masks_rewards = self.masks * self.rewards
self.loss_rewards = seq2seq.sequence_loss(
logits=decoder_logits_train,
targets=self.decoder_inputs,
weights=self.masks_rewards,
average_across_timesteps=True,
average_across_batch=True,
)
self.loss = seq2seq.sequence_loss(
logits=decoder_logits_train,
targets=self.decoder_inputs,
weights=self.masks,
average_across_timesteps=True,
average_across_batch=True,
)
self.loss_add = self.loss + self.add_loss
elif self.mode == 'decode':
# 预测模式，非训练
start_tokens = tf.tile([WordSequence.START], [self.batch_size])
end_token = WordSequence.END
def embed_and_input_proj(inputs):
"""输入层的投影层wrapper
"""
return tf.nn.embedding_lookup(self.decoder_embeddings,
inputs)
if not self.use_beamsearch_decode:
# Helper to feed inputs for greedy decoding:
# uses the argmax of the output
decoding_helper = seq2seq.GreedyEmbeddingHelper(
start_tokens=start_tokens,
end_token=end_token,
embedding=embed_and_input_proj)
# Basic decoder performs greedy decoding at each time step
# print("building greedy decoder..")
inference_decoder = seq2seq.BasicDecoder(
cell=self.decoder_cell,
helper=decoding_helper,
initial_state=self.decoder_initial_state,
output_layer=self.decoder_output_projection)
else:
# Beamsearch is used to approximately
# find the most likely translation
# print("building beamsearch decoder..")
inference_decoder = BeamSearchDecoder(
cell=self.decoder_cell,
embedding=embed_and_input_proj,
start_tokens=start_tokens,
end_token=end_token,
initial_state=self.decoder_initial_state,
beam_width=self.beam_width,
output_layer=self.decoder_output_projection,
)
# For GreedyDecoder, return
# decoder_outputs_decode: BasicDecoderOutput instance
# namedtuple(rnn_outputs, sample_id)
# decoder_outputs_decode.rnn_output:
# if output_time_major=False:
# [batch_size, max_time_step, num_decoder_symbols]
# if output_time_major=True
# [max_time_step, batch_size, num_decoder_symbols]
# decoder_outputs_decode.sample_id:
# if output_time_major=False
# [batch_size, max_time_step], tf.int32
# if output_time_major=True
# [max_time_step, batch_size], tf.int32
# For BeamSearchDecoder, return
# decoder_outputs_decode: FinalBeamSearchDecoderOutput instance
# namedtuple(predicted_ids, beam_search_decoder_output)
# decoder_outputs_decode.predicted_ids:
# if output_time_major=False:
# [batch_size, max_time_step, beam_width]
# if output_time_major=True
# [max_time_step, batch_size, beam_width]
# decoder_outputs_decode.beam_search_decoder_output:
# BeamSearchDecoderOutput instance
# namedtuple(scores, predicted_ids, parent_ids)
# 官方文档提到的一个潜在的最大长度选择
# 我这里改为 * 4
# maximum_iterations = tf.round(tf.reduce_max(source_sequence_length) * 2)
# https://www.tensorflow.org/tutorials/seq2seq
if self.max_decode_step is not None:
max_decode_step = self.max_decode_step
else:
# 默认 4 倍输入长度的输出解码
max_decode_step = tf.round(
tf.reduce_max(self.encoder_inputs_length) * 4)
(
self.decoder_outputs_decode,
self.final_state,
_ # self.decoder_outputs_length_decode
) = (
seq2seq.dynamic_decode(
decoder=inference_decoder,
output_time_major=self.time_major,
# impute_finished=True, # error occurs
maximum_iterations=max_decode_step,
parallel_iterations=self.parallel_iterations,
swap_memory=True,
scope=decoder_scope))
if not self.use_beamsearch_decode:
# decoder_outputs_decode.sample_id:
# [batch_size, max_time_step]
# Or use argmax to find decoder symbols to emit:
# self.decoder_pred_decode = tf.argmax(
# self.decoder_outputs_decode.rnn_output,
# axis=-1, name='decoder_pred_decode')
# Here, we use expand_dims to be compatible with
# the result of the beamsearch decoder
# decoder_pred_decode:
# [batch_size, max_time_step, 1] (output_major=False)
# self.decoder_pred_decode = tf.expand_dims(
# self.decoder_outputs_decode.sample_id,
# -1
# )
dod = self.decoder_outputs_decode
self.decoder_pred_decode = dod.sample_id
if self.time_major:
self.decoder_pred_decode = tf.transpose(
self.decoder_pred_decode, (1, 0))
else:
# Use beam search to approximately
# find the most likely translation
# decoder_pred_decode:
# [batch_size, max_time_step, beam_width] (output_major=False)
self.decoder_pred_decode = \
self.decoder_outputs_decode.predicted_ids
if self.time_major:
self.decoder_pred_decode = tf.transpose(
self.decoder_pred_decode, (1, 0, 2))
self.decoder_pred_decode = tf.transpose(
self.decoder_pred_decode, perm=[0, 2, 1])
dod = self.decoder_outputs_decode
self.beam_prob = dod.beam_search_decoder_output.scores
#plt.xlabel('breast')
#plt.ylabel('Class')
#plt.show
X = ordinal_encode[:, 1:]
y = ordinal_encode[:, 0]
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.2,
random_state=0)
print(X_train.shape)
print(y_train.shape)
input_shape = X_train[0].shape
print(input_shape)
model = tf.keras.Sequential([
layers.Dense(64, activation=tf.nn.relu, input_shape=input_shape),
layers.Dense(64, activation=tf.nn.relu),
layers.Dense(1, activation=tf.nn.sigmoid)
])
model.compile(optimizer=tf.keras.optimizers.Adam(),
loss=['binary_crossentropy'],
metrics=['accuracy'])
#history = model.fit(X_train, y_train, epochs=100, verbose=2, validation_split=0.2)
loss, score = model.evaluate(X_test, y_test)
#print(score)
predictor = model.predict(X_test)
#print(predictor)
#performance evaluation
def FC(self, units, use_bias=False):
return tl.Dense(units,
use_bias=use_bias,
kernel_initializer=self.w_initer)
X = []
y = []
for feature, label in training_dataset():
X.append(feature)
y.append(label)
X_train = np.array(X).reshape(-1, 150, 150, 1)
train_label = np.array(y).reshape(-1, 1)
label_encoded = OneHotEncoder().fit_transform(train_label)
y_train = label_encoded.A
X_train = X_train / 255.0
model = keras.Sequential([
layers.Conv2D(64, (3, 3),
activation=tf.nn.relu,
input_shape=X_train.shape[1:]),
layers.MaxPooling2D(pool_size=(2, 2), strides=(1, 1)),
layers.Conv2D(64, (3, 3), activation=tf.nn.relu),
layers.MaxPooling2D(pool_size=(2, 2), strides=(1, 1)),
layers.Flatten(),
layers.Dense(64, activation=tf.nn.relu),
layers.Dense(8, activation=tf.nn.softmax)
])
model.compile(loss='categorical_crossentropy',
optimizer=tf.keras.optimizers.Adam(),
metrics=['accuracy'])
model.fit(X_train, y_train, batch_size=4, validation_split=0.1, epoch=5)
def build_decoder(self, encoder_outputs, encoder_final_state):
"""
构建完整解码器
:return:
"""
with tf.variable_scope("decode"):
decoder_cell, decoder_initial_state = self.build_decoder_cell(
encoder_outputs, encoder_final_state, self.hidden_size,
self.cell_type, self.layer_size)
# 输出层投影
decoder_output_projection = layers.Dense(
self.decoder_vocab_size,
dtype=tf.float32,
use_bias=False,
kernel_initializer=tf.truncated_normal_initializer(mean=0.0,
stddev=0.1),
name='decoder_output_projection')
if self.mode == 'train':
# 训练模式
decoder_inputs_embdedded = tf.nn.embedding_lookup(
self.decoder_embeddings, self.decoder_inputs_train)
'''
TrainingHelper用于train阶段，next_inputs方法一样也接收outputs与sample_ids，但是只是从初始化时的inputs返回下一时刻的输入。
TrainingHelper
__init__( inputs, sequence_length, time_major=False, name=None )
- inputs: A (structure of) input tensors.
- sequence_length: An int32 vector tensor.
- time_major: Python bool. Whether the tensors in inputs are time major. If False (default), they are assumed to be batch major.
- name: Name scope for any created operations.
inputs：对应Decoder框架图中的embedded_input，time_major=False的时候，inputs的shape就是[batch_size, sequence_length, embedding_size] ，time_major=True时，inputs的shape为[sequence_length, batch_size, embedding_size]
sequence_length：这个文档写的太简略了，不过在源码中可以看出指的是当前batch中每个序列的长度(self._batch_size = array_ops.size(sequence_length))。
time_major：决定inputs Tensor前两个dim表示的含义
name：如文档所述
'''
training_helper = TrainingHelper(
inputs=decoder_inputs_embdedded,
sequence_length=self.decoder_inputs_length,
name='training_helper')
'''
BasicDecoder的作用就是定义一个封装了decoder应该有的功能的实例，根据Helper实例的不同，这个decoder可以实现不同的功能，比如在train的阶段，不把输出重新作为输入，而在inference阶段，将输出接到输入。
BasicDecoder
__init__( cell, helper, initial_state, output_layer=None )
- cell: An RNNCell instance.
- helper: A Helper instance.
- initial_state: A (possibly nested tuple of…) tensors and TensorArrays. The initial state of the RNNCell.
- output_layer: (Optional) An instance of tf.layers.Layer, i.e., tf.layers.Dense. Optional layer to apply to the RNN output prior to storing the result or sampling.
cell：在这里就是一个多层LSTM的实例，与定义encoder时无异
helper：这里只是简单说明是一个Helper实例，第一次看文档的时候肯定还不知道这个Helper是什么，不用着急，看到具体的Helper实例就明白了
initial_state：encoder的final state，类型要一致，也就是说如果encoder的final state是tuple类型(如LSTM的包含了cell state与hidden state)，那么这里的输入也必须是tuple。直接将encoder的final_state作为这个参数输入即可
output_layer：对应的就是框架图中的Dense_Layer，只不过文档里写tf.layers.Dense，但是tf.layers下只有dense方法，Dense的实例还需要from tensorflow.python.layers.core import Dense。
'''
training_decoder = BasicDecoder(decoder_cell, training_helper,
decoder_initial_state,
decoder_output_projection)
max_decoder_length = tf.reduce_max(self.decoder_inputs_length)
'''
首先tf.contrib.seq2seq.dynamic_decode主要作用是接收一个Decoder类，然后依据Encoder进行解码，实现序列的生成（映射）。
其中，这个函数主要的一个思想是一步一步地调用Decoder的step函数（该函数接收当前的输入和隐层状态会生成下一个词），实现最后的一句话的生成。该函数类似tf.nn.dynamic_rnn。
'''
training_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(
training_decoder, maximum_iterations=max_decoder_length)
'''
tf.sequence_mask函数返回的一个mask张量。经过tf.Session()打印可以得到一个array数据。
decoder_inputs_length范围内的数据用1填充，[decoder_inputs_length,max_decoder_length]区间用0填充
'''
self.masks = tf.sequence_mask(self.decoder_inputs_length,
maxlen=max_decoder_length,
dtype=tf.float32,
name='masks')
'''
tf.contrib.seq2seq.sequence_loss可以直接计算序列的损失函数，重要参数：
logits：尺寸[batch_size, sequence_length, num_decoder_symbols]
targets：尺寸[batch_size, sequence_length]，不用做one_hot。
weights：[batch_size, sequence_length]，即mask，滤去padding的loss计算，使loss计算更准确。
'''
self.loss = tf.contrib.seq2seq.sequence_loss(
logits=training_decoder_output.rnn_output,
targets=self.decoder_inputs,
weights=self.masks, # mask，滤去padding的loss计算，使loss计算更准确。
average_across_timesteps=True,
average_across_batch=True)
else:
# 预测模式
start_token = [DataUnit.START_INDEX] * self.batch_size
end_token = DataUnit.END_INDEX
'''
BeamSearchDecoder
cell: An RNNCell instance.
embedding: A callable that takes a vector tensor of ids (argmax ids), or the params argument for embedding_lookup.
start_tokens: int32 vector shaped [batch_size], the start tokens.
end_token: int32 scalar, the token that marks end of decoding.
initial_state: A (possibly nested tuple of...) tensors and TensorArrays.
beam_width: Python integer, the number of beams.
output_layer: (Optional) An instance of tf.keras.layers.Layer, i.e., tf.keras.layers.Dense. Optional layer to apply to the RNN output prior to storing the result or sampling.
length_penalty_weight: Float weight to penalize length. Disabled with 0.0.
coverage_penalty_weight: Float weight to penalize the coverage of source sentence. Disabled with 0.0.
reorder_tensor_arrays: If True, TensorArrays' elements within the cell state will be reordered according to the beam search path.
If the TensorArray can be reordered, the stacked form will be returned. Otherwise,
the TensorArray will be returned as is. Set this flag to False if the cell state contains TensorArrays that are not amenable to reordering.
'''
inference_decoder = BeamSearchDecoder(
cell=decoder_cell,
embedding=lambda x: tf.nn.embedding_lookup(
self.decoder_embeddings, x),
start_tokens=start_token,
end_token=end_token,
initial_state=decoder_initial_state,
beam_width=self.beam_width,
output_layer=decoder_output_projection)
'''
首先tf.contrib.seq2seq.dynamic_decode主要作用是接收一个Decoder类，然后依据Encoder进行解码，实现序列的生成（映射）。
其中，这个函数主要的一个思想是一步一步地调用Decoder的step函数（该函数接收当前的输入和隐层状态会生成下一个词），实现最后的一句话的生成。该函数类似tf.nn.dynamic_rnn。
'''
inference_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(
inference_decoder, maximum_iterations=self.max_decode_step)
self.decoder_pred_decode = inference_decoder_output.predicted_ids
self.decoder_pred_decode = tf.transpose(
self.decoder_pred_decode, perm=[0, 2, 1])
plt.imshow(image_batch[n])
plt.title(predicted_class_names[n])
plt.axis('off')
_ = plt.suptitle("ImageNet predictions")
feature_extractor_layer = hub.KerasLayer(feature_extractor_url,
input_shape=(224,224,4))
feature_batch = feature_extractor_layer(image_batch)
print(feature_batch.shape)
feature_extractor_layer.trainable = False
model = tf.keras.Sequential([
feature_extractor_layer,
layers.Dense(image_data.num_classes)
])
model.summary()
predictions = model(image_batch)
predictions.shape
model.compile(
optimizer=tf.keras.optimizers.Adam(),
loss=tf.keras.losses.CategoricalCrossentropy(from_logits=True),
metrics=['acc'])
class CollectBatchStats(tf.keras.callbacks.Callback):
def __init__(self):
def build_decoder(self, encoder_outputs, encoder_state):
"""构建解码器
"""
with tf.variable_scope('decoder') as decoder_scope:
(self.decoder_cell,
self.decoder_initial_state) = self.build_decoder_cell(
encoder_outputs, encoder_state)
# 解码器embedding
with tf.device(_get_embed_device(self.target_vocab_size)):
if self.share_embedding:
self.decoder_embeddings = self.encoder_embeddings
elif self.pretrained_embedding:
self.decoder_embeddings = tf.Variable(tf.constant(
0.0,
shape=(self.target_vocab_size, self.embedding_size)),
trainable=True,
name='embeddings')
self.decoder_embeddings_placeholder = tf.placeholder(
tf.float32,
(self.target_vocab_size, self.embedding_size))
self.decoder_embeddings_init = \
self.decoder_embeddings.assign(
self.decoder_embeddings_placeholder)
else:
self.decoder_embeddings = tf.get_variable(
name='embeddings',
shape=(self.target_vocab_size, self.embedding_size),
initializer=self.initializer,
dtype=tf.float32)
self.decoder_output_projection = layers.Dense(
self.target_vocab_size,
dtype=tf.float32,
use_bias=False,
name='decoder_output_projection')
if self.mode == 'train':
self.decoder_inputs_embedded = tf.nn.embedding_lookup(
params=self.decoder_embeddings,
ids=self.decoder_inputs_train)
inputs = self.decoder_inputs_embedded
if self.time_major:
inputs = tf.transpose(inputs, (1, 0, 2))
training_helper = seq2seq.TrainingHelper(
inputs=inputs,
sequence_length=self.decoder_inputs_length,
time_major=self.time_major,
name='training_helper')
# 训练的时候不在这里应用 output_layer
# 因为这里会每个 time_step 的进行 output_layer 的投影计算，比较慢
# 注意这个trick要成功必须设置 dynamic_decode 的 scope 参数
training_decoder = seq2seq.BasicDecoder(
cell=self.decoder_cell,
helper=training_helper,
initial_state=self.decoder_initial_state,
)
# Maximum decoder time_steps in current batch
max_decoder_length = tf.reduce_max(self.decoder_inputs_length)
(
outputs,
self.final_state, # contain attention
_ # self.final_sequence_lengths
) = seq2seq.dynamic_decode(
decoder=training_decoder,
output_time_major=self.time_major,
impute_finished=True,
maximum_iterations=max_decoder_length,
parallel_iterations=self.parallel_iterations,
swap_memory=True,
scope=decoder_scope)
self.decoder_logits_train = self.decoder_output_projection(
outputs.rnn_output)
# masks: masking for valid and padded time steps,
# [batch_size, max_time_step + 1]
self.masks = tf.sequence_mask(
lengths=self.decoder_inputs_length,
maxlen=max_decoder_length,
dtype=tf.float32,
name='masks')
decoder_logits_train = self.decoder_logits_train
if self.time_major:
decoder_logits_train = tf.transpose(
decoder_logits_train, (1, 0, 2))
self.decoder_pred_train = tf.argmax(decoder_logits_train,
axis=-1,
name='decoder_pred_train')
# 下面的一些变量用于特殊的学习训练
# 自定义rewards，其实我这里是修改了masks
# train_entropy = cross entropy
self.train_entropy = \
tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=self.decoder_inputs,
logits=decoder_logits_train)
self.masks_rewards = self.masks * self.rewards
self.loss_rewards = seq2seq.sequence_loss(
logits=decoder_logits_train,
targets=self.decoder_inputs,
weights=self.masks_rewards,
average_across_timesteps=True,
average_across_batch=True,
)
self.loss = seq2seq.sequence_loss(
logits=decoder_logits_train,
targets=self.decoder_inputs,
weights=self.masks,
average_across_timesteps=True,
average_across_batch=True,
)
self.loss_add = self.loss + self.add_loss
elif self.mode == 'decode':
# 预测模式，非训练
start_tokens = tf.tile([WordSequence.START], [self.batch_size])
end_token = WordSequence.END
def embed_and_input_proj(inputs):
"""输入层的投影层wrapper
"""
return tf.nn.embedding_lookup(self.decoder_embeddings,
inputs)
if not self.use_beamsearch_decode:
# Helper to feed inputs for greedy decoding:
# uses the argmax of the output
decoding_helper = seq2seq.GreedyEmbeddingHelper(
start_tokens=start_tokens,
end_token=end_token,
embedding=embed_and_input_proj)
# Basic decoder performs greedy decoding at each time step
# print("building greedy decoder..")
inference_decoder = seq2seq.BasicDecoder(
cell=self.decoder_cell,
helper=decoding_helper,
initial_state=self.decoder_initial_state,
output_layer=self.decoder_output_projection)
else:
# Beamsearch is used to approximately
# find the most likely translation
# print("building beamsearch decoder..")
inference_decoder = BeamSearchDecoder(
cell=self.decoder_cell,
embedding=embed_and_input_proj,
start_tokens=start_tokens,
end_token=end_token,
initial_state=self.decoder_initial_state,
beam_width=self.beam_width,
output_layer=self.decoder_output_projection,
)
if self.max_decode_step is not None:
max_decode_step = self.max_decode_step
else:
# 默认 4 倍输入长度的输出解码
max_decode_step = tf.round(
tf.reduce_max(self.encoder_inputs_length) * 4)
(
self.decoder_outputs_decode,
self.final_state,
_ # self.decoder_outputs_length_decode
) = (
seq2seq.dynamic_decode(
decoder=inference_decoder,
output_time_major=self.time_major,
# impute_finished=True, # error occurs
maximum_iterations=max_decode_step,
parallel_iterations=self.parallel_iterations,
swap_memory=True,
scope=decoder_scope))
if not self.use_beamsearch_decode:
dod = self.decoder_outputs_decode
self.decoder_pred_decode = dod.sample_id
if self.time_major:
self.decoder_pred_decode = tf.transpose(
self.decoder_pred_decode, (1, 0))
else:
self.decoder_pred_decode = \
self.decoder_outputs_decode.predicted_ids
if self.time_major:
self.decoder_pred_decode = tf.transpose(
self.decoder_pred_decode, (1, 0, 2))
self.decoder_pred_decode = tf.transpose(
self.decoder_pred_decode, perm=[0, 2, 1])
dod = self.decoder_outputs_decode
self.beam_prob = dod.beam_search_decoder_output.scores
def __init__(self,
name,
state_size,
action_size,
opt,
feature_layers=None,
critic_layers=None,
actor_layers=None):
self.name = name
self.state_size = state_size
self.action_size = action_size
self.optimizer = opt
self.feature_layers = [
layers.Conv2D(filters=16,
kernel_size=(8, 8),
strides=(4, 4),
activation=tf.nn.leaky_relu),
layers.Conv2D(filters=32,
kernel_size=(4, 4),
strides=(2, 2),
activation=tf.nn.leaky_relu),
layers.Flatten(),
layers.Dense(256, activation=tf.nn.leaky_relu, name="features"),
] if (feature_layers is None
or not isinstance(feature_layers, Iterable)) else feature_layers
critic_layers = [layers.Dense(1, name='value')] if (
critic_layers is None
or not isinstance(critic_layers, Iterable)) else critic_layers
actor_layers = [layers.Dense(action_size, name='logits')] if (
actor_layers is None
or not isinstance(actor_layers, Iterable)) else actor_layers
self.selected_action = tf.placeholder(tf.uint8, [None], name="labels")
self.actions_onehot = tf.one_hot(self.selected_action,
self.action_size,
dtype=tf.float32)
self.advantages = tf.placeholder(tf.float32, [None])
self.discounted_reward = tf.placeholder(tf.float32, [None])
self.state = tf.placeholder(tf.float32,
shape=[None, *state_size],
name="states")
with tf.variable_scope(self.name):
self.feature = self._layers_output(self.feature_layers, self.state)
self.value = self._layers_output(critic_layers, self.feature)
self.logits = self._layers_output(actor_layers, self.feature)
self.policy = tf.nn.softmax(self.logits, name='policy')
# self.value_loss, self.policy_loss, self.entropy, self.total_loss = self._compute_loss()
# self.target = tf.placeholder(tf.float32, [None])
responsible_outputs = tf.reduce_sum(
self.policy * self.actions_onehot, 1)
self.entropy = 0.005 * tf.reduce_sum(
-self.policy * tf.log(self.policy + 1e-7), 1)
self.policy_loss = -tf.reduce_mean(
(tf.log(responsible_outputs + 1e-7)) * self.advantages +
self.entropy)
self.value_loss = tf.losses.mean_squared_error(
self.advantages, tf.squeeze(self.value))
self.total_loss = 0.5 * self.value_loss + self.policy_loss # - entropy * 0.005
self.optimizer = tf.train.RMSPropOptimizer(learning_rate=0.01,
decay=.99)
# if name != 'global':
var_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope=self.name)
self.gradients = self.optimizer.compute_gradients(
self.total_loss, var_list)
self.gradients_placeholders = []
for grad, var in self.gradients:
self.gradients_placeholders.append(
(tf.placeholder(var.dtype, shape=var.get_shape()), var))
self.apply_gradients = self.optimizer.apply_gradients(
self.gradients_placeholders)
# self.gradients = tf.gradients(self.total_loss,
# tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=self.name))
# self.grads, self.grad_norms = tf.clip_by_global_norm(self.gradients, 40.0)
# self.apply_grads = opt.apply_gradients(
# zip(self.grads, tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, 'global')))
# else:
self.reward_summary_ph = tf.placeholder(tf.float32,
name="reward_summary")
self.reward_summary = tf.summary.scalar(name='reward_summary',
tensor=self.reward_summary_ph)
self.merged_summary_op = tf.summary.merge_all()
self.writer = tf.summary.FileWriter('./graphs', tf.get_default_graph())
self.test = tf.get_default_graph().get_tensor_by_name(
os.path.split(self.value.name)[0] + '/kernel:0')
