def __graph__():
"""Build the inference graph"""
with tf.name_scope('input'):
# [BATCH_SIZE, NUM_FEATURES]
x_input = tf.placeholder(dtype=tf.float32,
shape=[None, self.num_features],
name='x_input')
# [BATCH_SIZE, NUM_CLASSES]
y_input = tf.placeholder(dtype=tf.float32,
shape=[None, self.num_classes],
name='y_input')
learning_rate = tf.placeholder(dtype=tf.float32,
name='learning_rate')
first_hidden_layer = {
'weights':
self.weight_variable('h1_w_layer',
[self.num_features, self.node_size[0]]),
'biases':
self.bias_variable('h1_b_layer', [self.node_size[0]])
}
second_hidden_layer = {
'weights':
self.weight_variable('h2_w_layer',
[self.node_size[0], self.node_size[1]]),
'biases':
self.bias_variable('h2_b_layer', [self.node_size[1]])
}
third_hidden_layer = {
'weights':
self.weight_variable('h3_w_layer',
[self.node_size[1], self.node_size[2]]),
'biases':
self.bias_variable('h3_b_layer', [self.node_size[2]])
}
last_hidden_layer = {
'weights':
self.weight_variable('output_w_layer',
[self.node_size[2], self.num_classes]),
'biases':
self.bias_variable('output_b_layer', [self.num_classes])
}
first_layer = tf.matmul(
x_input,
first_hidden_layer['weights']) + first_hidden_layer['biases']
first_layer = tf.nn.leaky_relu(first_layer)
second_layer = tf.matmul(
first_layer,
second_hidden_layer['weights']) + second_hidden_layer['biases']
second_layer = tf.nn.leaky_relu(second_layer)
third_layer = tf.matmul(
second_layer,
third_hidden_layer['weights']) + third_hidden_layer['biases']
third_layer = tf.nn.leaky_relu(third_layer)
output_layer = tf.matmul(
third_layer,
last_hidden_layer['weights']) + last_hidden_layer['biases']
tf.summary.histogram('pre-activations', output_layer)
with tf.name_scope('loss'):
loss = svm_loss(labels=y_input,
logits=output_layer,
num_classes=self.num_classes,
penalty_parameter=self.penalty_parameter,
weight=last_hidden_layer['weights'])
tf.summary.scalar('loss', loss)
optimizer_op = tf.train.AdamOptimizer(
learning_rate=learning_rate).minimize(loss)
with tf.name_scope('accuracy'):
predicted_class = tf.sign(output_layer)
predicted_class = tf.identity(predicted_class,
name='predicted_class')
with tf.name_scope('correct_prediction'):
correct_prediction = tf.equal(
tf.argmax(predicted_class, 1), tf.argmax(y_input, 1))
with tf.name_scope('accuracy'):
accuracy = tf.reduce_mean(
tf.cast(correct_prediction, 'float'))
tf.summary.scalar('accuracy', accuracy)
merged = tf.summary.merge_all()
self.x_input = x_input
self.y_input = y_input
self.learning_rate = learning_rate
self.loss = loss
self.optimizer_op = optimizer_op
self.predicted_class = predicted_class
self.accuracy = accuracy
self.merged = merged
def __graph__():
"""Build the inference graph"""
with tf.name_scope("input"):
# [BATCH_SIZE, SEQUENCE_WIDTH, SEQUENCE_HEIGHT]
x_input = tf.placeholder(
dtype=tf.float32,
shape=[None, self.sequence_width, sequence_height],
name="x_input",
)
# [BATCH_SIZE, NUM_CLASSES]
y_input = tf.placeholder(
dtype=tf.float32, shape=[None, self.num_classes], name="y_input"
)
state = tf.placeholder(
dtype=tf.float32,
shape=[None, self.cell_size * self.num_layers],
name="initial_state",
)
p_keep = tf.placeholder(dtype=tf.float32, name="p_keep")
learning_rate = tf.placeholder(dtype=tf.float32, name="learning_rate")
cells = [
tf.contrib.rnn.GRUCell(self.cell_size) for _ in range(self.num_layers)
]
drop_cells = [
tf.contrib.rnn.DropoutWrapper(cell, input_keep_prob=p_keep)
for cell in cells
]
multi_cell = tf.contrib.rnn.MultiRNNCell(drop_cells, state_is_tuple=False)
multi_cell = tf.contrib.rnn.DropoutWrapper(
multi_cell, output_keep_prob=p_keep
)
# outputs: [BATCH_SIZE, SEQUENCE_LENGTH, CELL_SIZE]
# states: [BATCH_SIZE, CELL_SIZE]
outputs, states = tf.nn.dynamic_rnn(
multi_cell, x_input, initial_state=state, dtype=tf.float32
)
states = tf.identity(states, name="H")
with tf.name_scope("final_training_ops"):
with tf.name_scope("weights"):
xav_init = tf.contrib.layers.xavier_initializer
weight = tf.get_variable(
name="weights",
shape=[self.cell_size, self.num_classes],
initializer=xav_init(),
)
self.variable_summaries(weight)
with tf.name_scope("biases"):
bias = tf.get_variable(
name="biases",
initializer=tf.constant(0.1, shape=[self.num_classes]),
)
self.variable_summaries(bias)
hf = tf.transpose(outputs, [1, 0, 2])
last = tf.gather(hf, int(hf.get_shape()[0]) - 1)
with tf.name_scope("Wx_plus_b"):
output = tf.matmul(last, weight) + bias
tf.summary.histogram("pre-activations", output)
# L2-SVM
with tf.name_scope("loss"):
loss = svm_loss(
labels=y_input,
logits=output,
num_classes=num_classes,
penalty_parameter=self.svm_c,
weight=weight,
)
tf.summary.scalar("loss", loss)
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(
loss
)
with tf.name_scope("accuracy"):
predicted_class = tf.sign(output)
predicted_class = tf.identity(predicted_class, name="predicted_class")
with tf.name_scope("correct_prediction"):
correct = tf.equal(
tf.argmax(predicted_class, 1), tf.argmax(y_input, 1)
)
with tf.name_scope("accuracy"):
accuracy = tf.reduce_mean(tf.cast(correct, "float"))
tf.summary.scalar("accuracy", accuracy)
# merge all the summaries collected from the TF graph
merged = tf.summary.merge_all()
# set class properties
self.x_input = x_input
self.y_input = y_input
self.p_keep = p_keep
self.loss = loss
self.optimizer = optimizer
self.state = state
self.states = states
self.learning_rate = learning_rate
self.predicted_class = predicted_class
self.accuracy = accuracy
self.merged = merged
def __graph__():
"""Build the inference graph"""
with tf.name_scope("input"):
# [BATCH_SIZE, NUM_FEATURES]
x_input = tf.placeholder(dtype=tf.float32,
shape=[None, self.num_features],
name="x_input")
# [BATCH_SIZE, NUM_CLASSES]
y_input = tf.placeholder(dtype=tf.float32,
shape=[None, self.num_classes],
name="y_input")
learning_rate = tf.placeholder(dtype=tf.float32,
name="learning_rate")
first_hidden_layer = {
"weights":
self.weight_variable("h1_w_layer",
[self.num_features, self.node_size[0]]),
"biases":
self.bias_variable("h1_b_layer", [self.node_size[0]]),
}
second_hidden_layer = {
"weights":
self.weight_variable("h2_w_layer",
[self.node_size[0], self.node_size[1]]),
"biases":
self.bias_variable("h2_b_layer", [self.node_size[1]]),
}
third_hidden_layer = {
"weights":
self.weight_variable("h3_w_layer",
[self.node_size[1], self.node_size[2]]),
"biases":
self.bias_variable("h3_b_layer", [self.node_size[2]]),
}
last_hidden_layer = {
"weights":
self.weight_variable("output_w_layer",
[self.node_size[2], self.num_classes]),
"biases":
self.bias_variable("output_b_layer", [self.num_classes]),
}
first_layer = (tf.matmul(x_input, first_hidden_layer["weights"]) +
first_hidden_layer["biases"])
first_layer = tf.nn.leaky_relu(first_layer)
second_layer = (
tf.matmul(first_layer, second_hidden_layer["weights"]) +
second_hidden_layer["biases"])
second_layer = tf.nn.leaky_relu(second_layer)
third_layer = (
tf.matmul(second_layer, third_hidden_layer["weights"]) +
third_hidden_layer["biases"])
third_layer = tf.nn.leaky_relu(third_layer)
output_layer = (
tf.matmul(third_layer, last_hidden_layer["weights"]) +
last_hidden_layer["biases"])
tf.summary.histogram("pre-activations", output_layer)
with tf.name_scope("loss"):
loss = svm_loss(
labels=y_input,
logits=output_layer,
num_classes=self.num_classes,
penalty_parameter=self.penalty_parameter,
weight=last_hidden_layer["weights"],
)
tf.summary.scalar("loss", loss)
optimizer_op = tf.train.AdamOptimizer(
learning_rate=learning_rate).minimize(loss)
with tf.name_scope("accuracy"):
predicted_class = tf.sign(output_layer)
predicted_class = tf.identity(predicted_class,
name="predicted_class")
with tf.name_scope("correct_prediction"):
correct_prediction = tf.equal(
tf.argmax(predicted_class, 1), tf.argmax(y_input, 1))
with tf.name_scope("accuracy"):
accuracy = tf.reduce_mean(
tf.cast(correct_prediction, "float"))
tf.summary.scalar("accuracy", accuracy)
merged = tf.summary.merge_all()
self.x_input = x_input
self.y_input = y_input
self.learning_rate = learning_rate
self.loss = loss
self.optimizer_op = optimizer_op
self.predicted_class = predicted_class
self.accuracy = accuracy
self.merged = merged
def __graph__():
with tf.name_scope('input'):
x_input = tf.placeholder(
dtype=tf.float32,
shape=[None, sequence_width, sequence_height],
name='x_input')
y_input = tf.placeholder(dtype=tf.float32,
shape=[None, num_classes],
name='y_input')
# state = tf.placeholder(dtype=tf.float32, shape=[None, self.cell_size * self.num_layers],
# name='initial_state')
p_keep = tf.placeholder(dtype=tf.float32, name='p_keep')
learning_rate = tf.placeholder(dtype=tf.float32,
name='learning_rate')
hidden_size = int(sequence_width)
# seq_len = tf.Variable(tf.constant(hidden_size),name='seq_len')
rnn_outputs, _ = bi_rnn(GRUCell(hidden_size),
GRUCell(hidden_size),
inputs=x_input,
sequence_length=None,
dtype=tf.float32)
tf.summary.histogram('RNN_outputs', rnn_outputs)
# Attention layer
with tf.name_scope('Attention_layer'):
attention_output, alphas = attention(
input=rnn_outputs,
hidden_size=self.sequence_width,
attention_size=ATTENTION_SIZE,
return_alpha=True)
tf.summary.histogram('alphas', alphas)
# dropout
drop = tf.nn.dropout(attention_output, keep_prob=p_keep)
# fully connected layer
with tf.name_scope('Fully_connected_layer'):
W = tf.Variable(tf.truncated_normal(
[hidden_size * 2, self.num_classes], stddev=0.1),
name='W')
b = tf.Variable(tf.constant(0.0, shape=[self.num_classes]),
name='b')
y_hat = tf.nn.xw_plus_b(drop, W, b)
# y_hat=tf.squeeze(y_hat)
tf.summary.histogram('W', W)
with tf.name_scope('loss'):
loss = svm_loss(labels=y_input,
logits=y_hat,
num_classes=self.num_classes,
penalty_parameter=self.svm_c,
weight=W)
tf.summary.scalar('loss', loss)
optimizer = tf.train.AdamOptimizer(
learning_rate=learning_rate).minimize(loss=loss)
with tf.name_scope('accuracy'):
predicted_class = tf.sign(y_hat)
predicted_class = tf.identity(predicted_class,
name='predicted_class')
with tf.name_scope('correct_prediction'):
correct = tf.equal(tf.argmax(predicted_class, 1),
tf.argmax(y_input, 1))
with tf.name_scope('accuracy'):
accuracy = tf.reduce_mean(tf.cast(correct, 'float'))
tf.summary.scalar('accuracy', accuracy)
merged = tf.summary.merge_all()
# set class properties
self.x_input = x_input
self.y_input = y_input
self.p_keep = p_keep
self.loss = loss
self.optimizer = optimizer
# self.state=state
# self.states=states
self.learning_rate = learning_rate
self.predicted_class = predicted_class
self.accuracy = accuracy
self.merged = merged
def __graph__():
# [batch_size,sequence_length]
x_input = tf.placeholder(dtype=tf.float32,
shape=[None, num_features],
name="x_input")
# [batch_size,num_classes]
y_input = tf.placeholder(dtype=tf.float32,
shape=[None, num_classes],
name="y_input")
# first convolutional layer
first_conv_weight = self.weight_variable([5, 5, 1, 36])
first_conv_bias = self.bias_variable([36])
input_image = tf.reshape(x_input, [-1, 32, 32, 1])
first_conv_activation = tf.nn.leaky_relu(
self.conv2d(input_image, first_conv_weight) + first_conv_bias)
first_conv_pool = self.max_pool_2x2(first_conv_activation)
# second convolutional layer
second_conv_weight = self.weight_variable([5, 5, 36, 72])
second_conv_bias = self.bias_variable([72])
second_conv_activation = tf.nn.leaky_relu(
self.conv2d(first_conv_pool, second_conv_weight) +
second_conv_bias)
second_conv_pool = self.max_pool_2x2(second_conv_activation)
# fully-connected layer
dense_layer_weight = self.weight_variable([8 * 8 * 72, 1024])
dense_layer_bias = self.bias_variable([1024])
second_conv_pool_flatten = tf.reshape(second_conv_pool,
[-1, 8 * 8 * 72])
dense_layer_activation = tf.nn.leaky_relu(
tf.matmul(second_conv_pool_flatten,
dense_layer_weight + dense_layer_bias))
# Dropout ,to avoid overfitting
keep_prob = tf.placeholder(tf.float32, name="p_keep")
h_fc1_drop = tf.nn.dropout(dense_layer_activation,
keep_prob=keep_prob)
# readout layer
readout_weight = self.weight_variable([1024, num_classes])
readout_bias = self.bias_variable([num_classes])
logits = tf.matmul(h_fc1_drop, readout_weight) + readout_bias
with tf.name_scope('loss'):
loss = svm_loss(labels=y_input,
logits=logits,
num_classes=num_classes,
penalty_parameter=penalty_parameter,
weight=readout_weight)
tf.summary.scalar('loss', loss)
train_op = tf.train.AdamOptimizer(
learning_rate=alpha).minimize(loss)
with tf.name_scope('metrics'):
predicted_class = tf.sign(logits)
predicted_class = tf.identity(predicted_class,
name='predicted_class')
with tf.name_scope('correct_prediction'):
correct_prediction = tf.equal(
tf.argmax(predicted_class, 1), tf.argmax(y_input, 1))
with tf.name_scope('accuracy'):
accuracy = tf.reduce_mean(
tf.cast(correct_prediction, 'float'))
tf.summary.scalar('accuracy', accuracy)
merged = tf.summary.merge_all()
self.x_input = x_input
self.y_input = y_input
self.keep_prob = keep_prob
self.logits = logits
self.loss = loss
self.train_op = train_op
self.predicted_class = predicted_class
self.accuracy = accuracy
self.merged = merged