def __init__(self):
self.graph = tf.Graph()
with self.graph.as_default():
self.y = placeholder(tf.float32, [None, n_y])
self.z = placeholder(tf.float32, [None, n_z])
self.keep_prob = placeholder(tf.float32, [])
self.Wy1 = tf.get_variable('Wy1', [n_z, 512], tf.float32, glorot_normal_initializer())
self.by1 = tf.get_variable('by1', [512], tf.float32, zeros_initializer())
self.Wy2 = tf.get_variable('Wy2', [512, 512], tf.float32, glorot_normal_initializer())
self.by2 = tf.get_variable('by2', [512], tf.float32, zeros_initializer())
self.Wy3 = tf.get_variable('Wy3', [512, n_y], tf.float32, glorot_normal_initializer())
self.by3 = tf.get_variable('by3', [n_y], tf.float32, zeros_initializer())
z = dropout(self.z, self.keep_prob)
h = tf.nn.relu(tf.matmul(z, self.Wy1) + self.by1)
h = dropout(h, self.keep_prob)
h = tf.nn.relu(tf.matmul(h, self.Wy2) + self.by2)
h = dropout(h, self.keep_prob)
self.pred = tf.sigmoid(tf.matmul(h, self.Wy3) + self.by3)
cost = -self.y * tf.log(self.pred + 1e-6) - (1. - self.y) * tf.log(1. - self.pred + 1e-6)
self.cost = tf.reduce_mean(tf.reduce_sum(cost, 1))
self.pred_mask = tf.cast(self.pred >= 0.5, tf.int32)
self.tmp = tf.cast(self.y, tf.int32)
self.acc_mask = tf.cast(tf.equal(self.tmp, self.pred_mask), tf.float32)
self.acc = tf.reduce_mean(self.acc_mask)
def cnn_graph(x, keep_prob, size, captcha_list = CAPTCHA_LIST, captcha_len = CAPTCHA_LEN):
x_image = reshape(x, shape = [-1, size[0], size[1], 1])
w_conv1 = weight_variable([3, 3, 1, 32])
b_conv1 = bias_variable([32])
h_conv1 = relu(conv2d(x_image, w_conv1) + b_conv1)
h_pool1 = max_pool2d(h_conv1)
h_drop1 = dropout(h_pool1, rate = 1 - keep_prob)
w_conv2 = weight_variable([3, 3, 32, 64])
b_conv2 = bias_variable([64])
h_conv2 = relu(conv2d(h_drop1, w_conv2) + b_conv2)
h_pool2 = max_pool2d(h_conv2)
h_drop2 = dropout(h_pool2, rate = 1 - keep_prob)
w_conv3 = weight_variable([3, 3, 64, 64])
b_conv3 = bias_variable([64])
h_conv3 = relu(conv2d(h_drop2, w_conv3) + b_conv3)
h_pool3 = max_pool2d(h_conv3)
h_drop3 = dropout(h_pool3, rate = 1 - keep_prob)
image_height = int(h_drop3.shape[1])
image_width = int(h_drop3.shape[2])
w_fc = weight_variable([image_height * image_width * 64, 1024])
b_fc = bias_variable([1024])
h_drop3_re = reshape(h_drop3, [-1, image_height * image_width * 64])
h_fc = relu(matmul(h_drop3_re, w_fc) + b_fc)
h_drop_fc = dropout(h_fc, rate = 1 - keep_prob)
w_out = weight_variable([1024, len(captcha_list) * captcha_len])
b_out = bias_variable([len(captcha_list) * captcha_len])
y_conv = matmul(h_drop_fc, w_out) + b_out
return y_conv
def build(self,
lstm_unit=256,
hidden_unit=16,
output_unit=1,
learning_rate=0.001,
encoder='lstm'):
word_embA = self.embedding_layer(
self.input_sentA) # (batch_size, num_step, emb_dim)
word_embB = self.embedding_layer(
self.input_sentB) # (batch_size, num_step, emb_dim)
if encoder == 'lstm':
repA = self.lstm(word_embA, self.input_seq_lenA, lstm_unit,
None) # (batch_size, lstm_unit)
repB = self.lstm(word_embB, self.input_seq_lenB, lstm_unit,
True) # (batch_size, lstm_unit)
input_dim = lstm_unit * 2
elif encoder == 'bilstm':
repA = self.bilstm(word_embA, self.input_seq_lenA, lstm_unit,
None) # (batch_size, num_step, lstm_unit * 2)
repB = self.bilstm(word_embA, self.input_seq_lenB, lstm_unit,
True) # (batch_size, num_step, lstm_unit * 2)
repA = tf.reduce_sum(repA, axis=1) # (batch_size, lstm_unit * 2)
repB = tf.reduce_sum(repB, axis=1) # (batch_size, lstm_unit * 2)
input_dim = lstm_unit * 4
rep = tf.concat(
[repA, repB], axis=1
) # lstm: (batch_size, lstm_unit * 2), bilstm: (batch_size, lstm_unit * 4)
rep = dropout(rep, keep_prob=0.5)
hidden = self.dense(rep, hidden_unit,
'hidden') # (batch_size, hidden_unit)
hidden = dropout(hidden, keep_prob=0.5)
self.output = self.dense(hidden, output_unit,
'output') # (batch_size, output_unit)
self.output = tf.reshape(self.output, (-1, ))
self.loss = self.loss_function(self.output) # ()
self.optimizer = tf.train.AdamOptimizer(
learning_rate=learning_rate).minimize(self.loss)
def add_layer_dropput(input, in_size, out_size,keep_prob = None,activation_function=None):
Weights = Variable(random_normal([out_size, in_size]))
Biases = Variable(zeros([out_size, 1]) + 0.1)
Wx_plus_b = matmul(Weights, input) + Biases
Wx_plus_b = nn.dropout(Wx_plus_b,keep_prob= keep_prob)
if activation_function == None:
output = Wx_plus_b
else:
output = activation_function(transpose(Wx_plus_b))
output = transpose(output)
return output
def forward_pass(self, input_data):
input_data = input_data / 255
pre_inception = InceptionNet._pre_inception_layer(
input_data, self.pre_inception_filters)
inception_1 = InceptionNet._inception_layer(pre_inception,
self.inception_1_filters)
inception_2 = InceptionNet._inception_layer(inception_1,
self.inception_2_filters)
# print(inception_2)
post_inception_pool = InceptionNet._max_pool(inception_2, (3, 3),
(2, 2))
inception_3 = InceptionNet._inception_layer(post_inception_pool,
self.inception_3_filters)
inception_4 = InceptionNet._inception_layer(inception_3,
self.inception_4_filters)
inception_5 = InceptionNet._inception_layer(inception_4,
self.inception_5_filters)
inception_6 = InceptionNet._inception_layer(inception_5,
self.inception_6_filters)
inception_7 = InceptionNet._inception_layer(inception_6,
self.inception_7_filters)
post_inception_pool_2 = InceptionNet._max_pool(inception_7, (3, 3),
(2, 2))
inception_8 = InceptionNet._inception_layer(post_inception_pool_2,
self.inception_8_filters)
inception_9 = InceptionNet._inception_layer(inception_8,
self.inception_9_filters)
post_inception_pool_3 = nn.avg_pool(inception_9, (7, 7),
strides=4,
padding="SAME")
flatten_layer = tf.reshape(
tf.keras.backend.flatten(post_inception_pool_3),
[1024, input_data.shape[0]])
relu_layer = nn.relu(
tf.matmul(tf.transpose(self.relu_weights), flatten_layer) +
self.relu_bias)
dropout_layer = nn.dropout(relu_layer, .4)
linear_layer = tf.matmul(tf.transpose(self.linear_weights),
dropout_layer) + self.linear_bias
return nn.softmax(tf.transpose(nn.tanh(linear_layer)))
def _construct_model(self):
n_steps = tf.shape(self.words)[0]
n_x, n_h, n_v = self.options['n_x'], self.options['n_h'], self.options[
'n_v']
batch_size = tf.shape(self.y)[0]
words_embed = embedding_lookup(self.embeddings, self.words)
# n_steps, batch_size, embed_dim
words_embed = dropout(words_embed, self.keep_prob,
(1, batch_size, n_x))
# convert video feature from None * n_z to None * n_x
# vid_feat_proj shape: (1, batch_size, embed_dim)
vid_feat_proj = expand_dims(matmul(self.z, self.c0), 0)
# use video feature as the input for the first step
# state_below shape: (n_steps, batch_size, embed_dim)
state_below = concat([vid_feat_proj, words_embed[:-1]], 0)
# h_list shape: (n_steps, batch_size, h_dims)
self.h_list = self._decoder_layer(state_below)
self.h_list_reshape = reshape(self.h_list, [-1, n_h])
# logits shape: (n_steps*batch_size, n_vocabulary)
self.logits = matmul(self.h_list_reshape, self.output_w) + self.bhid
logits_reshaped = reshape(self.logits, [-1, batch_size, n_v])
self.sents = tf.argmax(logits_reshaped, -1)
# w_reshape shape: (n_steps, batch_size)
weighted_mask = self.mask / (tf.reduce_sum(self.mask, 0, keepdims=True)
**0.7)
self.loss = sparse_softmax_cross_entropy(
logits=logits_reshaped + 1e-8,
labels=self.words,
weights=weighted_mask,
reduction=tf.losses.Reduction.SUM)
self.train_loss = self.loss / tf.cast(batch_size, tf.float32)
test_h_list = self._test_layer(matmul(self.z, self.c0))
test_h_list = reshape(test_h_list, (-1, n_h))
test_logits = matmul(test_h_list, self.output_w) + self.bhid
test_logits = reshape(test_logits, (-1, batch_size, n_v))
self.test_sents = tf.argmax(test_logits, -1)
def build_preprocess(input, embedding, training):
cell_input = nn.embedding_lookup(embedding, input)
if training:
cell_input = nn.dropout(cell_input, DROPOUT)
return cell_input
def train(self, input, h, ffd_drop, hid_units, adj, number):
# self.input = tf.placeholder(dtype=tf.float64, shape=(self.batch_size, 3025, 1870))
# self.h = tf.placeholder(dtype=tf.float64, shape=(self.batch_size, 64))
embed_list = []
"""g"""
input = tf.expand_dims(input, 0)
# print(input.shape, 'fdfdfdfdfdfd')
# exit()
if ffd_drop != 0.0:
seq = tf.nn.dropout(input, 1.0 - ffd_drop)
input = tf.layers.conv1d(seq, hid_units, 1, use_bias=False)
# print(input.shape, 'kokokoko')
# print(h.shape)
# print(ffd_drop)
# print(hid_units)
# print(adj)
# exit()
# adj = adj.astype(np.float32)
# input1 = tf.matmul(adj, input) + input
# print(input_1.shape)
# exit()
print(number)
print(self.batch_size)
for i in range(self.batch_size):
one_adj = adj[0][i + number*self.batch_size].reshape(adj[0].shape[0], 1)
input1 = one_adj * input
# embed_list.append(input1)
# input = tf.concat(embed_list, axis=1)
# print(multi_embed.shape, 'lplplplpllp')
# print(input1.shape)
# exit()
# input1 = tf.reshape(input1, [55, 55, 8])
# print(input1.shape)
# exit()
# input1 = tf.tile(input=input1, multiples=[3025, 1, 1])
# # self.input = input
# # self.h = h
# input1 = tf.nn.conv2d(input1, self.w3, strides=[1, 3, 3, 1], padding='SAME')
# input1 = tf.nn.conv2d(input1, self.w4, strides=[1, 3, 3, 1], padding='SAME')
# input1 = tf.nn.conv2d(input1, self.w5, strides=[1, 3, 3, 1], padding='SAME')
# input1 = tf.nn.conv2d(input1, self.w6, strides=[1, 3, 3, 1], padding='SAME')
# # print(input.shape, 'ffff')
# # exit()
# # input1 = tf.squeeze(input1)
# # print(input1.shape)
input1 = tf.reshape(input1, [1, -1, 8])
# print(input1.shape)
embed_list.append(input1)
input = tf.concat(embed_list, axis=0)
print(input.shape)
input1 = tf.expand_dims(input, 0)
#
input1 = tf.nn.conv2d(input1, self.w, strides=[1, 1, 3, 1], padding='SAME')
input1 = tf.nn.conv2d(input1, self.w1, strides=[1, 1, 3, 1], padding='SAME')
input1 = tf.nn.conv2d(input1, self.w2, strides=[1, 1, 3, 1], padding='SAME')
input1 = tf.nn.conv2d(input1, self.w3, strides=[1, 1, 3, 1], padding='SAME')
# print(input1.shape)
# exit()
input = tf.squeeze(input1)
h = h[number*self.batch_size : self.batch_size + number*self.batch_size]
# print(input.shape, 'ffff')
# exit()
# input = tf.reshape(input, [3025, -1, 8])
# input = tf.expand_dims(input, 0)
# input = tf.tile(input=input, multiples=[self.batch_size, 1, 1])
mb = input.shape[0]
# print(mb, 'ssssss')
n_channels = input.shape[2]
dd = input.shape[1]
# print(input.shape, 'jijijjj')
# exit()
# print(mb, n_channels, dd, "opopopopooopopoop")
# print(input.shape,'fffffff')
x_flat = tf.concat([input, self.coord_tensor], 2)
# print(h, 'qqqqq')
# h = tf.squeeze(h)
# self.h = tf.expand_dims(self.h, 0)
print(h.shape, 'jojojoj')
qst = tf.expand_dims(h, 1)
qst = tf.tile(input=qst, multiples=[1, 38, 1])
qst = tf.expand_dims(qst, 2)
x_i = tf.expand_dims(x_flat, 1) # (64x1x25x26+5)
x_i = tf.tile(input=x_i, multiples=[1, 38, 1, 1]) # (64x25x25x26+5)
x_j = tf.expand_dims(x_flat, 2) # (64x25x1x26+5)
x_j = tf.concat([x_j, qst], 3)
x_j = tf.tile(input=x_j, multiples=[1, 1, 38, 1]) # (64x25x25x26+5)
# concatenate all together
x_full = tf.concat([x_i, x_j], 3) # (64x25x25x2*26+5)
# print(x_f)
# print(x_full.shape, 'uiuiuiuiu')
# exit()
# reshape for passing through network
x_ = tf.reshape(x_full, [mb*dd*dd, 84])
# print(x_.shape, 'kokokkoko')
# exit()
x_ = tf.layers.dense(x_, 128)
x_ = nn.relu(x_)
x_ = tf.layers.dense(x_, 128)
x_ = nn.relu(x_)
x_ = tf.layers.dense(x_, 128)
x_ = nn.relu(x_)
x_ = tf.layers.dense(x_, 128)
x_ = nn.relu(x_)
# reshape again and sum
# print(x_.shape, 'jijijijijiji')
# exit()
# print(mb, 'fdfdfdfdfffd')
# print(dd*dd, 'jijijijijj')
x_g = tf.reshape(x_, [mb, dd*dd, 128])
# print(x_g.shape, 'llolooollololo')
# exit()
# print(x_g.shape, 'jiijijjji')
x_g = tf.reduce_sum(x_g, 1)
# print(x_g.shape, 'kokokokok')
# exit()
# print(x_g.shape, 'huuhuhuhu')
x_g = tf.squeeze(x_g)
# print(x_g.shape, 'huuhuhuhu')
# exit()
"""f"""
x_f = tf.layers.dense(x_g, 128)
x_f = nn.relu(x_f)
x = tf.layers.dense(x_f, 128)
x = nn.relu(x)
x = nn.dropout(x, keep_prob=0.5)
x = tf.layers.dense(x, 64)
return x
def __init__(self, **kwargs):
'''The following arguments are accepted:
Parameters
----------
vocab_size : int
Size of the vocabulary for creating embeddings
embedding_matrix : int
Dimensionality of the embedding space
memory_size : int
LSTM memory size
keep_prob : float
Inverse of dropout percentage for embedding and LSTM
subsequence_length : int
Length of the subsequences (all embeddings are padded to this
length)
optimizer : OptimizerSpec
'''
############################################################################################
# Get all hyperparameters #
############################################################################################
vocab_size = kwargs['vocab_size']
embedding_size = kwargs['embedding_size']
memory_size = kwargs['memory_size']
keep_prob = kwargs['keep_prob']
subsequence_length = kwargs['subsequence_length']
optimizer_spec = kwargs['optimizer']
optimizer = optimizer_spec.create()
self.learning_rate = optimizer_spec.learning_rate
self.step_counter = optimizer_spec.step_counter
############################################################################################
# Net inputs #
############################################################################################
self.batch_size = placeholder(tf.int32, shape=[], name='batch_size')
self.is_training = placeholder(tf.bool, shape=[], name='is_training')
self.word_ids = placeholder(tf.int32,
shape=(None, subsequence_length),
name='word_ids')
self.labels = placeholder(tf.int32, shape=(None, ), name='labels')
self.hidden_state = placeholder(tf.float32,
shape=(None, memory_size),
name='hidden_state')
self.cell_state = placeholder(tf.float32,
shape=(None, memory_size),
name='cell_state')
lengths = sequence_lengths(self.word_ids)
############################################################################################
# Embedding #
############################################################################################
self.embedding_matrix, _bias = get_weights_and_bias(
(vocab_size, embedding_size))
embeddings = cond(
self.is_training, lambda: nn.dropout(nn.embedding_lookup(
self.embedding_matrix, self.word_ids),
keep_prob=keep_prob),
lambda: nn.embedding_lookup(self.embedding_matrix, self.word_ids))
############################################################################################
# LSTM layer #
############################################################################################
cell = BasicLSTMCell(memory_size, activation=tf.nn.tanh)
# during inference, use entire ensemble
keep_prob = cond(self.is_training, lambda: constant(keep_prob),
lambda: constant(1.0))
cell = DropoutWrapper(cell, output_keep_prob=keep_prob)
# what's the difference to just creating a zero-filled tensor tuple?
self.zero_state = cell.zero_state(self.batch_size, tf.float32)
state = LSTMStateTuple(h=self.cell_state, c=self.hidden_state)
# A dynamic rnn creates the graph on the fly, so it can deal with embeddings of different
# lengths. We do not need to unstack the embedding tensor to get rows, instead we compute
# the actual sequence lengths and pass that
# We are not sure how any of this works. Do we need to mask the cost function so the cell
# outputs for _NOT_A_WORD_ inputs are ignored? Is the final cell state really relevant if it
# was last updated with _NOT_A_WORD_ input? Does static_rnn absolve us of any of those
# issues?
outputs, self.state = nn.dynamic_rnn(cell,
embeddings,
sequence_length=lengths,
initial_state=state)
# Recreate tensor from list
outputs = reshape(concat(outputs, 1),
[-1, subsequence_length * memory_size])
self.outputs = reduce_mean(outputs)
############################################################################################
# Fully connected layer, loss, and training #
############################################################################################
ff1 = fully_connected(outputs, 2, with_activation=False, use_bias=True)
loss = reduce_mean(
nn.sparse_softmax_cross_entropy_with_logits(labels=self.labels,
logits=ff1))
self.train_step = optimizer.minimize(loss,
global_step=self.step_counter)
self.predictions = nn.softmax(ff1)
correct_prediction = equal(cast(argmax(self.predictions, 1), tf.int32),
self.labels)
self.accuracy = reduce_mean(cast(correct_prediction, tf.float32))
############################################################################################
# Create summaraies #
############################################################################################
with tf.variable_scope('summary'):
self.summary_loss = tf.summary.scalar('loss', loss)
self.summary_accuracy = tf.summary.scalar('accuracy',
self.accuracy)
def _decoder_layer(self, state_below):
"""state below: size of (n_steps, batch_size, n_x)
"""
n_x = self.options['n_x']
n_h = self.options['n_h']
n_f = self.options['n_f']
# batch_size * n_f
y = dropout(self.y, self.keep_prob)
tmp2_i = matmul(y, self.Wb_i)
tmp2_f = matmul(y, self.Wb_f)
tmp2_o = matmul(y, self.Wb_o)
tmp2_c = matmul(y, self.Wb_c)
# batch_size * n_f
z = dropout(self.z, self.keep_prob)
tmp3_i = matmul(z, self.Ca_i)
tmp3_f = matmul(z, self.Ca_f)
tmp3_o = matmul(z, self.Ca_o)
tmp3_c = matmul(z, self.Ca_c)
# batch_size * n_f
tmp4_i = matmul(y, self.Cb_i)
tmp4_f = matmul(y, self.Cb_f)
tmp4_o = matmul(y, self.Cb_o)
tmp4_c = matmul(y, self.Cb_c)
def _state_below(tmp1, tmp2, tmp3, tmp4, Wc, Cc, b):
# print('tmp1:', tmp1, 'tmp2:', tmp2, 'tmp3:', tmp3, 'tmp4:', tmp4)
state_b = matmul(tmp1 * tmp2, Wc) + matmul(tmp3 * tmp4, Cc) + b
return state_b
def _step(a, b):
print('in decoder layer', a, b)
word_embed1, step = b[0], b[1]
def _preactivate(a, y, w1, w2, w3, x):
p = matmul(matmul(a, w1) * matmul(y, w2), w3) + x
return p
def _get_word_embed(h):
word_logit = matmul(h, self.output_w) + self.bhid
word_chosen1 = tf.argmax(word_logit, 1)
word_chosen2 = tf.multinomial(word_logit, 1)
word_chosen2 = tf.squeeze(word_chosen2)
word_chosen = tf.cond(self.if_argmax, lambda: word_chosen1,
lambda: word_chosen2)
return embedding_lookup(self.embeddings, word_chosen)
word_embed = tf.cond((tf.random_uniform([]) >= self.sample_prob)
| tf.equal(step, 0), lambda: word_embed1,
lambda: _get_word_embed(a[0]))
# batch_size = tf.shape(word_embed)[0]
word_embed = tf.reshape(word_embed, (-1, n_x))
# batch_size * n_f
tmp1_i = matmul(word_embed, self.Wa_i)
tmp1_f = matmul(word_embed, self.Wa_f)
tmp1_o = matmul(word_embed, self.Wa_o)
tmp1_c = matmul(word_embed, self.Wa_c)
# batch_size * n_h
input_i = _state_below(tmp1_i, tmp2_i, tmp3_i, tmp4_i, self.Wc_i,
self.Cc_i, self.b_i)
input_f = _state_below(tmp1_f, tmp2_f, tmp3_f, tmp4_f, self.Wc_f,
self.Cc_f, self.b_f)
input_o = _state_below(tmp1_o, tmp2_o, tmp3_o, tmp4_o, self.Wc_o,
self.Cc_o, self.b_o)
input_c = _state_below(tmp1_c, tmp2_c, tmp3_c, tmp4_c, self.Wc_c,
self.Cc_c, self.b_c)
# batch_size * n_h
preact_i = _preactivate(a[0], y, self.Ua_i, self.Ub_i, self.Uc_i,
input_i)
preact_f = _preactivate(a[0], y, self.Ua_f, self.Ub_f, self.Uc_f,
input_f)
preact_o = _preactivate(a[0], y, self.Ua_o, self.Ub_o, self.Uc_o,
input_o)
preact_c = _preactivate(a[0], y, self.Ua_c, self.Ub_c, self.Uc_c,
input_c)
i = tf.sigmoid(preact_i)
f = tf.sigmoid(preact_f)
o = tf.sigmoid(preact_o)
c = tf.tanh(preact_c)
c = f * a[1] + i * c
h = o * tf.tanh(c)
return h, c
# self.mask: shape n_steps * batch_size
# state_below: shape n_steps * batch_size * n_x
n_steps = tf.shape(state_below)[0]
steps = tf.range(n_steps, dtype=tf.int32)
elems = [state_below, steps]
batch_size = tf.shape(state_below)[1]
init_t = (tf.zeros([batch_size, n_h]), tf.zeros([batch_size, n_h]))
h_list, c_list = tf.scan(_step, elems, init_t)
h_list = dropout(h_list, self.keep_prob, [1, batch_size, n_h])
return h_list
def train_layers(self, train_x, train_y, test_x, test_y):
params = {}
X = tf.placeholder(tf.float32, [None, self.opt['n_dim']])
Y = tf.placeholder(tf.float32, [None, self.opt['n_classes']])
keep_prob = tf.placeholder(tf.float32) #for dropout
params['W1'] = tf.Variable(
tf.random_normal([self.opt['n_dim'], self.opt['num_hidden1']],
mean=0,
stddev=self.opt['std']))
params['b1'] = tf.Variable(
tf.random_normal([self.opt['num_hidden1']],
mean=0,
stddev=self.opt['std']))
params['a1'] = nn.sigmoid(tf.matmul(X, params['W1']) + params['b1'])
params['dropout1'] = nn.dropout(params['a1'], keep_prob)
params['W2'] = tf.Variable(
tf.random_normal(
[self.opt['num_hidden1'], self.opt['num_hidden2']],
mean=0,
stddev=self.opt['std']))
params['b2'] = tf.Variable(
tf.random_normal([self.opt['num_hidden2']],
mean=0,
stddev=self.opt['std']))
params['a2'] = nn.relu(
tf.matmul(params['dropout1'], params['W2']) + params['b2'])
params['dropout2'] = nn.dropout(params['a2'], keep_prob)
params['W3'] = tf.Variable(
tf.random_normal(
[self.opt['num_hidden2'], self.opt['num_hidden3']],
mean=0,
stddev=self.opt['std']))
params['b3'] = tf.Variable(
tf.random_normal([self.opt['num_hidden3']],
mean=0,
stddev=self.opt['std']))
params['a3'] = nn.tanh(
tf.matmul(params['dropout2'], params['W3']) + params['b3'])
params['dropout3'] = nn.dropout(params['a3'], keep_prob)
params['outW'] = tf.Variable(
tf.random_normal([self.opt['num_hidden3'], self.opt['n_classes']],
mean=0,
stddev=self.opt['std']))
params['outb'] = tf.Variable(
tf.random_normal([self.opt['n_classes']],
mean=0,
stddev=self.opt['std']))
out = nn.softmax(
tf.matmul(params['dropout3'], params['outW']) + params['outb'])
cost = tf.reduce_mean(
-tf.reduce_sum(Y * tf.log(out), reduction_indices=[1]))
optimizer = tf.train.AdamOptimizer(
self.opt['learning_rate']).minimize(cost)
correct_pred = tf.equal(tf.argmax(out, 1), tf.argmax(Y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
cost_history = np.empty(shape=[1], dtype=float)
y, y_pred = None, None
#reshape labels into a one hot vector
f = FeatureParser()
train_y = f.one_hot_encode(train_y)
test_y = f.one_hot_encode(test_y)
print('TRAIN_ONE_HOT_LABEL{}'.format(train_y))
print('Training...')
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for epoch in range(training_epochs):
_, loss, acc = sess.run([optimizer, cost, accuracy],
feed_dict={
X: train_x,
Y: train_y,
keep_prob: 0.5
})
cost_history = np.append(cost_history, loss)
if epoch % 50 == 0:
print('Epoch#', epoch, 'Cost:', loss, 'Train acc.:', acc)
y_pred = sess.run(tf.argmax(out, 1),
feed_dict={
X: test_x,
keep_prob: 1.0
})
y = sess.run(tf.argmax(test_y, 1))
print(
"Test accuracy: ",
round(
sess.run(accuracy,
feed_dict={
X: test_x,
Y: test_y,
keep_prob: 1.0
}), 3))
fig = plt.figure(figsize=(10, 8))
plt.plot(cost_history)
plt.xlabel('Iterations')
plt.ylabel('Cost')
plt.axis([0, training_epochs, 0, np.max(cost_history)])
plt.show()
precision, recall, f_score, s = precision_recall_fscore_support(
y, y_pred, average='micro')
print('F score:', round(f_score, 3))
