def create_conv_layer(input_data, input_channels_count, filters_count,
filter_shape, pool_shape, name):
conv_shape = [
filter_shape[0], filter_shape[1], input_channels_count, filters_count
]
weights = tf.Variable(tf.truncated_normal(conv_shape, stddev=0.03),
name=name + '_weights')
bias = tf.Variable(tf.truncated_normal([filters_count]),
name=name + '_bias')
out_layer = tf.nn.conv2d(input_data, weights, [1, 1, 1, 1], padding='SAME')
out_layer += bias
out_layer = tf.nn.relu(out_layer)
if pool_shape is None:
return out_layer
ksize = [1, pool_shape[0], pool_shape[1], 1]
strides = [1, 2, 2, 1]
out_layer = tf.nn.max_pool(out_layer,
ksize=ksize,
strides=strides,
padding='SAME')
return out_layer
def __init__(self):
self.init_lrate = 0.001
self.nHidden = 256
self.seqLen = 75
self.batchSize = 32
self.maxClipping = 5
self.pKeep = 0.8
self.maxEpoch = 8
self.displayStep = 100
self.former = 0.0
self.patience = 5
self.maxPatience = 5
self.maxTF1_validF1 = 0
self.globalStep = tf.Variable(0, trainable=False)
self.maxF1 = self.pre = self.rec = self.acc = self.maxTF1 = self.maxTpre = self.maxTrec = self.maxTacc = 0
self.texti = loadData.TextIterator(self.batchSize, self.seqLen)
self.validStep = int(self.texti.threshold / 3)
self.lrate = tf.train.exponential_decay(
self.init_lrate,
global_step=self.globalStep,
decay_steps=self.texti.threshold,
decay_rate=0.8)
self.addGlobal = self.globalStep.assign_add(1)
self.net = NNManager.Net(self.nHidden, self.seqLen)
self.loss, self.PredAcc, self.p = self.net.predicting(self.texti.rate)
updates = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
tVars = tf.trainable_variables()
gradients, _ = tf.clip_by_global_norm(tf.gradients(self.loss, tVars),
self.maxClipping)
optimizer = tf.train.AdamOptimizer(learning_rate=self.lrate)
self.trainOP = optimizer.apply_gradients(zip(gradients, tVars))
self.fileNameOutput = []
self.start = time.time()
def __conv(self,
input,
filter_width,
filter_height,
filters_count,
stride_x,
stride_y,
padding='VALID',
init_biases_with_the_constant_1=False,
name='conv'):
with tf.name_scope(name):
input_channels = input.get_shape().as_list()[-1]
filters = tf.Variable(self.__random_values(shape=[
filter_height, filter_width, input_channels, filters_count
]),
name='filters')
convs = tf.nn.conv2d(input=input,
filter=filters,
strides=[1, stride_y, stride_x, 1],
padding=padding,
name='convs')
if init_biases_with_the_constant_1:
biases = tf.Variable(tf.ones(shape=[filters_count],
dtype=tf.float32),
name='biases')
else:
biases = tf.Variable(tf.zeros(shape=[filters_count],
dtype=tf.float32),
name='biases')
preactivations = tf.nn.bias_add(convs,
biases,
name='preactivations')
activations = tf.nn.relu(preactivations, name='activations')
with tf.name_scope('filter_summaries'):
self.__variable_summaries(filters)
with tf.name_scope('bias_summaries'):
self.__variable_summaries(biases)
with tf.name_scope('preactivations_histogram'):
tf.summary.histogram('preactivations', preactivations)
with tf.name_scope('activations_histogram'):
tf.summary.histogram('activations', activations)
return activations
def __fully_connected(self,
input,
inputs_count,
outputs_count,
relu=True,
init_biases_with_the_constant_1=False,
name='fully_connected'):
with tf.name_scope(name):
wights = tf.Variable(
self.__random_values(shape=[inputs_count, outputs_count]),
name='wights')
if init_biases_with_the_constant_1:
biases = tf.Variable(tf.ones(shape=[outputs_count],
dtype=tf.float32),
name='biases')
else:
biases = tf.Variable(tf.zeros(shape=[outputs_count],
dtype=tf.float32),
name='biases')
preactivations = tf.nn.bias_add(tf.matmul(input, wights),
biases,
name='preactivations')
if relu:
activations = tf.nn.relu(preactivations, name='activations')
with tf.name_scope('wight_summaries'):
self.__variable_summaries(wights)
with tf.name_scope('bias_summaries'):
self.__variable_summaries(biases)
with tf.name_scope('preactivations_histogram'):
tf.summary.histogram('preactivations', preactivations)
if relu:
with tf.name_scope('activations_histogram'):
tf.summary.histogram('activations', activations)
if relu:
return activations
else:
return preactivations
def choose_action(self, state, epsilon=1.):
# 计算q值. epsilon为随机度. 为1时, 则为全随机选择action
M_s = np.zeros([self.n_action, self.fai_s_size])
for i in range(self.n_action):
M_s[i] = self.sess.run(self.eval_M[i],
feed_dict={self.state, state})
w = self.sess.graph.get_tensor_by_name('eval_net/R_s/r/w:0')
# fai = self.sess.graph.get_tensor_by_name('eval_net/f_theta/fai_s:0')
q_ = tf.matmul(w, tf.Variable(M_s))
if np.random.uniform(0, 1) < epsilon:
action = np.random.randint(self.n_action)
else:
action = np.argmax(q_)
return action
def train(x_train, y_train):
n_samples, n_features = x_train.shape
w = tf.Variable(np.random.rand(input_dim, 1).astype(dtype='float32'),
name="weight")
b = tf.Variable(0.0, dtype=tf.float32, name="bias")
x = tf.placeholder(dtype=tf.float32, name='x')
y = tf.placeholder(dtype=tf.float32, name='y')
predictions = tf.matmul(x, w) + b
loss = tf.reduce_mean(
tf.log(1 + tf.exp(tf.multiply(-1.0 * y, predictions))))
# optimizer = tf.train.GradientDescentOptimizer(learn_rate).minimize(loss)
optimizer = tf.train.ProximalGradientDescentOptimizer(
learning_rate=learn_rate,
l1_regularization_strength=0.1).minimize(loss)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for epoch in range(n_epochs):
for idx in range(0, n_samples, batch_size):
iE = min(n_samples, idx + batch_size)
x_batch = x_train[idx:iE, :]
y_batch = y_train[idx:iE, :]
sess.run([optimizer], feed_dict={x: x_batch, y: y_batch})
curr_w, curr_b = sess.run([w, b])
for idx in range(len(curr_w)):
if curr_w[idx] < threshold * -1:
curr_w[idx] += threshold
else:
curr_w[idx] -= threshold
sess.run([tf.assign(w, curr_w)])
return curr_w, curr_b
def __init__(self):
self.embedding = self.getEmb()
self.embSize = self.embedding.shape[1]
self.vocabSize = self.embedding.shape[0]
self.x = tf.placeholder(tf.int32, [None, 5])
with tf.variable_scope("training_variable"):
self.weights = {
"MLP1":
tf.Variable(
tf.truncated_normal(
shape=[self.embSize,
int(self.embSize / 2)],
stddev=0.08)),
"MLP2":
tf.Variable(
tf.truncated_normal(shape=[int(self.embSize / 2), 1],
stddev=0.08))
}
self.biases = {
"MLP1":
tf.Variable(
tf.constant(0.01,
shape=[int(self.embSize / 2)],
dtype=tf.float32)),
"MLP2":
tf.Variable(tf.constant(0.01, shape=[1], dtype=tf.float32))
}
self.inputEmb = tf.nn.embedding_lookup(self.embedding, self.x)
p1 = tf.matmul(tf.reshape(self.inputEmb, [-1, self.embSize]),
self.weights["MLP1"]) + self.biases["MLP1"]
p1 = tf.matmul(tf.nn.relu(p1),
self.weights["MLP2"]) + self.biases["MLP2"]
p1 = tf.reshape(p1, [-1, 5])
p1 = tf.reshape(tf.nn.softmax(p1), [-1, 1, 5])
self.finalState = tf.reshape(tf.matmul(p1, self.inputEmb),
[-1, self.embSize])
if os.path.exists("./tmp/beginner-export"):
shutil.rmtree("./tmp/beginner-export")
# Import data
# from tensorflow.examples.tutorials.mnist import input_data
import input_data
mnist = input_data.read_data_sets("tmp/data/", one_hot=True)
g = tf.Graph()
with g.as_default():
# Create the model
# x = tf.placeholder("float", [None, 784])
x = tf.compat.v1.placeholder("float", [None, 784])
W = tf.Variable(tf.zeros([784, 10]), name="vaiable_W")
b = tf.Variable(tf.zeros([10]), name="variable_b")
y = tf.nn.softmax(tf.matmul(x, W) + b)
# Define loss and optimizer
y_ = tf.compat.v1.placeholder("float", [None, 10])
cross_entropy = -tf.reduce_sum(y_ * tf.log(y))
train_step = tf.train.GradientDescentOptimizer(0.01).minimize(
cross_entropy)
sess = tf.Session()
# Train
init = tf.initialize_all_variables()
sess.run(init)
def gain(data_x, gain_parameters):
'''Impute missing values in data_x
Args:
- data_x: original data with missing values
- gain_parameters: GAIN network parameters:
- batch_size: Batch size
- hint_rate: Hint rate
- alpha: Hyperparameter
- iterations: Iterations
Returns:
- imputed_data: imputed data
'''
# Define mask matrix
data_m = 1 - np.isnan(data_x)
# System parameters
batch_size = gain_parameters['batch_size']
hint_rate = gain_parameters['hint_rate']
alpha = gain_parameters['alpha']
iterations = gain_parameters['iterations']
# Other parameters
no, dim = data_x.shape
# Hidden state dimensions
h_dim = int(dim)
# Normalization
norm_data, norm_parameters = normalization(data_x)
norm_data_x = np.nan_to_num(norm_data, 0)
## GAIN architecture
# Input placeholders
# Data vector
tf.disable_v2_behavior()
X = tf.placeholder(tf.float32, shape=[None, dim])
# Mask vector
M = tf.placeholder(tf.float32, shape=[None, dim])
# Hint vector
H = tf.placeholder(tf.float32, shape=[None, dim])
# Discriminator variables
D_W1 = tf.Variable(xavier_init([dim * 2, h_dim])) # Data + Hint as inputs
D_b1 = tf.Variable(tf.zeros(shape=[h_dim]))
D_W2 = tf.Variable(xavier_init([h_dim, h_dim]))
D_b2 = tf.Variable(tf.zeros(shape=[h_dim]))
D_W3 = tf.Variable(xavier_init([h_dim, dim]))
D_b3 = tf.Variable(tf.zeros(shape=[dim])) # Multi-variate outputs
theta_D = [D_W1, D_W2, D_W3, D_b1, D_b2, D_b3]
#Generator variables
# Data + Mask as inputs (Random noise is in missing components)
G_W1 = tf.Variable(xavier_init([dim * 2, h_dim]))
G_b1 = tf.Variable(tf.zeros(shape=[h_dim]))
G_W2 = tf.Variable(xavier_init([h_dim, h_dim]))
G_b2 = tf.Variable(tf.zeros(shape=[h_dim]))
G_W3 = tf.Variable(xavier_init([h_dim, dim]))
G_b3 = tf.Variable(tf.zeros(shape=[dim]))
theta_G = [G_W1, G_W2, G_W3, G_b1, G_b2, G_b3]
## GAIN functions
# Generator
def generator(x, m):
# Concatenate Mask and Data
inputs = tf.concat(values=[x, m], axis=1)
G_h1 = tf.nn.relu(tf.matmul(inputs, G_W1) + G_b1)
G_h2 = tf.nn.relu(tf.matmul(G_h1, G_W2) + G_b2)
# MinMax normalized output
G_prob = tf.nn.sigmoid(tf.matmul(G_h2, G_W3) + G_b3)
return G_prob
# Discriminator
def discriminator(x, h):
# Concatenate Data and Hint
inputs = tf.concat(values=[x, h], axis=1)
D_h1 = tf.nn.relu(tf.matmul(inputs, D_W1) + D_b1)
D_h2 = tf.nn.relu(tf.matmul(D_h1, D_W2) + D_b2)
D_logit = tf.matmul(D_h2, D_W3) + D_b3
D_prob = tf.nn.sigmoid(D_logit)
return D_prob
## GAIN structure
# Generator
G_sample = generator(X, M)
# Combine with observed data
Hat_X = X * M + G_sample * (1 - M)
# Discriminator
D_prob = discriminator(Hat_X, H)
## GAIN loss
D_loss_temp = -tf.reduce_mean(M * tf.log(D_prob + 1e-8) \
+ (1-M) * tf.log(1. - D_prob + 1e-8))
G_loss_temp = -tf.reduce_mean((1 - M) * tf.log(D_prob + 1e-8))
MSE_loss = \
tf.reduce_mean((M * X - M * G_sample)**2) / tf.reduce_mean(M)
D_loss = D_loss_temp
G_loss = G_loss_temp + alpha * MSE_loss
## GAIN solver
D_solver = tf.train.AdamOptimizer().minimize(D_loss, var_list=theta_D)
G_solver = tf.train.AdamOptimizer().minimize(G_loss, var_list=theta_G)
## Iterations
sess = tf.Session()
sess.run(tf.global_variables_initializer())
# Start Iterations
for it in tqdm(range(iterations)):
# Sample batch
batch_idx = sample_batch_index(no, batch_size)
X_mb = norm_data_x[batch_idx, :]
M_mb = data_m[batch_idx, :]
# Sample random vectors
Z_mb = uniform_sampler(0, 0.01, batch_size, dim)
# Sample hint vectors
H_mb_temp = binary_sampler(hint_rate, batch_size, dim)
H_mb = M_mb * H_mb_temp
# Combine random vectors with observed vectors
X_mb = M_mb * X_mb + (1 - M_mb) * Z_mb
_, D_loss_curr = sess.run([D_solver, D_loss_temp],
feed_dict={
M: M_mb,
X: X_mb,
H: H_mb
})
_, G_loss_curr, MSE_loss_curr = \
sess.run([G_solver, G_loss_temp, MSE_loss],
feed_dict = {X: X_mb, M: M_mb, H: H_mb})
## Return imputed data
Z_mb = uniform_sampler(0, 0.01, no, dim)
M_mb = data_m
X_mb = norm_data_x
X_mb = M_mb * X_mb + (1 - M_mb) * Z_mb
imputed_data = sess.run([G_sample], feed_dict={X: X_mb, M: M_mb})[0]
imputed_data = data_m * norm_data_x + (1 - data_m) * imputed_data
# Renormalization
imputed_data = renormalization(imputed_data, norm_parameters)
# Rounding
imputed_data = rounding(imputed_data, data_x)
return imputed_data
def __init__(self, nHidden, seqLen):
self.representation_score = {}
self.y = tf.placeholder(tf.float32, shape=[None, 1])
self.extractFeature = ExtractFeature.ExtractFeature()
self.imageFeature = ImageFeature.ImageFeature()
newNet = tf.reduce_mean(self.imageFeature.outputLS, axis=0)
self.textFeature = TextFeature.TextFeature(
nHidden, seqLen, self.extractFeature.finalState, newNet)
self.l2_para = 1e-7
with tf.variable_scope("training_variable"):
self.weights = {
"MLP1":
tf.Variable(
tf.truncated_normal(shape=[512, 256],
stddev=0.08,
name="MLP1_W")),
"MLP2":
tf.Variable(
tf.truncated_normal(shape=[256, 1],
stddev=0.08,
name="MLP2_W")),
"ATT_attr1_1":
tf.Variable(
tf.truncated_normal(shape=[
self.imageFeature.defaultFeatureSize +
self.extractFeature.embSize,
int(self.imageFeature.defaultFeatureSize / 2 +
self.extractFeature.embSize / 2)
],
stddev=0.08,
name="ATT_attr1_1")),
"ATT_attr1_2":
tf.Variable(
tf.truncated_normal(shape=[
self.textFeature.nHidden * 2 +
self.extractFeature.embSize,
int(self.textFeature.nHidden +
self.extractFeature.embSize / 2)
],
stddev=0.08,
name="ATT_attr1_2")),
"ATT_attr1_3":
tf.Variable(
tf.truncated_normal(shape=[
2 * self.extractFeature.embSize,
self.extractFeature.embSize
],
stddev=0.08,
name="ATT_attr1_3")),
"ATT_attr2_1":
tf.Variable(
tf.truncated_normal(shape=[
int(self.imageFeature.defaultFeatureSize / 2 +
self.extractFeature.embSize / 2), 1
],
stddev=0.08,
name="ATT_attr2_1")),
"ATT_attr2_2":
tf.Variable(
tf.truncated_normal(shape=[
int(self.textFeature.nHidden +
self.extractFeature.embSize / 2), 1
],
stddev=0.08,
name="ATT_attr2_2")),
"ATT_attr2_3":
tf.Variable(
tf.truncated_normal(shape=[self.extractFeature.embSize, 1],
stddev=0.08,
name="ATT_attr2_3")),
"ATT_img1_1":
tf.Variable(
tf.truncated_normal(shape=[
self.imageFeature.defaultFeatureSize +
self.textFeature.nHidden * 2,
int(self.imageFeature.defaultFeatureSize / 2 +
self.textFeature.nHidden)
],
stddev=0.08,
name="ATT_image1_1")),
"ATT_img1_2":
tf.Variable(
tf.truncated_normal(shape=[
self.imageFeature.defaultFeatureSize +
self.extractFeature.embSize,
int(self.imageFeature.defaultFeatureSize / 2 +
self.extractFeature.embSize / 2)
],
stddev=0.08,
name="ATT_image1_2")),
"ATT_img1_3":
tf.Variable(
tf.truncated_normal(shape=[
self.imageFeature.defaultFeatureSize * 2,
self.imageFeature.defaultFeatureSize
],
stddev=0.08,
name="ATT_image1_3")),
"ATT_img2_1":
tf.Variable(
tf.truncated_normal(shape=[
int(self.imageFeature.defaultFeatureSize / 2 +
self.textFeature.nHidden), 1
],
stddev=0.08,
name="ATT_image2_1")),
"ATT_img2_2":
tf.Variable(
tf.truncated_normal(shape=[
int(self.imageFeature.defaultFeatureSize / 2 +
self.extractFeature.embSize / 2), 1
],
stddev=0.08,
name="ATT_image2_2")),
"ATT_img2_3":
tf.Variable(
tf.truncated_normal(
shape=[self.imageFeature.defaultFeatureSize, 1],
stddev=0.08,
name="ATT_image2_3")),
"ATT_text1_1":
tf.Variable(
tf.truncated_normal(shape=[
self.imageFeature.defaultFeatureSize +
self.textFeature.nHidden * 2,
int(self.imageFeature.defaultFeatureSize / 2 +
self.textFeature.nHidden)
],
stddev=0.08,
name="ATT_text1_1")),
"ATT_text1_2":
tf.Variable(
tf.truncated_normal(shape=[
self.textFeature.nHidden * 2 +
self.extractFeature.embSize,
int(self.textFeature.nHidden +
self.extractFeature.embSize / 2)
],
stddev=0.08,
name="ATT_text1_2")),
"ATT_text1_3":
tf.Variable(
tf.truncated_normal(shape=[
self.textFeature.nHidden * 4,
self.textFeature.nHidden * 2
],
stddev=0.08,
name="ATT_text1_3")),
"ATT_text2_1":
tf.Variable(
tf.truncated_normal(shape=[
int(self.imageFeature.defaultFeatureSize / 2 +
self.textFeature.nHidden), 1
],
stddev=0.08,
name="ATT_text2_1")),
"ATT_text2_2":
tf.Variable(
tf.truncated_normal(shape=[
int(self.textFeature.nHidden +
self.extractFeature.embSize / 2), 1
],
stddev=0.08,
name="ATT_text2_2")),
"ATT_text2_3":
tf.Variable(
tf.truncated_normal(
shape=[self.textFeature.nHidden * 2, 1],
stddev=0.08,
name="ATT_text2_3")),
"ATT_WI1":
tf.Variable(
tf.truncated_normal(
shape=[self.imageFeature.defaultFeatureSize, 512],
stddev=0.08,
name="ATT_WI")),
"ATT_WT1":
tf.Variable(
tf.truncated_normal(shape=[2 * nHidden, 512],
stddev=0.08,
name="ATT_WT")),
"ATT_WA1":
tf.Variable(
tf.truncated_normal(shape=[200, 512],
stddev=0.08,
name="ATT_WA")),
"ATT_WI2":
tf.Variable(
tf.truncated_normal(
shape=[self.imageFeature.defaultFeatureSize, 512],
stddev=0.08,
name="ATT_WI2")),
"ATT_WT2":
tf.Variable(
tf.truncated_normal(shape=[2 * nHidden, 512],
stddev=0.08,
name="ATT_WT2")),
"ATT_WA2":
tf.Variable(
tf.truncated_normal(shape=[200, 512],
stddev=0.08,
name="ATT_WA2")),
"ATT_WF_1":
tf.Variable(
tf.truncated_normal(shape=[512, 1],
stddev=0.08,
name="ATT_WF_1")),
"ATT_WF_2":
tf.Variable(
tf.truncated_normal(shape=[512, 1],
stddev=0.08,
name="ATT_WF_2")),
"ATT_WF_3":
tf.Variable(
tf.truncated_normal(shape=[512, 1],
stddev=0.08,
name="ATT_WF_3")),
}
self.biases = {
"MLP1":
tf.Variable(
tf.constant(0.01,
shape=[256],
dtype=tf.float32,
name="MLP1_b")),
"MLP2":
tf.Variable(
tf.constant(0.01,
shape=[1],
dtype=tf.float32,
name="MLP2_b")),
"ATT_attr1_1":
tf.Variable(
tf.constant(
0.01,
shape=[
int(self.imageFeature.defaultFeatureSize / 2 +
self.extractFeature.embSize / 2)
],
name="ATT_attr1_1")),
"ATT_attr1_2":
tf.Variable(
tf.constant(0.01,
shape=[
int(self.textFeature.nHidden +
self.extractFeature.embSize / 2)
],
name="ATT_attr1_2")),
"ATT_attr1_3":
tf.Variable(
tf.constant(0.01,
shape=[self.extractFeature.embSize],
name="ATT_attr1_3")),
"ATT_attr2_1":
tf.Variable(tf.constant(0.01, shape=[1], name="ATT_attr2_1")),
"ATT_attr2_2":
tf.Variable(tf.constant(0.01, shape=[1], name="ATT_attr2_2")),
"ATT_attr2_3":
tf.Variable(tf.constant(0.01, shape=[1], name="ATT_attr2_3")),
"ATT_img1_1":
tf.Variable(
tf.constant(
0.01,
shape=[
int(self.imageFeature.defaultFeatureSize / 2 +
self.textFeature.nHidden)
],
name="ATT_image1_1")),
"ATT_img1_2":
tf.Variable(
tf.constant(
0.01,
shape=[
int(self.imageFeature.defaultFeatureSize / 2 +
self.extractFeature.embSize / 2)
],
name="ATT_image1_2")),
"ATT_img1_3":
tf.Variable(
tf.constant(0.01,
shape=[self.imageFeature.defaultFeatureSize],
name="ATT_image1_3")),
"ATT_img2_1":
tf.Variable(tf.constant(0.01, shape=[1], name="ATT_image2_1")),
"ATT_img2_2":
tf.Variable(tf.constant(0.01, shape=[1], name="ATT_image2_2")),
"ATT_img2_3":
tf.Variable(tf.constant(0.01, shape=[1], name="ATT_image2_3")),
"ATT_text1_1":
tf.Variable(
tf.constant(
0.01,
shape=[
int(self.imageFeature.defaultFeatureSize / 2 +
self.textFeature.nHidden)
],
name="ATT_text1_1")),
"ATT_text1_2":
tf.Variable(
tf.constant(0.01,
shape=[
int(self.textFeature.nHidden +
self.extractFeature.embSize / 2)
],
name="ATT_text1_2")),
"ATT_text1_3":
tf.Variable(
tf.constant(0.01,
shape=[self.textFeature.nHidden * 2],
name="ATT_text1_3")),
"ATT_text2_1":
tf.Variable(tf.constant(0.01, shape=[1], name="ATT_text2_1")),
"ATT_text2_2":
tf.Variable(tf.constant(0.01, shape=[1], name="ATT_text2_2")),
"ATT_text2_3":
tf.Variable(tf.constant(0.01, shape=[1], name="ATT_text2_3")),
"ATT_WW":
tf.Variable(tf.constant(0.01, shape=[512], name="ATT_WW")),
"ATT_WI":
tf.Variable(tf.constant(0.01, shape=[512], name="ATT_WI")),
"ATT_WT":
tf.Variable(tf.constant(0.01, shape=[512], name="ATT_WT")),
"ATT_WI1":
tf.Variable(tf.constant(0.01, shape=[512], name="ATT_WI1")),
"ATT_WT1":
tf.Variable(tf.constant(0.01, shape=[512], name="ATT_WT1")),
"ATT_WA":
tf.Variable(tf.constant(0.01, shape=[512], name="ATT_WA")),
"ATT_WF_1":
tf.Variable(tf.constant(0.01, shape=[1], name="ATT_WF_1")),
"ATT_WF_2":
tf.Variable(tf.constant(0.01, shape=[1], name="ATT_WF_2")),
"ATT_WF_3":
tf.Variable(tf.constant(0.01, shape=[1], name="ATT_WF_3")),
}
print("newnet dimension :", newNet)
imageVec = self.Attention(newNet, self.imageFeature.outputLS,
self.textFeature.RNNState,
self.extractFeature.finalState, "ATT_img1",
"ATT_img2", 196, True)
textVec = self.Attention(self.textFeature.RNNState,
self.textFeature.outputs, newNet,
self.extractFeature.finalState, "ATT_text1",
"ATT_text2", self.textFeature.seqLen, False)
attrVec = self.Attention(self.extractFeature.finalState,
self.extractFeature.inputEmb, newNet,
self.textFeature.RNNState, "ATT_attr1",
"ATT_attr2", 5, False)
attHidden = tf.tanh(
tf.matmul(imageVec, self.weights["ATT_WI1"]) +
self.biases["ATT_WI1"])
attHidden2 = tf.tanh(
tf.matmul(textVec, self.weights["ATT_WT1"]) +
self.biases["ATT_WT1"])
attHidden3 = tf.tanh(
tf.matmul(attrVec, self.weights["ATT_WA1"]) +
self.biases["ATT_WW"])
scores1 = tf.matmul(attHidden,
self.weights["ATT_WF_1"]) + self.biases["ATT_WF_1"]
scores2 = tf.matmul(attHidden2,
self.weights["ATT_WF_2"]) + self.biases["ATT_WF_2"]
scores3 = tf.matmul(attHidden3,
self.weights["ATT_WF_3"]) + self.biases["ATT_WF_3"]
scoreLS = [scores1, scores2, scores3]
scoreLS = tf.nn.softmax(scoreLS, dim=0)
imageVec = tf.tanh(
tf.matmul(imageVec, self.weights["ATT_WI2"]) +
self.biases["ATT_WI"])
textVec = tf.tanh(
tf.matmul(textVec, self.weights["ATT_WT2"]) +
self.biases["ATT_WT"])
attrVec = tf.tanh(
tf.matmul(attrVec, self.weights["ATT_WA2"]) +
self.biases["ATT_WA"])
self.concatInput = scoreLS[0] * imageVec + scoreLS[
1] * textVec + scoreLS[2] * attrVec
def weight_variable(shape):
initial = tf.truncated_normal(shape, stddev=0.1)
return tf.Variable(initial)
"c3_filter": [3, 3, 256, 384],
"c4_filter": [3, 3, 192, 384],
"c5_filter": [3, 3, 192, 256]
}
# Fully connected shapes
fc_connection_shapes = {
"f1_shape": [13 * 13 * 256, 4096],
"f2_shape": [4096, 4096],
"f3_shape": [4096, dataset_dict["num_labels"]]
}
# Weights for each layer
conv_weights = {
"c1_weights":
tf.Variable(tf.truncated_normal(conv_filter_shapes["c1_filter"]),
name="c1_weights"),
"c2_weights":
tf.Variable(tf.truncated_normal(conv_filter_shapes["c2_filter"]),
name="c2_weights"),
"c3_weights":
tf.Variable(tf.truncated_normal(conv_filter_shapes["c3_filter"]),
name="c3_weights"),
"c4_weights":
tf.Variable(tf.truncated_normal(conv_filter_shapes["c4_filter"]),
name="c4_weights"),
"c5_weights":
tf.Variable(tf.truncated_normal(conv_filter_shapes["c5_filter"]),
name="c5_weights"),
"f1_weights":
tf.Variable(tf.truncated_normal(fc_connection_shapes["f1_shape"]),
name="f1_weights"),
def main(trainModel=True,
buildConfusionMatrix=True,
restore=False,
buildClassifiedMatrix=True):
tf.disable_v2_behavior()
input_images = tf.placeholder(tf.float32, [None, 28, 28], name="Input")
real = tf.placeholder(tf.float32, [None, CLASSES], name="real_classes")
layer1 = create_conv_layer(tf.reshape(input_images, [-1, 28, 28, 1]),
1,
28, [5, 5], [2, 2],
name="conv_no_pool")
layer2 = create_conv_layer(layer1,
28,
56, [5, 5], [2, 2],
name='conv_with_pool')
conv_result = tf.reshape(layer2, [-1, 7 * 7 * 56])
relu_layer_weight = tf.Variable(tf.truncated_normal([7 * 7 * 56, 1000],
stddev=STDDEV * 2),
name='relu_layer_weight')
rely_layer_bias = tf.Variable(tf.truncated_normal([1000],
stddev=STDDEV / 2),
name='rely_layer_bias')
relu_layer = tf.matmul(conv_result, relu_layer_weight) + rely_layer_bias
relu_layer = tf.nn.relu(relu_layer)
relu_layer = tf.nn.dropout(relu_layer, DROPOUT)
final_layer_weight = tf.Variable(tf.truncated_normal([1000, CLASSES],
stddev=STDDEV * 2),
name='final_layer_weight')
final_layer_bias = tf.Variable(tf.truncated_normal([CLASSES],
stddev=STDDEV / 2),
name='final_layer_bias')
final_layer = tf.matmul(relu_layer, final_layer_weight) + final_layer_bias
predicts = tf.nn.softmax(final_layer)
predicts_for_log = tf.clip_by_value(predicts, 1e-9, 0.999999999)
#crossEntropy = -tf.reduce_mean(tf.reduce_sum(y * tf.log(y_clipped) + (1 - y) * tf.log(1 - y_clipped), axis=1))
loss = -tf.reduce_mean(
tf.reduce_sum(real * tf.log(predicts_for_log) +
(1 - real) * tf.log(1 - predicts_for_log),
axis=1),
axis=0)
#test = tf.reduce_sum(real * tf.log(predicts_for_log) + (1 - real) * tf.log(1 - predicts_for_log), axis=1)
#loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=final_layer, labels=real))
optimiser = tf.train.GradientDescentOptimizer(
learning_rate=LEARNING_RATE).minimize(loss)
correct_prediction = tf.equal(tf.argmax(real, axis=1),
tf.argmax(predicts, axis=1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
confusion_matrix = tf.confusion_matrix(labels=tf.argmax(real, axis=1),
predictions=tf.argmax(predicts,
axis=1),
num_classes=CLASSES)
saver = tf.train.Saver()
# dataset = get_mnist_dataset()
dataset = get_fashion_dataset()
with tf.Session() as session:
session.run(tf.global_variables_initializer())
if restore:
saver.restore(session, SAVE_PATH)
if trainModel:
train(input_images, real, session, optimiser, loss, accuracy,
saver, dataset)
if buildConfusionMatrix:
test_cm = session.run(confusion_matrix,
feed_dict={
input_images: dataset.test_x,
real: dataset.test_y
})
draw_confusion_matrix(test_cm)
if buildClassifiedMatrix:
all_probs = session.run(predicts,
feed_dict={
input_images: dataset.test_x,
real: dataset.test_y
})
max_failure_picture_index = [[(-1, -1.0)] * CLASSES
for _ in range(CLASSES)]
for i in range(len(all_probs)):
real = np.argmax(dataset.test_y[i])
for j in range(CLASSES):
if max_failure_picture_index[real][j][1] < all_probs[i][j]:
max_failure_picture_index[real][j] = (i,
all_probs[i][j])
draw_max_failure_pictures(dataset.test_x,
max_failure_picture_index)
def __init__(self, nHidden, seqLen, guidence, newNet):
self.nHidden = nHidden
self.seqLen = seqLen
tmp = self.getEmbedding()
self.embedding = tf.Variable(tmp)
with tf.variable_scope("training_variable"):
self.weights = {
"ATT":
tf.Variable(
tf.truncated_normal(shape=[2 * self.nHidden, self.nHidden],
stddev=0.08,
name="text_att")),
"ATTG":
tf.Variable(
tf.truncated_normal(shape=[200, self.nHidden],
stddev=0.08,
name="text_att2")),
"ATTS":
tf.Variable(
tf.truncated_normal(shape=[self.nHidden, 1],
stddev=0.08,
name="text_att3")),
"Fw1":
tf.Variable(
tf.truncated_normal(shape=[200, self.nHidden],
stddev=0.08,
name="init_fw1")),
"Fw2":
tf.Variable(
tf.truncated_normal(shape=[200, self.nHidden],
stddev=0.08,
name="init_fw2")),
"Bw1":
tf.Variable(
tf.truncated_normal(shape=[200, self.nHidden],
stddev=0.08,
name="init_bw1")),
"Bw2":
tf.Variable(
tf.truncated_normal(shape=[200, self.nHidden],
stddev=0.08,
name="init_bw2")),
}
self.biases = {
"Fw1":
tf.Variable(
tf.constant(0.01, shape=[self.nHidden], name="init_Fw1")),
"Fw2":
tf.Variable(
tf.constant(0.01, shape=[self.nHidden], name="init_Fw2")),
"Bw1":
tf.Variable(
tf.constant(0.01, shape=[self.nHidden], name="init_Bw1")),
"Bw2":
tf.Variable(
tf.constant(0.01, shape=[self.nHidden], name="init_Bw2")),
}
self.X = tf.placeholder(tf.int32, [None, self.seqLen])
self.pKeep = tf.placeholder(tf.float32)
self.build(guidence, newNet)
import tensorflow._api.v2.compat.v1 as tf
import numpy as np
tf.reset_default_graph()
tf.compat.v1.disable_eager_execution()
tf.compat.v1.disable_v2_behavior()
tf.global_variables_initializer()
x = tf.placeholder(tf.float32, shape=[None, 4])
y = tf.placeholder(tf.float32, shape=[None, 1])
w = tf.Variable(tf.random_normal([4, 1]), name="weight")
b = tf.Variable(tf.random_normal([1]), name="bias")
hypo = tf.matmul(x, w) + b
saver = tf.train.Saver()
test_arr = [[12, 6.5, 15.7, 10.8]]
sess2 = tf.Session()
saver.restore(sess2, "./saved.ckpt")
predict = sess2.run(hypo, feed_dict={x: test_arr})
print(predict[0])
def bias_variable(shape):
initial = tf.constant(0.1, shape=shape)
return tf.Variable(initial)
# Create a simple TF Graph # By Omid Alemi - Jan 2017 # Works with TF r1.0 # import tensorflow as tf import tensorflow._api.v2.compat.v1 as tf tf.disable_v2_behavior() I = tf.placeholder(tf.float32, shape=[None,3], name='I') # input W = tf.Variable(tf.zeros(shape=[3,2]), dtype=tf.float32, name='W') # weights b = tf.Variable(tf.zeros(shape=[2]), dtype=tf.float32, name='b') # biases O = tf.nn.relu(tf.matmul(I, W) + b, name='O') # activation / output saver = tf.train.Saver() init_op = tf.global_variables_initializer() with tf.Session() as sess: sess.run(init_op) # save the graph tf.train.write_graph(sess.graph_def, '.', 'tfdroid.pbtxt') # normally you would do some training here # but fornow we will just assign something to W sess.run(tf.assign(W, [[1, 2],[4,5],[7,8]])) sess.run(tf.assign(b, [1,1])) #save a checkpoint file, which will store the above assignment saver.save(sess, 'tfdroid.ckpt')