def create_conv_layer(input_data, input_channels_count, filters_count, filter_shape, pool_shape, name): conv_shape = [ filter_shape[0], filter_shape[1], input_channels_count, filters_count ] weights = tf.Variable(tf.truncated_normal(conv_shape, stddev=0.03), name=name + '_weights') bias = tf.Variable(tf.truncated_normal([filters_count]), name=name + '_bias') out_layer = tf.nn.conv2d(input_data, weights, [1, 1, 1, 1], padding='SAME') out_layer += bias out_layer = tf.nn.relu(out_layer) if pool_shape is None: return out_layer ksize = [1, pool_shape[0], pool_shape[1], 1] strides = [1, 2, 2, 1] out_layer = tf.nn.max_pool(out_layer, ksize=ksize, strides=strides, padding='SAME') return out_layer
def __init__(self): self.init_lrate = 0.001 self.nHidden = 256 self.seqLen = 75 self.batchSize = 32 self.maxClipping = 5 self.pKeep = 0.8 self.maxEpoch = 8 self.displayStep = 100 self.former = 0.0 self.patience = 5 self.maxPatience = 5 self.maxTF1_validF1 = 0 self.globalStep = tf.Variable(0, trainable=False) self.maxF1 = self.pre = self.rec = self.acc = self.maxTF1 = self.maxTpre = self.maxTrec = self.maxTacc = 0 self.texti = loadData.TextIterator(self.batchSize, self.seqLen) self.validStep = int(self.texti.threshold / 3) self.lrate = tf.train.exponential_decay( self.init_lrate, global_step=self.globalStep, decay_steps=self.texti.threshold, decay_rate=0.8) self.addGlobal = self.globalStep.assign_add(1) self.net = NNManager.Net(self.nHidden, self.seqLen) self.loss, self.PredAcc, self.p = self.net.predicting(self.texti.rate) updates = tf.get_collection(tf.GraphKeys.UPDATE_OPS) tVars = tf.trainable_variables() gradients, _ = tf.clip_by_global_norm(tf.gradients(self.loss, tVars), self.maxClipping) optimizer = tf.train.AdamOptimizer(learning_rate=self.lrate) self.trainOP = optimizer.apply_gradients(zip(gradients, tVars)) self.fileNameOutput = [] self.start = time.time()
def __conv(self, input, filter_width, filter_height, filters_count, stride_x, stride_y, padding='VALID', init_biases_with_the_constant_1=False, name='conv'): with tf.name_scope(name): input_channels = input.get_shape().as_list()[-1] filters = tf.Variable(self.__random_values(shape=[ filter_height, filter_width, input_channels, filters_count ]), name='filters') convs = tf.nn.conv2d(input=input, filter=filters, strides=[1, stride_y, stride_x, 1], padding=padding, name='convs') if init_biases_with_the_constant_1: biases = tf.Variable(tf.ones(shape=[filters_count], dtype=tf.float32), name='biases') else: biases = tf.Variable(tf.zeros(shape=[filters_count], dtype=tf.float32), name='biases') preactivations = tf.nn.bias_add(convs, biases, name='preactivations') activations = tf.nn.relu(preactivations, name='activations') with tf.name_scope('filter_summaries'): self.__variable_summaries(filters) with tf.name_scope('bias_summaries'): self.__variable_summaries(biases) with tf.name_scope('preactivations_histogram'): tf.summary.histogram('preactivations', preactivations) with tf.name_scope('activations_histogram'): tf.summary.histogram('activations', activations) return activations
def __fully_connected(self, input, inputs_count, outputs_count, relu=True, init_biases_with_the_constant_1=False, name='fully_connected'): with tf.name_scope(name): wights = tf.Variable( self.__random_values(shape=[inputs_count, outputs_count]), name='wights') if init_biases_with_the_constant_1: biases = tf.Variable(tf.ones(shape=[outputs_count], dtype=tf.float32), name='biases') else: biases = tf.Variable(tf.zeros(shape=[outputs_count], dtype=tf.float32), name='biases') preactivations = tf.nn.bias_add(tf.matmul(input, wights), biases, name='preactivations') if relu: activations = tf.nn.relu(preactivations, name='activations') with tf.name_scope('wight_summaries'): self.__variable_summaries(wights) with tf.name_scope('bias_summaries'): self.__variable_summaries(biases) with tf.name_scope('preactivations_histogram'): tf.summary.histogram('preactivations', preactivations) if relu: with tf.name_scope('activations_histogram'): tf.summary.histogram('activations', activations) if relu: return activations else: return preactivations
def choose_action(self, state, epsilon=1.): # 计算q值. epsilon为随机度. 为1时, 则为全随机选择action M_s = np.zeros([self.n_action, self.fai_s_size]) for i in range(self.n_action): M_s[i] = self.sess.run(self.eval_M[i], feed_dict={self.state, state}) w = self.sess.graph.get_tensor_by_name('eval_net/R_s/r/w:0') # fai = self.sess.graph.get_tensor_by_name('eval_net/f_theta/fai_s:0') q_ = tf.matmul(w, tf.Variable(M_s)) if np.random.uniform(0, 1) < epsilon: action = np.random.randint(self.n_action) else: action = np.argmax(q_) return action
def train(x_train, y_train): n_samples, n_features = x_train.shape w = tf.Variable(np.random.rand(input_dim, 1).astype(dtype='float32'), name="weight") b = tf.Variable(0.0, dtype=tf.float32, name="bias") x = tf.placeholder(dtype=tf.float32, name='x') y = tf.placeholder(dtype=tf.float32, name='y') predictions = tf.matmul(x, w) + b loss = tf.reduce_mean( tf.log(1 + tf.exp(tf.multiply(-1.0 * y, predictions)))) # optimizer = tf.train.GradientDescentOptimizer(learn_rate).minimize(loss) optimizer = tf.train.ProximalGradientDescentOptimizer( learning_rate=learn_rate, l1_regularization_strength=0.1).minimize(loss) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) for epoch in range(n_epochs): for idx in range(0, n_samples, batch_size): iE = min(n_samples, idx + batch_size) x_batch = x_train[idx:iE, :] y_batch = y_train[idx:iE, :] sess.run([optimizer], feed_dict={x: x_batch, y: y_batch}) curr_w, curr_b = sess.run([w, b]) for idx in range(len(curr_w)): if curr_w[idx] < threshold * -1: curr_w[idx] += threshold else: curr_w[idx] -= threshold sess.run([tf.assign(w, curr_w)]) return curr_w, curr_b
def __init__(self): self.embedding = self.getEmb() self.embSize = self.embedding.shape[1] self.vocabSize = self.embedding.shape[0] self.x = tf.placeholder(tf.int32, [None, 5]) with tf.variable_scope("training_variable"): self.weights = { "MLP1": tf.Variable( tf.truncated_normal( shape=[self.embSize, int(self.embSize / 2)], stddev=0.08)), "MLP2": tf.Variable( tf.truncated_normal(shape=[int(self.embSize / 2), 1], stddev=0.08)) } self.biases = { "MLP1": tf.Variable( tf.constant(0.01, shape=[int(self.embSize / 2)], dtype=tf.float32)), "MLP2": tf.Variable(tf.constant(0.01, shape=[1], dtype=tf.float32)) } self.inputEmb = tf.nn.embedding_lookup(self.embedding, self.x) p1 = tf.matmul(tf.reshape(self.inputEmb, [-1, self.embSize]), self.weights["MLP1"]) + self.biases["MLP1"] p1 = tf.matmul(tf.nn.relu(p1), self.weights["MLP2"]) + self.biases["MLP2"] p1 = tf.reshape(p1, [-1, 5]) p1 = tf.reshape(tf.nn.softmax(p1), [-1, 1, 5]) self.finalState = tf.reshape(tf.matmul(p1, self.inputEmb), [-1, self.embSize])
if os.path.exists("./tmp/beginner-export"): shutil.rmtree("./tmp/beginner-export") # Import data # from tensorflow.examples.tutorials.mnist import input_data import input_data mnist = input_data.read_data_sets("tmp/data/", one_hot=True) g = tf.Graph() with g.as_default(): # Create the model # x = tf.placeholder("float", [None, 784]) x = tf.compat.v1.placeholder("float", [None, 784]) W = tf.Variable(tf.zeros([784, 10]), name="vaiable_W") b = tf.Variable(tf.zeros([10]), name="variable_b") y = tf.nn.softmax(tf.matmul(x, W) + b) # Define loss and optimizer y_ = tf.compat.v1.placeholder("float", [None, 10]) cross_entropy = -tf.reduce_sum(y_ * tf.log(y)) train_step = tf.train.GradientDescentOptimizer(0.01).minimize( cross_entropy) sess = tf.Session() # Train init = tf.initialize_all_variables() sess.run(init)
def gain(data_x, gain_parameters): '''Impute missing values in data_x Args: - data_x: original data with missing values - gain_parameters: GAIN network parameters: - batch_size: Batch size - hint_rate: Hint rate - alpha: Hyperparameter - iterations: Iterations Returns: - imputed_data: imputed data ''' # Define mask matrix data_m = 1 - np.isnan(data_x) # System parameters batch_size = gain_parameters['batch_size'] hint_rate = gain_parameters['hint_rate'] alpha = gain_parameters['alpha'] iterations = gain_parameters['iterations'] # Other parameters no, dim = data_x.shape # Hidden state dimensions h_dim = int(dim) # Normalization norm_data, norm_parameters = normalization(data_x) norm_data_x = np.nan_to_num(norm_data, 0) ## GAIN architecture # Input placeholders # Data vector tf.disable_v2_behavior() X = tf.placeholder(tf.float32, shape=[None, dim]) # Mask vector M = tf.placeholder(tf.float32, shape=[None, dim]) # Hint vector H = tf.placeholder(tf.float32, shape=[None, dim]) # Discriminator variables D_W1 = tf.Variable(xavier_init([dim * 2, h_dim])) # Data + Hint as inputs D_b1 = tf.Variable(tf.zeros(shape=[h_dim])) D_W2 = tf.Variable(xavier_init([h_dim, h_dim])) D_b2 = tf.Variable(tf.zeros(shape=[h_dim])) D_W3 = tf.Variable(xavier_init([h_dim, dim])) D_b3 = tf.Variable(tf.zeros(shape=[dim])) # Multi-variate outputs theta_D = [D_W1, D_W2, D_W3, D_b1, D_b2, D_b3] #Generator variables # Data + Mask as inputs (Random noise is in missing components) G_W1 = tf.Variable(xavier_init([dim * 2, h_dim])) G_b1 = tf.Variable(tf.zeros(shape=[h_dim])) G_W2 = tf.Variable(xavier_init([h_dim, h_dim])) G_b2 = tf.Variable(tf.zeros(shape=[h_dim])) G_W3 = tf.Variable(xavier_init([h_dim, dim])) G_b3 = tf.Variable(tf.zeros(shape=[dim])) theta_G = [G_W1, G_W2, G_W3, G_b1, G_b2, G_b3] ## GAIN functions # Generator def generator(x, m): # Concatenate Mask and Data inputs = tf.concat(values=[x, m], axis=1) G_h1 = tf.nn.relu(tf.matmul(inputs, G_W1) + G_b1) G_h2 = tf.nn.relu(tf.matmul(G_h1, G_W2) + G_b2) # MinMax normalized output G_prob = tf.nn.sigmoid(tf.matmul(G_h2, G_W3) + G_b3) return G_prob # Discriminator def discriminator(x, h): # Concatenate Data and Hint inputs = tf.concat(values=[x, h], axis=1) D_h1 = tf.nn.relu(tf.matmul(inputs, D_W1) + D_b1) D_h2 = tf.nn.relu(tf.matmul(D_h1, D_W2) + D_b2) D_logit = tf.matmul(D_h2, D_W3) + D_b3 D_prob = tf.nn.sigmoid(D_logit) return D_prob ## GAIN structure # Generator G_sample = generator(X, M) # Combine with observed data Hat_X = X * M + G_sample * (1 - M) # Discriminator D_prob = discriminator(Hat_X, H) ## GAIN loss D_loss_temp = -tf.reduce_mean(M * tf.log(D_prob + 1e-8) \ + (1-M) * tf.log(1. - D_prob + 1e-8)) G_loss_temp = -tf.reduce_mean((1 - M) * tf.log(D_prob + 1e-8)) MSE_loss = \ tf.reduce_mean((M * X - M * G_sample)**2) / tf.reduce_mean(M) D_loss = D_loss_temp G_loss = G_loss_temp + alpha * MSE_loss ## GAIN solver D_solver = tf.train.AdamOptimizer().minimize(D_loss, var_list=theta_D) G_solver = tf.train.AdamOptimizer().minimize(G_loss, var_list=theta_G) ## Iterations sess = tf.Session() sess.run(tf.global_variables_initializer()) # Start Iterations for it in tqdm(range(iterations)): # Sample batch batch_idx = sample_batch_index(no, batch_size) X_mb = norm_data_x[batch_idx, :] M_mb = data_m[batch_idx, :] # Sample random vectors Z_mb = uniform_sampler(0, 0.01, batch_size, dim) # Sample hint vectors H_mb_temp = binary_sampler(hint_rate, batch_size, dim) H_mb = M_mb * H_mb_temp # Combine random vectors with observed vectors X_mb = M_mb * X_mb + (1 - M_mb) * Z_mb _, D_loss_curr = sess.run([D_solver, D_loss_temp], feed_dict={ M: M_mb, X: X_mb, H: H_mb }) _, G_loss_curr, MSE_loss_curr = \ sess.run([G_solver, G_loss_temp, MSE_loss], feed_dict = {X: X_mb, M: M_mb, H: H_mb}) ## Return imputed data Z_mb = uniform_sampler(0, 0.01, no, dim) M_mb = data_m X_mb = norm_data_x X_mb = M_mb * X_mb + (1 - M_mb) * Z_mb imputed_data = sess.run([G_sample], feed_dict={X: X_mb, M: M_mb})[0] imputed_data = data_m * norm_data_x + (1 - data_m) * imputed_data # Renormalization imputed_data = renormalization(imputed_data, norm_parameters) # Rounding imputed_data = rounding(imputed_data, data_x) return imputed_data
def __init__(self, nHidden, seqLen): self.representation_score = {} self.y = tf.placeholder(tf.float32, shape=[None, 1]) self.extractFeature = ExtractFeature.ExtractFeature() self.imageFeature = ImageFeature.ImageFeature() newNet = tf.reduce_mean(self.imageFeature.outputLS, axis=0) self.textFeature = TextFeature.TextFeature( nHidden, seqLen, self.extractFeature.finalState, newNet) self.l2_para = 1e-7 with tf.variable_scope("training_variable"): self.weights = { "MLP1": tf.Variable( tf.truncated_normal(shape=[512, 256], stddev=0.08, name="MLP1_W")), "MLP2": tf.Variable( tf.truncated_normal(shape=[256, 1], stddev=0.08, name="MLP2_W")), "ATT_attr1_1": tf.Variable( tf.truncated_normal(shape=[ self.imageFeature.defaultFeatureSize + self.extractFeature.embSize, int(self.imageFeature.defaultFeatureSize / 2 + self.extractFeature.embSize / 2) ], stddev=0.08, name="ATT_attr1_1")), "ATT_attr1_2": tf.Variable( tf.truncated_normal(shape=[ self.textFeature.nHidden * 2 + self.extractFeature.embSize, int(self.textFeature.nHidden + self.extractFeature.embSize / 2) ], stddev=0.08, name="ATT_attr1_2")), "ATT_attr1_3": tf.Variable( tf.truncated_normal(shape=[ 2 * self.extractFeature.embSize, self.extractFeature.embSize ], stddev=0.08, name="ATT_attr1_3")), "ATT_attr2_1": tf.Variable( tf.truncated_normal(shape=[ int(self.imageFeature.defaultFeatureSize / 2 + self.extractFeature.embSize / 2), 1 ], stddev=0.08, name="ATT_attr2_1")), "ATT_attr2_2": tf.Variable( tf.truncated_normal(shape=[ int(self.textFeature.nHidden + self.extractFeature.embSize / 2), 1 ], stddev=0.08, name="ATT_attr2_2")), "ATT_attr2_3": tf.Variable( tf.truncated_normal(shape=[self.extractFeature.embSize, 1], stddev=0.08, name="ATT_attr2_3")), "ATT_img1_1": tf.Variable( tf.truncated_normal(shape=[ self.imageFeature.defaultFeatureSize + self.textFeature.nHidden * 2, int(self.imageFeature.defaultFeatureSize / 2 + self.textFeature.nHidden) ], stddev=0.08, name="ATT_image1_1")), "ATT_img1_2": tf.Variable( tf.truncated_normal(shape=[ self.imageFeature.defaultFeatureSize + self.extractFeature.embSize, int(self.imageFeature.defaultFeatureSize / 2 + self.extractFeature.embSize / 2) ], stddev=0.08, name="ATT_image1_2")), "ATT_img1_3": tf.Variable( tf.truncated_normal(shape=[ self.imageFeature.defaultFeatureSize * 2, self.imageFeature.defaultFeatureSize ], stddev=0.08, name="ATT_image1_3")), "ATT_img2_1": tf.Variable( tf.truncated_normal(shape=[ int(self.imageFeature.defaultFeatureSize / 2 + self.textFeature.nHidden), 1 ], stddev=0.08, name="ATT_image2_1")), "ATT_img2_2": tf.Variable( tf.truncated_normal(shape=[ int(self.imageFeature.defaultFeatureSize / 2 + self.extractFeature.embSize / 2), 1 ], stddev=0.08, name="ATT_image2_2")), "ATT_img2_3": tf.Variable( tf.truncated_normal( shape=[self.imageFeature.defaultFeatureSize, 1], stddev=0.08, name="ATT_image2_3")), "ATT_text1_1": tf.Variable( tf.truncated_normal(shape=[ self.imageFeature.defaultFeatureSize + self.textFeature.nHidden * 2, int(self.imageFeature.defaultFeatureSize / 2 + self.textFeature.nHidden) ], stddev=0.08, name="ATT_text1_1")), "ATT_text1_2": tf.Variable( tf.truncated_normal(shape=[ self.textFeature.nHidden * 2 + self.extractFeature.embSize, int(self.textFeature.nHidden + self.extractFeature.embSize / 2) ], stddev=0.08, name="ATT_text1_2")), "ATT_text1_3": tf.Variable( tf.truncated_normal(shape=[ self.textFeature.nHidden * 4, self.textFeature.nHidden * 2 ], stddev=0.08, name="ATT_text1_3")), "ATT_text2_1": tf.Variable( tf.truncated_normal(shape=[ int(self.imageFeature.defaultFeatureSize / 2 + self.textFeature.nHidden), 1 ], stddev=0.08, name="ATT_text2_1")), "ATT_text2_2": tf.Variable( tf.truncated_normal(shape=[ int(self.textFeature.nHidden + self.extractFeature.embSize / 2), 1 ], stddev=0.08, name="ATT_text2_2")), "ATT_text2_3": tf.Variable( tf.truncated_normal( shape=[self.textFeature.nHidden * 2, 1], stddev=0.08, name="ATT_text2_3")), "ATT_WI1": tf.Variable( tf.truncated_normal( shape=[self.imageFeature.defaultFeatureSize, 512], stddev=0.08, name="ATT_WI")), "ATT_WT1": tf.Variable( tf.truncated_normal(shape=[2 * nHidden, 512], stddev=0.08, name="ATT_WT")), "ATT_WA1": tf.Variable( tf.truncated_normal(shape=[200, 512], stddev=0.08, name="ATT_WA")), "ATT_WI2": tf.Variable( tf.truncated_normal( shape=[self.imageFeature.defaultFeatureSize, 512], stddev=0.08, name="ATT_WI2")), "ATT_WT2": tf.Variable( tf.truncated_normal(shape=[2 * nHidden, 512], stddev=0.08, name="ATT_WT2")), "ATT_WA2": tf.Variable( tf.truncated_normal(shape=[200, 512], stddev=0.08, name="ATT_WA2")), "ATT_WF_1": tf.Variable( tf.truncated_normal(shape=[512, 1], stddev=0.08, name="ATT_WF_1")), "ATT_WF_2": tf.Variable( tf.truncated_normal(shape=[512, 1], stddev=0.08, name="ATT_WF_2")), "ATT_WF_3": tf.Variable( tf.truncated_normal(shape=[512, 1], stddev=0.08, name="ATT_WF_3")), } self.biases = { "MLP1": tf.Variable( tf.constant(0.01, shape=[256], dtype=tf.float32, name="MLP1_b")), "MLP2": tf.Variable( tf.constant(0.01, shape=[1], dtype=tf.float32, name="MLP2_b")), "ATT_attr1_1": tf.Variable( tf.constant( 0.01, shape=[ int(self.imageFeature.defaultFeatureSize / 2 + self.extractFeature.embSize / 2) ], name="ATT_attr1_1")), "ATT_attr1_2": tf.Variable( tf.constant(0.01, shape=[ int(self.textFeature.nHidden + self.extractFeature.embSize / 2) ], name="ATT_attr1_2")), "ATT_attr1_3": tf.Variable( tf.constant(0.01, shape=[self.extractFeature.embSize], name="ATT_attr1_3")), "ATT_attr2_1": tf.Variable(tf.constant(0.01, shape=[1], name="ATT_attr2_1")), "ATT_attr2_2": tf.Variable(tf.constant(0.01, shape=[1], name="ATT_attr2_2")), "ATT_attr2_3": tf.Variable(tf.constant(0.01, shape=[1], name="ATT_attr2_3")), "ATT_img1_1": tf.Variable( tf.constant( 0.01, shape=[ int(self.imageFeature.defaultFeatureSize / 2 + self.textFeature.nHidden) ], name="ATT_image1_1")), "ATT_img1_2": tf.Variable( tf.constant( 0.01, shape=[ int(self.imageFeature.defaultFeatureSize / 2 + self.extractFeature.embSize / 2) ], name="ATT_image1_2")), "ATT_img1_3": tf.Variable( tf.constant(0.01, shape=[self.imageFeature.defaultFeatureSize], name="ATT_image1_3")), "ATT_img2_1": tf.Variable(tf.constant(0.01, shape=[1], name="ATT_image2_1")), "ATT_img2_2": tf.Variable(tf.constant(0.01, shape=[1], name="ATT_image2_2")), "ATT_img2_3": tf.Variable(tf.constant(0.01, shape=[1], name="ATT_image2_3")), "ATT_text1_1": tf.Variable( tf.constant( 0.01, shape=[ int(self.imageFeature.defaultFeatureSize / 2 + self.textFeature.nHidden) ], name="ATT_text1_1")), "ATT_text1_2": tf.Variable( tf.constant(0.01, shape=[ int(self.textFeature.nHidden + self.extractFeature.embSize / 2) ], name="ATT_text1_2")), "ATT_text1_3": tf.Variable( tf.constant(0.01, shape=[self.textFeature.nHidden * 2], name="ATT_text1_3")), "ATT_text2_1": tf.Variable(tf.constant(0.01, shape=[1], name="ATT_text2_1")), "ATT_text2_2": tf.Variable(tf.constant(0.01, shape=[1], name="ATT_text2_2")), "ATT_text2_3": tf.Variable(tf.constant(0.01, shape=[1], name="ATT_text2_3")), "ATT_WW": tf.Variable(tf.constant(0.01, shape=[512], name="ATT_WW")), "ATT_WI": tf.Variable(tf.constant(0.01, shape=[512], name="ATT_WI")), "ATT_WT": tf.Variable(tf.constant(0.01, shape=[512], name="ATT_WT")), "ATT_WI1": tf.Variable(tf.constant(0.01, shape=[512], name="ATT_WI1")), "ATT_WT1": tf.Variable(tf.constant(0.01, shape=[512], name="ATT_WT1")), "ATT_WA": tf.Variable(tf.constant(0.01, shape=[512], name="ATT_WA")), "ATT_WF_1": tf.Variable(tf.constant(0.01, shape=[1], name="ATT_WF_1")), "ATT_WF_2": tf.Variable(tf.constant(0.01, shape=[1], name="ATT_WF_2")), "ATT_WF_3": tf.Variable(tf.constant(0.01, shape=[1], name="ATT_WF_3")), } print("newnet dimension :", newNet) imageVec = self.Attention(newNet, self.imageFeature.outputLS, self.textFeature.RNNState, self.extractFeature.finalState, "ATT_img1", "ATT_img2", 196, True) textVec = self.Attention(self.textFeature.RNNState, self.textFeature.outputs, newNet, self.extractFeature.finalState, "ATT_text1", "ATT_text2", self.textFeature.seqLen, False) attrVec = self.Attention(self.extractFeature.finalState, self.extractFeature.inputEmb, newNet, self.textFeature.RNNState, "ATT_attr1", "ATT_attr2", 5, False) attHidden = tf.tanh( tf.matmul(imageVec, self.weights["ATT_WI1"]) + self.biases["ATT_WI1"]) attHidden2 = tf.tanh( tf.matmul(textVec, self.weights["ATT_WT1"]) + self.biases["ATT_WT1"]) attHidden3 = tf.tanh( tf.matmul(attrVec, self.weights["ATT_WA1"]) + self.biases["ATT_WW"]) scores1 = tf.matmul(attHidden, self.weights["ATT_WF_1"]) + self.biases["ATT_WF_1"] scores2 = tf.matmul(attHidden2, self.weights["ATT_WF_2"]) + self.biases["ATT_WF_2"] scores3 = tf.matmul(attHidden3, self.weights["ATT_WF_3"]) + self.biases["ATT_WF_3"] scoreLS = [scores1, scores2, scores3] scoreLS = tf.nn.softmax(scoreLS, dim=0) imageVec = tf.tanh( tf.matmul(imageVec, self.weights["ATT_WI2"]) + self.biases["ATT_WI"]) textVec = tf.tanh( tf.matmul(textVec, self.weights["ATT_WT2"]) + self.biases["ATT_WT"]) attrVec = tf.tanh( tf.matmul(attrVec, self.weights["ATT_WA2"]) + self.biases["ATT_WA"]) self.concatInput = scoreLS[0] * imageVec + scoreLS[ 1] * textVec + scoreLS[2] * attrVec
def weight_variable(shape): initial = tf.truncated_normal(shape, stddev=0.1) return tf.Variable(initial)
"c3_filter": [3, 3, 256, 384], "c4_filter": [3, 3, 192, 384], "c5_filter": [3, 3, 192, 256] } # Fully connected shapes fc_connection_shapes = { "f1_shape": [13 * 13 * 256, 4096], "f2_shape": [4096, 4096], "f3_shape": [4096, dataset_dict["num_labels"]] } # Weights for each layer conv_weights = { "c1_weights": tf.Variable(tf.truncated_normal(conv_filter_shapes["c1_filter"]), name="c1_weights"), "c2_weights": tf.Variable(tf.truncated_normal(conv_filter_shapes["c2_filter"]), name="c2_weights"), "c3_weights": tf.Variable(tf.truncated_normal(conv_filter_shapes["c3_filter"]), name="c3_weights"), "c4_weights": tf.Variable(tf.truncated_normal(conv_filter_shapes["c4_filter"]), name="c4_weights"), "c5_weights": tf.Variable(tf.truncated_normal(conv_filter_shapes["c5_filter"]), name="c5_weights"), "f1_weights": tf.Variable(tf.truncated_normal(fc_connection_shapes["f1_shape"]), name="f1_weights"),
def main(trainModel=True, buildConfusionMatrix=True, restore=False, buildClassifiedMatrix=True): tf.disable_v2_behavior() input_images = tf.placeholder(tf.float32, [None, 28, 28], name="Input") real = tf.placeholder(tf.float32, [None, CLASSES], name="real_classes") layer1 = create_conv_layer(tf.reshape(input_images, [-1, 28, 28, 1]), 1, 28, [5, 5], [2, 2], name="conv_no_pool") layer2 = create_conv_layer(layer1, 28, 56, [5, 5], [2, 2], name='conv_with_pool') conv_result = tf.reshape(layer2, [-1, 7 * 7 * 56]) relu_layer_weight = tf.Variable(tf.truncated_normal([7 * 7 * 56, 1000], stddev=STDDEV * 2), name='relu_layer_weight') rely_layer_bias = tf.Variable(tf.truncated_normal([1000], stddev=STDDEV / 2), name='rely_layer_bias') relu_layer = tf.matmul(conv_result, relu_layer_weight) + rely_layer_bias relu_layer = tf.nn.relu(relu_layer) relu_layer = tf.nn.dropout(relu_layer, DROPOUT) final_layer_weight = tf.Variable(tf.truncated_normal([1000, CLASSES], stddev=STDDEV * 2), name='final_layer_weight') final_layer_bias = tf.Variable(tf.truncated_normal([CLASSES], stddev=STDDEV / 2), name='final_layer_bias') final_layer = tf.matmul(relu_layer, final_layer_weight) + final_layer_bias predicts = tf.nn.softmax(final_layer) predicts_for_log = tf.clip_by_value(predicts, 1e-9, 0.999999999) #crossEntropy = -tf.reduce_mean(tf.reduce_sum(y * tf.log(y_clipped) + (1 - y) * tf.log(1 - y_clipped), axis=1)) loss = -tf.reduce_mean( tf.reduce_sum(real * tf.log(predicts_for_log) + (1 - real) * tf.log(1 - predicts_for_log), axis=1), axis=0) #test = tf.reduce_sum(real * tf.log(predicts_for_log) + (1 - real) * tf.log(1 - predicts_for_log), axis=1) #loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=final_layer, labels=real)) optimiser = tf.train.GradientDescentOptimizer( learning_rate=LEARNING_RATE).minimize(loss) correct_prediction = tf.equal(tf.argmax(real, axis=1), tf.argmax(predicts, axis=1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) confusion_matrix = tf.confusion_matrix(labels=tf.argmax(real, axis=1), predictions=tf.argmax(predicts, axis=1), num_classes=CLASSES) saver = tf.train.Saver() # dataset = get_mnist_dataset() dataset = get_fashion_dataset() with tf.Session() as session: session.run(tf.global_variables_initializer()) if restore: saver.restore(session, SAVE_PATH) if trainModel: train(input_images, real, session, optimiser, loss, accuracy, saver, dataset) if buildConfusionMatrix: test_cm = session.run(confusion_matrix, feed_dict={ input_images: dataset.test_x, real: dataset.test_y }) draw_confusion_matrix(test_cm) if buildClassifiedMatrix: all_probs = session.run(predicts, feed_dict={ input_images: dataset.test_x, real: dataset.test_y }) max_failure_picture_index = [[(-1, -1.0)] * CLASSES for _ in range(CLASSES)] for i in range(len(all_probs)): real = np.argmax(dataset.test_y[i]) for j in range(CLASSES): if max_failure_picture_index[real][j][1] < all_probs[i][j]: max_failure_picture_index[real][j] = (i, all_probs[i][j]) draw_max_failure_pictures(dataset.test_x, max_failure_picture_index)
def __init__(self, nHidden, seqLen, guidence, newNet): self.nHidden = nHidden self.seqLen = seqLen tmp = self.getEmbedding() self.embedding = tf.Variable(tmp) with tf.variable_scope("training_variable"): self.weights = { "ATT": tf.Variable( tf.truncated_normal(shape=[2 * self.nHidden, self.nHidden], stddev=0.08, name="text_att")), "ATTG": tf.Variable( tf.truncated_normal(shape=[200, self.nHidden], stddev=0.08, name="text_att2")), "ATTS": tf.Variable( tf.truncated_normal(shape=[self.nHidden, 1], stddev=0.08, name="text_att3")), "Fw1": tf.Variable( tf.truncated_normal(shape=[200, self.nHidden], stddev=0.08, name="init_fw1")), "Fw2": tf.Variable( tf.truncated_normal(shape=[200, self.nHidden], stddev=0.08, name="init_fw2")), "Bw1": tf.Variable( tf.truncated_normal(shape=[200, self.nHidden], stddev=0.08, name="init_bw1")), "Bw2": tf.Variable( tf.truncated_normal(shape=[200, self.nHidden], stddev=0.08, name="init_bw2")), } self.biases = { "Fw1": tf.Variable( tf.constant(0.01, shape=[self.nHidden], name="init_Fw1")), "Fw2": tf.Variable( tf.constant(0.01, shape=[self.nHidden], name="init_Fw2")), "Bw1": tf.Variable( tf.constant(0.01, shape=[self.nHidden], name="init_Bw1")), "Bw2": tf.Variable( tf.constant(0.01, shape=[self.nHidden], name="init_Bw2")), } self.X = tf.placeholder(tf.int32, [None, self.seqLen]) self.pKeep = tf.placeholder(tf.float32) self.build(guidence, newNet)
import tensorflow._api.v2.compat.v1 as tf import numpy as np tf.reset_default_graph() tf.compat.v1.disable_eager_execution() tf.compat.v1.disable_v2_behavior() tf.global_variables_initializer() x = tf.placeholder(tf.float32, shape=[None, 4]) y = tf.placeholder(tf.float32, shape=[None, 1]) w = tf.Variable(tf.random_normal([4, 1]), name="weight") b = tf.Variable(tf.random_normal([1]), name="bias") hypo = tf.matmul(x, w) + b saver = tf.train.Saver() test_arr = [[12, 6.5, 15.7, 10.8]] sess2 = tf.Session() saver.restore(sess2, "./saved.ckpt") predict = sess2.run(hypo, feed_dict={x: test_arr}) print(predict[0])
def bias_variable(shape): initial = tf.constant(0.1, shape=shape) return tf.Variable(initial)
# Create a simple TF Graph # By Omid Alemi - Jan 2017 # Works with TF r1.0 # import tensorflow as tf import tensorflow._api.v2.compat.v1 as tf tf.disable_v2_behavior() I = tf.placeholder(tf.float32, shape=[None,3], name='I') # input W = tf.Variable(tf.zeros(shape=[3,2]), dtype=tf.float32, name='W') # weights b = tf.Variable(tf.zeros(shape=[2]), dtype=tf.float32, name='b') # biases O = tf.nn.relu(tf.matmul(I, W) + b, name='O') # activation / output saver = tf.train.Saver() init_op = tf.global_variables_initializer() with tf.Session() as sess: sess.run(init_op) # save the graph tf.train.write_graph(sess.graph_def, '.', 'tfdroid.pbtxt') # normally you would do some training here # but fornow we will just assign something to W sess.run(tf.assign(W, [[1, 2],[4,5],[7,8]])) sess.run(tf.assign(b, [1,1])) #save a checkpoint file, which will store the above assignment saver.save(sess, 'tfdroid.ckpt')