def two_sample_t(A,B,expected_diff=0): diff = (np.mean(A) - np.mean(B) - expected_diff) na = len(A) nb = len(B) df = na + nb - 2 sum_sq = (var(A)*(na-1) + var(B)*(nb-1)) f = (1/na + 1/nb)/df t = diff/np.sqrt(sum_sq*f) return (t, stdtr(df,t))
def two_sample_t(A, B, expected_diff=0): diff = (np.mean(A) - np.mean(B) - expected_diff) na = len(A) nb = len(B) df = na + nb - 2 sum_sq = (var(A) * (na - 1) + var(B) * (nb - 1)) f = (1 / na + 1 / nb) / df t = diff / np.sqrt(sum_sq * f) return (t, stdtr(df, t))
def forward(self, input, gamma, beta): in_mean, in_var = reduce_mean(input, dim=[2, 3], keep_dim=True), var(input, dim=[2, 3], keep_dim=True) out_in = (input - in_mean) / sqrt(in_var + self.eps) ln_mean, ln_var = reduce_mean(input, dim=[1, 2, 3], keep_dim=True), var(input, dim=[1, 2, 3], keep_dim=True) out_ln = (input - ln_mean) / sqrt(ln_var + self.eps) out = expand(self.rho, expand_times=[ input.shape[0], 1, 1, 1 ]) * out_in + (1 - expand( self.rho, expand_times=[input.shape[0], 1, 1, 1])) * out_ln out = out * unsqueeze(unsqueeze(gamma, 2), 3) + unsqueeze( unsqueeze(beta, 2), 3) return out
def test_var(): x1 = var("x1") y = x1 grad_x1, = gradients(y, [x1]) executor = Executor([y, grad_x1]) x1_val = 2 * np.ones(3) y_val, grad_x1_val= executor.run(feed_dict = {x1 : x1_val}) assert isinstance(y, Node) assert np.array_equal(y_val, x1_val) assert np.array_equal(grad_x1_val, np.ones_like(x1_val))
def __summarize(self,blocks): ns_len = max([block.str_len() for block in blocks]) # 最长文本块的长度 for block in blocks: self.score_li.append( self.scorer.score(block,self.tws,ns_len) ) sim_li = [ score['sim_title'] for score in self.score_li ] self.sim_var = var( sim_li ) if self.sim_var > SIM_TITLE_VAR: self.confidence_li = [ score['sim_title']*score['link_rate']*score['text_rate'] \ for score in self.score_li ] else: # title相似度普遍低,说明title不具说明性,不计该指标 self.confidence_li = [ score['link_rate']*score['text_rate'] for score in self.score_li ] return self.confidence_li
import gensim import utils from current_words_list import lst model = gensim.models.Word2Vec.load( "./word2vec models/heb 56 epochs 300 dim/w2v_56_300dim_heb") print("model details: ") utils.model_details(model) print("the average vector of the words list", utils.avg_vec_model(lst, model)) print("result without change to the vectors or the words: ", utils.one_mean(lst, model)) print("the variance of every dim in the vectors of the words: ", utils.var(lst, model)) print("the variance of every dim sorted by the variance value") variance = (utils.var(lst, model)) variance.sort(key=lambda k: k[1]) print(variance) model_after_remove_dim = utils.remove_dim(lst, model, -30) print("after remove 30 dimensions the length of the vectors is: ", model_after_remove_dim.vector_size) print("result after remove 30 dimensions: ", utils.one_mean(lst, model)) # print("create xls file with the result for the lst with remove 0 - 10 words and remove 0 , 30 , 60 ,90 dimension") # utils.output_res(lst,model,savepath ="output.xls")
def train(self, X_train, Y_train, X_val, Y_val, max_epochs, batch_size, learning_rate_init, reg_param=0, learning_rate_decay_type='inverse', learning_rate_decay_parameter=10, keep_prob=[], early_stopping=True, save_path=Path('./UNet'), reset_parameters=False, val_checks_per_epoch=10, seed=None, data_on_GPU=True): ''' Trains the network on the given data, provided as numpy arrays. It is assumed all preprocessing has already been done, including shuffling and splitting of the data into training/validation sets. ''' assert type(save_path) == type(Path('.')), 'save_path needs to be a pathlib Path' check_val_every_n_batches = math.ceil(X_train.shape[0]/batch_size/val_checks_per_epoch) # (Re)load base graph from file print('Loading graph...') saver = self.load_graph(save_path) with self.G.as_default(): # Set the seed if seed is None: seed = int(time.time()) tf.set_random_seed(seed) np.random.seed(seed) print('Inserting data augmentation operations...') # Get dataset size/statistics m_train, height, width, n_channels = X_train.shape m_val = X_val.shape[0] m = m_train + m_val n_classes = Y_train.shape[-1] X_train_mean = np.mean(X_train) X_val_mean = np.mean(X_val) X_mean = ((m_train*X_train_mean + m_val*X_val_mean)/m) X_train_var = u.var(X_train, X_mean) X_val_var = u.var(X_val, X_mean) X_std = np.sqrt((m_train*X_train_var + m_val*X_val_var)/m) # Load data onto GPU, replace the input placeholder with an index into the data on the GPU (if applicable) if data_on_GPU: print('Loading X_train to GPU...') X_train_t = tf.constant(X_train, dtype=tf.uint8) del X_train print('Loading X_val to GPU...') X_val_t = tf.constant(X_val, dtype=tf.uint8) del X_val print('Loading Y_train to GPU...') Y_train_t = tf.constant(Y_train, dtype=tf.bool) del Y_train print('Loading Y_val to GPU...') Y_val_t = tf.constant(Y_val, dtype=tf.bool) del Y_val train_idx = tf.placeholder_with_default([0], shape=[None]) X_train_t = tf.gather(X_train_t, train_idx, axis=0) Y_train_t = tf.gather(Y_train_t, train_idx, axis=0) else: X_train_t = tf.placeholder_with_default(np.zeros([0,height,width,n_channels], dtype=np.uint8), shape=self.input.shape, name='X_train_input') X_val_t = tf.placeholder_with_default(np.zeros([0,height,width,n_channels], dtype=np.uint8), shape=self.input.shape, name='X_val_input') Y_train_t = tf.placeholder_with_default(np.zeros([0,height,width,n_classes], dtype=np.bool), shape=self.labels.shape, name='Y_train_input') Y_val_t = tf.placeholder_with_default(np.zeros([0,height,width,n_classes], dtype=np.bool), shape=self.labels.shape, name='Y_val_input') # Insert data augmentation steps to graph train_or_val_idx = tf.placeholder(dtype=tf.int32, shape=[None]) X_train_aug, Y_train_aug, X_val_aug, Y_val_aug = self.data_augmentation(X_train_t, Y_train_t, X_val_t, Y_val_t) X = (tf.cast(tf.gather(tf.concat([X_train_aug, X_val_aug], axis=0), train_or_val_idx), tf.float32) - X_mean)/X_std Y = tf.cast(tf.gather(tf.concat([Y_train_aug, Y_val_aug], axis=0), train_or_val_idx), tf.float32) ge.swap_ts([X, Y], [self.input, self.labels]) # Use X and Y from now on! # Add metrics prob = tf.sigmoid(self.output[:,:,:,0]) Y_bool = tf.cast(Y[:,:,:,0], bool) is_over_thresh = (prob>0.5) is_equal = tf.equal(is_over_thresh, Y_bool) intersection = tf.reduce_sum(tf.cast(tf.logical_and(is_over_thresh, Y_bool), tf.float32)) union = tf.reduce_sum(tf.cast(tf.logical_or(is_over_thresh, Y_bool), tf.float32)) acc = tf.reduce_mean(tf.cast(is_equal, tf.float32)) conf = tf.reduce_mean(2*tf.abs(prob-0.5)*tf.cast(is_equal, tf.float32)) IOU = intersection/union # Write to log file with open(str(save_path)+'.log', 'w+') as fo: fo.write('Training log\n\n') fo.write('Dataset metrics:\n') fo.write('Training data shape: {}\n'.format(X_train.shape)) fo.write('Validation set size: {}\n'.format(m_val)) fo.write('X_mean: {}\n'.format(X_mean)) fo.write('X_std: {}\n\n'.format(X_std)) fo.write('Hyperparameters:\n') fo.write('Batch size: {}\n'.format(batch_size)) fo.write('Learning rate: {}\n'.format(learning_rate_init)) fo.write('Learning rate decay parameter: {}\n'.format(learning_rate_decay_parameter)) fo.write('Learning rate decay type: {}\n'.format(learning_rate_decay_type)) # fo.write('Stepped anneal: {}\n'.format(STEPPED_ANNEAL)) # fo.write('Regularization type: {}\n'.format(REGULARIZATION_TYPE)) fo.write('Regularization parameter: {}\n'.format(reg_param)) # fo.write('Input noise variance: {:.2f}\n'.format(INPUT_NOISE_MAGNITUDE**2)) # fo.write('Weight noise variance: {:.2f}\n'.format(WEIGHT_NOISE_MAGNITUDE**2)) for n in range(len(keep_prob)): fo.write('Dropout keep prob. group {}: {:.2f}\n'.format(n, keep_prob[n])) fo.write('Logging frequency: {} global steps\n'.format(check_val_every_n_batches)) fo.write('Random seed: {}\n'.format(seed)) # Initialize control flow variables and logs # best_val_accuracy = 0 best_val_conf = 0 # best_val_loss = np.inf global_step = 0 if reset_parameters: io_mode = 'w+' else: io_mode = 'a+' with open(str(save_path)+'_val_accuracy.log', io_mode) as fo: fo.write('') with open(str(save_path)+'_val_loss.log', io_mode) as fo: fo.write('') with open(str(save_path)+'_val_confidence.log', io_mode) as fo: fo.write('') with open(str(save_path)+'_val_IOU.log', io_mode) as fo: fo.write('') with open(str(save_path)+'_train_accuracy.log', io_mode) as fo: fo.write('') with open(str(save_path)+'_train_loss.log', io_mode) as fo: fo.write('') with open(str(save_path)+'_train_confidence.log', io_mode) as fo: fo.write('') with open(str(save_path)+'_train_IOU.log', io_mode) as fo: fo.write('') with open(str(save_path)+'_learning_rate.log', io_mode) as fo: fo.write('') # Start tensorflow session, reset_parameters or reload checkpoint print('Starting tensorflow session...') with tf.Session() as sess: if reset_parameters: # saver = tf.train.Saver() sess.run(tf.global_variables_initializer()) else: try: saver.restore(sess, save_path) except: # saver = tf.train.Saver() sess.run(tf.global_variables_initializer()) uninitialized_vars = [] for var in tf.global_variables(): try: sess.run(var) except tf.errors.FailedPreconditionError: uninitialized_vars.append(var) sess.run(tf.variables_initializer(uninitialized_vars)) # Iterate over training epochs for epoch in range(max_epochs): if learning_rate_decay_type == 'inverse': learning_rate = learning_rate_init/(1+epoch/learning_rate_decay_parameter) elif learning_rate_decay_type == 'constant': learning_rate = learning_rate_init elif learning_rate_decay_type == 'exponential': learning_rate = learning_rate_init*np.exp(-epoch/learning_rate_decay_parameter) else: raise Exception('Unknown learning rate decay function') # Iterate over batches: n_batches = int(math.ceil(m_train/batch_size)) for b in range(n_batches): train_idx_i = b*batch_size train_idx_f = min((b+1)*batch_size, m_train) if data_on_GPU: feed_dict = {train_idx:range(train_idx_i, train_idx_f+1), train_or_val_idx:range(train_idx_f-train_idx_i), self.learning_rate:learning_rate, self.reg_param:reg_param} else: feed_dict = {X_train_t:X_train[train_idx_i:train_idx_f], Y_train_t:Y_train[train_idx_i:train_idx_f], train_or_val_idx:range(train_idx_f-train_idx_i), self.learning_rate:learning_rate, self.reg_param:reg_param} feed_dict = {**feed_dict, **{self.keep_prob_dict[i]:kp for i, kp in enumerate(keep_prob)}} train_loss, train_acc, train_conf, train_IOU, _ = sess.run([self.loss, acc, conf, IOU, self.train_op], feed_dict=feed_dict) print('Epoch {}, batch {}/{}: loss={:.3e}, acc={:.3f}, IOU={:.3f}'.format(epoch+1, b, n_batches, train_loss, train_acc, train_IOU)) if np.isnan(train_loss) or np.isinf(train_loss): print('Detected nan, exiting training') quit() exit() break if ((global_step % check_val_every_n_batches) == 0) and (global_step != 0): if data_on_GPU: feed_dict = {train_or_val_idx:range(1,m_val+1), self.reg_param:reg_param} else: feed_dict = {X_val_t:X_val, Y_val_t:Y_val, train_or_val_idx:range(m_val), self.reg_param:reg_param} val_loss, val_acc, val_conf, val_IOU = sess.run([self.loss, acc, conf, IOU], feed_dict=feed_dict) print('Validation set: loss={:.3e}, acc={:.3f}, IOU={:.3f}'.format(val_loss, val_acc, val_IOU)) if early_stopping and (val_conf > best_val_conf): best_val_conf = val_conf print('New best validation confidence: {:.3e}! Saving...'.format(val_conf)) saver.save(sess, str(save_path), write_meta_graph=False) # Write to logs everytime validation set is run with open(str(save_path)+'_train_loss.log', 'a') as fo: fo.write(str(train_loss)+'\n') with open(str(save_path)+'_train_accuracy.log', 'a') as fo: fo.write(str(train_acc)+'\n') with open(str(save_path)+'_train_confidence.log', 'a') as fo: fo.write(str(train_conf)+'\n') with open(str(save_path)+'_train_IOU.log', 'a') as fo: fo.write(str(train_IOU)+'\n') with open(str(save_path)+'_val_accuracy.log', 'a') as fo: fo.write(str(val_acc)+'\n') with open(str(save_path)+'_val_loss.log', 'a') as fo: fo.write(str(val_loss)+'\n') with open(str(save_path)+'_val_confidence.log', 'a') as fo: fo.write(str(val_conf)+'\n') with open(str(save_path)+'_val_IOU.log', 'a') as fo: fo.write(str(val_IOU)+'\n') with open(str(save_path)+'_learning_rate.log', 'a') as fo: fo.write(str(learning_rate)+'\n') # Plot metrics (actually, I can just run this manually) # u.plot_metrics(str(save_path)) # Iterate global step global_step += 1 # Save if not using early stopping if not early_stopping: saver.save(sess, str(save_path), write_meta_graph=False) # Plot an example of how the algorithm is doing on the task if data_on_GPU: pass else: x = X_val[0] feed_dict = {X_val_t:np.expand_dims(x, axis=0), Y_val_t:np.expand_dims(Y_val[0], axis=0), train_or_val_idx:range(1), self.reg_param:reg_param} x, y = sess.run([X_val_aug, prob], feed_dict=feed_dict).squeeze() plt.ioff() plt.figure('Progress pic') plt.clf() plt.imshow((x-np.min(x))/(np.max(x)-np.min(x))) plt.imshow(y, alpha=0.4) (save_path.parent/'ProgressPics').mkdir(exist_ok=True) plt.savefig(str(save_path.parent)+'/ProgressPics/ProgressPic{}.png'.format(epoch))