def predict(self): train_pred = self.model.predict(self.X_train, batch_size=self.batch_size) dev_pred = self.model.predict(self.X_dev, batch_size=self.batch_size) test_pred = self.model.predict(self.X_test, batch_size=self.batch_size) train_pred = np.round(train_pred) dev_pred = np.round(dev_pred) test_pred = np.round(test_pred) self.dev_acc = accuracy_score(self.Y_dev, dev_pred) self.dev_precision = precision(self.Y_dev, dev_pred) self.dev_recall = recall(self.Y_dev, dev_pred) self.dev_f1 = f1_score(self.Y_dev, dev_pred) self.dev_false_pos_rate = false_positive_rate(self.Y_dev, dev_pred) self.test_acc = accuracy_score(self.Y_test, test_pred) self.test_precision = precision(self.Y_test, test_pred) self.test_recall = recall(self.Y_test, test_pred) self.test_f1 = f1_score(self.Y_test, test_pred) self.test_false_pos_rate = false_positive_rate(self.Y_test, test_pred) if self.dev_acc > self.best_dev_acc: self.best_dev_acc = self.dev_acc self.best_dev_precision = self.dev_precision self.best_dev_recall = self.dev_recall self.best_dev_f1 = self.dev_f1 self.best_dev_false_pos_rate = self.dev_false_pos_rate self.best_test_acc = self.test_acc self.best_test_precision = self.test_precision self.best_test_recall = self.test_recall self.best_test_f1 = self.test_f1 self.best_test_false_pos_rate = self.test_false_pos_rate
def logistic_regression(train_data, train_labels, test_data, test_labels): print(f'{LogisticRegression.__name__}:') # Create and train model lr_model = LogisticRegression(train_data.shape[1], eta=0.001, epochs=50) model = OneVersusRest(lr_model) model.train(train_data, train_labels) # Predict 2000 validation set samples and calculate accuracy test_data_2k = test_data[:len(test_labels)] test_pred = model.predict(test_data_2k) # Print metrics print('\nTest Accuracy: {:.02f}%\n'.format( 100 * accuracy(test_pred, test_labels))) mat, classes = confusion_matrix(test_pred, test_labels) print('Precision:\n{}\n'.format( np.round(precision(test_pred, test_labels), 2))) print('Recall:\n{}\n'.format(np.round(recall(test_pred, test_labels), 2))) print('F1:\n{}\n'.format(np.round(f1_score(test_pred, test_labels), 2))) print('Confusion Matrix:') print(mat) # Predict 10000 test set samples and save predictions print('Predicting 10k samples...') test_pred = model.predict(test_data) save_predictions(logistic_regression.__name__, test_pred) print('Saved 10k predictions.\n')
def linear_svm(train_data, train_labels, test_data, test_labels): print(f'{LinearSVM.__name__}:') # Create and train model lsvm_model = LinearSVM(alpha=0.01, features=180) model = OneVersusRest(lsvm_model) model.train(train_data, train_labels) # Predict 2000 validation set samples and calculate accuracy test_data_2k = test_data[:len(test_labels)] test_pred = model.predict(test_data_2k) # Print metrics print('\nTest Accuracy: {:.02f}%\n'.format( 100 * accuracy(test_pred, test_labels))) mat, classes = confusion_matrix(test_pred, test_labels) print('Precision:\n{}\n'.format( np.round(precision(test_pred, test_labels), 2))) print('Recall:\n{}\n'.format(np.round(recall(test_pred, test_labels), 2))) print('F1:\n{}\n'.format(np.round(f1_score(test_pred, test_labels), 2))) print('Confusion Matrix:') print(mat) # Predict 10000 test set samples and save predictions print('Predicting 10k samples...') test_pred = model.predict(test_data) save_predictions(linear_svm.__name__, test_pred) print('Saved 10k predictions.\n')
def nearest_neighbour(train_data, train_labels, test_data, test_labels): print(f'{NearestNeighbour.__name__}:') # Create and train model model = NearestNeighbour(5, dist=manhattan) model.train(train_data, train_labels) # Predict 2000 validation set samples and calculate accuracy test_data_2k = test_data[:len(test_labels)] test_pred = model.predict(test_data_2k) # Print metrics print('\nTest Accuracy: {:.02f}%\n'.format( 100 * accuracy(test_pred, test_labels))) mat, classes = confusion_matrix(test_pred, test_labels) print('Precision:\n{}\n'.format( np.round(precision(test_pred, test_labels), 2))) print('Recall:\n{}\n'.format(np.round(recall(test_pred, test_labels), 2))) print('F1:\n{}\n'.format(np.round(f1_score(test_pred, test_labels), 2))) print('Confusion Matrix:') print(mat) # Predict 10000 test set samples and save predictions print('Predicting 10k samples...') test_pred = model.predict(test_data) save_predictions(nearest_neighbour.__name__, test_pred) print('Saved 10k predictions.\n')
def calc_metric(pred, true): tmp_jac = jaccard(pred, true) tmp_ham = hamming(pred, true) tmp_prec = precision(pred, true) tmp_rec = recall(pred, true) tmp_f1 = f1_score(pred, true) tmp_acc = accuracy(pred, true) return tmp_jac, tmp_ham, tmp_prec, tmp_rec, tmp_f1, tmp_acc
def card(corpus): global vectorizer jac_list = [] ham_list = [] prec_list = [] rec_list = [] f1_list = [] acc_list = [] id2term = {} for k, v in vectorizer.vocabulary_.items(): id2term[v] = k for idx, item in enumerate(corpus): docstring = vectorizer.transform([item[-1]])[0] true_types = item[3] terms = [id2term[i] for i in docstring.indices] pred_types = [] # try hint terms of each basic type whether match a type # term pattern `\w+(\_)?name|\w+(\_)?method` # is more likely to be a `str` type for basic_type in basic_types: h_terms = hint_terms[basic_type] if set(terms) & set(h_terms): pred_types.append('List' if basic_type == 'Tuple' else basic_type) elif basic_type == 'str': for term in terms: if type_to_regexp[basic_type].match(term): pred_types.append(basic_type) break # if type `Dict` and `str` in pred_types at same time # remove `str` as type hardly never occur `Dict` and `str` at same time if 'Dict' in pred_types and 'str' in pred_types: pred_types.remove('str') # type `Type` often occur independently if 'Type' in pred_types and len(pred_types) > 1: pred_types.remove('Type') # if pred_types is empty # then find whether exists a term whose part of speech(pos) is NNS or NNPS # if found, the pos of this variable is likely to be `List` if not pred_types: pos_tags = [tag for w, tag in nltk.pos_tag(terms)] if set(['NNS', 'NNPS']) & set(pos_tags): pred_types = ['List'] pred_types = set(pred_types) included_types = set(true_types) - set(['NoneType']) if 'Tuple' in included_types: included_types -= set(['Tuple']) included_types.add('List') # calculate Jaccard Coefficient jac_list.append(jaccard(pred_types, included_types)) ham_list.append(hamming(pred_types, included_types)) prec_list.append(precision(pred_types, included_types)) rec_list.append(recall(pred_types, included_types)) f1_list.append(f1_score(pred_types, included_types)) acc_list.append(accuracy(pred_types, included_types)) return jac_list, ham_list, prec_list, rec_list, f1_list, acc_list
def predict(self): train_pred = self.model.predict(self.X_train, batch_size=self.batch_size) dev_pred = self.model.predict(self.X_dev, batch_size=self.batch_size) test_pred = self.model.predict(self.X_test, batch_size=self.batch_size) train_pred = np.round(train_pred) dev_pred = np.round(dev_pred) test_pred = np.round(test_pred) self.dev_acc = accuracy_score(self.Y_dev, dev_pred) self.dev_precision = precision(self.Y_dev, dev_pred) self.dev_recall = recall(self.Y_dev, dev_pred) self.dev_f1 = f1_score(self.Y_dev, dev_pred) self.dev_false_pos_rate = false_positive_rate(self.Y_dev, dev_pred) self.test_acc = accuracy_score(self.Y_test, test_pred) self.test_precision = precision(self.Y_test, test_pred) self.test_recall = recall(self.Y_test, test_pred) self.test_f1 = f1_score(self.Y_test, test_pred) self.test_false_pos_rate = false_positive_rate(self.Y_test, test_pred) if self.dev_acc > self.best_dev_acc: self.best_dev_acc = self.dev_acc self.best_dev_precision = self.dev_precision self.best_dev_recall = self.dev_recall self.best_dev_f1 = self.dev_f1 self.best_dev_false_pos_rate = self.dev_false_pos_rate self.best_test_acc = self.test_acc self.best_test_precision = self.test_precision self.best_test_recall = self.test_recall self.best_test_f1 = self.test_f1 self.best_test_false_pos_rate = self.test_false_pos_rate # SAVE MODE TO JSON model_json = self.model.to_json() with open(self.save_path + '.json', 'w') as json_file: json_file.write(model_json) # SAVE WEIGHTS self.model.save_weights(self.save_path + '.h5') logger.info("Model saved")
def eval(self, epoch): # Eval SPM self.SPM.eval() # Validation statistics val_pred_loss, val_count = 0.0, 0 metrics = {'mae': 0.0, 'precision': 0.0, 'recall': 0.0, 'f1': 0.0} # Run Batch with torch.no_grad(): with tqdm(self.valset) as val_bar: for batch_idx, data in enumerate(val_bar): # Prepare data x = data['image'].to(self.device) y = data['label'].to(self.device) y_pred = self.SPM(x)['out'] # Compute loss pred_loss = self.loss_fn(y_pred, y) # Save batch losses val_count += self.batch_size val_pred_loss += pred_loss * self.batch_size # Compute metrics y_pred = to_numpy(y_pred) y = to_numpy(y) metrics['mae'] += mae(y_pred, y) * self.batch_size metrics['precision'] += precision(y_pred, y) * self.batch_size metrics['recall'] += recall(y_pred, y) * self.batch_size # Update tdqm val_bar.set_description( f'Validation [{epoch}/{self.epochs}] Loss: {val_pred_loss / val_count:.4f}' ) # Remove data from GPU memory del x, y, y_pred, data, batch_idx, pred_loss torch.cuda.empty_cache() # Save epoch losses and metrics val_pred_loss = val_pred_loss / val_count metrics['mae'] = metrics['mae'] / val_count metrics['precision'] = metrics['precision'] / val_count metrics['recall'] = metrics['recall'] / val_count metrics['f1'] = f1(metrics['precision'] / val_count, metrics['recall'] / val_count, beta2=0.3) return val_pred_loss, metrics['mae'], metrics['precision'], metrics[ 'recall'], metrics['f1']
def evaluate(self, data, labels): """ Args: data: vector of embeddings labels: ground truth Returns: Metrics on the result. """ predictions = self.predict(data) return { "Accuracy": metrics.accuracy(labels, predictions), "Recall": metrics.recall(labels, predictions), "Precision": metrics.precision(labels, predictions), "F1": metrics.f1(labels, predictions), "Predictions": predictions }
def test_final(self, epoch): # Test SPM self.SPM.eval() # Test statistics test_count = 0 metrics = {'mae': 0.0, 'precision': 0.0, 'recall': 0.0, 'f1': 0.0} # Run Batch with torch.no_grad(): with tqdm(self.testset) as test_bar: for batch_idx, data in enumerate(test_bar): # Prepare data x = data['image'].to(self.device) y = data['label'].to(self.device) y_pred = self.SPM(x)['out'] # Save batch losses test_count += self.batch_size # Compute metrics y_pred = to_numpy(y_pred) y = to_numpy(y) metrics['mae'] += mae(y_pred, y) * self.batch_size metrics['precision'] += precision(y_pred, y) * self.batch_size metrics['recall'] += recall(y_pred, y) * self.batch_size metrics['f1'] = f1(metrics['precision'] / test_count, metrics['recall'] / test_count, beta2=0.3) # Update tdqm F1 = metrics['f1'] MAE = metrics['mae'] test_bar.set_description( f'Test: F1-score: {F1:.4f} MAE: {MAE / test_count:.4f}' ) # Save statistcs metrics['mae'] /= test_count metrics['precision'] /= test_count metrics['recall'] /= test_count df = pd.DataFrame(data=metrics, index=[0]) df.to_csv( f'output/{self.experiment}/{self.optim.__name__}/{self.scheduler.__name__}/{self.optim.__name__}_{self.scheduler.__name__}_{self.epochs}_test_{epoch}_stats.csv', index_label='epoch') return True
def test_single_image(parser, threshold, save_random=True): # obtain groundtruth data metadata = parser.fetch_metadata() # filter to get people who only wear mask idxs = np.where(metadata["placement"] != "none")[0] metadata["bboxes"] = metadata["bboxes"][idxs] # do inference on mtcnn image = cv2.imread(parser.image_path) bboxes, _ = fetch_faces(image, return_landmarks=False) recall = metrics.recall(metadata["bboxes"], bboxes, threshold=threshold) # save results for ablation study if save_random and np.random.uniform() <= 0.10: filename = os.path.basename(parser.image_path) save_image(image, filename, metadata["bboxes"], bboxes) # recall for single inference image return recall
def precision_recall_curve(x, y, start, end): """A function draws precision_recall_curve. Args: x (list or numpy array): 2D inputs, personalized recommendation list, sorted by the probabilities y (list or numpy array): 2D inputs, actual selection list, sorted by the probabilities start (int): start length of the recommendation list end (int): end length of the recommendation list """ if not isinstance(start, int) or not isinstance(end, int): raise TypeError("start and end must be an positive integer") if start > end: raise ValueError("start must be less than or equal to end") if not isinstance(x, np.ndarray): x = np.array(x) if not isinstance(y, np.ndarray): y = np.array(y) if x.shape[0] != y.shape[0]: raise ValueError("inputs must be same length") if (x.shape[0] == 0) or (y.shape[0] == 0): raise ValueError("inputs must not be empty") precisions, recalls = [], [] for i in range(start, end + 1): precisions.append(precision(x, y, i)) recalls.append(recall(x, y, i)) fig = go.Figure(go.Scatter(x=recalls, y=precisions, mode="lines+markers")) fig.update_layout( title="precision-recall curve", xaxis_title="recall", yaxis_title="precision", ) fig.update_xaxes(range=(0.0, 1.1)) fig.update_yaxes(range=(0.0, 1.1)) fig.show()
def neural_net(train_data, train_labels, test_data, test_labels): print(f'{NeuralNetwork.__name__}:') # Create and train model model = NeuralNetwork([ FlatDenseLayer((784, ), activation=tanh), FlatDenseLayer((100, ), activation=tanh), FlatDenseLayer((20, ), activation=tanh), FlatDenseLayer((10, ), activation=sigmoid), ], eta=0.01, batch_size=64, epochs=250) model.train(train_data, train_labels) # Predict 2000 validation set samples and calculate accuracy test_data_2k = test_data[:len(test_labels)] test_activations, test_pred = model.predict(test_data_2k) # Print metrics print('\nTest Accuracy: {:.02f}%\n'.format( 100 * accuracy(test_pred, test_labels))) mat, classes = confusion_matrix(test_pred, test_labels) print('Precision:\n{}\n'.format( np.round(precision(test_pred, test_labels), 2))) print('Recall:\n{}\n'.format(np.round(recall(test_pred, test_labels), 2))) print('F1:\n{}\n'.format(np.round(f1_score(test_pred, test_labels), 2))) print('Confusion Matrix:') print(mat) # Predict 10000 test set samples and save predictions print('Predicting 10k samples...') test_activations, test_pred = model.predict(test_data) print(len(test_pred)) save_predictions(neural_net.__name__, test_pred) print('Saved 10k predictions.\n')
def train( self, X, Y, max_epochs=100, batch_size=256, seed=42, verbose=0, # training vars use_imblearn=False, imblearn_class=SMOTE(random_state=42, ratio=1.0), # imblearn vars early_stopping_callback='default', test_split=0.2, # early stopping vars testing=False, kfold_function=KFold, kfold_splits=5, ): # testing vars (returns predictions from kfold) ''' trains self.model w/ selected parameters @param X w/ shape (num_examples,num_error,num_sites) - where each number is integer of number of errors happened at that site @param Y w/ shape (num_examples,) - where each number is integer that represents one class @param max_epochs - maximum learning epochs, if not using kfold to approximate num_epochs: then this is num_epochs used for training @param batch_size - neural network parameter @param seed - random seed that is used everywhere for reproducability @param verbose - if 0: no print output, else: print output @param use_imblearn - boolean that decides to use resampling for training or not @param imblearn_class - class from iblearn library used for resampling (doesn't do anything if use_imblearn = False) @param early_stopping_callback - early stopping keras callback used to find num_epochs @param test_split - test split used for early stopping @param testing - if True: stops training after cross validation and returns predictions of all data across all kfolds @param kfold_function - cross validation function from sklearn library (doesn't do anything if testing == False) @param kfold_splits - number of cross validation splits, used in kfold_function (doesn't do anything if testing == False) return if not testing: return keras history.history dict else : list of len = kfold_splits, where each element is keras history.history dict ''' data = DataPlaceholder() if testing: enum = enumerate( kfold_function(n_splits=kfold_splits, shuffle=True, random_state=seed).split(X, Y)) if verbose != 0: enum = tqdm(enum, total=kfold_splits, desc='kfold', leave=False, initial=0) histories = [] for i, (index_train, index_valid) in enum: data.train.x, data.val.x = X[index_train], X[index_valid] data.train.y, data.val.y = Y[index_train], Y[index_valid] if use_imblearn: data.train.x, data.train.y = imblearn_sample( data.train.x, data.train.y, imblearn_class, verbose=verbose) self.class_weights = get_class_weights(data.train.y, self.num_classes) data.train.x, data.train.y = self.change_inputs( data.train.x, data.train.y) data.val.x, data.val.y = self.change_inputs( data.val.x, data.val.y) self.create_model(**self.model_params) keras_callbacks = [ PredictDataCallback(self.model, data.train.x, data.train.y, ''), PredictDataCallback(self.model, data.val.x, data.val.y, 'val_') ] history = self.model.fit(x=data.train.x, y=data.train.y, validation_data=(data.val.x, data.val.y), epochs=max_epochs, batch_size=batch_size, callbacks=[] + keras_callbacks, verbose=verbose) histories.append(history.history) return histories else: if early_stopping_callback == 'default': self.class_weights = get_class_weights(Y, self.num_classes) early_stopping_functions = [ confusion_mse(), recall(average='macro'), precision(average='macro') ] modes = ['min', 'max', 'max'] early_stopping_callback = MultipleMetricsEarlyStopping( early_stopping_functions, modes=modes) data.train.x, data.val.x, data.train.y, data.val.y = train_test_split( X, Y, test_size=test_split, random_state=seed) if use_imblearn: data.train.x, data.train.y = imblearn_sample(data.train.x, data.train.y, imblearn_class, verbose=verbose) self.class_weights = get_class_weights(data.train.y, self.num_classes) data.train.x, data.train.y = self.change_inputs( data.train.x, data.train.y) data.val.x, data.val.y = self.change_inputs(data.val.x, data.val.y) self.create_model(**self.model_params) history = self.model.fit(x=data.train.x, y=data.train.y, validation_data=(data.val.x, data.val.y), epochs=max_epochs, batch_size=batch_size, callbacks=[early_stopping_callback], verbose=verbose) return history.history
def evaluating(self, model, dataset, split): """ input: model: (object) pytorch model dataset: (object) dataset split: (str) split of dataset in ['train', 'val', 'test'] return [overall_accuracy, precision, recall, f1-score, jaccard, kappa] """ args = self.args oa, precision, recall, f1, jac, kappa = 0, 0, 0, 0, 0, 0 model.eval() data_loader = DataLoader(dataset, args.batch_size, num_workers=4, shuffle=False) batch_iterator = iter(data_loader) steps = len(dataset) // args.batch_size start = time.time() for step in range(steps): x, y = next(batch_iterator) x = Variable(x, volatile=True) y = Variable(y, volatile=True) if args.cuda: x = x.cuda() y = y.cuda() # calculate pixel accuracy of generator gen_y = model(x) if self.is_multi: gen_y = gen_y[0] oa += metrics.overall_accuracy(gen_y.data, y.data) precision += metrics.precision(gen_y.data, y.data) recall += metrics.recall(gen_y.data, y.data) f1 += metrics.f1_score(gen_y.data, y.data) jac += metrics.jaccard(gen_y.data, y.data) kappa += metrics.kappa(gen_y.data, y.data) _time = time.time() - start if not os.path.exists(os.path.join(Logs_DIR, 'statistic')): os.makedirs(os.path.join(Logs_DIR, 'statistic')) # recording performance of the model nb_samples = steps * args.batch_size basic_info = [ self.date, self.method, self.epoch, self.iter, nb_samples, _time ] basic_info_names = [ 'date', 'method', 'epochs', 'iters', 'nb_samples', 'time(sec)' ] perform = [ round(idx / steps, 3) for idx in [oa, precision, recall, f1, jac, kappa] ] perform_names = [ "overall_accuracy", "precision", "recall", "f1-score", "jaccard", "kappa" ] cur_log = pd.DataFrame([basic_info + perform], columns=basic_info_names + perform_names) # save performance if os.path.exists( os.path.join(Logs_DIR, 'statistic', "{}.csv".format(split))): logs = pd.read_csv( os.path.join(Logs_DIR, 'statistic', "{}.csv".format(split))) else: logs = pd.DataFrame([]) logs = logs.append(cur_log, ignore_index=True) logs.to_csv(os.path.join(Logs_DIR, 'statistic', "{}.csv".format(split)), index=False, float_format='%.3f')
def train_final(self, epoch): # Train SPM self.SPM.train() # Statistics train_pred_loss, train_noise_loss, train_count = 0.0, 0.0, 0 metrics = {'mae': 0.0, 'precision': 0.0, 'recall': 0.0, 'f1': 0.0} with tqdm(self.trainset) as train_bar: for batch_idx, data in enumerate(train_bar): # x: Input data, (batch_size, 3, 256, 256) x = data['image'].to(self.device) # y: GT label (batch_size, 1, 256, 256) y = data['label'].to(self.device) # Keep original x to compute metrics upon true data original_x = x # Prepare unsupervised labels to experiments if self.experiment == 'Final-Average': y_noise = data['unsup_labels'].to(self.device) # y_noise: Taking mean unsupervised labels (batch_size, 1, 256, 256) y_noise = torch.mean(y_noise, dim=1, keepdim=True) elif self.experiment == 'Final-Noise': y_noise = data['unsup_labels'].to(self.device) # Normalizing y_noise between (0, 1) y_n_min, y_n_max = min2d(y_noise), max2d(y_noise) y_noise = (y_noise - y_n_min) / (y_n_max - y_n_min) # pred: (batch_size, 1, 256, 256) pred = self.SPM(x)['out'] y_pred = pred # Noise training in Complete experiment # Round 1 if epoch <= 20: if self.experiment == 'Final-Noise': repeat = 4 elif self.experiment == 'Final-Average': repeat = 1 # pred: In round 1 repeat along dim=1, (batch_size, NUM_MAPS(4), 256, 256) pred = torch.repeat_interleave(pred, repeats=repeat, dim=1) y_pred = pred # Round > 1 else: # noise_prior: noise from NMM (batch_size, NUM_MAPS, 256, 256) noise_prior = self.NMM.sample(data['index']).to( self.device) # noisy_pred: Noisy predictions after adding noise to predictions, (batch_size, NUM_MAPS, 256, 256) noisy_pred = pred + noise_prior # Range inside [0, 1] (see 3.2 after Eq 4) noisy_min, noisy_max = min2d(noisy_pred), max2d(noisy_pred) noisy_pred = (noisy_pred - noisy_min) / (noisy_max - noisy_min) y_pred = noisy_pred # Compute BCE loss (Eq. 4) pred_loss = self.loss_fn(y_pred, y_noise) # Noise loss noise_loss = 0 if self.experiment == 'Final-Average': y_noise = data['unsup_labels'].to(self.device) if epoch > 20: # compute batch noise loss (Eq 6) emp_var = torch.var(y_noise - pred, 1).reshape( self.batch_size, -1) + 1e-16 prior_var, var_idx = self.NMM.get_index_multiple( img_indexes=data['index']) prior_var = torch.from_numpy(prior_var).float() # Important Order for loss var needs to get close to emp_var noise_loss = self.NMM.loss(prior_var, emp_var) # Backprogation self.optim.zero_grad() # Total loss computed (Eq. 2) total_loss = pred_loss + 0.01 * noise_loss # lambda: 0.01 total_loss.backward() self.optim.step() # Save batch losses train_count += self.batch_size train_pred_loss += to_numpy(pred_loss) * self.batch_size if epoch <= 20: train_noise_loss += noise_loss * self.batch_size else: train_noise_loss += to_numpy(noise_loss) * self.batch_size # Compute metrics y_pred_t = self.SPM(original_x)['out'] y_pred_t = to_numpy(y_pred_t) y = to_numpy(y) metrics['mae'] += mae(y_pred_t, y) * self.batch_size metrics['precision'] += precision(y_pred_t, y) * self.batch_size metrics['recall'] += recall(y_pred_t, y) * self.batch_size # Update tdqm train_bar.set_description( f'Train Epoch [{epoch}/{self.epochs}] Loss: { (train_noise_loss + train_pred_loss) / train_count:.4f} Noise: {train_noise_loss/train_count:.4f} Pred: {train_pred_loss / train_count:.4f}' ) # Remove data from GPU memory del x, y, y_noise, y_pred, y_pred_t, pred, data torch.cuda.empty_cache() # Prepare statistics total_train_loss = (train_noise_loss + train_pred_loss) / train_count train_noise_loss /= train_count train_pred_loss /= train_count metrics['mae'] /= train_count metrics['precision'] /= train_count metrics['recall'] /= train_count metrics['f1'] = f1(metrics['precision'], metrics['recall'], beta2=0.3) return total_train_loss, train_pred_loss, train_noise_loss, metrics[ 'mae'], metrics['precision'], metrics['recall'], metrics['f1']
def train(model, epoch_num, start_epoch, optimizer, criterion, exp_lr_scheduler, data_set, data_loader, save_dir, print_inter=200, val_inter=3500, ): writer = SummaryWriter(save_dir) best_model_wts = model.state_dict() best_f1 = 0 val_loss = 0 train_loss = 0 # running_loss = 20 step = -1 for epoch in range(start_epoch,epoch_num): # train phase # exp_lr_scheduler.step(epoch) model.train(True) # Set model to training mode for batch_cnt, data in enumerate(data_loader['train']): step += 1 if step % val_inter == 0: # val phase model.eval() loss_fn = weighted_mse_loss # loss_fn = cross_entropy_loss_RCF # loss_fn = criterion t0 = time.time() test_precisions, test_recalls, test_f1_scores, val_loss = predict(loss_fn, model, data_set['val'], data_loader['val'], counting=False) t1 = time.time() since = t1 - t0 logging.info('--' * 30) # logging.info('current lr:%s' % exp_lr_scheduler.get_lr()) logging.info('%s epoch[%d] | val_loss: %.4f | precisions: %.4f | recalls: %.4f | f1_scores: %.4f | time: %d' % (dt(), epoch, val_loss, test_precisions, test_recalls, test_f1_scores, since)) if test_f1_scores > best_f1: best_f1 = test_f1_scores best_model_wts = deepcopy(model.state_dict()) # save model save_path1 = os.path.join(save_dir, 'weights-%d-%d-[%.3f].pth' % (epoch, batch_cnt, test_f1_scores)) torch.save(model.state_dict(), save_path1) save_path2 = os.path.join(save_dir, 'optimizer-state.pth') torch.save(optimizer.state_dict(), save_path2) logging.info('saved model to %s' % (save_path1)) logging.info('--' * 30) model.train(True) imgs, masks, _ = data imgs = Variable(imgs.cuda()) masks = Variable(masks.cuda(),requires_grad=False) # zero the parameter gradients optimizer.zero_grad() outputs = model(imgs) # outputs = outputs.view(-1, outputs.size()[2], outputs.size()[3]) # # print outputs.size(), masks.size() # if outputs.size() != masks.size(): # outputs = F.upsample(outputs, size=masks.size()[-2:], mode='bilinear') mask_loss = torch.zeros(1).cuda() for o in outputs: o = o.view(-1, o.size()[2], o.size()[3]) mask_loss = mask_loss + loss_fn(o, masks) mask_loss = mask_loss # loss.backward() # print outputs.size() # print masks.size() # mask_loss = criterion(outputs, masks) # mask_loss = cross_entropy_loss_RCF(outputs, masks) # mask_loss = weighted_mse_loss(outputs, masks) # loss = F.mean_absolute_error(outputs, masks) ###############################################cross entropy loss train_loss = mask_loss ############################################### train_loss.backward() optimizer.step() # running_loss = running_loss*0.95 + 0.05*loss.data[0] # running_loss = loss.data[0] # cal pixel acc # _, preds = torch.max(outputs,1) # (bs, H, W) # # preds = F.softmax(outputs,dim=1).round()[:, 1, :].long() # batch_corrects = torch.sum((preds==masks).long()).data[0] # batch_acc = 1.*batch_corrects / (masks.size(0)*masks.size(1)*masks.size(2)) output = outputs[-1] output = output.view(-1, output.size()[2], output.size()[3]) true_positives, predicted_positives, possible_positives, union_areas = metrics_pred(output.data.cpu().numpy(),\ imgs.cpu().data.numpy(), masks.cpu().data.numpy()) train_precisions = precision(true_positives, predicted_positives) train_recalls = recall(true_positives, possible_positives) train_f1_scores = f1_score(train_recalls, train_precisions) if step % print_inter == 0: logging.info('%s [%d-%d] | train_loss: %.4f | precisions: %.4f | recalls: %.4f | f1_scores: %.4f' % (dt(), epoch, batch_cnt, train_loss, train_precisions, train_recalls, train_f1_scores)) # plot image if step % (print_inter) == 0: smp_img = imgs[0] # (3, H, W) true_hm = masks[0] #(H,W) pred_hm = output[0] imgs_to_plot = getPlotImg(smp_img, pred_hm, true_hm) # for TensorBoard imgs_to_plot = torch.from_numpy(imgs_to_plot.transpose((0,3,1,2))/255.0) grid_image = make_grid(imgs_to_plot, 2) writer.add_image('plotting',grid_image, step) writer.add_scalar('train_loss', train_loss , step) writer.add_scalar('val_loss', val_loss, step) # save best model save_path = os.path.join(save_dir, 'bestweights-[%.3f].pth' % (best_f1)) torch.save(best_model_wts, save_path) logging.info('saved model to %s' % (save_path)) return best_f1, best_model_wts
def run(dtype): """ opens specific dataset, splits 75-25 percent for train-test and runs extraction """ print("DATASET ", dtype) global END_TRAINING data = dataset.read('extraction', dtype)['intents'] intents = [] for case in data: intent = [] for part in case['parts']: intent.append(part) intents.append(intent) print("DATASET CASES #", len(intents)) highest_precision = 0 highest_recall = 0 highest_f1 = 0 highest_try = 0 num_tries = 0 while num_tries < 30: num_tries += 1 END_TRAINING = None n_samples = int(ceil(len(intents) * 0.75)) training = sample(intents, n_samples) validation = sample(intents, len(intents) - n_samples) diag = Dialogflow(evaluation=True) diag.update_intent(INTENT_ID, training, False) training_begin = diag.train_agent(training_callback) time_elapsed = None while True: if END_TRAINING: time_elapsed = (END_TRAINING - training_begin) print("Training time: ", time_elapsed) break # time.sleep(50) print("Testing...") results = diag.detect_intent_texts(validation) with open(config.EXTRACTION_RESULTS_PATH.format(dtype, num_tries), 'w') as csvfile: csv_writer = csv.writer(csvfile, delimiter=',') csv_writer.writerow([ "text", "recognized_entities", "expected_entities", "training_time", "recall", "precision", "f1_score" ]) mean_precision = 0 mean_recall = 0 num_entries = len(results) for result in results: rec = metrics.recall(result['tp'], result['fn']) prec = metrics.precision(result['tp'], result['fp']) f1_sc = metrics.f1_score(prec, rec) mean_precision += prec mean_recall += rec print(result['text']) print('recall: ', rec) print('precision: ', prec) print('f1_score: ', f1_sc) csv_writer.writerow([ result['text'], result['recognized_entities'], result['expected_entities'], time_elapsed, rec, prec, f1_sc ]) mean_precision /= num_entries mean_recall /= num_entries mean_f1 = metrics.f1_score(mean_precision, mean_recall) csv_writer.writerow(["Mean Precision", mean_precision]) csv_writer.writerow(["Mean Recall", mean_recall]) csv_writer.writerow(["Mean F1", mean_f1]) print("Mean Precision", mean_precision) print("Mean Recall", mean_recall) print("Mean F1", mean_f1) if mean_f1 > highest_f1: highest_f1 = mean_f1 highest_precision = mean_precision highest_recall = mean_recall highest_try = num_tries print("Highest Precision", highest_precision) print("Highest Recall", highest_recall) print("Highest F1", highest_f1) print("Highest Try", highest_try)
def feedback(): """ opens alpha dataset, splits 75-25 percent for train-feedback """ print("FEEDBACK") global END_TRAINING diag = Dialogflow(evaluation=True) all_data = dataset.read('extraction', 'both')['intents'] all_intents = [] for case in all_data: intent = [] for part in case['parts']: intent.append(part) all_intents.append(intent) num_repeats = 1 with open(config.EXTRACTION_RESULTS_PATH.format('feedback', 'single'), 'w') as csvfile: csv_writer = csv.writer(csvfile, delimiter=',') csv_writer.writerow([ "repeat", "feedback_round", "text", "recognized_entities", "expected_entities", "tp", "fp", "fn", "recall", "precision", "f1_score" ]) for repeat in range(num_repeats): n_samples = int(floor(len(all_intents) * 0.25)) training = sample(all_intents, n_samples) feedback = sample(all_intents, len(all_intents) - n_samples) print("DATASET CASES TRAIN #", len(training)) print("DATASET CASES FEEDBACK #", len(feedback)) diag.update_intent(INTENT_ID, training, False) training_begin = diag.train_agent(training_callback) time_elapsed = None while True: if END_TRAINING: time_elapsed = (END_TRAINING - training_begin) print("Training time: ", time_elapsed) break time.sleep(60) print("Testing...") results = [] shuffle(feedback) for idx, feedback_case in enumerate(feedback): print("intent", idx) result = diag.detect_intent_texts([feedback_case])[0] rec = metrics.recall(result['tp'], result['fn']) prec = metrics.precision(result['tp'], result['fp']) f1_sc = metrics.f1_score(prec, rec) print(result['text']) print('recall: ', rec) print('precision: ', prec) print('f1_score: ', f1_sc) csv_writer.writerow([ repeat, idx, result['text'], result['recognized_entities'], result['expected_entities'], result['tp'], result['fp'], result['fn'], rec, prec, f1_sc ]) if result['fp'] != 0 or result['fn'] != 0: training.append(feedback_case) print("DATASET CASES TRAIN #", len(training)) diag.update_intent(INTENT_ID, training, False) END_TRAINING = None training_begin = diag.train_agent(training_callback) time_elapsed = None while True: if END_TRAINING: time_elapsed = (END_TRAINING - training_begin) print("Training time: ", time_elapsed) break time.sleep(60) csv_writer.writerow(["DATASET CASES TRAIN #", len(training)])