def predict(self):
train_pred = self.model.predict(self.X_train,
batch_size=self.batch_size)
dev_pred = self.model.predict(self.X_dev, batch_size=self.batch_size)
test_pred = self.model.predict(self.X_test, batch_size=self.batch_size)
train_pred = np.round(train_pred)
dev_pred = np.round(dev_pred)
test_pred = np.round(test_pred)
self.dev_acc = accuracy_score(self.Y_dev, dev_pred)
self.dev_precision = precision(self.Y_dev, dev_pred)
self.dev_recall = recall(self.Y_dev, dev_pred)
self.dev_f1 = f1_score(self.Y_dev, dev_pred)
self.dev_false_pos_rate = false_positive_rate(self.Y_dev, dev_pred)
self.test_acc = accuracy_score(self.Y_test, test_pred)
self.test_precision = precision(self.Y_test, test_pred)
self.test_recall = recall(self.Y_test, test_pred)
self.test_f1 = f1_score(self.Y_test, test_pred)
self.test_false_pos_rate = false_positive_rate(self.Y_test, test_pred)
if self.dev_acc > self.best_dev_acc:
self.best_dev_acc = self.dev_acc
self.best_dev_precision = self.dev_precision
self.best_dev_recall = self.dev_recall
self.best_dev_f1 = self.dev_f1
self.best_dev_false_pos_rate = self.dev_false_pos_rate
self.best_test_acc = self.test_acc
self.best_test_precision = self.test_precision
self.best_test_recall = self.test_recall
self.best_test_f1 = self.test_f1
self.best_test_false_pos_rate = self.test_false_pos_rate
def logistic_regression(train_data, train_labels, test_data, test_labels):
print(f'{LogisticRegression.__name__}:')
# Create and train model
lr_model = LogisticRegression(train_data.shape[1], eta=0.001, epochs=50)
model = OneVersusRest(lr_model)
model.train(train_data, train_labels)
# Predict 2000 validation set samples and calculate accuracy
test_data_2k = test_data[:len(test_labels)]
test_pred = model.predict(test_data_2k)
# Print metrics
print('\nTest Accuracy: {:.02f}%\n'.format(
100 * accuracy(test_pred, test_labels)))
mat, classes = confusion_matrix(test_pred, test_labels)
print('Precision:\n{}\n'.format(
np.round(precision(test_pred, test_labels), 2)))
print('Recall:\n{}\n'.format(np.round(recall(test_pred, test_labels), 2)))
print('F1:\n{}\n'.format(np.round(f1_score(test_pred, test_labels), 2)))
print('Confusion Matrix:')
print(mat)
# Predict 10000 test set samples and save predictions
print('Predicting 10k samples...')
test_pred = model.predict(test_data)
save_predictions(logistic_regression.__name__, test_pred)
print('Saved 10k predictions.\n')
def linear_svm(train_data, train_labels, test_data, test_labels):
print(f'{LinearSVM.__name__}:')
# Create and train model
lsvm_model = LinearSVM(alpha=0.01, features=180)
model = OneVersusRest(lsvm_model)
model.train(train_data, train_labels)
# Predict 2000 validation set samples and calculate accuracy
test_data_2k = test_data[:len(test_labels)]
test_pred = model.predict(test_data_2k)
# Print metrics
print('\nTest Accuracy: {:.02f}%\n'.format(
100 * accuracy(test_pred, test_labels)))
mat, classes = confusion_matrix(test_pred, test_labels)
print('Precision:\n{}\n'.format(
np.round(precision(test_pred, test_labels), 2)))
print('Recall:\n{}\n'.format(np.round(recall(test_pred, test_labels), 2)))
print('F1:\n{}\n'.format(np.round(f1_score(test_pred, test_labels), 2)))
print('Confusion Matrix:')
print(mat)
# Predict 10000 test set samples and save predictions
print('Predicting 10k samples...')
test_pred = model.predict(test_data)
save_predictions(linear_svm.__name__, test_pred)
print('Saved 10k predictions.\n')
def nearest_neighbour(train_data, train_labels, test_data, test_labels):
print(f'{NearestNeighbour.__name__}:')
# Create and train model
model = NearestNeighbour(5, dist=manhattan)
model.train(train_data, train_labels)
# Predict 2000 validation set samples and calculate accuracy
test_data_2k = test_data[:len(test_labels)]
test_pred = model.predict(test_data_2k)
# Print metrics
print('\nTest Accuracy: {:.02f}%\n'.format(
100 * accuracy(test_pred, test_labels)))
mat, classes = confusion_matrix(test_pred, test_labels)
print('Precision:\n{}\n'.format(
np.round(precision(test_pred, test_labels), 2)))
print('Recall:\n{}\n'.format(np.round(recall(test_pred, test_labels), 2)))
print('F1:\n{}\n'.format(np.round(f1_score(test_pred, test_labels), 2)))
print('Confusion Matrix:')
print(mat)
# Predict 10000 test set samples and save predictions
print('Predicting 10k samples...')
test_pred = model.predict(test_data)
save_predictions(nearest_neighbour.__name__, test_pred)
print('Saved 10k predictions.\n')
def calc_metric(pred, true):
tmp_jac = jaccard(pred, true)
tmp_ham = hamming(pred, true)
tmp_prec = precision(pred, true)
tmp_rec = recall(pred, true)
tmp_f1 = f1_score(pred, true)
tmp_acc = accuracy(pred, true)
return tmp_jac, tmp_ham, tmp_prec, tmp_rec, tmp_f1, tmp_acc
def card(corpus):
global vectorizer
jac_list = []
ham_list = []
prec_list = []
rec_list = []
f1_list = []
acc_list = []
id2term = {}
for k, v in vectorizer.vocabulary_.items():
id2term[v] = k
for idx, item in enumerate(corpus):
docstring = vectorizer.transform([item[-1]])[0]
true_types = item[3]
terms = [id2term[i] for i in docstring.indices]
pred_types = []
# try hint terms of each basic type whether match a type
# term pattern `\w+(\_)?name|\w+(\_)?method`
# is more likely to be a `str` type
for basic_type in basic_types:
h_terms = hint_terms[basic_type]
if set(terms) & set(h_terms):
pred_types.append('List' if basic_type ==
'Tuple' else basic_type)
elif basic_type == 'str':
for term in terms:
if type_to_regexp[basic_type].match(term):
pred_types.append(basic_type)
break
# if type `Dict` and `str` in pred_types at same time
# remove `str` as type hardly never occur `Dict` and `str` at same time
if 'Dict' in pred_types and 'str' in pred_types:
pred_types.remove('str')
# type `Type` often occur independently
if 'Type' in pred_types and len(pred_types) > 1:
pred_types.remove('Type')
# if pred_types is empty
# then find whether exists a term whose part of speech(pos) is NNS or NNPS
# if found, the pos of this variable is likely to be `List`
if not pred_types:
pos_tags = [tag for w, tag in nltk.pos_tag(terms)]
if set(['NNS', 'NNPS']) & set(pos_tags):
pred_types = ['List']
pred_types = set(pred_types)
included_types = set(true_types) - set(['NoneType'])
if 'Tuple' in included_types:
included_types -= set(['Tuple'])
included_types.add('List')
# calculate Jaccard Coefficient
jac_list.append(jaccard(pred_types, included_types))
ham_list.append(hamming(pred_types, included_types))
prec_list.append(precision(pred_types, included_types))
rec_list.append(recall(pred_types, included_types))
f1_list.append(f1_score(pred_types, included_types))
acc_list.append(accuracy(pred_types, included_types))
return jac_list, ham_list, prec_list, rec_list, f1_list, acc_list
def predict(self):
train_pred = self.model.predict(self.X_train,
batch_size=self.batch_size)
dev_pred = self.model.predict(self.X_dev, batch_size=self.batch_size)
test_pred = self.model.predict(self.X_test, batch_size=self.batch_size)
train_pred = np.round(train_pred)
dev_pred = np.round(dev_pred)
test_pred = np.round(test_pred)
self.dev_acc = accuracy_score(self.Y_dev, dev_pred)
self.dev_precision = precision(self.Y_dev, dev_pred)
self.dev_recall = recall(self.Y_dev, dev_pred)
self.dev_f1 = f1_score(self.Y_dev, dev_pred)
self.dev_false_pos_rate = false_positive_rate(self.Y_dev, dev_pred)
self.test_acc = accuracy_score(self.Y_test, test_pred)
self.test_precision = precision(self.Y_test, test_pred)
self.test_recall = recall(self.Y_test, test_pred)
self.test_f1 = f1_score(self.Y_test, test_pred)
self.test_false_pos_rate = false_positive_rate(self.Y_test, test_pred)
if self.dev_acc > self.best_dev_acc:
self.best_dev_acc = self.dev_acc
self.best_dev_precision = self.dev_precision
self.best_dev_recall = self.dev_recall
self.best_dev_f1 = self.dev_f1
self.best_dev_false_pos_rate = self.dev_false_pos_rate
self.best_test_acc = self.test_acc
self.best_test_precision = self.test_precision
self.best_test_recall = self.test_recall
self.best_test_f1 = self.test_f1
self.best_test_false_pos_rate = self.test_false_pos_rate
# SAVE MODE TO JSON
model_json = self.model.to_json()
with open(self.save_path + '.json', 'w') as json_file:
json_file.write(model_json)
# SAVE WEIGHTS
self.model.save_weights(self.save_path + '.h5')
logger.info("Model saved")
def eval(self, epoch):
# Eval SPM
self.SPM.eval()
# Validation statistics
val_pred_loss, val_count = 0.0, 0
metrics = {'mae': 0.0, 'precision': 0.0, 'recall': 0.0, 'f1': 0.0}
# Run Batch
with torch.no_grad():
with tqdm(self.valset) as val_bar:
for batch_idx, data in enumerate(val_bar):
# Prepare data
x = data['image'].to(self.device)
y = data['label'].to(self.device)
y_pred = self.SPM(x)['out']
# Compute loss
pred_loss = self.loss_fn(y_pred, y)
# Save batch losses
val_count += self.batch_size
val_pred_loss += pred_loss * self.batch_size
# Compute metrics
y_pred = to_numpy(y_pred)
y = to_numpy(y)
metrics['mae'] += mae(y_pred, y) * self.batch_size
metrics['precision'] += precision(y_pred,
y) * self.batch_size
metrics['recall'] += recall(y_pred, y) * self.batch_size
# Update tdqm
val_bar.set_description(
f'Validation [{epoch}/{self.epochs}] Loss: {val_pred_loss / val_count:.4f}'
)
# Remove data from GPU memory
del x, y, y_pred, data, batch_idx, pred_loss
torch.cuda.empty_cache()
# Save epoch losses and metrics
val_pred_loss = val_pred_loss / val_count
metrics['mae'] = metrics['mae'] / val_count
metrics['precision'] = metrics['precision'] / val_count
metrics['recall'] = metrics['recall'] / val_count
metrics['f1'] = f1(metrics['precision'] / val_count,
metrics['recall'] / val_count,
beta2=0.3)
return val_pred_loss, metrics['mae'], metrics['precision'], metrics[
'recall'], metrics['f1']
def evaluate(self, data, labels):
"""
Args:
data: vector of embeddings
labels: ground truth
Returns: Metrics on the result.
"""
predictions = self.predict(data)
return {
"Accuracy": metrics.accuracy(labels, predictions),
"Recall": metrics.recall(labels, predictions),
"Precision": metrics.precision(labels, predictions),
"F1": metrics.f1(labels, predictions),
"Predictions": predictions
}
def test_final(self, epoch):
# Test SPM
self.SPM.eval()
# Test statistics
test_count = 0
metrics = {'mae': 0.0, 'precision': 0.0, 'recall': 0.0, 'f1': 0.0}
# Run Batch
with torch.no_grad():
with tqdm(self.testset) as test_bar:
for batch_idx, data in enumerate(test_bar):
# Prepare data
x = data['image'].to(self.device)
y = data['label'].to(self.device)
y_pred = self.SPM(x)['out']
# Save batch losses
test_count += self.batch_size
# Compute metrics
y_pred = to_numpy(y_pred)
y = to_numpy(y)
metrics['mae'] += mae(y_pred, y) * self.batch_size
metrics['precision'] += precision(y_pred,
y) * self.batch_size
metrics['recall'] += recall(y_pred, y) * self.batch_size
metrics['f1'] = f1(metrics['precision'] / test_count,
metrics['recall'] / test_count,
beta2=0.3)
# Update tdqm
F1 = metrics['f1']
MAE = metrics['mae']
test_bar.set_description(
f'Test: F1-score: {F1:.4f} MAE: {MAE / test_count:.4f}'
)
# Save statistcs
metrics['mae'] /= test_count
metrics['precision'] /= test_count
metrics['recall'] /= test_count
df = pd.DataFrame(data=metrics, index=[0])
df.to_csv(
f'output/{self.experiment}/{self.optim.__name__}/{self.scheduler.__name__}/{self.optim.__name__}_{self.scheduler.__name__}_{self.epochs}_test_{epoch}_stats.csv',
index_label='epoch')
return True
def test_single_image(parser, threshold, save_random=True):
# obtain groundtruth data
metadata = parser.fetch_metadata()
# filter to get people who only wear mask
idxs = np.where(metadata["placement"] != "none")[0]
metadata["bboxes"] = metadata["bboxes"][idxs]
# do inference on mtcnn
image = cv2.imread(parser.image_path)
bboxes, _ = fetch_faces(image, return_landmarks=False)
recall = metrics.recall(metadata["bboxes"], bboxes, threshold=threshold)
# save results for ablation study
if save_random and np.random.uniform() <= 0.10:
filename = os.path.basename(parser.image_path)
save_image(image, filename, metadata["bboxes"], bboxes)
# recall for single inference image
return recall
def precision_recall_curve(x, y, start, end):
"""A function draws precision_recall_curve.
Args:
x (list or numpy array): 2D inputs, personalized recommendation list, sorted by the probabilities
y (list or numpy array): 2D inputs, actual selection list, sorted by the probabilities
start (int): start length of the recommendation list
end (int): end length of the recommendation list
"""
if not isinstance(start, int) or not isinstance(end, int):
raise TypeError("start and end must be an positive integer")
if start > end:
raise ValueError("start must be less than or equal to end")
if not isinstance(x, np.ndarray):
x = np.array(x)
if not isinstance(y, np.ndarray):
y = np.array(y)
if x.shape[0] != y.shape[0]:
raise ValueError("inputs must be same length")
if (x.shape[0] == 0) or (y.shape[0] == 0):
raise ValueError("inputs must not be empty")
precisions, recalls = [], []
for i in range(start, end + 1):
precisions.append(precision(x, y, i))
recalls.append(recall(x, y, i))
fig = go.Figure(go.Scatter(x=recalls, y=precisions, mode="lines+markers"))
fig.update_layout(
title="precision-recall curve",
xaxis_title="recall",
yaxis_title="precision",
)
fig.update_xaxes(range=(0.0, 1.1))
fig.update_yaxes(range=(0.0, 1.1))
fig.show()
def neural_net(train_data, train_labels, test_data, test_labels):
print(f'{NeuralNetwork.__name__}:')
# Create and train model
model = NeuralNetwork([
FlatDenseLayer((784, ), activation=tanh),
FlatDenseLayer((100, ), activation=tanh),
FlatDenseLayer((20, ), activation=tanh),
FlatDenseLayer((10, ), activation=sigmoid),
],
eta=0.01,
batch_size=64,
epochs=250)
model.train(train_data, train_labels)
# Predict 2000 validation set samples and calculate accuracy
test_data_2k = test_data[:len(test_labels)]
test_activations, test_pred = model.predict(test_data_2k)
# Print metrics
print('\nTest Accuracy: {:.02f}%\n'.format(
100 * accuracy(test_pred, test_labels)))
mat, classes = confusion_matrix(test_pred, test_labels)
print('Precision:\n{}\n'.format(
np.round(precision(test_pred, test_labels), 2)))
print('Recall:\n{}\n'.format(np.round(recall(test_pred, test_labels), 2)))
print('F1:\n{}\n'.format(np.round(f1_score(test_pred, test_labels), 2)))
print('Confusion Matrix:')
print(mat)
# Predict 10000 test set samples and save predictions
print('Predicting 10k samples...')
test_activations, test_pred = model.predict(test_data)
print(len(test_pred))
save_predictions(neural_net.__name__, test_pred)
print('Saved 10k predictions.\n')
def train(
self,
X,
Y,
max_epochs=100,
batch_size=256,
seed=42,
verbose=0, # training vars
use_imblearn=False,
imblearn_class=SMOTE(random_state=42, ratio=1.0), # imblearn vars
early_stopping_callback='default',
test_split=0.2, # early stopping vars
testing=False,
kfold_function=KFold,
kfold_splits=5,
): # testing vars (returns predictions from kfold)
''' trains self.model w/ selected parameters
@param X w/ shape (num_examples,num_error,num_sites) - where each number is integer of number of errors happened at that site
@param Y w/ shape (num_examples,) - where each number is integer that represents one class
@param max_epochs - maximum learning epochs, if not using kfold to approximate num_epochs: then this is num_epochs used for training
@param batch_size - neural network parameter
@param seed - random seed that is used everywhere for reproducability
@param verbose - if 0: no print output, else: print output
@param use_imblearn - boolean that decides to use resampling for training or not
@param imblearn_class - class from iblearn library used for resampling (doesn't do anything if use_imblearn = False)
@param early_stopping_callback - early stopping keras callback used to find num_epochs
@param test_split - test split used for early stopping
@param testing - if True: stops training after cross validation and returns predictions of all data across all kfolds
@param kfold_function - cross validation function from sklearn library (doesn't do anything if testing == False)
@param kfold_splits - number of cross validation splits, used in kfold_function (doesn't do anything if testing == False)
return if not testing: return keras history.history dict
else : list of len = kfold_splits, where each element is keras history.history dict
'''
data = DataPlaceholder()
if testing:
enum = enumerate(
kfold_function(n_splits=kfold_splits,
shuffle=True,
random_state=seed).split(X, Y))
if verbose != 0:
enum = tqdm(enum,
total=kfold_splits,
desc='kfold',
leave=False,
initial=0)
histories = []
for i, (index_train, index_valid) in enum:
data.train.x, data.val.x = X[index_train], X[index_valid]
data.train.y, data.val.y = Y[index_train], Y[index_valid]
if use_imblearn:
data.train.x, data.train.y = imblearn_sample(
data.train.x,
data.train.y,
imblearn_class,
verbose=verbose)
self.class_weights = get_class_weights(data.train.y,
self.num_classes)
data.train.x, data.train.y = self.change_inputs(
data.train.x, data.train.y)
data.val.x, data.val.y = self.change_inputs(
data.val.x, data.val.y)
self.create_model(**self.model_params)
keras_callbacks = [
PredictDataCallback(self.model, data.train.x, data.train.y,
''),
PredictDataCallback(self.model, data.val.x, data.val.y,
'val_')
]
history = self.model.fit(x=data.train.x,
y=data.train.y,
validation_data=(data.val.x,
data.val.y),
epochs=max_epochs,
batch_size=batch_size,
callbacks=[] + keras_callbacks,
verbose=verbose)
histories.append(history.history)
return histories
else:
if early_stopping_callback == 'default':
self.class_weights = get_class_weights(Y, self.num_classes)
early_stopping_functions = [
confusion_mse(),
recall(average='macro'),
precision(average='macro')
]
modes = ['min', 'max', 'max']
early_stopping_callback = MultipleMetricsEarlyStopping(
early_stopping_functions, modes=modes)
data.train.x, data.val.x, data.train.y, data.val.y = train_test_split(
X, Y, test_size=test_split, random_state=seed)
if use_imblearn:
data.train.x, data.train.y = imblearn_sample(data.train.x,
data.train.y,
imblearn_class,
verbose=verbose)
self.class_weights = get_class_weights(data.train.y,
self.num_classes)
data.train.x, data.train.y = self.change_inputs(
data.train.x, data.train.y)
data.val.x, data.val.y = self.change_inputs(data.val.x, data.val.y)
self.create_model(**self.model_params)
history = self.model.fit(x=data.train.x,
y=data.train.y,
validation_data=(data.val.x, data.val.y),
epochs=max_epochs,
batch_size=batch_size,
callbacks=[early_stopping_callback],
verbose=verbose)
return history.history
def evaluating(self, model, dataset, split):
"""
input:
model: (object) pytorch model
dataset: (object) dataset
split: (str) split of dataset in ['train', 'val', 'test']
return [overall_accuracy, precision, recall, f1-score, jaccard, kappa]
"""
args = self.args
oa, precision, recall, f1, jac, kappa = 0, 0, 0, 0, 0, 0
model.eval()
data_loader = DataLoader(dataset,
args.batch_size,
num_workers=4,
shuffle=False)
batch_iterator = iter(data_loader)
steps = len(dataset) // args.batch_size
start = time.time()
for step in range(steps):
x, y = next(batch_iterator)
x = Variable(x, volatile=True)
y = Variable(y, volatile=True)
if args.cuda:
x = x.cuda()
y = y.cuda()
# calculate pixel accuracy of generator
gen_y = model(x)
if self.is_multi:
gen_y = gen_y[0]
oa += metrics.overall_accuracy(gen_y.data, y.data)
precision += metrics.precision(gen_y.data, y.data)
recall += metrics.recall(gen_y.data, y.data)
f1 += metrics.f1_score(gen_y.data, y.data)
jac += metrics.jaccard(gen_y.data, y.data)
kappa += metrics.kappa(gen_y.data, y.data)
_time = time.time() - start
if not os.path.exists(os.path.join(Logs_DIR, 'statistic')):
os.makedirs(os.path.join(Logs_DIR, 'statistic'))
# recording performance of the model
nb_samples = steps * args.batch_size
basic_info = [
self.date, self.method, self.epoch, self.iter, nb_samples, _time
]
basic_info_names = [
'date', 'method', 'epochs', 'iters', 'nb_samples', 'time(sec)'
]
perform = [
round(idx / steps, 3)
for idx in [oa, precision, recall, f1, jac, kappa]
]
perform_names = [
"overall_accuracy", "precision", "recall", "f1-score", "jaccard",
"kappa"
]
cur_log = pd.DataFrame([basic_info + perform],
columns=basic_info_names + perform_names)
# save performance
if os.path.exists(
os.path.join(Logs_DIR, 'statistic', "{}.csv".format(split))):
logs = pd.read_csv(
os.path.join(Logs_DIR, 'statistic', "{}.csv".format(split)))
else:
logs = pd.DataFrame([])
logs = logs.append(cur_log, ignore_index=True)
logs.to_csv(os.path.join(Logs_DIR, 'statistic',
"{}.csv".format(split)),
index=False,
float_format='%.3f')
def train_final(self, epoch):
# Train SPM
self.SPM.train()
# Statistics
train_pred_loss, train_noise_loss, train_count = 0.0, 0.0, 0
metrics = {'mae': 0.0, 'precision': 0.0, 'recall': 0.0, 'f1': 0.0}
with tqdm(self.trainset) as train_bar:
for batch_idx, data in enumerate(train_bar):
# x: Input data, (batch_size, 3, 256, 256)
x = data['image'].to(self.device)
# y: GT label (batch_size, 1, 256, 256)
y = data['label'].to(self.device)
# Keep original x to compute metrics upon true data
original_x = x
# Prepare unsupervised labels to experiments
if self.experiment == 'Final-Average':
y_noise = data['unsup_labels'].to(self.device)
# y_noise: Taking mean unsupervised labels (batch_size, 1, 256, 256)
y_noise = torch.mean(y_noise, dim=1, keepdim=True)
elif self.experiment == 'Final-Noise':
y_noise = data['unsup_labels'].to(self.device)
# Normalizing y_noise between (0, 1)
y_n_min, y_n_max = min2d(y_noise), max2d(y_noise)
y_noise = (y_noise - y_n_min) / (y_n_max - y_n_min)
# pred: (batch_size, 1, 256, 256)
pred = self.SPM(x)['out']
y_pred = pred
# Noise training in Complete experiment
# Round 1
if epoch <= 20:
if self.experiment == 'Final-Noise':
repeat = 4
elif self.experiment == 'Final-Average':
repeat = 1
# pred: In round 1 repeat along dim=1, (batch_size, NUM_MAPS(4), 256, 256)
pred = torch.repeat_interleave(pred, repeats=repeat, dim=1)
y_pred = pred
# Round > 1
else:
# noise_prior: noise from NMM (batch_size, NUM_MAPS, 256, 256)
noise_prior = self.NMM.sample(data['index']).to(
self.device)
# noisy_pred: Noisy predictions after adding noise to predictions, (batch_size, NUM_MAPS, 256, 256)
noisy_pred = pred + noise_prior
# Range inside [0, 1] (see 3.2 after Eq 4)
noisy_min, noisy_max = min2d(noisy_pred), max2d(noisy_pred)
noisy_pred = (noisy_pred - noisy_min) / (noisy_max -
noisy_min)
y_pred = noisy_pred
# Compute BCE loss (Eq. 4)
pred_loss = self.loss_fn(y_pred, y_noise)
# Noise loss
noise_loss = 0
if self.experiment == 'Final-Average':
y_noise = data['unsup_labels'].to(self.device)
if epoch > 20:
# compute batch noise loss (Eq 6)
emp_var = torch.var(y_noise - pred, 1).reshape(
self.batch_size, -1) + 1e-16
prior_var, var_idx = self.NMM.get_index_multiple(
img_indexes=data['index'])
prior_var = torch.from_numpy(prior_var).float()
# Important Order for loss var needs to get close to emp_var
noise_loss = self.NMM.loss(prior_var, emp_var)
# Backprogation
self.optim.zero_grad()
# Total loss computed (Eq. 2)
total_loss = pred_loss + 0.01 * noise_loss # lambda: 0.01
total_loss.backward()
self.optim.step()
# Save batch losses
train_count += self.batch_size
train_pred_loss += to_numpy(pred_loss) * self.batch_size
if epoch <= 20:
train_noise_loss += noise_loss * self.batch_size
else:
train_noise_loss += to_numpy(noise_loss) * self.batch_size
# Compute metrics
y_pred_t = self.SPM(original_x)['out']
y_pred_t = to_numpy(y_pred_t)
y = to_numpy(y)
metrics['mae'] += mae(y_pred_t, y) * self.batch_size
metrics['precision'] += precision(y_pred_t,
y) * self.batch_size
metrics['recall'] += recall(y_pred_t, y) * self.batch_size
# Update tdqm
train_bar.set_description(
f'Train Epoch [{epoch}/{self.epochs}] Loss: { (train_noise_loss + train_pred_loss) / train_count:.4f} Noise: {train_noise_loss/train_count:.4f} Pred: {train_pred_loss / train_count:.4f}'
)
# Remove data from GPU memory
del x, y, y_noise, y_pred, y_pred_t, pred, data
torch.cuda.empty_cache()
# Prepare statistics
total_train_loss = (train_noise_loss + train_pred_loss) / train_count
train_noise_loss /= train_count
train_pred_loss /= train_count
metrics['mae'] /= train_count
metrics['precision'] /= train_count
metrics['recall'] /= train_count
metrics['f1'] = f1(metrics['precision'], metrics['recall'], beta2=0.3)
return total_train_loss, train_pred_loss, train_noise_loss, metrics[
'mae'], metrics['precision'], metrics['recall'], metrics['f1']
def train(model,
epoch_num,
start_epoch,
optimizer,
criterion,
exp_lr_scheduler,
data_set,
data_loader,
save_dir,
print_inter=200,
val_inter=3500,
):
writer = SummaryWriter(save_dir)
best_model_wts = model.state_dict()
best_f1 = 0
val_loss = 0
train_loss = 0
# running_loss = 20
step = -1
for epoch in range(start_epoch,epoch_num):
# train phase
# exp_lr_scheduler.step(epoch)
model.train(True) # Set model to training mode
for batch_cnt, data in enumerate(data_loader['train']):
step += 1
if step % val_inter == 0:
# val phase
model.eval()
loss_fn = weighted_mse_loss
# loss_fn = cross_entropy_loss_RCF
# loss_fn = criterion
t0 = time.time()
test_precisions, test_recalls, test_f1_scores, val_loss = predict(loss_fn, model, data_set['val'], data_loader['val'], counting=False)
t1 = time.time()
since = t1 - t0
logging.info('--' * 30)
# logging.info('current lr:%s' % exp_lr_scheduler.get_lr())
logging.info('%s epoch[%d] | val_loss: %.4f | precisions: %.4f | recalls: %.4f | f1_scores: %.4f | time: %d'
% (dt(), epoch, val_loss, test_precisions, test_recalls, test_f1_scores, since))
if test_f1_scores > best_f1:
best_f1 = test_f1_scores
best_model_wts = deepcopy(model.state_dict())
# save model
save_path1 = os.path.join(save_dir,
'weights-%d-%d-[%.3f].pth' % (epoch, batch_cnt, test_f1_scores))
torch.save(model.state_dict(), save_path1)
save_path2 = os.path.join(save_dir,
'optimizer-state.pth')
torch.save(optimizer.state_dict(), save_path2)
logging.info('saved model to %s' % (save_path1))
logging.info('--' * 30)
model.train(True)
imgs, masks, _ = data
imgs = Variable(imgs.cuda())
masks = Variable(masks.cuda(),requires_grad=False)
# zero the parameter gradients
optimizer.zero_grad()
outputs = model(imgs)
# outputs = outputs.view(-1, outputs.size()[2], outputs.size()[3])
# # print outputs.size(), masks.size()
# if outputs.size() != masks.size():
# outputs = F.upsample(outputs, size=masks.size()[-2:], mode='bilinear')
mask_loss = torch.zeros(1).cuda()
for o in outputs:
o = o.view(-1, o.size()[2], o.size()[3])
mask_loss = mask_loss + loss_fn(o, masks)
mask_loss = mask_loss
# loss.backward()
# print outputs.size()
# print masks.size()
# mask_loss = criterion(outputs, masks)
# mask_loss = cross_entropy_loss_RCF(outputs, masks)
# mask_loss = weighted_mse_loss(outputs, masks)
# loss = F.mean_absolute_error(outputs, masks)
###############################################cross entropy loss
train_loss = mask_loss
###############################################
train_loss.backward()
optimizer.step()
# running_loss = running_loss*0.95 + 0.05*loss.data[0]
# running_loss = loss.data[0]
# cal pixel acc
# _, preds = torch.max(outputs,1) # (bs, H, W)
# # preds = F.softmax(outputs,dim=1).round()[:, 1, :].long()
# batch_corrects = torch.sum((preds==masks).long()).data[0]
# batch_acc = 1.*batch_corrects / (masks.size(0)*masks.size(1)*masks.size(2))
output = outputs[-1]
output = output.view(-1, output.size()[2], output.size()[3])
true_positives, predicted_positives, possible_positives, union_areas = metrics_pred(output.data.cpu().numpy(),\
imgs.cpu().data.numpy(), masks.cpu().data.numpy())
train_precisions = precision(true_positives, predicted_positives)
train_recalls = recall(true_positives, possible_positives)
train_f1_scores = f1_score(train_recalls, train_precisions)
if step % print_inter == 0:
logging.info('%s [%d-%d] | train_loss: %.4f | precisions: %.4f | recalls: %.4f | f1_scores: %.4f'
% (dt(), epoch, batch_cnt, train_loss, train_precisions, train_recalls, train_f1_scores))
# plot image
if step % (print_inter) == 0:
smp_img = imgs[0] # (3, H, W)
true_hm = masks[0] #(H,W)
pred_hm = output[0]
imgs_to_plot = getPlotImg(smp_img, pred_hm, true_hm)
# for TensorBoard
imgs_to_plot = torch.from_numpy(imgs_to_plot.transpose((0,3,1,2))/255.0)
grid_image = make_grid(imgs_to_plot, 2)
writer.add_image('plotting',grid_image, step)
writer.add_scalar('train_loss', train_loss , step)
writer.add_scalar('val_loss', val_loss, step)
# save best model
save_path = os.path.join(save_dir,
'bestweights-[%.3f].pth' % (best_f1))
torch.save(best_model_wts, save_path)
logging.info('saved model to %s' % (save_path))
return best_f1, best_model_wts
def run(dtype):
""" opens specific dataset, splits 75-25 percent for train-test and runs extraction """
print("DATASET ", dtype)
global END_TRAINING
data = dataset.read('extraction', dtype)['intents']
intents = []
for case in data:
intent = []
for part in case['parts']:
intent.append(part)
intents.append(intent)
print("DATASET CASES #", len(intents))
highest_precision = 0
highest_recall = 0
highest_f1 = 0
highest_try = 0
num_tries = 0
while num_tries < 30:
num_tries += 1
END_TRAINING = None
n_samples = int(ceil(len(intents) * 0.75))
training = sample(intents, n_samples)
validation = sample(intents, len(intents) - n_samples)
diag = Dialogflow(evaluation=True)
diag.update_intent(INTENT_ID, training, False)
training_begin = diag.train_agent(training_callback)
time_elapsed = None
while True:
if END_TRAINING:
time_elapsed = (END_TRAINING - training_begin)
print("Training time: ", time_elapsed)
break
# time.sleep(50)
print("Testing...")
results = diag.detect_intent_texts(validation)
with open(config.EXTRACTION_RESULTS_PATH.format(dtype, num_tries),
'w') as csvfile:
csv_writer = csv.writer(csvfile, delimiter=',')
csv_writer.writerow([
"text", "recognized_entities", "expected_entities",
"training_time", "recall", "precision", "f1_score"
])
mean_precision = 0
mean_recall = 0
num_entries = len(results)
for result in results:
rec = metrics.recall(result['tp'], result['fn'])
prec = metrics.precision(result['tp'], result['fp'])
f1_sc = metrics.f1_score(prec, rec)
mean_precision += prec
mean_recall += rec
print(result['text'])
print('recall: ', rec)
print('precision: ', prec)
print('f1_score: ', f1_sc)
csv_writer.writerow([
result['text'], result['recognized_entities'],
result['expected_entities'], time_elapsed, rec, prec, f1_sc
])
mean_precision /= num_entries
mean_recall /= num_entries
mean_f1 = metrics.f1_score(mean_precision, mean_recall)
csv_writer.writerow(["Mean Precision", mean_precision])
csv_writer.writerow(["Mean Recall", mean_recall])
csv_writer.writerow(["Mean F1", mean_f1])
print("Mean Precision", mean_precision)
print("Mean Recall", mean_recall)
print("Mean F1", mean_f1)
if mean_f1 > highest_f1:
highest_f1 = mean_f1
highest_precision = mean_precision
highest_recall = mean_recall
highest_try = num_tries
print("Highest Precision", highest_precision)
print("Highest Recall", highest_recall)
print("Highest F1", highest_f1)
print("Highest Try", highest_try)
def feedback():
""" opens alpha dataset, splits 75-25 percent for train-feedback """
print("FEEDBACK")
global END_TRAINING
diag = Dialogflow(evaluation=True)
all_data = dataset.read('extraction', 'both')['intents']
all_intents = []
for case in all_data:
intent = []
for part in case['parts']:
intent.append(part)
all_intents.append(intent)
num_repeats = 1
with open(config.EXTRACTION_RESULTS_PATH.format('feedback', 'single'),
'w') as csvfile:
csv_writer = csv.writer(csvfile, delimiter=',')
csv_writer.writerow([
"repeat", "feedback_round", "text", "recognized_entities",
"expected_entities", "tp", "fp", "fn", "recall", "precision",
"f1_score"
])
for repeat in range(num_repeats):
n_samples = int(floor(len(all_intents) * 0.25))
training = sample(all_intents, n_samples)
feedback = sample(all_intents, len(all_intents) - n_samples)
print("DATASET CASES TRAIN #", len(training))
print("DATASET CASES FEEDBACK #", len(feedback))
diag.update_intent(INTENT_ID, training, False)
training_begin = diag.train_agent(training_callback)
time_elapsed = None
while True:
if END_TRAINING:
time_elapsed = (END_TRAINING - training_begin)
print("Training time: ", time_elapsed)
break
time.sleep(60)
print("Testing...")
results = []
shuffle(feedback)
for idx, feedback_case in enumerate(feedback):
print("intent", idx)
result = diag.detect_intent_texts([feedback_case])[0]
rec = metrics.recall(result['tp'], result['fn'])
prec = metrics.precision(result['tp'], result['fp'])
f1_sc = metrics.f1_score(prec, rec)
print(result['text'])
print('recall: ', rec)
print('precision: ', prec)
print('f1_score: ', f1_sc)
csv_writer.writerow([
repeat, idx, result['text'], result['recognized_entities'],
result['expected_entities'], result['tp'], result['fp'],
result['fn'], rec, prec, f1_sc
])
if result['fp'] != 0 or result['fn'] != 0:
training.append(feedback_case)
print("DATASET CASES TRAIN #", len(training))
diag.update_intent(INTENT_ID, training, False)
END_TRAINING = None
training_begin = diag.train_agent(training_callback)
time_elapsed = None
while True:
if END_TRAINING:
time_elapsed = (END_TRAINING - training_begin)
print("Training time: ", time_elapsed)
break
time.sleep(60)
csv_writer.writerow(["DATASET CASES TRAIN #", len(training)])
