def ensemble_results(folder_path, output_file_path, weights=None):
# intialise results size
instances = 0
models = 0
for root, dirs, files in os.walk(folder_path):
models = len(files)
for file in files:
result = pd.read_csv(os.path.join(folder_path, file), sep='\t')
instances = len(result)
break
break
predictions = np.zeros((instances, models))
# read model predictions
for root, dirs, files in os.walk(folder_path):
i = 0
files.sort()
for file in files:
result = pd.read_csv(os.path.join(folder_path, file), sep='\t')
preds = encode(result['predictions'])
predictions[:, i] = preds
i += 1
final_predictions = majority_class_ensemble(predictions)
# final_predictions = weighted_ensemble(predictions, weights)
# decode final predictions
final_predictions = decode(final_predictions)
output_file = open(output_file_path, 'w', encoding='utf-8')
for pred in final_predictions:
output_file.write(pred + '\n')
output_file.close()
if INCLUDE_RAW_PREDICTIONS:
# calculate average of raw prediction
avg_raw_preds, final_raw_preds = average_predictions(dev_raw_preds)
dev['raw-predictions'] = final_raw_preds
avg_raw_preds, final_raw_preds = average_predictions(test_raw_preds)
test['raw-predictions'] = final_raw_preds
else:
model.train_model(train, f1=f1, accuracy=sklearn.metrics.accuracy_score)
predictions, raw_outputs = model.predict(dev_sentences)
dev['predictions'] = predictions
test_predictions, test_raw_outputs = model.predict(test_sentences)
test['predictions'] = test_predictions
dev['predictions'] = decode(dev['predictions'])
dev['class'] = decode(dev['class'])
test['predictions'] = decode(test['predictions'])
if INCLUDE_RAW_PREDICTIONS:
dev['raw-predictions'] = decode(dev['raw-predictions'])
test['raw-predictions'] = decode(test['raw-predictions'])
time.sleep(5)
print("Started Evaluation")
results = evaluatation_scores(dev, 'class', 'predictions', labels, pos_label)
print_results(results)
save_eval_results(results, os.path.join(TEMP_DIRECTORY, DEV_EVAL_FILE))
def precision(y_true, y_pred):
y_true = decode(y_true)
y_pred = decode(y_pred)
return precision_score(y_true, y_pred, labels=labels, pos_label=pos_label)
def f1(y_true, y_pred):
y_true = decode(y_true)
y_pred = decode(y_pred)
return f1_score(y_true, y_pred, labels=labels, pos_label=pos_label)
def recall(y_true, y_pred):
y_true = decode(y_true)
y_pred = decode(y_pred)
return recall_score(y_true, y_pred, labels=labels, pos_label=pos_label)