def get_values(data_input):
"""
For each given depth, calculate the maximum depth
and accuracies of the unpruned and pruned tree
"""
data = data_input.copy()
np.random.shuffle(data)
t_split = 0.8
v_split = 0.9
train = data[:int(len(data) * t_split)]
validation = data[int(len(data) * t_split):(int(len(data) * v_split))]
test = data[int(len(data) * v_split):]
depths = []
pruned_depths = []
accuracies = []
pruned_accuracies = []
for i in range(1, 20):
model = binarySearchTree(train, limit=i)
depth = model.get_max_depth()
y_pred = [x.item() for x in model.predict(test[:, :-1])]
y_true = [int(val.item()) for val in test[:, -1]]
depths.append(depth)
cm = ev.confusion_matrix(y_true, y_pred)
accuracies.append(ev.get_class_rate(cm))
model.prune_tree(validation)
depth2 = model.get_max_depth()
y_pred2 = [x.item() for x in model.predict(test[:, :-1])]
y_true2 = [int(val.item()) for val in test[:, -1]]
pruned_depths.append(depth2)
cm2 = ev.confusion_matrix(y_true2, y_pred2)
pruned_accuracies.append(ev.get_class_rate(cm2))
return depths, accuracies, pruned_depths, pruned_accuracies
def grow_binary_trees(data_input, stratified=False, pruning=False):
'''
Splits the data set into 10 folds and uses each fold as a testing set once.
Calculates metrics for each fold to get standard errors.
Data split - stratified or fully random. #TODO
'''
# Create 10 Folds: Group indices into 10 folds
data = data_input.copy()
np.random.shuffle(data)
data = data.reshape((10, -1, 8))
if pruning:
results, results_pruned = {}, {}
for i, test_fold in enumerate(data):
results[i] = np.zeros((4, 4))
results_pruned[i] = np.zeros((4, 4))
train_val_data = np.delete(data, i, axis=0)
for j, val_fold in enumerate(train_val_data):
train_data = np.vstack(np.delete(train_val_data, j, axis=0))
tree = binarySearchTree(train_data)
results[i] += ev.confusion_matrix(test_fold[:, -1].astype(int),
tree.predict(test_fold),
normalised=True)
tree.prune_tree(val_fold)
results_pruned[i] += ev.confusion_matrix(
test_fold[:, -1].astype(int),
tree.predict(test_fold),
normalised=True)
results[i] /= 9
results_pruned[i] /= 9
return results, results_pruned
else:
results = {}
for i, test_fold in enumerate(data):
train_data = np.vstack(np.delete(data, i, axis=0))
tree = binarySearchTree(train_data)
results[i] = ev.confusion_matrix(test_fold[:, -1].astype(int),
tree.predict(test_fold),
normalised=True)
return results
def get_f1(self, data):
"""
Calculates the F1 score
"""
pred = self.predict(data[:, :-1])
cm = ev.confusion_matrix(data[:, -1], pred)
return ev.get_f1_scores(cm)
def predict(self):
correct_labels = []
predicted_labels = []
# Perform MAP classification
with open(DATA_DIR + self.test_file) as f:
for line in f:
doc = line.split()
correct_labels.append(self.classes[doc[0]])
map_classifier = np.zeros(self.num_classes)
# Calculate decision function on test doc features for each class
for model_class in range(self.num_classes):
map_classifier[model_class] = np.array(
[math.log(float(self.doc_counts[model_class]/np.sum(self.doc_counts)))])
if self.runmode == 'multinomial':
for word in doc[1:]:
word = word.split(':')
if word[0] in self.model:
map_classifier[model_class] += math.log(self.model[word[0]][model_class])
elif self.runmode == 'bernoulli':
words = [word.split(':')[0] for word in doc[1:]]
for word in self.model:
if word in words:
map_classifier[model_class] += math.log(self.model[word][model_class])
else:
map_classifier[model_class] += math.log(1. - self.model[word][model_class])
# Get best classified class
predicted_labels.append(np.argmax(map_classifier))
# Get total accuracy
correct_labels = np.array(correct_labels)
predicted_labels = np.array(predicted_labels)
accuracy = calc_accuracy(correct_labels, predicted_labels)
logger.info('NB model is {0:.2f}% accurate on the {1} data with k = {2}.'
.format(accuracy, self.runmode, self.k))
# Get confusion matrix with class accuracies
cm = confusion_matrix(correct_labels, predicted_labels, self.num_classes)
class_accuracies = [cm[n][n] for n in range(self.num_classes)]
for n, x in enumerate(class_accuracies):
logger.info('Class {0} has an accuracy of {1:.2f}%'.format(self.class_names[n], 100 * x))
# Plot confusion matrix
plt.figure(figsize=(30,30))
plt.imshow(cm, cmap=plt.get_cmap('Greens'), interpolation='nearest')
plt.title('Confusion Matrix')
plt.xticks(np.arange(self.num_classes), self.class_names, fontsize = 8)
plt.yticks(np.arange(self.num_classes), self.class_names, fontsize = 10)
plt.xlabel('Predictions')
plt.ylabel('Truths')
plt.colorbar()
plt.show()
def task_3_logistic(x, y, x_test, y_test, args):
accuracies = []
sizes = np.linspace(10, 200, num=20)
N = y.shape[0]
for size in sizes:
acc = 0
for i in range(50):
rand = np.random.randint(int(N), size=int(size))
m = LogisticRegression(x[rand], y[rand])
m.fit(lr=args[0], eps=args[1], regularization=args[2])
pred = m.predict(x_test)
cm = evaluation.confusion_matrix(y_test, pred)
acc += evaluation.accuracy(cm)
accuracies.append(acc/50)
return accuracies, sizes
def evaluate_link(class_match_set, class_nonmatch_set, true_match_set,
all_comparisons):
# Linkage evaluation
linkage_result = evaluation.confusion_matrix(class_match_set,
class_nonmatch_set,
true_match_set,
all_comparisons)
accuracy = evaluation.accuracy(linkage_result)
precision = evaluation.precision(linkage_result)
recall = evaluation.recall(linkage_result)
fmeasure = evaluation.fmeasure(linkage_result)
print('Linkage evaluation:')
print(' Accuracy: %.6f' % (accuracy))
print(' Precision: %.6f' % (precision))
print(' Recall: %.6f' % (recall))
print(' F-measure: %.6f' % (fmeasure))
print('')
def evaluate(dataset, args, verbose=True):
"""Evaluate the audio tagging predictions and record the results.
Args:
dataset (Dataset): Information about the dataset.
args: Named tuple of configuration arguments.
verbose (bool): Whether to print the results.
Returns:
pd.DataFrame: The results of the evaluation.
"""
import evaluation
# Load grouth truth data and predictions
path = os.path.join(args.prediction_path, args.training_id,
f'{dataset.name}.csv')
y_pred = pd.read_csv(path, index_col=0)
df_true = pd.read_csv(dataset.metadata_path, index_col=0)
y_true = pd.get_dummies(df_true.loc[y_pred.index].label).values
y_pred = y_pred.values
# Evaluate audio tagging performance
scores = evaluation.evaluate(y_true, y_pred)
C = evaluation.confusion_matrix(y_true, y_pred)
# Ensure output directory exist
result_path = os.path.join(args.result_path, args.training_id)
os.makedirs(result_path, exist_ok=True)
# Write results to disk
output_path = os.path.join(result_path, '%s_{}.csv' % dataset.name)
scores.to_csv(output_path.format('scores'))
C.to_csv(output_path.format('cmatrix'))
# Print results (optional)
if verbose:
print('Confusion Matrix:\n', C.values, '\n')
pd.options.display.float_format = '{:,.3f}'.format
print(str(scores))
return scores
def task_3_naive(df, test_df, label, cont=[], cat=[], bin=[]):
accuracies = []
sizes = np.linspace(10, 200, num=20)
N = df.shape[0]
for size in sizes:
acc = 0
for i in range(25):
print(size, i)
rand = np.random.randint(int(N), size=int(size))
m = NaiveBayes(df.loc[rand], label, continuous=cont, categorical=cat, binary=bin)
pred = test_df.apply(m.predict, axis=1)
cm = evaluation.confusion_matrix(test_df[label].to_numpy(), pred.to_numpy())
acc += evaluation.accuracy(cm)
accuracies.append(acc/25)
return accuracies, sizes
def new_model(data1, data2):
"""
Training and testing a tree on the noisy dataset without those observations above
Results in a 97% accuracy
"""
diff_obs = check_signals_only(data1, data2)
df2 = pd.DataFrame(data2)
clean_removed = pd.concat([df2, diff_obs,
diff_obs]).drop_duplicates(keep=False)
clean_removed_dataset = clean_removed.to_numpy()
np.random.shuffle(clean_removed_dataset)
split = 0.7
train = clean_removed_dataset[:int(len(clean_removed_dataset) * split)]
test = clean_removed_dataset[int(len(clean_removed_dataset) * split):]
model = trees.binarySearchTree(train)
print('Max depth is', model.get_max_depth())
y_pred = model.predict(test[:, :-1])
cm = ev.confusion_matrix(test[:, -1], y_pred)
i = ev.get_metrics(cm, printout=True)
ev.plot_conf_matrix(cm)
def train(self, epochs=10):
if self.__xtrain and self.__ytrain and self.__xtest and self.__ytest:
pass
else:
self.__load_dataset()
# Open a writer to write summaries.
self.__writer = tf.summary.FileWriter(self.__TMP_DIR,
self.__session.graph)
for epoch in range(epochs):
#learning_rate = self.__session.run(self.__lr)
#print('Learning rate', learning_rate)
average_loss = 0
num_steps = len(self.__flow)
for step in tqdm.tqdm(range(num_steps),
desc='Epoch ' +
str(epoch + 1 + self.__GLOBAL_EPOCH) + '/' +
str(epochs + self.__GLOBAL_EPOCH)):
batch, label = self.__flow.next()
run_metadata = tf.RunMetadata()
_, l = self.__session.run([self.__train_op, self.__loss],
feed_dict={
self.__images: batch,
self.__labels: label
},
run_metadata=run_metadata)
average_loss += l
# print loss and accuracy on test set at the and of each epoch
if step == num_steps - 1:
y_true = []
y_pred = []
for i in range(len(self.__xtest)):
prediction = self.__session.run(
self.__labels_predicted,
feed_dict={self.__images: [self.__xtest[i]]},
run_metadata=run_metadata)
y_true.append(self.__ytest[i])
y_pred.append(prediction[0])
accuracy = ev.accuracy(y_true, y_pred)
print('Loss:', str(average_loss / step), '\tAccuracy:',
accuracy)
with open(self.__TMP_DIR + '/log.txt',
'a',
encoding='utf8') as f:
f.write(
str(accuracy) + ' ' + str(average_loss / step) +
'\n')
if step == (num_steps - 1) and epoch + 1 == epochs:
s = self.__session.run(self.__global_step)
self.__writer.add_run_metadata(run_metadata,
'step%d' % step,
global_step=s)
self.__saver.save(self.__session,
os.path.join(self.__TMP_DIR, 'model.ckpt'))
dp.global_epoch(self.__TMP_DIR + 'epoch.txt',
update=self.__GLOBAL_EPOCH + epochs)
self.__writer.close()
pg.generate_accuracy_plot(data_dir=self.__TMP_DIR)
pg.generate_loss_plot(data_dir=self.__TMP_DIR)
conf_mat = ev.confusion_matrix(y_true, y_pred, len(self.__SENTIMENTS))
pg.generate_confusion_matrix_plot(conf_mat,
self.__SENTIMENTS,
data_dir=self.__TMP_DIR)
pg.generate_confusion_matrix_plot(conf_mat,
self.__SENTIMENTS,
normalize=True,
data_dir=self.__TMP_DIR)
estimator.fit(X_train_normalized, y_train_row)
#print(estimator.cv_results_)
print(estimator.best_score_)
print(estimator.best_params_)
### Plot the best number of components ###
pca.fit(X_train_normalized)
plt.figure(1, figsize=(4, 3))
plt.clf()
plt.axes([.2, .2, .7, .7])
plt.plot(pca.explained_variance_, linewidth=2)
plt.axis('tight')
plt.xlabel('n_components')
plt.ylabel('explained_variance')
plt.axvline(estimator.best_estimator_.named_steps['pca'].n_components,
linestyle=':',
label='n_components chosen')
plt.legend(prop=dict(size=12))
plt.show()
### Evaluation ###
prediction = estimator.predict(X_train_normalized)
evaluation.visualize_errors_by_genre(y_train_row, prediction, y_label_names)
evaluation.confusion_matrix(y_train_row, prediction, y_label_names)
###Visualize grid search
results = estimator.cv_results_
evaluation.visualize_gridsearch(results, 'logistic__C', 'C')
#evaluation.visualize_gridsearch(results,'pca__n_components', 'n_components')
# Blocking evaluation
#
rr = evaluation.reduction_ratio(num_comparisons, all_comparisons)
pc = evaluation.pairs_completeness(cand_rec_id_pair_list, true_match_set)
pq = evaluation.pairs_quality(cand_rec_id_pair_list, true_match_set)
print('Blocking evaluation:')
print(' Reduction ratio: %.3f' % (rr))
print(' Pairs completeness: %.3f' % (pc))
print(' Pairs quality: %.3f' % (pq))
print('')
# Linkage evaluation
#
linkage_result = evaluation.confusion_matrix(class_match_set,
class_nonmatch_set,
true_match_set, all_comparisons)
accuracy = evaluation.accuracy(linkage_result)
precision = evaluation.precision(linkage_result)
recall = evaluation.recall(linkage_result)
fmeasure = evaluation.fmeasure(linkage_result)
print('Linkage evaluation:')
print(' Accuracy: %.3f' % (accuracy))
print(' Precision: %.3f' % (precision))
print(' Recall: %.3f' % (recall))
print(' F-measure: %.3f' % (fmeasure))
print('')
linkage_time = loading_time + blocking_time + comparison_time + \
def main(blocking_fn,
classification_fn,
threshold,
minthresh,
weightvec,
blocking_attrs,
func_list,
save=False):
# ******** In lab 3, explore different attribute sets for blocking ************
# The list of attributes to use for blocking (all must occur in the above
# attribute lists)
blocking_attrA_list = blocking_attrs
blocking_attrB_list = blocking_attrs
# ******** In lab 4, explore different comparison functions for different ****
# ******** attributes ****
# The list of tuples (comparison function, attribute number in record A,
# attribute number in record B)
#
exact_comp_funct_list = [
(comparison.exact_comp, 1, 1), # First name
(comparison.exact_comp, 2, 2), # Middle name
(comparison.exact_comp, 3, 3), # Last name
(comparison.exact_comp, 8, 8), # Suburb
(comparison.exact_comp, 10, 10), # State
]
approx_comp_funct_list = [
(func_list[0], 1, 1), # First name
(func_list[1], 2, 2), # Middle name
(func_list[2], 3, 3), # Last name
(func_list[3], 7, 7), # Address
(func_list[4], 8, 8), # Suburb
(func_list[5], 10, 10), # State
]
# =============================================================================
#
# Step 1: Load the two datasets from CSV files
start_time = time.time()
recA_dict = loadDataset.load_data_set(datasetA_name, rec_idA_col, \
attrA_list, headerA_line)
recB_dict = loadDataset.load_data_set(datasetB_name, rec_idB_col, \
attrB_list, headerB_line)
# Load data set of true matching pairs
#
true_match_set = loadDataset.load_truth_data(truthfile_name)
loading_time = time.time() - start_time
# -----------------------------------------------------------------------------
# Step 2: Block the datasets
def genericBlock(block_function='none',
recA_dict=recA_dict,
recB_dict=recB_dict,
blocking_attrA_list=blocking_attrA_list,
blocking_attrB_list=blocking_attrB_list):
start_time = time.time()
# Select one blocking technique
if block_function == 'none':
# No blocking (all records in one block)
#
resultA = blocking.noBlocking(recA_dict)
resultB = blocking.noBlocking(recB_dict)
if block_function == 'attr':
# Simple attribute-based blocking
#
resultA = blocking.simpleBlocking(recA_dict, blocking_attrA_list)
resultB = blocking.simpleBlocking(recB_dict, blocking_attrB_list)
if block_function == 'soundex':
# Phonetic (Soundex) based blocking
#
resultA = blocking.phoneticBlocking(recA_dict, blocking_attrA_list)
resultB = blocking.phoneticBlocking(recB_dict, blocking_attrB_list)
if block_function == 'slk':
# Statistical linkage key (SLK-581) based blocking
#
fam_name_attr_ind = 3
giv_name_attr_ind = 1
dob_attr_ind = 6
gender_attr_ind = 4
resultA = blocking.slkBlocking(recA_dict, fam_name_attr_ind, \
giv_name_attr_ind, dob_attr_ind, \
gender_attr_ind)
resultB = blocking.slkBlocking(recB_dict, fam_name_attr_ind, \
giv_name_attr_ind, dob_attr_ind, \
gender_attr_ind)
block_time = time.time() - start_time
# Print blocking statistics
#
# blocking.printBlockStatistics(resultA, resultB)
return resultA, resultB, block_time
blockA_dict, blockB_dict, blocking_time = genericBlock(
block_function=blocking_fn)
# -----------------------------------------------------------------------------
# Step 3: Compare the candidate pairs
start_time = time.time()
sim_vec_dict = comparison.compareBlocks(blockA_dict, blockB_dict, \
recA_dict, recB_dict, \
approx_comp_funct_list)
comparison_time = time.time() - start_time
# -----------------------------------------------------------------------------
# Step 4: Classify the candidate pairs
def genericClassification(classification_function='exact',
sim_vec_dict=sim_vec_dict,
sim_threshold=threshold,
min_sim_threshold=minthresh,
weight_vec=weightvec,
true_match_set=true_match_set):
start_time = time.time()
if classification_function == 'exact':
# Exact matching based classification
class_match_set1, class_nonmatch_set1 = \
classification.exactClassify(sim_vec_dict)
if classification_function == 'simthresh':
# Similarity threshold based classification
#
class_match_set1, class_nonmatch_set1 = \
classification.thresholdClassify(sim_vec_dict, sim_threshold)
if classification_function == 'minsim':
# Minimum similarity threshold based classification
#
class_match_set1, class_nonmatch_set1 = \
classification.minThresholdClassify(sim_vec_dict,
min_sim_threshold)
if classification_function == 'weightsim':
# Weighted similarity threshold based classification
#
# weight_vec = [1.0] * len(approx_comp_funct_list)
# Lower weights for middle name and state
#
# weight_vec = [2.0, 1.0, 2.0, 2.0, 2.0, 1.0]
class_match_set1, class_nonmatch_set1 = \
classification.weightedSimilarityClassify(sim_vec_dict,
weight_vec,
sim_threshold)
if classification_function == 'dt':
# A supervised decision tree classifier
#
class_match_set1, class_nonmatch_set1 = \
classification.supervisedMLClassify(sim_vec_dict, true_match_set)
class_time = time.time() - start_time
return class_match_set1, class_nonmatch_set1, class_time
threshold = minthresh
class_match_set, class_nonmatch_set, classification_time = genericClassification(
classification_fn)
# -----------------------------------------------------------------------------
# Step 5: Evaluate the classification
# Initialise dictionary of results
dict = {}
# Get the number of record pairs compared
#
num_comparisons = len(sim_vec_dict)
# Get the number of total record pairs to compared if no blocking used
#
all_comparisons = len(recA_dict) * len(recB_dict)
# Get the list of identifiers of the compared record pairs
#
cand_rec_id_pair_list = sim_vec_dict.keys()
# Blocking evaluation
#
rr = evaluation.reduction_ratio(num_comparisons, all_comparisons)
pc = evaluation.pairs_completeness(cand_rec_id_pair_list, true_match_set)
pq = evaluation.pairs_quality(cand_rec_id_pair_list, true_match_set)
# Linkage evaluation
#
linkage_result = evaluation.confusion_matrix(class_match_set,
class_nonmatch_set,
true_match_set,
all_comparisons)
accuracy = evaluation.accuracy(linkage_result)
precision = evaluation.precision(linkage_result)
recall = evaluation.recall(linkage_result)
fmeasure = evaluation.fmeasure(linkage_result)
# print('Linkage evaluation:')
# print(' Accuracy: %.3f' % (accuracy))
# print(' Precision: %.3f' % (precision))
# print(' Recall: %.3f' % (recall))
# print(' F-measure: %.3f' % (fmeasure))
# print('')
linkage_time = loading_time + blocking_time + comparison_time + \
classification_time
# print('Total runtime required for linkage: %.3f sec' % (linkage_time))
# Export blocking metrics
dict['blocking_fn'] = blocking_fn
dict['classification_fn'] = classification_fn
dict['threshold'] = threshold
dict['min_thresh'] = minthresh
dict['weight_vec'] = weightvec
dict['blocking_attrs'] = blocking_attrs
dict['comp_funcs'] = func_list
dict['num_comparisons'] = num_comparisons
dict['all_comparisons'] = all_comparisons
# dict['cand_rec_id_pair_list'] = cand_rec_id_pair_list
dict['rr'] = rr
dict['pc'] = pc
dict['pq'] = pq
dict['blocking_time'] = blocking_time
# dict['linkage_result'] = linkage_result
dict['accuracy'] = accuracy
dict['precision'] = precision
dict['recall'] = recall
dict['fmeasure'] = fmeasure
dict['linkage_time'] = linkage_time
# Save results
if save:
saveLinkResult.save_linkage_set('final_results.txt', class_match_set)
# Return results
return dict
)
if limit == '':
print('No limit entered')
limit = None
else:
limit = int(limit)
np.random.shuffle(data)
train = data[:int(len(data) * split)]
test = data[int(len(data) * split):]
model = binarySearchTree(train, limit=limit)
print('Max depth of tree is', model.get_max_depth())
y_pred = model.predict(test[:, :-1])
cm = ev.confusion_matrix(test[:, -1], y_pred)
i = ev.get_metrics(cm, printout=True)
print('To continue, you may need to close the plot windows first')
ev.plot_conf_matrix(cm)
print('Visualising the pruned trees')
model.visualise_tree()
input('\nTo restart, hit enter\n')
if model == '2':
split = float(input('Enter training data split value, eg 0.7\n'))
while True:
if split < 0 or split > 1:
print('Invalid split entered')
else:
break
# 1. Compare accuracy of naive bayes and logistic regression
# Get cross validation accuracy for 5-fold cv
print("Ionosphere validation accuracy (default parameters):")
evaluation.cross_validation(5, ionosphere_train_features, ionosphere_train_labels, model=LogisticRegression)
# Grid search for optimal hyperparameters
print("Ionosphere grid search hyperparameters:")
ionosphere_max_val_acc, ionosphere_arg_max = evaluation.grid_search(learning_rates=lrs, epsilons=eps, lambdas=lamdas, x=ionosphere_train_features, y=ionosphere_train_labels, model=LogisticRegression)
# Accuracy on test split - train with best hyperparameters
print("Ionosphere test accuracy:")
logistic_ionosphere = LogisticRegression(ionosphere_train_features, ionosphere_train_labels)
logistic_ionosphere.fit(lr=ionosphere_arg_max[0], eps=ionosphere_arg_max[1], regularization=ionosphere_arg_max[2])
ionosphere_prediction = logistic_ionosphere.predict(ionosphere_test_features)
cm_ionosphere = evaluation.confusion_matrix(ionosphere_test_labels, ionosphere_prediction)
print("Accuracy:", evaluation.accuracy(cm_ionosphere), "Precision:", evaluation.precision(cm_ionosphere), "Recall:", evaluation.true_positive(cm_ionosphere), "F1:", evaluation.f_score(cm_ionosphere))
# 5-fold CV for naive bayes
print("Ionosphere validation accuracy (naive bayes):")
evaluation.cross_validation_naive(5, ionosphere_dataset.train_data, NaiveBayes, ionosphere_dataset.label_column, ionosphere_dataset.feature_columns)
naive_ionosphere = NaiveBayes(ionosphere_dataset.train_data, ionosphere_dataset.label_column, continuous=ionosphere_dataset.feature_columns)
print("Ionosphere test accuracy (naive bayes):")
ionosphere_pred_naive = ionosphere_dataset.test_data.apply(naive_ionosphere.predict, axis=1)
cm_ionosphere_naive = evaluation.confusion_matrix(ionosphere_test_labels, ionosphere_pred_naive.to_numpy())
print("Accuracy:", evaluation.accuracy(cm_ionosphere_naive), "Precision:", evaluation.precision(cm_ionosphere_naive), "Recall:", evaluation.true_positive(cm_ionosphere_naive), "F1:", evaluation.f_score(cm_ionosphere_naive))
def predict(self, info=True):
if self.runmode == 'digits':
test_label_path = DATA_DIR + '/testlabels'
test_images_path = DATA_DIR + '/testimages'
elif self.runmode == 'faces':
test_label_path = DATA_DIR + '/facedatatestlabels'
test_images_path = DATA_DIR + '/facedatatest'
correct_labels = []
with open(test_label_path) as f:
for line in f:
correct_labels.append(int(line))
num_images = len(correct_labels)
# Using python list instead of np since np chararrays replace spaces with empty string
test_images = [[None for _ in range(self.row)] for _ in range(num_images)]
with open(test_images_path) as f:
for n in range(num_images):
for y in range(self.row):
test_images[n][y] = list(f.readline().rstrip('\n'))
predicted_labels = []
for n in range(num_images):
map_classifier = np.zeros(self.num_classes)
for num in range(self.num_classes):
map_classifier[num] = np.array([math.log(self.num_counts[num]/np.sum(self.num_counts))])
for y in range(self.row):
for x in range(self.col):
if test_images[n][y][x] == ' ':
map_classifier[num] += math.log(self.model[num][y][x][0])
if self.num_features == 3:
if test_images[n][y][x] == '+':
map_classifier[num] += math.log(self.model[num][y][x][1])
elif test_images[n][y][x] == '#':
map_classifier[num] += math.log(self.model[num][y][x][2])
else:
if test_images[n][y][x] in ['+', '#']:
map_classifier[num] += math.log(self.model[num][y][x][1])
predicted_label = np.argmax(map_classifier)
predicted_labels.append((predicted_label, map_classifier[predicted_label], n))
truths = np.array(correct_labels)
predictions = np.array([x[0] for x in predicted_labels])
accuracy = calc_accuracy(truths, predictions)
logger.info('NB model is {0:.2f}% accurate on the {1} data with k = {2}.'.format(accuracy, self.runmode, self.k))
if info:
cm = confusion_matrix(truths, predictions, self.num_classes)
class_accuracies = [cm[n][n] for n in range(self.num_classes)]
# Class accuracies
for n, x in enumerate(class_accuracies):
logger.info('Class {0} has an accuracy of {1:.2f}%'.format(n, 100 * x))
# Confusion matrix
plt.figure()
plt.imshow(cm, cmap=plt.get_cmap('Greens'), interpolation='nearest')
plt.title('Confusion Matrix')
plt.xticks(np.arange(self.num_classes))
plt.yticks(np.arange(self.num_classes))
plt.xlabel('Predictions')
plt.ylabel('Truths')
# Test images with the highest and lowest posterior probability
# Sorts from lowest to highest by class, then by posterior probability
sorted_predictions = sorted(predicted_labels)
class_indices = []
for x in range(len(sorted_predictions)):
if sorted_predictions[x][0] != sorted_predictions[x-1][0]:
class_indices.append(x)
for x in range(len(class_indices)):
curr_class = sorted_predictions[class_indices[x]][0]
lowest_idex = sorted_predictions[class_indices[x]][2]
try:
highest_idx = sorted_predictions[class_indices[x+1]-1][2]
except IndexError:
highest_idx = sorted_predictions[len(sorted_predictions)-1][2]
best_test_image = [[0 if x in ['#', '+'] else 1 for x in y] for y in test_images[highest_idx]]
worst_test_image = [[0 if x in ['#', '+'] else 1 for x in y] for y in test_images[lowest_idex]]
plt.figure()
plt.suptitle('Class {0}'.format(curr_class))
plt.subplot(1, 2, 1)
plt.imshow(best_test_image, cmap=plt.get_cmap('Greys_r'))
plt.title('Highest')
plt.xticks([])
plt.yticks([])
plt.subplot(1, 2, 2)
plt.title('Lowest')
plt.xticks([])
plt.yticks([])
plt.imshow(worst_test_image, cmap=plt.get_cmap('Greys_r'))
# Odds ratio for the four worst classes
cm_ravel = np.ravel(cm)
least_accurate_pairs = cm_ravel.argsort()[:4]
least_accurate_pairs = [(x % self.num_classes, math.floor(x / self.num_classes)) for x in least_accurate_pairs]
if self.num_features == 2 and self.runmode == 'digits':
for i, j in least_accurate_pairs:
log_likelihood_one = np.zeros((self.col, self.row))
log_likelihood_two = np.zeros((self.col, self.row))
odds_ratio = np.zeros((self.col, self.row))
for y in range(self.row):
for x in range(self.col):
log_likelihood_one[y][x] = math.log(self.model[i][y][x][1])
log_likelihood_two[y][x] = math.log(self.model[j][y][x][1])
odds_ratio[y][x] = math.log(self.model[i][y][x][1] / self.model[j][y][x][1])
plt.figure()
plt.subplot(1, 3, 1)
plt.imshow(log_likelihood_one, interpolation='nearest')
plt.title('Likelihood of {0}'.format(i))
plt.xticks([])
plt.yticks([])
cbar = plt.colorbar(shrink=0.35)
cbar.set_ticks(np.arange(np.amin(log_likelihood_one), np.amax(log_likelihood_one), step=2, dtype=np.int8))
for t in cbar.ax.get_yticklabels():
t.set_horizontalalignment('right')
t.set_x(4)
plt.subplot(1, 3, 2)
plt.imshow(log_likelihood_two, interpolation='nearest')
plt.title('Likelihood of {0}'.format(j))
plt.xticks([])
plt.yticks([])
cbar = plt.colorbar(shrink=0.35)
cbar.set_ticks(np.arange(np.amin(log_likelihood_two), np.amax(log_likelihood_two), step=2, dtype=np.int8))
for t in cbar.ax.get_yticklabels():
t.set_horizontalalignment('right')
t.set_x(4)
plt.subplot(1, 3, 3)
plt.imshow(odds_ratio, interpolation='nearest')
plt.title('Odds ratio')
plt.xticks([])
plt.yticks([])
cbar = plt.colorbar(shrink=0.35)
cbar.set_ticks(np.arange(np.amin(odds_ratio), np.amax(odds_ratio), step=2, dtype=np.int8))
for t in cbar.ax.get_yticklabels():
t.set_horizontalalignment('right')
t.set_x(4)
plt.show()
def predict(self, info=True):
test_label_path = DATA_DIR + '/testlabels'
test_images_path = DATA_DIR + '/testimages'
correct_labels = []
with open(test_label_path) as f:
for line in f:
correct_labels.append(int(line))
num_images = len(correct_labels)
test_images = [[None for _ in range(self.row)] for _ in range(num_images)]
with open(test_images_path) as f:
for n in range(num_images):
for y in range(self.row):
test_images[n][y] = list(f.readline().rstrip('\n'))
predicted_labels = []
for n in range(num_images):
model = np.zeros((self.row,self.col))
for y in range(self.row):
for x in range(self.col):
if test_images[n][y][x] in ['+', '#']:
model[y][x] = 1
decision = self.train_decision(model)
predicted_labels.append(decision)
truths = np.array(correct_labels)
predictions = np.array(predicted_labels)
accuracy = calc_accuracy(truths, predictions)
logger.info('NB model is {0:.2f}% accurate on the digit data'.format(accuracy))
X = np.array(range(self.col))
Y = np.fliplr(np.atleast_2d(np.array(range(self.row))))[0]
if info:
confm = confusion_matrix(truths, predictions, self.num_classes)
class_accuracies = [confm[n][n] for n in range(self.num_classes)]
# Class accuracies
for n, x in enumerate(class_accuracies):
logger.info('Class {0} has an accuracy of {1:.2f}%'.format(n, 100 * x))
# Confusion matrixx
plt.figure()
plt.imshow(confm, cmap=plt.get_cmap('Greens'), interpolation='nearest')
plt.title('Confusion Matrix')
plt.xticks(np.arange(self.num_classes))
plt.yticks(np.arange(self.num_classes))
plt.xlabel('Predictions')
plt.ylabel('Truths')
X, Y = np.meshgrid(range(self.col), range(self.row))
Y = Y[::-1]
for i in range(self.num_classes):
hf = plt.figure()
ha = hf.gca(projection = '3d')
ha.plot_surface(X, Y, self.feature_weight_vectors[i], rstride=1, cstride=1,
linewidth=0, cmap=cm.coolwarm, antialiased = False)
ha.set_xlabel('X')
ha.set_ylabel('Y')
ha.set_zlabel('weigh')
plt.show()
if __name__=="__main__":
args = parse_args()
print "Reading data..."
titles, bodies, tags_sets, _ = da.read_data(args.data, args.maxRows)
tags = [list(t)[0] for t in tags_sets]
X_train, X_test, y_train, y_test = evaluation.cross_validation(zip(titles, bodies), tags)
X_train_t, X_train_b = zip(*X_train)
print "Generating features..."
if args.feat == "bow":
X, extractor = fe.bag_of_words(X_train_t, X_train_b)
elif args.feat == "tfidf":
X, extractor = fe.tfidf(X_train_t, X_train_b)
elif args.feat == "bigram":
X, extractor = fe.ngrams(X_train_t, X_train_b, n_upper=2)
else:
X, extractor = fe.ngrams(X_train_t, X_train_b, n_upper=3)
print "Train..."
if args.classifier == "naive":
classifier = MultinomialNB()
classifier.fit(X, y_train)
print "Test..."
predictions = [classifier.predict(extractor.transform(t, b))[0] for t,b in X_test]
evaluation.confusion_matrix(y_test, predictions)
def measure_all_coefficients(data_test,
data_gt,
calc_coef={
'matthews': True,
'dice': True
},
save_coef=True,
filename='coefficients.csv'):
"""Measures all comparison coefficients between two input data.
Parameters
----------
data_test : (M, N, P) array
The input test data.
data_gt : (M, N, P) array
The gold standard data.
calc_coef : dict, optional (default : {'matthews': True, 'dice': True})
Determines what coefficients to calculate.
save_coef : boolean, optional (default : True)
If True, saves the coefficients to a file in the disk.
filename : str, optional (default : 'coefficients.csv')
If save_coef is True, where to save the calculated coefficients.
Returns
-------
all_matthews : array
Array containing all Matthews coefficient values for the input data.
all_dice : array
Array containing all Dice coefficient values for the input data.
Example
-------
>>> from skimage import io
>>> data_bin = io.ImageCollection(load_pattern='res_figures/binary/Test_TIRR_0_1p5_B0p2_*_bin.png',
plugin=None)[1000:2000]
>>> data_gt = io.ImageCollection(load_pattern='gt_figures/19_Gray_*.tif',
plugin=None)
>>> all_matthews, all_dice = measure_all_coefficients(data_bin,
data_gt,
save_coef=True)
"""
_assert_same_length(data_test, data_gt)
all_matthews, all_dice = ['matthews'], ['dice']
for idx, img_test in enumerate(data_test):
aux_gt = process_goldstd_images(data_gt[idx])
_assert_compatible(img_test, aux_gt)
aux_confusion = evaluation.confusion_matrix(aux_gt, img_test)
if calc_coef['matthews']:
all_matthews.append(evaluation.measure_matthews(aux_confusion))
if calc_coef['dice']:
all_dice.append(evaluation.measure_dice(aux_confusion))
if save_coef:
if calc_coef['matthews']:
with open(filename, 'a+') as file_coef:
coef_writer = csv.writer(file_coef, delimiter=',')
coef_writer.writerow(all_matthews)
if calc_coef['dice']:
with open(filename, 'a+') as file_coef:
coef_writer = csv.writer(file_coef, delimiter=',')
coef_writer.writerow(all_dice)
return all_matthews, all_dice
X_train, X_test, y_train, y_test = evaluation.cross_validation(zip(titles, bodies), tags)
X_train_t, X_train_b = zip(*X_train)
print "Generating features..."
if args.feat == "bow":
X, extractor = fe.bag_of_words(X_train_t, X_train_b)
elif args.feat == "tfidf":
X, extractor = fe.tfidf(X_train_t, X_train_b)
elif args.feat == "bigram":
X, extractor = fe.ngrams(X_train_t, X_train_b, n_upper=2)
else:
X, extractor = fe.ngrams(X_train_t, X_train_b, n_upper=3)
print "Train..."
if args.classifier == "knn":
classifier = KNeighborsClassifier(n_neighbors=3)
elif args.classifier == "log-reg":
classifier = LogisticRegression(C=1e5)
elif args.classifier == "dec-tree":
classifier = DecisionTreeClassifier()
else:
classifier = svm.SVC()
classifier.fit(X, y_train)
print "Test..."
predictions = [classifier.predict(extractor.transform(t, b))[0] for t,b in X_test]
evaluation.confusion_matrix(y_test, predictions)
def main():
model = "pca_svc"
## Read data sets
header = pd.read_csv('Datasets/header.txt', delimiter=",", header=None)
X_train = pd.read_csv('Datasets/train_data.csv',
delimiter=",",
header=None)
X_train.columns = list(header.values)
X_test = pd.read_csv('Datasets/test_data.csv', delimiter=",", header=None)
X_test.columns = list(header.values)
y_train = pd.read_csv('Datasets/train_labels.csv',
delimiter=",",
header=None)
y_label_names = [
'Pop_Rock', 'Electronic', 'Rap', 'Jazz', 'Latin', 'RnB',
'International', 'Country', 'Reggae', 'Blues'
]
labels = y_train.values
c, r = labels.shape
labels = labels.reshape(c, )
## Feature normalization
X_train = preprocessing.scale(X_train)
X_test = preprocessing.scale(X_test)
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score
clf = RandomForestClassifier()
scores = cross_val_score(clf, X_train, labels)
scores.mean()
clf = None
if (model == "pca_svc"):
## Defining pipeline with PCA and SVC rbf hyperparameter tuning
svm = SVC(kernel="rbf")
pca = decomposition.PCA()
pipe = Pipeline(steps=[('pca', pca), ('svc', svm)])
n_components = [30]
C_range = np.logspace(2, 6, 5)
gamma_range = np.logspace(-9, -1, 5)
svc_pipe = GridSearchCV(pipe,
dict(pca__n_components=n_components,
svc__C=C_range,
svc__gamma=gamma_range),
verbose=10,
n_jobs=-1)
svc_pipe.fit(X_train, labels)
print(svc_pipe.best_score_)
print(svc_pipe.best_params_)
clf = svc_pipe.best_estimator_
if (model == "rfe_log"):
log = linear_model.LogisticRegressionCV(multi_class='multinomial',
class_weight=None,
cv=3)
rfecv = RFECV(estimator=log,
step=5,
cv=2,
scoring='accuracy',
verbose=100,
n_jobs=-1)
rfecv.fit(X_train, labels)
print("Optimal number of features : %d" % rfecv.n_features_)
X_train = rfecv.transform(X_train)
# Plot number of features VS. cross-validation scores
plt.figure()
plt.xlabel("Number of features selected")
plt.ylabel("Cross validation score (nb of correct classifications)")
plt.plot(range(1, len(rfecv.grid_scores_) + 1), rfecv.grid_scores_)
plt.show()
log_reg = linear_model.LogisticRegressionCV(multi_class='multinomial',
class_weight=None,
cv=3)
log_reg.fit(X_train, labels)
X_test = rfecv.transform(X_test)
clf = log_reg
### Evaluation ###
accuracy = clf.score(X_train, labels)
print("Accuracy: {0}", format(accuracy))
prediction_training = clf.predict(X_train)
precision = evaluation.precision(prediction_training, labels)
recall = evaluation.recall(prediction_training, labels)
print("Precision: {0}", format(precision))
print("Recall: {0}", format(recall))
evaluation.confusion_matrix(labels, prediction_training, y_label_names)
# ### Output ###
prediction_test = clf.predict(X_test)
output.accuracy(prediction_test)
