def experiment2c():
image_dim = 28
n_classes = 10
n_trials = 5
epochs = 20
batch_size = 500
db = getMACResultsDB()
collection = db['experiment2c']
x_train, y_train, x_train_flattened, x_test, y_test = loadMNIST()
y_train = np_utils.to_categorical(y_train, n_classes)
y_test = np_utils.to_categorical(y_test, n_classes)
clustering = clusterData(x_train_flattened)
fractions = [0.00, 0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 1.0]
recovery = []
results = []
for fraction in fractions:
x_labelled, labels, labelled_indices, unlabelled_indices, \
sampled_dominant_cluster_classes, dominant_cluster_classes = \
experiment2cLabelleingMethod(x_train, y_train, clustering,
n_classes, fraction)
result = \
runTrials(x_labelled, labels, x_test, y_test, n_trials, n_classes, \
data='mnist_experiment2c_fraction%.2lf'%(fraction), \
epochs=epochs)
results.append(result)
recovery.append(
calculateLabellingAccuracy(
np_utils.to_categorical(dominant_cluster_classes),
np_utils.to_categorical(sampled_dominant_cluster_classes)) /
100.0)
doc = {
'name': 'majority class recovery by cluster subsample',
'm': labels.shape[0],
'fraction sampled': fraction,
'accuracy': result[0],
'error': result[1],
'trials accuracy': result[2],
'labelling accuracy': \
calculateLabellingAccuracy(y_train[labelled_indices], labels),
'training set class distribution': \
calculateClassDistribution(labels).tolist(),
'cluster majority class recovery rate': calculateLabellingAccuracy(
np_utils.to_categorical(dominant_cluster_classes),
np_utils.to_categorical(sampled_dominant_cluster_classes))/100.0
}
collection.insert_one(doc)
return fractions, results, recovery
def experiment2b():
image_dim = 28
n_classes = 10
n_trials = 5
epochs = 20
batch_size = 500
db = getMACResultsDB()
collection = db['experiment2b']
x_train, y_train, x_train_flattened, x_test, y_test = loadMNIST()
y_train = np_utils.to_categorical(y_train, n_classes)
y_test = np_utils.to_categorical(y_test, n_classes)
clustering = clusterData(x_train_flattened)
fractions = [0.11, 0.2, 0.25, 0.3, 0.4, 0.5, \
0.6, 0.7, 0.8, 0.9, 0.95, 0.99, 1.0]
results = []
for fraction in fractions:
x_labelled, labels, labelled_indices, unlabelled_indices = \
experiment2bLabelleingMethod(x_train, y_train, clustering,
n_classes, fraction)
print(labels.shape[0])
result = \
runTrials(x_labelled, labels, x_test, y_test, n_trials, n_classes, \
data='mnist_experiment2b_fraction%.2lf'%(fraction), \
epochs=epochs)
results.append(result)
doc = {
'name': 'majority class label assignment by fraction',
'm': labels.shape[0],
'fraction': fraction,
'accuracy': result[0],
'error': result[1],
'trials accuracy': result[2],
'labelling accuracy': \
calculateLabellingAccuracy(y_train[labelled_indices], labels),
'training set class distribution': \
calculateClassDistribution(labels).tolist()
}
collection.insert_one(doc)
return fractions, results
def experiment2aNoise(filename):
image_dim = 28
n_classes = 10
n_trials = 5
epochs = 20
batch_size = 500
db = getMACResultsDB()
x_train, y_train, x_train_flattened, x_test, y_test = loadMNIST()
# calculate performance on gold labels
y_train = np_utils.to_categorical(y_train, n_classes)
y_test = np_utils.to_categorical(y_test, n_classes)
clustering = clusterData(x_train_flattened)
# experiment 2a labelling accuracy
labelling_accuracies_2a = []
labelling_accuracies_2a_errors = []
labelling_accuracies_2a_2 = []
labelling_accuracies_2a_2_errors = []
# experiment 2a machine accuracy
machine_accuracies_2a = []
machine_accuracies_2a_errors = []
machine_accuracies_2a_2 = []
machine_accuracies_2a_2_errors = []
noise_levels = np.arange(0, 1.1, 0.1)
collection_name = 'experiment2d'
collection = db[collection_name]
for noise in noise_levels:
l_results = []
l_results_2 = []
m_results = []
m_results_2 = []
for i in range(n_trials):
# experiment 2a with random noise
experiment_name = \
'experiment 2d - experiment2a random noise %.2lf trial %d' % (noise, i)
try:
assert collection_name in db.collection_names()
doc = db[collection_name].find({'name': experiment_name})[0]
l_results.append(doc['labelling accuracy'])
m_results.append(doc['machine accuracy'])
except (AssertionError, IndexError):
x_labelled, labels, labelled_indices, unlabelled_indices = \
experiment2aNoiseLabelling(x_train, y_train, clustering, \
n_classes, noise=noise)
l_results.append(
calculateLabellingAccuracy(y_train[labelled_indices],
labels))
r = runTrials(x_labelled, labels, x_test, y_test, 1, n_classes, \
data='mnist_experiment2d_noise%.2lf_trial%d'%(noise, i))
m_results.append(r[0])
doc = {
'name':experiment_name,
'm': labels.shape[0],
'noise': noise,
'trial': i,
'labelling accuracy': \
calculateLabellingAccuracy(y_train[labelled_indices], labels),
'machine accuracy': r[0],
'training set class distribution': \
calculateClassDistribution(labels).tolist()
}
collection.insert_one(doc)
# experiment 2a with weighted noise
experiment_name = \
'experiment 2d - experiment2a class weighted noise %.2lf trial %d' \
% (noise, i)
try:
assert collection_name in db.collection_names()
doc = db[collection_name].find({'name': experiment_name})[0]
l_results_2.append(doc['labelling accuracy'])
m_results_2.append(doc['machine accuracy'])
except (AssertionError, IndexError):
x_labelled, labels, labelled_indices, unlabelled_indices = \
experiment2aNoiseLabelling(x_train, y_train, clustering, \
n_classes, intelligent_noise=noise)
l_results_2.append(
calculateLabellingAccuracy(y_train[labelled_indices],
labels))
r = runTrials(x_labelled, labels, x_test, y_test, 1, n_classes, \
data='mnist_experiment2d_class_weighted_noise%.2lf_trial%d'%(noise, i))
m_results_2.append(r[0])
doc = {
'name':experiment_name,
'm': labels.shape[0],
'noise': noise,
'trial': i,
'labelling accuracy': \
calculateLabellingAccuracy(y_train[labelled_indices], labels),
'machine accuracy': r[0],
'training set class distribution': \
calculateClassDistribution(labels).tolist()
}
collection.insert_one(doc)
labelling_accuracies_2a.append(np.mean(l_results))
labelling_accuracies_2a_errors.append(np.std(l_results))
labelling_accuracies_2a_2.append(np.mean(l_results_2))
labelling_accuracies_2a_2_errors.append(np.std(l_results_2))
machine_accuracies_2a.append(np.mean(m_results))
machine_accuracies_2a_errors.append(np.std(m_results))
machine_accuracies_2a_2.append(np.mean(m_results_2))
machine_accuracies_2a_2_errors.append(np.std(m_results_2))
cursor = db['experiment2a'].find(
{'name': 'majority class label assignment'})
for doc in cursor:
majority_class_benchmark = doc['labelling accuracy']
fig, ax = plt.subplots()
ax.plot(np.arange(-0.02,1.03,0.01), \
np.ones((105))*majority_class_benchmark,'k--')
ax.errorbar(noise_levels, labelling_accuracies_2a, \
yerr=labelling_accuracies_2a_errors, fmt='o', mfc='None', \
color='#B8336A', label='majority class - random noise')
ax.errorbar(noise_levels, labelling_accuracies_2a_2, \
yerr=labelling_accuracies_2a_2_errors, fmt='o', mfc='None', \
color='#726DA8', label='majority class - class weighted noise', zorder=100)
ax.set_xlabel('labelling noise')
ax.set_ylabel('labelling accuracy')
ax.set_ylim(-2, 100)
ax.set_xlim(-0.02, 1.03)
plt.legend(loc='lower left')
#plt.show()
plt.savefig(filename + 'labelling_noise.pdf')
plt.savefig(filename + 'labelling_noise.png')
cursor = db['experiment2a'].find({'name': 'gold benchmark'})
for doc in cursor:
gold_benchmark = doc['accuracy']
gold_benchmark_error = doc['error']
cursor = db['experiment2a'].find(
{'name': 'majority class label assignment'})
for doc in cursor:
majority_class_benchmark = doc['accuracy']
majority_class_error = doc['error']
fig, ax = plt.subplots()
ax.plot(np.arange(-0.02,1.03,0.01), np.ones((105,))*gold_benchmark, \
color='#726DA8', label='gold benchmark')
ax.axhspan(gold_benchmark-gold_benchmark_error, \
gold_benchmark+gold_benchmark_error, \
facecolor='#726DA8', alpha=0.5)
ax.plot(np.arange(-0.02,1.03,0.01), np.ones((105,))*majority_class_benchmark, \
color='#A0D2DB', label='majority class benchmark')
ax.axhspan(majority_class_benchmark-majority_class_error, \
majority_class_benchmark+majority_class_error, \
facecolor='#A0D2DB', alpha=0.5)
ax.errorbar(noise_levels, machine_accuracies_2a, \
yerr=machine_accuracies_2a_errors, fmt='o', mfc='None', \
color='#B8336A', label='majority class - random noise')
ax.errorbar(noise_levels, machine_accuracies_2a_2, \
yerr=machine_accuracies_2a_2_errors, fmt='o', mfc='None', \
color='#726DA8', label='majority class - class weighted noise', zorder=100)
ax.set_xlabel('labelling noise')
ax.set_ylabel('machine accuracy')
ax.set_ylim(-2, 100)
ax.set_xlim(-0.02, 1.03)
plt.legend(loc='lower left')
#plt.show()
plt.savefig(filename + '_machine_accuracy.pdf')
plt.savefig(filename + '_machine_accuracy.png')
def experiment2a():
image_dim = 28
n_classes = 10
n_trials = 5
epochs = 20
batch_size = 500
db = getMACResultsDB()
collection = db['experiment2a']
x_train, y_train, x_train_flattened, x_test, y_test = loadMNIST()
# calculate performance on gold labels
y_train = np_utils.to_categorical(y_train, n_classes)
y_test = np_utils.to_categorical(y_test, n_classes)
gold_benchmark = \
runTrials(x_train, y_train, x_test, y_test, n_trials, n_classes, \
data='mnist_gold', epochs=epochs)
print(gold_benchmark)
doc = {
'name': 'gold benchmark',
'm': y_train.shape[0],
'accuracy': gold_benchmark[0],
'error': gold_benchmark[1],
'trials accuracy': gold_benchmark[2],
'labelling accuracy': calculateLabellingAccuracy(y_train, y_train),
'training set class distribution': \
calculateClassDistribution(y_train).tolist()
}
collection.insert_one(doc)
clustering = clusterData(x_train_flattened)
x_labelled, labels, labelled_indices, unlabelled_indices = \
experiment2aLabelleingMethod(x_train, y_train, \
clustering, n_classes)
result = \
runTrials(x_labelled, labels, x_test, y_test, n_trials, n_classes, \
data='mnist_experiment2a', epochs=epochs)
print(result)
print(calculateLabellingAccuracy(y_train[labelled_indices], labels))
doc = {
'name': 'majority class label assignment',
'm': labels.shape[0],
'accuracy': result[0],
'error': result[1],
'trials accuracy': result[2],
'labelling accuracy': \
calculateLabellingAccuracy(y_train[labelled_indices], labels),
'training set class distribution': \
calculateClassDistribution(labels).tolist()
}
collection.insert_one(doc)