def evaluate_batch(network, reader_eval, samples_per_epoch_eval, classes,
output_directory, number_of_samples):
features = network['features']
label = network['label']
model = network['model']
map_input_to_streams = {
features: reader_eval.streams.features,
label: reader_eval.streams.labels
}
data = reader_eval.next_minibatch(samples_per_epoch_eval,
input_map=map_input_to_streams)
labels = data[label].as_sequences(label)
images = data[features].as_sequences(features)
predictions = [model.eval(image) for image in images]
# Find the index with the maximum value for both predicted as well as the ground truth
y_pred = [np.argmax(prediction) for prediction in predictions]
y_true = [np.argmax(label) for label in labels]
helpers.plot_confusion_matrix_with_acc_and_fbeta(
y_true, y_pred, classes, save_path='outputs/yoo.png', normalize=True)
fscore = fbeta_score(y_true, y_pred, beta=0.5, average='macro')
run.log('f_score', np.float(fscore))
# save model to outputs folder
model.save(
f'{output_directory}/cntk-{features.shape}-{number_of_samples}.model')
def train(X_train, y_train, X_test, y_test, classifier_name, number_of_samples,
shape, output_directory, beta, color_insensitive, is_local, run):
classifier = get_classifier(classifier_name)
start = time.time()
# classifier.fit(X_train, y_train)
accuracy, f_score, y_preds, y_tests = k_fold_crossvalidation(
classifier, np.concatenate((X_train, X_test)),
np.asarray(y_train + y_test))
training_time = time.time() - start
# y_pred = classifier.predict(X_test)
# accuracy = accuracy_score(y_test, y_pred)
# fscore = fbeta_score(y_test, y_pred, beta=beta, average='macro')
color_insensitive = 'color_insensitive' if color_insensitive else 'color_sensitive'
helpers.plot_confusion_matrix_with_acc_and_fbeta(
y_tests,
y_preds,
classes=set(y_test),
normalize=True,
save_path=f"{output_directory}/{classifier_name}-hog(ppc="
f"{(shape[0] / 16, shape[0] / 16)},"
f"cpb={(16, 16)})"
f"-{color_insensitive}-{shape}-{number_of_samples}-cm.png",
fbeta_beta=beta)
if not is_local and run:
run.log('classifier_name', np.float(training_time))
run.log('training_time', np.float(training_time))
run.log('accuracy', np.float(accuracy))
run.log('f_score', np.float(f_score))
run.log('number_of_samples', number_of_samples)
run.log('shape_and_samples',
int(str(number_of_samples) + str(shape[0])))
run.log('color_insensitive', color_insensitive)
with open(f'{output_directory}/results-{datetime.now().date()}.csv',
'a',
newline='') as csvfile:
results = {
'classifier': classifier_name,
'number_of_samples': number_of_samples,
'shape': shape,
'color_insensitive': color_insensitive,
'accuracy': accuracy,
'f_score': f_score
}
writer = csv.DictWriter(csvfile, fieldnames=results.keys())
writer.writerow(results)
joblib.dump(
value=classifier,
filename=
f'{output_directory}/{classifier_name}-hog(ppc={(shape[0] / 16, shape[0] / 16)},'
f'cpb={(16, 16)})-{color_insensitive}-{shape}-{number_of_samples}.joblib'
)
def train(data_dir, test_dir, classifier_name, number_of_samples, shape,
output_directory, beta, color_insensitive, is_local, run):
start = time.time()
if test_dir:
X_train, y_train = get_data(data_dir, int(number_of_samples * 0.7),
shape, color_insensitive)
X_test, y_test = get_data(test_dir, int(number_of_samples * 0.3),
shape, color_insensitive)
X_train, y_train, = shuffle(X_train, y_train)
X_test, y_test = shuffle(X_test, y_test)
else:
X, y = get_data(data_dir, number_of_samples, shape, color_insensitive)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.3,
random_state=0)
endtime = time.time() - start
classifier = get_classifier(classifier_name)
start = time.time()
classifier.fit(X_train, y_train)
endtime = time.time() - start
y_pred = classifier.predict(X_test)
accuracy = accuracy_score(y_test, y_pred)
fscore = fbeta_score(y_test, y_pred, beta=beta, average='macro')
training_data = data_dir.split("\\")[-1]
output_dir = f'{output_directory}/{training_data}'
color_insensitive = 'color_insensitive' if color_insensitive else 'color_sensitive'
if not os.path.exists(output_dir):
os.makedirs(output_dir)
helpers.plot_confusion_matrix_with_acc_and_fbeta(
y_test,
y_pred,
classes=set(y_test),
normalize=True,
save_path=f"{output_dir}/{classifier_name}-hog(ppc="
f"{(shape[0] / 16, shape[0] / 16)},"
f"cpb={(16, 16)})"
f"-{color_insensitive}-{shape}-{number_of_samples}-cm.png",
fbeta_beta=beta)
if not is_local and run:
run.log('data_loading_time', np.float(endtime))
run.log('training_time', np.float(endtime))
run.log('accuracy', np.float(accuracy))
run.log('f_score', np.float(fscore))
joblib.dump(
value=classifier,
filename=
f'{output_dir}/{classifier_name}-hog(ppc={(shape[0] / 16, shape[0] / 16)},'
f'cpb={(16, 16)})-{color_insensitive}-{shape}-{number_of_samples}.joblib'
)
def train(data_dir, test_dir, classifier_name, number_of_samples, shape,
output_directory, beta, with_dominant_color_attached):
start = time.time()
if test_dir:
X_train, y_train = get_data(data_dir, int(number_of_samples * 0.7),
shape, with_dominant_color_attached)
X_test, y_test = get_data(test_dir, int(number_of_samples * 0.3),
shape, with_dominant_color_attached)
X_train, y_train, = shuffle(X_train, y_train)
X_test, y_test = shuffle(X_test, y_test)
else:
X, y = get_data(data_dir, number_of_samples, shape,
with_dominant_color_attached)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.3,
random_state=0)
endtime = time.time() - start
run.log('data_loading_time', np.float(endtime))
classifier = get_classifier(classifier_name)
start = time.time()
classifier.fit(X_train, y_train)
endtime = time.time() - start
run.log('training_time', np.float(endtime))
y_pred = classifier.predict(X_test)
accuracy = accuracy_score(y_test, y_pred)
run.log('accuracy', np.float(accuracy))
fscore = fbeta_score(y_test, y_pred, beta=beta, average='macro')
run.log('f_score', np.float(fscore))
rgb_or_dom = 'dom' if with_dominant_color_attached else 'rgb'
helpers.plot_confusion_matrix_with_acc_and_fbeta(
y_test,
y_pred,
classes=set(y_test),
normalize=True,
save_path=f"{output_directory}/{classifier_name}-hog(ppc="
f"{(shape[0] / 16, shape[0] / 16)},"
f"cpb={(16, 16)})"
f"-{rgb_or_dom}-{shape}-{number_of_samples}-cm.png",
fbeta_beta=beta)
joblib.dump(
value=classifier,
filename=
f'{output_directory}/{classifier_name}-hog(ppc={(shape[0] / 16, shape[0] / 16)},'
f'cpb={(16, 16)})'
f'-{rgb_or_dom}-{shape}-{number_of_samples}.joblib')