def evaluate_batch(network, reader_eval, samples_per_epoch_eval, classes, output_directory, number_of_samples): features = network['features'] label = network['label'] model = network['model'] map_input_to_streams = { features: reader_eval.streams.features, label: reader_eval.streams.labels } data = reader_eval.next_minibatch(samples_per_epoch_eval, input_map=map_input_to_streams) labels = data[label].as_sequences(label) images = data[features].as_sequences(features) predictions = [model.eval(image) for image in images] # Find the index with the maximum value for both predicted as well as the ground truth y_pred = [np.argmax(prediction) for prediction in predictions] y_true = [np.argmax(label) for label in labels] helpers.plot_confusion_matrix_with_acc_and_fbeta( y_true, y_pred, classes, save_path='outputs/yoo.png', normalize=True) fscore = fbeta_score(y_true, y_pred, beta=0.5, average='macro') run.log('f_score', np.float(fscore)) # save model to outputs folder model.save( f'{output_directory}/cntk-{features.shape}-{number_of_samples}.model')
def train(X_train, y_train, X_test, y_test, classifier_name, number_of_samples, shape, output_directory, beta, color_insensitive, is_local, run): classifier = get_classifier(classifier_name) start = time.time() # classifier.fit(X_train, y_train) accuracy, f_score, y_preds, y_tests = k_fold_crossvalidation( classifier, np.concatenate((X_train, X_test)), np.asarray(y_train + y_test)) training_time = time.time() - start # y_pred = classifier.predict(X_test) # accuracy = accuracy_score(y_test, y_pred) # fscore = fbeta_score(y_test, y_pred, beta=beta, average='macro') color_insensitive = 'color_insensitive' if color_insensitive else 'color_sensitive' helpers.plot_confusion_matrix_with_acc_and_fbeta( y_tests, y_preds, classes=set(y_test), normalize=True, save_path=f"{output_directory}/{classifier_name}-hog(ppc=" f"{(shape[0] / 16, shape[0] / 16)}," f"cpb={(16, 16)})" f"-{color_insensitive}-{shape}-{number_of_samples}-cm.png", fbeta_beta=beta) if not is_local and run: run.log('classifier_name', np.float(training_time)) run.log('training_time', np.float(training_time)) run.log('accuracy', np.float(accuracy)) run.log('f_score', np.float(f_score)) run.log('number_of_samples', number_of_samples) run.log('shape_and_samples', int(str(number_of_samples) + str(shape[0]))) run.log('color_insensitive', color_insensitive) with open(f'{output_directory}/results-{datetime.now().date()}.csv', 'a', newline='') as csvfile: results = { 'classifier': classifier_name, 'number_of_samples': number_of_samples, 'shape': shape, 'color_insensitive': color_insensitive, 'accuracy': accuracy, 'f_score': f_score } writer = csv.DictWriter(csvfile, fieldnames=results.keys()) writer.writerow(results) joblib.dump( value=classifier, filename= f'{output_directory}/{classifier_name}-hog(ppc={(shape[0] / 16, shape[0] / 16)},' f'cpb={(16, 16)})-{color_insensitive}-{shape}-{number_of_samples}.joblib' )
def train(data_dir, test_dir, classifier_name, number_of_samples, shape, output_directory, beta, color_insensitive, is_local, run): start = time.time() if test_dir: X_train, y_train = get_data(data_dir, int(number_of_samples * 0.7), shape, color_insensitive) X_test, y_test = get_data(test_dir, int(number_of_samples * 0.3), shape, color_insensitive) X_train, y_train, = shuffle(X_train, y_train) X_test, y_test = shuffle(X_test, y_test) else: X, y = get_data(data_dir, number_of_samples, shape, color_insensitive) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0) endtime = time.time() - start classifier = get_classifier(classifier_name) start = time.time() classifier.fit(X_train, y_train) endtime = time.time() - start y_pred = classifier.predict(X_test) accuracy = accuracy_score(y_test, y_pred) fscore = fbeta_score(y_test, y_pred, beta=beta, average='macro') training_data = data_dir.split("\\")[-1] output_dir = f'{output_directory}/{training_data}' color_insensitive = 'color_insensitive' if color_insensitive else 'color_sensitive' if not os.path.exists(output_dir): os.makedirs(output_dir) helpers.plot_confusion_matrix_with_acc_and_fbeta( y_test, y_pred, classes=set(y_test), normalize=True, save_path=f"{output_dir}/{classifier_name}-hog(ppc=" f"{(shape[0] / 16, shape[0] / 16)}," f"cpb={(16, 16)})" f"-{color_insensitive}-{shape}-{number_of_samples}-cm.png", fbeta_beta=beta) if not is_local and run: run.log('data_loading_time', np.float(endtime)) run.log('training_time', np.float(endtime)) run.log('accuracy', np.float(accuracy)) run.log('f_score', np.float(fscore)) joblib.dump( value=classifier, filename= f'{output_dir}/{classifier_name}-hog(ppc={(shape[0] / 16, shape[0] / 16)},' f'cpb={(16, 16)})-{color_insensitive}-{shape}-{number_of_samples}.joblib' )
def train(data_dir, test_dir, classifier_name, number_of_samples, shape, output_directory, beta, with_dominant_color_attached): start = time.time() if test_dir: X_train, y_train = get_data(data_dir, int(number_of_samples * 0.7), shape, with_dominant_color_attached) X_test, y_test = get_data(test_dir, int(number_of_samples * 0.3), shape, with_dominant_color_attached) X_train, y_train, = shuffle(X_train, y_train) X_test, y_test = shuffle(X_test, y_test) else: X, y = get_data(data_dir, number_of_samples, shape, with_dominant_color_attached) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0) endtime = time.time() - start run.log('data_loading_time', np.float(endtime)) classifier = get_classifier(classifier_name) start = time.time() classifier.fit(X_train, y_train) endtime = time.time() - start run.log('training_time', np.float(endtime)) y_pred = classifier.predict(X_test) accuracy = accuracy_score(y_test, y_pred) run.log('accuracy', np.float(accuracy)) fscore = fbeta_score(y_test, y_pred, beta=beta, average='macro') run.log('f_score', np.float(fscore)) rgb_or_dom = 'dom' if with_dominant_color_attached else 'rgb' helpers.plot_confusion_matrix_with_acc_and_fbeta( y_test, y_pred, classes=set(y_test), normalize=True, save_path=f"{output_directory}/{classifier_name}-hog(ppc=" f"{(shape[0] / 16, shape[0] / 16)}," f"cpb={(16, 16)})" f"-{rgb_or_dom}-{shape}-{number_of_samples}-cm.png", fbeta_beta=beta) joblib.dump( value=classifier, filename= f'{output_directory}/{classifier_name}-hog(ppc={(shape[0] / 16, shape[0] / 16)},' f'cpb={(16, 16)})' f'-{rgb_or_dom}-{shape}-{number_of_samples}.joblib')