def conv_model(input_shape, num_classes, kernel_size=(3, 3), pool_size=(2, 2)): model = Sequential() model.add(Conv2D(32, kernel_size, padding='same', input_shape=input_shape)) model.add(Activation('relu')) model.add(Dropout(0.25)) model.add(Conv2D(64, kernel_size, padding='same')) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=pool_size)) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(512, kernel_regularizer=regularizers.l2(0.01))) model.add(Activation('relu')) model.add(Dropout(0.25)) model.add(Dense(num_classes, kernel_regularizer=regularizers.l2(0.01))) model.add(Dropout(0.25)) model.add(Activation('softmax')) top2 = lambda x, y: top_k_categorical_accuracy(x, y, k=2) top3 = lambda x, y: top_k_categorical_accuracy(x, y, k=3) top4 = lambda x, y: top_k_categorical_accuracy(x, y, k=4) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy', top2, top3, top4]) return model
def test_top_k_categorical_accuracy(): y_pred = K.variable(np.array([[0.3, 0.2, 0.1], [0.1, 0.2, 0.7]])) y_true = K.variable(np.array([[0, 1, 0], [1, 0, 0]])) success_result = K.eval(metrics.top_k_categorical_accuracy(y_true, y_pred, k=3)) assert success_result == 1 partial_result = K.eval(metrics.top_k_categorical_accuracy(y_true, y_pred, k=2)) assert partial_result == 0.5 failure_result = K.eval(metrics.top_k_categorical_accuracy(y_true, y_pred, k=1)) assert failure_result == 0
def test_top_k_categorical_accuracy(): y_pred = K.variable(np.array([[0.3, 0.2, 0.1], [0.1, 0.2, 0.7]])) y_true = K.variable(np.array([[0, 1, 0], [1, 0, 0]])) success_result = K.eval( metrics.top_k_categorical_accuracy(y_true, y_pred, k=3)) assert success_result == 1 partial_result = K.eval( metrics.top_k_categorical_accuracy(y_true, y_pred, k=2)) assert partial_result == 0.5 failure_result = K.eval( metrics.top_k_categorical_accuracy(y_true, y_pred, k=1)) assert failure_result == 0
def test_top_k_categorical_accuracy(self): with self.cached_session(): y_pred = K.variable(np.array([[0.3, 0.2, 0.1], [0.1, 0.2, 0.7]])) y_true = K.variable(np.array([[0, 1, 0], [1, 0, 0]])) result = K.eval( metrics.top_k_categorical_accuracy(y_true, y_pred, k=3)) self.assertEqual(np.mean(result), 1) result = K.eval( metrics.top_k_categorical_accuracy(y_true, y_pred, k=2)) self.assertEqual(np.mean(result), 0.5) result = K.eval( metrics.top_k_categorical_accuracy(y_true, y_pred, k=1)) self.assertEqual(np.mean(result), 0.)
def _mrr_metric(self, y_true, y_pred): mrr = 0 current_percentage = 0 for i in range(1, 26, 1): if i == 1: mrr = metrics.top_k_categorical_accuracy(y_true, y_pred, k=i) current_percentage = metrics.top_k_categorical_accuracy(y_true, y_pred, k=i) else: t = metrics.top_k_categorical_accuracy(y_true, y_pred, k=i) mrr += (t - current_percentage) * (1 / i) current_percentage = t return mrr
def evaluate_model(model, image_gen, classes, output_metrics, output_image, top_n=None, beta=1.): # Iterate all batches for testing for batch_num in range(len(image_gen)): logger.info('Processing batch %d of %d' % (batch_num, len(image_gen))) cur_x, cur_y = image_gen.next() cur_y_pred = model.predict(cur_x) if batch_num == 0: y_actual = cur_y y_pred = cur_y_pred else: y_actual = np.concatenate((y_actual, cur_y)) y_pred = np.concatenate((y_pred, cur_y_pred)) y_v = np.argmax(y_actual, axis=1) y_p_v = np.argmax(y_pred, axis=1) metrics = {} if output_image: logger.info('Writing confusion matrix to %s' % output_image) plot_confusion_matrix(confusion_matrix(y_v, y_p_v), classes, output_image) if output_metrics: logger.info('Writing metrics to %s' % output_metrics) precision, recall, fscore, support = precision_recall_fscore_support( y_v, y_p_v, beta=beta) metrics = { 'Precision': precision, 'Recall': recall, 'F-Score': fscore, 'Support': support } if top_n: top_n_value = K.get_value( top_k_categorical_accuracy(y_actual, y_pred, k=top_n)) metrics['Top_{}'.format(top_n)] = float(top_n_value) logger.info('Precision: {}, Recall: {}, F-Score: {}, Support: {}, Top_{}: {}'\ .format(precision, recall, fscore, support, top_n, top_n_value)) else: logger.info( 'Precision: {}, Recall: {}, F-Score: {}, Support: {}'.format( precision, recall, fscore, support)) os.makedirs(os.path.dirname(output_metrics), exist_ok=True) with open(output_metrics, 'w', encoding='utf-8') as fp: if top_n: fp.write( 'Precision, Recall, F_Score, Top_{}, Support\n'.format( top_n)) fp.write('{}, {}, {}, {}, {}\n'.format(precision, recall, fscore, top_n_value, support)) else: fp.write('Precision, Recall, F_Score, Support\n') fp.write('{}, {}, {}, {}\n'.format(precision, recall, fscore, support)) return metrics, y_actual, y_pred
def acc_top2(y_true, y_pred): """ :param y_true: 真实值 :param y_pred: 训练值 :return: # 计算top-k正确率,当预测值的前k个值中存在目标类别即认为预测正确 """ return top_k_categorical_accuracy(y_true, y_pred, k=2)
def top_3_categorical_accuracy(y_true, y_pred): return top_k_categorical_accuracy(K.reshape(y_true, shape=(-1, K.shape(y_true)[-1])), K.reshape(y_pred, shape=(-1, K.shape(y_pred)[-1])), k=3)
def top_5_accuracy(y_true, y_pred): """ Calculates top-5 accuracy of the predictions. To be used as evaluation metric in model.compile(). Arguments: y_true -- array-like, true labels y_pred -- array-like, predicted labels Returns: top-5 accuracy """ return top_k_categorical_accuracy(y_true, y_pred, k=5)
def test_top_k_categorical_accuracy(): # 2d tests - shape: (batch_size, number_of_categories) y_pred = K.variable(np.array([[0.3, 0.2, 0.1], [0.1, 0.2, 0.7]])) y_true = K.variable(np.array([[0, 1, 0], [1, 0, 0]])) success_result = K.eval( metrics.top_k_categorical_accuracy(y_true, y_pred, k=3)) assert success_result == 1 partial_result = K.eval( metrics.top_k_categorical_accuracy(y_true, y_pred, k=2)) assert partial_result == 0.5 failure_result = K.eval( metrics.top_k_categorical_accuracy(y_true, y_pred, k=1)) assert failure_result == 0 # 3d tests - example shape: (batch_size, sequence_size, number_of_categories) y_pred = K.variable( np.array([[[0.3, 0.2, 0.1], [0.1, 0.2, 0.7]], [[0.3, 0.2, 0.1], [0.1, 0.2, 0.7]]])) y_true = K.variable( np.array([[[0, 1, 0], [1, 0, 0]], [[0, 1, 0], [1, 0, 0]]])) success_result = K.eval( metrics.top_k_categorical_accuracy(y_true, y_pred, k=3)) assert success_result == 1 partial_result = K.eval( metrics.top_k_categorical_accuracy(y_true, y_pred, k=2)) assert partial_result == 0.5 failure_result = K.eval( metrics.top_k_categorical_accuracy(y_true, y_pred, k=1)) assert failure_result == 0
def lstm_model(input_shape_conv, time_distributed_input_shape, num_classes, kernel_size=(3, 3), pool_size=(2, 2), mode=None): model = Sequential() model.add(TimeDistributed(Conv2D(32, kernel_size, padding='same', input_shape=input_shape_conv), input_shape=time_distributed_input_shape)) model.add(TimeDistributed(Activation('relu'))) model.add(TimeDistributed(Dropout(0.25))) model.add(TimeDistributed(Conv2D(64, kernel_size, padding='same'))) model.add(TimeDistributed(Activation('relu'))) model.add(TimeDistributed(MaxPooling2D(pool_size=pool_size))) model.add(TimeDistributed(Dropout(0.25))) model.add(TimeDistributed(Flatten())) model.add(TimeDistributed(Dense(512, kernel_regularizer=regularizers.l2(0.01)))) model.add(TimeDistributed(Activation('relu'))) if mode == None: model.add(LSTM(512, return_sequences=False)) elif mode == 'max': model.add(LSTM(512, return_sequences=True)) model.add(GlobalMaxPooling1D()) else: model.add(LSTM(512, return_sequences=True)) model.add(GlobalAveragePooling1D()) model.add(Dense(512, kernel_regularizer=regularizers.l2(0.01))) model.add(Activation('relu')) model.add(Dropout(0.25)) model.add(Dense(num_classes, kernel_regularizer=regularizers.l2(0.01))) model.add(Dropout(0.25)) model.add(Activation('softmax')) top2 = lambda x, y: top_k_categorical_accuracy(x, y, k=2) top3 = lambda x, y: top_k_categorical_accuracy(x, y, k=3) top4 = lambda x, y: top_k_categorical_accuracy(x, y, k=4) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy', top2, top3, top4]) return model
def _generator_test(): # Single thread generator test generator = sample_generator('../Imagenet5W') data, anno = generator.next() for i in range(500): _data, _anno = generator.next() data = np.vstack((data, _data)) anno = np.vstack((anno, _anno)) res50 = Net("ResNet50") predict_result = res50.full_inference(data) print("Kr: %s" % K.get_session().run(top_k_categorical_accuracy(anno, predict_result, k=5))) print("My: %s" % top_k_acc(anno, predict_result, 5))
def _multithread_parser_test(): samplenames = [item.split('.')[0] for item in os.listdir('../Imagenet5W/Data/CLS-LOC/val/')] pool = Pool(processes=10) data_anno = pool.map(sample_parser, samplenames) data = np.vstack([item[0] for item in data_anno]) anno = np.vstack([item[1] for item in data_anno]) res50 = Net("ResNet50") predict_result = res50.full_inference(data) print("Kr: %s" % K.get_session().run(top_k_categorical_accuracy(anno, predict_result, k=5))) print("My: %s" % top_k_acc(anno, predict_result, 5))
def top_accuracy(true_word_indices, image_vectors): l2 = lambda x, axis: K.sqrt( K.sum(K.square(x), axis=axis, keepdims=True)) l2norm = lambda x, axis: x / l2(x, axis) l2_words = l2norm(ALL_word_embeds, axis=1) l2_images = l2norm(image_vectors, axis=1) tiled_words = K.tile(K.expand_dims(l2_words, axis=1), (1, 200, 1)) tiled_images = K.tile(K.expand_dims(l2_images, axis=1), (1, 20, 1)) diff = K.squeeze(l2(l2_words - l2_images, axis=2)) # slice_top3 = lambda x: x[:, 0:3] # slice_top1 = lambda x: x[:, 0:1] diff_top5 = metrics.top_k_categorical_accuracy(tiled_images, diff) return diff_top5
def top_5_acc(y_true, y_pred): return met.top_k_categorical_accuracy(y_true, tf.cast(y_pred, dtype='float32'), k=5)
def top_3_acc(y_true, y_pred): """returns proportion of time that the actual was in top 3 predicted""" return top_k_categorical_accuracy(y_true, y_pred, k=3)
def top_2_categorical_accuracy(self, y_true, y_pred): return top_k_categorical_accuracy(y_true, y_pred, k=2)
def top_3_accuracy(x,y): return top_k_categorical_accuracy(x,y, 3) from keras.callbacks import ModelCheckpoint, LearningRateScheduler, EarlyStopping, ReduceLROnPlateau
def acc_top2(y_true, y_pred): return top_k_categorical_accuracy(y_true, y_pred, k=1)
data_shape = [224, 224, 3] elif args.network == "GoogLeNet": net = getattr(keras_helpers, args.network)() data_shape = [224, 224, 3] else: sys.exit("Unknown Network") fake_data = np.random.rand(args.train_batch, data_shape[0], data_shape[1], data_shape[2]) tmp_fake_labels = np.random.randint(0, high=1000, size=args.train_batch) fake_labels = np.zeros([args.train_batch, 1000]) for i in range(args.train_batch): fake_labels[i, tmp_fake_labels[i]] = 1 loss = categorical_crossentropy(net.y_, net.y) top1 = categorical_accuracy(net.y_, net.y) top5 = top_k_categorical_accuracy(net.y_, net.y, 5) base_lr = 0.02 step = tf.Variable(0, trainable=False, name="Step") learning_rate = tf.train.exponential_decay(base_lr, step, 1, 0.999964) weight_list = [v for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES) if v.name[-3:] == "W:0"] bias_list = [v for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES) if v.name[-3:] == "b:0"] optimizer1 = tf.train.MomentumOptimizer(learning_rate, 0.9) optimizer2 = tf.train.MomentumOptimizer(tf.scalar_mul(2.0, learning_rate), 0.9) grads = optimizer1.compute_gradients(loss, var_list=weight_list+bias_list) w_grads = grads[:len(weight_list)] b_grads = grads[len(weight_list):] train1 = optimizer1.apply_gradients(w_grads, global_step=step)
callbacks=[checkpoint], validation_data=([sentence_test, countries_test, devices_test], labels_test)) # evaluate the results one last time model.load_weights(WEIGHTS_PATH) # scores = model.evaluate(x=[sentence_test, countries_test, devices_test], y=labels_test, batch_size=BATCHSIZE, verbose=1) # # print("Final result :") # for name, score in zip(model.metrics_names, scores): # print(name, score) predictions = model.predict(x=[sentence_test, countries_test, devices_test], batch_size=BATCHSIZE, verbose=1) k = 5 top_accuracy = top_k_categorical_accuracy(labels_test, predictions, k=k) top_accuracy = K.get_session().run(top_accuracy) print() print("Top %d Accuracy : " % k, top_accuracy) print() ground_labels = decode_predictions(labels_test) label_decoder = get_class_decoder() results = [] for pred in predictions: top_k_preds_indices = pred.argsort()[-k:][::-1] result = [(label_decoder.classes_[i], pred[i]) for i in top_k_preds_indices] result.sort(key=lambda x: x[-1], reverse=True)
def top_k_metric(y_true, y_pred, **kwargs): kk = kwargs.get('k', 3) return { 'top_%d_cat_acc' % kk: kmetrics.top_k_categorical_accuracy(y_true, y_pred, k=kk) }
def ctm_acck(y_true, y_pred): pred_list = tf.split(y_pred, 5, axis=-1) pred = pred_list[0] + pred_list[1] + 0.1 * pred_list[2] + pred_list[ 3] + 0.1 * pred_list[4] return top_k_categorical_accuracy(y_true, pred, k=3)
def top_3_accuracy(x,y): return top_k_categorical_accuracy(x,y,3) from sklearn.metrics import accuracy_score
def top_3_accuracy(y, z): return top_k_categorical_accuracy(y, z, k=3)
def top5(y1, y2): return metrics.top_k_categorical_accuracy(y1, y2, k=5)
def top_3_categorical_accuracy(y_true, y_pred): """A metric function that is used to judge the top-3 performance of our model. """ return top_k_categorical_accuracy(y_true, y_pred, k=3)
def top_5_accuracy(y_true, y_pred): return metrics.top_k_categorical_accuracy(y_true, y_pred, k=5)
def calculate_test_accuracies(network_name, test_data, one_file, write_to_file, is_LSTM, segment_size=15): with open(get_speaker_pickle(test_data), 'rb') as f: (X, y, s_list) = pickle.load(f) if one_file: model = load_model(get_experiment_nets(network_name + '.h5')) else: json_file = open(get_experiment_nets('cnn_speaker02.json'), 'r') loaded_model_json = json_file.read() json_file.close() model = model_from_json(loaded_model_json) # load weights into new model model.load_weights(get_experiment_nets('cnn_speaker.02h5')) model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=['accuracy', 'categorical_accuracy', ]) print("Data extraction...") X_test, y_test = dg.generate_test_data(X, y, segment_size) n_classes = np.amax(y_test) + 1 print("Data extraction done!") if is_LSTM: X_test = X_test.reshape(X_test.shape[0], X_test.shape[3], X_test.shape[2]) print("Test output...") im_model = Model(input=model.input, output=model.layers[2].output) data_out = im_model.predict(X_test, batch_size=128) print(data_out) da = np.asarray(data_out) np.savetxt("foo.csv", da, delimiter=",") with open(test_data + "cluster_out_01", 'wb') as f: pickle.dump((da, y, s_list), f, -1) output = model.predict(X_test, batch_size=128, verbose=1) y_t = np_utils.to_categorical(y_test, n_classes) eva = model.evaluate(X_test, y_t, batch_size=128, verbose=2) k_nearest2 = K.eval(metrics.top_k_categorical_accuracy(tf.stack(y_t), tf.stack(output), k=2)) k_nearest3 = K.eval(metrics.top_k_categorical_accuracy(tf.stack(y_t), tf.stack(output), k=3)) k_nearest5 = K.eval(metrics.top_k_categorical_accuracy(tf.stack(y_t), tf.stack(output), k=5)) k_nearest10 = K.eval(metrics.top_k_categorical_accuracy(tf.stack(y_t), tf.stack(output), k=10)) print(output.shape) output_sum = np.zeros((n_classes, n_classes)) output_geom = np.zeros((n_classes, n_classes)) y_pred_max = np.zeros(n_classes) y_pred_median = np.zeros(n_classes) for i in range(n_classes): indices = np.where(y_test == i)[0] speaker_output = np.take(output, indices, axis=0) max_val = 0 for o in speaker_output: output_sum[i] = np.add(output_sum[i], o) output_geom[i] = np.multiply(output_geom[i], o) if np.max(o) > max_val: max_val = np.max(o) y_pred_max[i] = np.argmax(o) output_geom[i] = np.power(output_geom[i], 1 / len(speaker_output)) y_pred_mean = np.zeros(n_classes) y_pred_geom = np.zeros(n_classes) for i in range(len(output_sum)): y_pred_mean[i] = np.argmax(output_sum[i]) y_pred_geom[i] = np.argmax(output_sum[i]) y_correct = np.arange(n_classes) print("geometric wrong") for j in range(len(y_correct)): if y_correct[j] != y_pred_geom[j]: print("Speaker: " + str(y_correct[j]) + ", Pred: " + str(y_pred_geom[j])) ind = np.argpartition(output_sum[j], -5)[-5:] print(np.argmax(output_sum[j])) print(ind[np.argsort(output_sum[j][ind])]) print("mean wrong") for j in range(len(y_correct)): if y_correct[j] != y_pred_mean[j]: print("Speaker: " + str(y_correct[j]) + ", Pred: " + str(y_pred_mean[j])) ind = np.argpartition(output_sum[j], -5)[-5:] print(np.argmax(output_sum[j])) print(ind[np.argsort(output_sum[j][ind])]) print(model.metrics_names) print(eva) print("Acc: %.4f" % eva[2]) print("k2: %.4f" % k_nearest2) print("k3: %.4f" % k_nearest3) print("k5: %.4f" % k_nearest5) print("k10: %.4f" % k_nearest10) print("Accuracy (Max.): %.4f" % accuracy_score(y_correct, y_pred_max)) print("Accuracy (Mean): %.4f" % accuracy_score(y_correct, y_pred_mean)) print("Accuracy (Geom): %.4f" % accuracy_score(y_correct, y_pred_geom)) if write_to_file == True: with open(get_experiment_logs('test_scores.txt'), 'ab') as f: f.write('---------- ' + network_name + '---------------\n') f.write("Accuracy: %.4f \n" % eva[2]) f.write("Accuracy (Max.): %.4f \n" % accuracy_score(y_correct, y_pred_max)) f.write("Accuracy (Mean): %.4f \n" % accuracy_score(y_correct, y_pred_mean)) f.write("Accuracy (Geom): %.4f \n" % accuracy_score(y_correct, y_pred_geom)) f.write("K2: {:.4}, K5: {:.4}, K10: {:.4} \n\n".format(k_nearest2, k_nearest5, k_nearest10))
def top_3_accuracy(x, y): return top_k_categorical_accuracy(x, y, 3)
def top_5_accuracy(y_true, y_pred): return top_k_categorical_accuracy(y_true, y_pred, 5)
def top_3_categ_acc(y_pred,y_true): return metrics.top_k_categorical_accuracy(y_true, y_pred, k=3)
def top_2_accuracy(in_gt, in_pred): return top_k_categorical_accuracy(in_gt, in_pred, k=2)
def top_3_accuracy(y_true, y_pred): return top_k_categorical_accuracy(y_true, y_pred, k=3)
def evaluate_attr(category_size, attribute_size, y_true, y_pred): return top_k_categorical_accuracy(y_true[:, category_size:], y_pred[:, category_size:])