def test(data_dir, results_dir, base_model, start_fold=None, end_fold=10):
if not start_fold:
start_fold = current_fold(results_dir, base_model.name + '.fold')
backend = 'tf' if K.backend() == 'tensorflow' else 'th'
target_size = (base_model.img_height, base_model.img_width)
test_datagen = ImageDataGenerator(rescale=1. / 255)
folds = [str(fold).zfill(2) for fold in range(start_fold, end_fold + 1)]
for fold in folds:
print 'Testing fold {} for {}'.format(fold, base_model.name)
weights = base_model.best_weights.format(fold)
model = base_model.load(weights=weights)
results = list()
test_dir = os.path.join(data_dir, fold, 'test')
test_images = read_fold_dir(test_dir)
for label, img_path in test_images:
img = load_image(test_datagen, img_path, target_size)
predictions = model.predict(img)
results.append((img_path, label, predictions))
results_fname = "{}.fold_{}.{}.csv".format(base_model.name, fold, backend)
results_filepath = os.path.join(results_dir, results_fname)
write_results(results, results_filepath)
del model
if K.backend() == 'tensorflow':
K.clear_session()
def test_on_features_original(data_dir,
results_dir,
features_filepath,
rf_model,
start_fold=None,
end_fold=10,
progress_percent=.01,
num_estimators=None):
backend = 'tf' if K.backend() == 'tensorflow' else 'th'
if not start_fold:
start_fold = current_fold(results_dir, rf_model.name + '.fold')
folds = [str(fold).zfill(2) for fold in range(start_fold, end_fold + 1)]
for fold in folds:
print "Testing fold {} on layers {}".format(fold,
", ".join(rf_model.layers))
weights = rf_model.weights.format(fold)
rf = load_random_forest(weights)
test_dir = os.path.join(data_dir, fold, 'validation')
with open(features_filepath.format(fold), 'r') as f:
users = pickle.load(f)
ind_by_img_path = get_images_indices(users)
images = list()
for image in images_from_fold_dir(test_dir):
images.append(image)
num_test_images = len(images)
test_progress_percent = int(num_test_images * progress_percent)
results = list()
for i, (user_id, date, relative_path) in enumerate(images):
ind = ind_by_img_path[relative_path]
features = users[user_id][date].images[ind].features
label = users[user_id][date].images[ind].label
prediction = rf.predict(features)[0].astype(np.int)
results.append((relative_path, label, prediction))
if progress_percent and (i + 1) % test_progress_percent == 0:
print("Progress %3.2f%% (%d/%d)" % (
(i + 1) / num_test_images * 100, i + 1, num_test_images))
if num_estimators is None:
results_fname = "validation.{}.fold_{}.{}.csv".format(
rf_model.name, fold, backend)
else:
results_fname = "validation.{}.num_estimators_{}.fold_{}.{}.csv".format(
rf_model.name, num_estimators, fold, backend)
results_filepath = os.path.join(results_dir, results_fname)
write_results(results, results_filepath)
def test_on_cnn(data_dir, results_dir, cnn_model, rf_model, start_fold=None, end_fold=10, progress_percent=.1):
if not start_fold:
start_fold = current_fold(results_dir, rf_model.name + '.fold')
backend = 'tf' if K.backend() == 'tensorflow' else 'th'
target_size = (cnn_model.img_height, cnn_model.img_width)
test_datagen = ImageDataGenerator(rescale=1. / 255)
folds = [str(fold).zfill(2) for fold in range(start_fold, end_fold + 1)]
for fold in folds:
weights = cnn_model.best_weights.format(fold)
base_model = cnn_model.load(weights=weights)
layers_by_name = {l.name: l for l in base_model.layers}
outputs = [layers_by_name[l].output for l in rf_model.layers]
model = Model(inputs=base_model.input, outputs=outputs)
weights = rf_model.weights.format(fold)
rf = load_random_forest(weights)
results = list()
test_dir = os.path.join(data_dir, fold, 'test')
test_images = read_fold_dir(test_dir)
num_test_images = len(test_images)
test_progress_percent = int(num_test_images * progress_percent)
print 'Testing fold {} for {} + RF on layers {}'.format(fold, cnn_model.name, ', '.join(rf_model.layers))
for i, (label, img_path) in enumerate(test_images):
img = load_image(test_datagen, img_path, target_size)
predictions = model.predict(img)
if len(rf_model.layers) == 1:
predictions = [predictions]
# Concatenating features
features = np.array([])
for p in predictions:
features = np.append(features, p[0].copy())
prediction = rf.predict([features])[0].astype(np.int)
results.append((img_path, label, prediction))
if progress_percent and (i + 1) % test_progress_percent == 0:
print("Progress %3.2f%% (%d/%d)" % ((i + 1) / num_test_images * 100, i + 1, num_test_images))
results_fname = "{}.fold_{}.{}.csv".format(rf_model.name, fold, backend)
results_filepath = os.path.join(results_dir, results_fname)
write_results(results, results_filepath)
del model
if K.backend() == 'tensorflow':
K.clear_session()
def test_on_features(data_dir, results_dir, features_filepath, rf_model, start_fold=None, end_fold=10, progress_percent=.01, num_estimators=None):
backend = 'tf' if K.backend() == 'tensorflow' else 'th'
if not start_fold:
start_fold = current_fold(results_dir, rf_model.name + '.fold')
folds = [str(fold).zfill(2) for fold in range(start_fold, end_fold + 1)]
for fold in folds:
print "Testing fold {} on layers {}".format(fold, ", ".join(rf_model.layers))
weights = rf_model.weights.format(fold)
rf = load_random_forest(weights)
test_dir = os.path.join(data_dir, fold, 'test')
with open(features_filepath.format(fold), 'r') as f:
users = pickle.load(f)
ind_by_img_path = get_images_indices(users)
images = list()
for image in images_from_fold_dir(test_dir):
images.append(image)
num_test_images = len(images)
test_progress_percent = int(num_test_images * progress_percent)
# Determining the number of features
user_id, date, relative_path = images[0]
ind = ind_by_img_path[relative_path]
num_features = users[user_id][date].images[ind].features.size
relative_paths = list()
labels = list()
features = np.zeros((num_test_images, num_features))
for i, (user_id, date, relative_path) in enumerate(images):
ind = ind_by_img_path[relative_path]
relative_paths.append(relative_path)
labels.append(users[user_id][date].images[ind].label)
features[i,:] = users[user_id][date].images[ind].features
predictions = rf.predict(features)
results = list()
for i, (relative_path, label) in enumerate(zip(relative_paths, labels)):
results.append((relative_path, label, predictions[i]))
if num_estimators is None:
results_fname = "{}.fold_{}.{}.csv".format(rf_model.name, fold, backend)
else:
results_fname = "{}.num_estimators_{}.fold_{}.{}.csv".format(rf_model.name, num_estimators, fold, backend)
results_filepath = os.path.join(results_dir, results_fname)
write_results(results, results_filepath)
def test(features_filepath,
results_dir,
base_model,
start_fold,
end_fold,
timestep,
iccv_epic=False):
users = IO.load_annotations(ntcir.filepaths)
sorted_users = ntcir.utils.sort(users)
num_frames_per_day = 2880
sequences = ntcir.get_sequences(sorted_users, num_frames_per_day)
backend = 'tf' if K.backend() == 'tensorflow' else 'th'
if not start_fold:
start_fold = current_fold(results_dir, base_model.name + '.fold')
folds = [str(fold).zfill(2) for fold in range(start_fold, end_fold + 1)]
for fold in folds:
with open(features_filepath.format(fold), 'r') as f:
features = pickle.load(f)
if iccv_epic:
test_split = ntcir.read_split('datasets/ntcir/test_split.txt')
else:
test_split = ntcir.get_split_fold(sorted_users,
int(fold) - 1, False)
test_batches = ntcir.get_batches(test_split,
sequences,
timestep=timestep,
include_last=True)
K.set_learning_phase(False)
weights = base_model.best_weights.format(fold)
model = base_model.load(
feature_vector_length=base_model.feature_vector_length,
weights=weights,
timestep=timestep)
frames = list()
groundtruth = list()
predictions = list()
for i, batch in enumerate(test_batches):
x, y = load_batch(
features,
batch,
feature_vector_length=base_model.feature_vector_length,
batch_size=1,
timestep=timestep)
prediction = model.predict_on_batch(x)
prediction = np.argmax(prediction, axis=2).squeeze()[0:batch.size]
predictions.extend(prediction)
groundtruth.extend(np.argmax(y, axis=2).squeeze()[0:batch.size])
for j, ind in enumerate(batch.indices):
image = features[batch.user_id][batch.date].images[ind]
frames.append(image.path)
results_fname = "{}.fold_{}.{}.csv".format(base_model.name, fold,
backend)
results_filepath = os.path.join(results_dir, results_fname)
write_results(frames, groundtruth, predictions, results_filepath)
del model
if K.backend() == 'tensorflow':
K.clear_session()
def test(features_filepath,
results_dir,
base_model,
start_fold,
end_fold,
timestep,
iccv_epic=False,
progress_percent=0.05):
users = IO.load_annotations(ntcir.filepaths)
sorted_users = ntcir.utils.sort(users)
num_frames_per_day = 2880
sequences = ntcir.get_sequences(sorted_users, num_frames_per_day)
backend = 'tf' if K.backend() == 'tensorflow' else 'th'
if not start_fold:
start_fold = current_fold(results_dir, base_model.name + '.fold')
folds = [str(fold).zfill(2) for fold in range(start_fold, end_fold + 1)]
for fold in folds:
with open(features_filepath.format(fold), 'r') as f:
features = pickle.load(f)
if iccv_epic:
test_split = ntcir.read_split('datasets/ntcir/test_split.txt')
else:
test_split = ntcir.get_split_fold(sorted_users,
int(fold) - 1, False)
test_batches = ntcir.get_training_batches(test_split,
sequences,
timestep=timestep)
K.set_learning_phase(False)
weights = base_model.best_weights.format(fold)
model = base_model.load(
feature_vector_length=base_model.feature_vector_length,
weights=weights,
timestep=timestep)
num_test_batches = len(test_batches)
if progress_percent:
test_progress_percent = int(num_test_batches * progress_percent)
print "Testing fold {}".format(fold)
results = list()
for i, batch in enumerate(test_batches):
x, y = load_batch(
features,
batch,
feature_vector_length=base_model.feature_vector_length,
batch_size=1,
timestep=timestep)
prediction = model.predict_on_batch(x)
prediction = np.argmax(prediction, axis=2).squeeze()[-1]
ind = batch.indices[-1]
image = features[batch.user_id][batch.date].images[ind]
results.append((image.path, image.label, prediction))
if progress_percent and (i + 1) % test_progress_percent == 0:
print("Progress %3.2f%% (%d/%d)" % (
(i + 1) / num_test_batches * 100, i + 1, num_test_batches))
results_fname = "{}.many2one.fold_{}.{}.csv".format(
base_model.name, fold, backend)
results_filepath = os.path.join(results_dir, results_fname)
write_results(results, results_filepath)
del model
if K.backend() == 'tensorflow':
K.clear_session()