def telemetry(sid, data):
# The current steering angle of the car
steering_angle = data["steering_angle"]
# The current throttle of the car
throttle = data["throttle"]
# The current speed of the car
speed = data["speed"]
# The current image from the center camera of the car
imgString = data["image"]
image = Image.open(BytesIO(base64.b64decode(imgString)))
image_array = np.asarray(image)
transformed_image_array = image_array[None, :, :, :]
transformed_image_array = utils.preprocess_images(transformed_image_array, True)
#This model currently assumes that the features of the model are just the images. Feel free to change this.
steering_angle = float(model.predict(transformed_image_array, batch_size=1))
#steering_angle = np.round(steering_angle, 4)
#The driving model currently just outputs a constant throttle. Feel free to edit this.
if float(speed) < 20.0:
throttle = 0.5
else:
throttle = 0.2
#throttle = 0.5
print(steering_angle, throttle)
send_control(steering_angle, throttle)
def generate(files, steering, batch_size, augment_data=True):
#data = shuffle(data)
num_examples = len(files)
offset = num_examples
i = 1
while True:
if (offset + batch_size) >= num_examples:
offset = 0
i = 1
files_s, steering_s = shuffle(files, steering)
for offset in range(0, num_examples, batch_size):
i += 1
end = offset + batch_size
if end >= num_examples:
end = num_examples
filename_x, batch_y = files_s[offset:end], steering_s[offset:end]
if augment_data:
batch_x, batch_y = utils.augment_dataset_single(
filename_x, batch_y)
else:
batch_x = utils.read_images(filename_x)
# Rescale and resize only
batch_x = utils.preprocess_images(batch_x, False)
yield batch_x.astype('float32'), batch_y.astype('float32')
def rollout(controller, playback=False):
if playback:
ims = []
state = preprocess_images(env.reset())
M.reset_states()
h = np.zeros(256)
done = False
cumulative_reward = 0
while not done:
_state = np.zeros((128, 64, 64, 3))
_state[0] = state
if playback:
ims.append(state)
z = V(_state)[2][0] #extract z and first from sequence
# combine V latent space with M hidden space
combined = np.concatenate([z, h], axis=0)
a = controller(combined)
state, reward, done, info = env.step(a)
state = preprocess_images(state)
cumulative_reward += reward
# extract hidden state from LSTM
h = get_hidden(tf.expand_dims(tf.expand_dims(np.concatenate([z, a], 0), 0), 0)).squeeze()
# get factored gaussians
# by feeding current latent_state + action
z = M(tf.expand_dims(tf.expand_dims(np.concatenate([z, a]), 0), 0))
# sample from factored gaussians
# 32 = output_dims
# 5 = num_mixtures
z = np.apply_along_axis(mdn.sample_from_output, 1, z[0], 32, 5, temp=1.0).squeeze()
env.close()
if playback:
return cumulative_reward, ims
return cumulative_reward
def predict(self, test_ims):
descriptor_extractor = self.get_descriptor_extractor()
test_ims = utils.preprocess_images(test_ims, **self.preprocess_params)
test_histograms = \
utils.get_histograms(test_ims,
self.BOVW,
descriptor_extractor,
self.spatial_pyramid_levels)
test_histograms = self.transform_histograms(test_histograms)
return self.model.predict(test_histograms)
def train(self, train_ims, train_im_labels):
descriptor_extractor = self.get_descriptor_extractor()
train_ims = utils.preprocess_images(train_ims,
**self.preprocess_params)
train_im_histograms = \
utils.get_histograms(train_ims,
self.BOVW,
descriptor_extractor,
self.spatial_pyramid_levels)
self.generate_data_transformers()
train_im_histograms = self.fit_data_transformers(train_im_histograms)
self.train_model(train_im_histograms, train_im_labels)
def load_batch(batchsize, mode, gt_type):
"""
This function loads a batch for mlnet training.
:param batchsize: batchsize.
:param mode: choose among [`train`, `val`, `test`].
:param gt_type: choose among [`sal`, `fix`].
:return: X and Y as ndarray having shape (b, c, h, w).
"""
assert mode in ['train', 'val', 'test'
], 'Unknown mode {} for dreyeve batch loader'.format(mode)
assert gt_type in [
'sal', 'fix'
], 'Unknown gt_type {} for dreyeve batch loader'.format(gt_type)
if mode == 'train':
sequences = dreyeve_train_seq
allowed_frames = train_frame_range
allow_mirror = True
elif mode == 'val':
sequences = dreyeve_train_seq
allowed_frames = val_frame_range
allow_mirror = False
elif mode == 'test':
sequences = dreyeve_test_seq
allowed_frames = test_frame_range
allow_mirror = False
x_list = []
y_list = []
for b in xrange(0, batchsize):
seq = random.choice(sequences)
fr = random.choice(allowed_frames)
x_list.append(
join(DREYEVE_DIR, '{:02d}'.format(seq), 'frames',
'{:06d}.jpg'.format(fr)))
y_list.append(
join(DREYEVE_DIR, '{:02d}'.format(seq),
'saliency' if gt_type == 'sal' else 'saliency_fix',
'{:06d}.png'.format(fr + 1)))
X = preprocess_images(x_list, shape_r=shape_r, shape_c=shape_c)
Y = preprocess_maps(y_list, shape_r=shape_r_gt, shape_c=shape_c_gt)
# TODO apply random mirroring?
return np.float32(X), np.float32(Y)
def load_dreyeve_sample(sequence_dir, sample, shape_r, shape_c):
"""
Function to load a dreyeve sample.
:param sequence_dir: the directory of the sequence we want to sample from.
:param sample: the number of the sample.
:param shape_r: rows of the image to predict.
:param shape_c: columns of the image to predict.
:return: a ndarray having shape (1, 3, shape_r, shape_w)
"""
filename = join(sequence_dir, 'frames', '{:06d}.jpg'.format(sample))
X = preprocess_images([filename], shape_r, shape_c)
return X
def BOVW_create(self, ims, im_labels):
logging.debug('Total images to process for BOVW: %d' % len(ims))
ims = utils.preprocess_images(ims, **self.preprocess_params)
descriptor_extractor = self.get_descriptor_extractor()
keypoints, descriptors = utils.get_kp_and_des(ims,
descriptor_extractor)
all_descriptors = np.concatenate(descriptors)
bovw = clustering.get_clustering(all_descriptors,
self.cluster_model_type,
self.cluster_model_params)
self.BOVW = bovw
self.BOVW.ims = ims
self.BOVW.im_labels = im_labels
self.BOVW.kp = keypoints
self.BOVW.des = descriptors
self.BOVW.clusters = [self.BOVW.predict(des) for des in descriptors]
logging.debug('BOVW (k=%d) created.' % self.BOVW.n_clusters)
def main():
# Import data
(train_images,
train_labels), (test_images,
test_labels) = keras.datasets.mnist.load_data()
test_images = test_images.reshape(test_images.shape[0], 28, 28, 1)
test_images = preprocess_images(test_images)
# Create the model
caps_net = CapsNet()
# Build up architecture
caps_net.eval_architecture('test', cfg.FIG_DIR)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# Read check point file
ckpt = tf.train.get_checkpoint_state(cfg.TRAIN_DIR)
# Test the network
with tf.Session(config=config) as sess:
caps_net.saver.restore(sess, ckpt.model_checkpoint_path)
for i in range(80):
# Read images
filename = '../images/img'
filename += str(i)
filename += '.png'
img = cv2.imread(filename, 0).astype('float32')
height, width = img.shape
if (height != 720) | (width != 720):
img = cv2.resize(img, (720, 720))
img = cv2.resize(img, (28, 28))
img = np.reshape(img, [28, 28])
img = np.reshape(img, [1, 784])
print('prediction for image ', filename)
preds, perc = caps_net.test_eval(sess, img, 1)
def extract_cam_descriptors(model,
batch_size,
num_classes,
size,
mean_value,
image_train_list_path,
desc_wp,
chunk_index,
ind=0):
images = [0] * image_batch_size
image_names = [0] * image_batch_size
counter = 0
desc_count = 0
num_images = 0
t0 = time.time()
print 'Horizontal size: ', size[0]
print 'Vertical size: ', size[1]
for line in open(image_train_list_path):
if counter >= image_batch_size:
print 'Processing image batch: ', ind
t1 = time.time()
data = preprocess_images(images, size[0], size[1], mean_value)
if aggregation_type == 'Offline':
features, cams, cl = \
extract_feat_cam(model, layer, batch_size, data, num_classes)
if saving_CAMs:
save_data(cams, cam_path, 'cams_' + str(ind) + '.h5')
save_data(features, feature_path,
'features_' + str(ind) + '.h5')
d_wp = weighted_cam_pooling(features, cams)
desc_wp = np.concatenate((desc_wp, d_wp))
print 'Image batch processed, CAMs descriptors obtained!'
print 'Time elapsed: ', time.time() - t1
sys.stdout.flush()
counter = 0
desc_count += image_batch_size
if descriptors_batch_size == desc_count and aggregation_type == 'Offline':
print 'Saving ...' + descriptors_cams_path_wp + '_' + str(
chunk_index) + '.h5'
save_data(
desc_wp,
descriptors_cams_path_wp + '_' + str(chunk_index) + '.h5',
'')
desc_count = 0
chunk_index += 1
desc_wp = np.zeros((0, dim_descriptor), dtype=np.float32)
ind += 1
line = line.rstrip('\n')
images[counter] = imread(line)
image_names[counter] = line
counter += 1
num_images += 1
#Last batch
print 'Last Batch:'
data = np.zeros((counter, 3, size[1], size[0]), dtype=np.float32)
data[0:] = preprocess_images(images[0:counter], size[0], size[1],
mean_value)
if aggregation_type == 'Offline':
features, cams, cl = extract_feat_cam(model, layer, batch_size, data,
num_classes)
if saving_CAMs:
save_data(cams, cam_path, 'cams_' + str(ind) + '.h5')
save_data(features, feature_path, 'features_' + str(ind) + '.h5')
d_wp = weighted_cam_pooling(features, cams)
desc_wp = np.concatenate((desc_wp, d_wp))
save_data(desc_wp,
descriptors_cams_path_wp + '_' + str(chunk_index) + '.h5',
'')
chunk_index += 1
desc_wp = np.zeros((0, dim_descriptor), dtype=np.float32)
ind += 1
print desc_wp.shape
print 'Batch processed, CAMs descriptors obtained!'
print 'Total time elapsed: ', time.time() - t0
sys.stdout.flush()
return desc_wp, chunk_index
# In[ ]:
import gym
# In[ ]:
env = gym.make("CarRacing-v0")
# In[ ]:
state = preprocess_images(env.reset())
env.close()
# In[ ]:
def rollout(controller, playback=False):
if playback:
ims = []
state = preprocess_images(env.reset())
M.reset_states()
h = np.zeros(256)
done = False
cumulative_reward = 0
def re_ranking(desc_query, class_list, batch_size, image_names, indices,
dataset, top_n_ranking, pca_matrix, model, model_name):
if dataset == 'Oxford' or dataset == 'Oxford105k':
images_path = '/data/jim011/datasets_retrieval/Oxford5k/images/'
if dataset == 'Paris' or dataset == 'Paris106k':
images_path = '/data/jim011/datasets_retrieval/Paris6k/images/'
index_q = indices[0:top_n_ranking]
tt = time.time()
indexed_names = list()
i = 0
if top_n_ranking >= 1000:
image_batch = 300
else:
image_batch = top_n_ranking
n_iterations = int(math.floor(top_n_ranking / image_batch))
last_batch = top_n_ranking % image_batch
scores = np.zeros(top_n_ranking, dtype=np.float32)
scores_h = np.zeros(top_n_ranking, dtype=np.float32)
scores_v = np.zeros(top_n_ranking, dtype=np.float32)
final_desc_h = np.zeros(top_n_ranking, dtype=np.float32)
final_desc_v = np.zeros(top_n_ranking, dtype=np.float32)
print desc_query.shape
final_descriptors_all = np.zeros((top_n_ranking, desc_query.shape[1]),
dtype=np.float32)
image_ranked_names = image_names[index_q]
num_cams = class_list.shape[0]
for k in range(0, n_iterations + 1):
images_h = list()
images_v = list()
images_ver = False
images_hor = False
t1 = time.time()
if k == n_iterations:
#Last Batch
if last_batch != 0:
last_ind = image_batch * k + last_batch
else:
break
else:
last_ind = image_batch * (k + 1)
print image_names[index_q[k * image_batch:last_ind]]
# Separate the images in Horizontal/Vertical for faster processing
image_order = list()
for ind_im, name in enumerate(
image_names[index_q[k * image_batch:last_ind]]):
if name[0] == '/':
im = imread(name.replace('\n', ''))
else:
im = imread(images_path + name.replace('\n', '') + '.jpg')
if im.shape[0] >= im.shape[1]:
images_v.append(im)
images_ver = True
image_order.append(1)
else:
images_h.append(im)
images_hor = True
image_order.append(0)
# Extract Features/CAMs
print 'Time loading images: ', time.time() - t1
if images_hor:
size = size_h
t2 = time.time()
images_tensor = preprocess_images(images_h, size[0], size[1],
stats[0], stats[1])
images_tensor = torch.autograd.Variable(images_tensor,
volatile=True)
features_h, cams_h, roi_h = extract_feat_cam(model,
model_name,
layer,
batch_size,
images_tensor,
num_cams,
class_list,
roi=True)
print 'Time extracting features hor: ', time.time() - t2
t3 = time.time()
scores_h, final_desc_h = compute_scores_cams(
desc_query, features_h, cams_h, roi_h, pca_matrix)
print 'Time computing scores: ', time.time() - t3
print scores_h
if images_ver:
size = size_v
t2 = time.time()
images_tensor = preprocess_images(images_v, size[0], size[1],
stats[0], stats[1])
images_tensor = torch.autograd.Variable(images_tensor,
volatile=True)
features_v, cams_v, roi_v = extract_feat_cam(model,
model_name,
layer,
batch_size,
images_tensor,
num_cams,
class_list,
roi=True)
print 'Time extracting features ver: ', time.time() - t2
t3 = time.time()
scores_v, final_desc_v = compute_scores_cams(
desc_query, features_v, cams_v, roi_v, pca_matrix)
print 'Time computing scores: ', time.time() - t3
print scores_v
# Compute Scores
print image_order
# Re-order
scores[k * image_batch:last_ind] = re_order(image_order, scores_h,
scores_v)
final_descriptors_all[k * image_batch:last_ind] = re_order(
image_order, final_desc_h, final_desc_v)
print final_descriptors_all.shape
print scores[k * image_batch:image_batch * (k + 1)]
print 'Time loading computing scores: ', time.time() - t2
print 'Time elapsed x image:', time.time() - t1
print scores
ordered_sc = scores.argsort()[::-1]
print ordered_sc
print image_names[index_q]
print image_ranked_names[ordered_sc]
# Index of the in order of relevance
ordered_ind = index_q[ordered_sc]
indexed_names.append(np.copy(image_ranked_names[ordered_sc]))
indices[0:top_n_ranking] = ordered_ind
i += 1
print 'Time elapsed:', time.time() - tt
# Return indices and data ordered by similarity with the query
return indices, final_descriptors_all[ordered_sc]
D_fake = discriminator(G, reuse=True)
return G, D_real, D_fake
# Define constants
NUM_EPOCHS = 100
BATCH_SIZE = 128
LEARNING_RATE = 0.0002
BETA1 = 0.5
NOISE_DIM = 100
SAMPLE_SIZE = 100
# Load mnist data
X_train = utils.load_mnist_data()
utils.plot_sample(X_train[:SAMPLE_SIZE], "output/mnist_data.png")
X_train = utils.preprocess_images(X_train)
mini_batches = utils.random_mini_batches(X_train, BATCH_SIZE)
# Create DCGAN
X = tf.placeholder(tf.float32, shape=(None, X_train.shape[1], X_train.shape[2], X_train.shape[3]))
Z = tf.placeholder(tf.float32, [None, NOISE_DIM])
G, D_real, D_fake = create_gan(X, Z)
# Create training steps
G_loss_func, D_loss_func = utils.create_loss_funcs(D_real, D_fake)
G_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="Generator")
D_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="Discriminator")
G_train_step = tf.train.AdamOptimizer(learning_rate=LEARNING_RATE, beta1=BETA1).minimize(G_loss_func, var_list=G_vars)
D_train_step = tf.train.AdamOptimizer(learning_rate=LEARNING_RATE, beta1=BETA1).minimize(D_loss_func, var_list=D_vars)
# Start session
def train(hparams):
input_size = hparams["input_size"]
if hparams["network_type"] == "cnn":
ds_train = preprocess_images(hparams, 'train', 'training', 0.2)
ds_dev = preprocess_images(hparams, 'train', 'validation', 0.2)
ds_test = preprocess_images(hparams, 'test')
else:
x, y = prepare_data(hparams, 'train')
x_scaled = preprocessing.scale(x)
#x_dev, y_dev = prepare_data(hparams, 'dev')
#x_test, y_test = prepare_data(hparams, 'test')
x_train, x_test, y_train, y_test = train_test_split(x_scaled, y, test_size=0.15, random_state=42)
x_train, x_dev, y_train, y_dev = train_test_split(x_train, y_train, test_size=0.15, random_state=42)
if hparams['training_type'] == 'DL':
if hparams["network_type"] == "lstm":
inputs, outputs = lstm(hparams,
input_size,
hparams["network_config"]["lstm"])
elif hparams["network_type"] == "cnn":
inputs, outputs = cnn(hparams,
input_size,
hparams["network_config"]["cnn"])
elif hparams["network_type"] == "fully_connected":
inputs, outputs = fully_connected(hparams,
input_size,
hparams["network_config"]["fully_connected"])
else:
raise ValueError('Undefined {} network for DL'.format(hparams["network_type"]))
model = tf.keras.Model(inputs=inputs, outputs=outputs, name="fault_detector")
model.summary()
model.compile(optimizer=tf.keras.optimizers.Adam(hparams["learning_rate"]),
loss=hparams["loss"])
if hparams["network_type"] == "cnn":
model.fit(ds_train, verbose=1, epochs=hparams["epochs"], shuffle=True, validation_data=ds_dev)
else:
model.fit(x_train, y_train, verbose=1, shuffle=True, validation_data=(x_dev, y_dev),
epochs=hparams["epochs"], batch_size=hparams["batch_size"])
elif hparams["training_type"] == 'ML':
if hparams["network_type"] == 'NB':
model = NaiiveBayes(x_train, y_train, hparams["network_config"]["NB"]["type"])
elif hparams["network_type"] == 'KNN':
model = KNN(x_train, y_train, hparams["network_config"]["KNN"]["k"])
elif hparams["network_type"] == 'logistic_regression':
model = LR(x_train, y_train, hparams["network_config"]["logistic_regression"]["c"])
elif hparams["network_type"] == 'SVM':
model = SVM(x_train, y_train, hparams["network_config"]["SVM"]["c"],
hparams["network_config"]["SVM"]["kernel"])
else:
raise ValueError('Undefined {} network type for ML'.format(hparams["network_type"]))
if hparams["network_type"] == "cnn":
model.evaluate(ds_test)
else:
evaluate_model(model, x_test, y_test, hparams["training_type"])
if bool(hparams['save_model']):
if hparams['training_type']=='DL':
model.save(hparams['model_path']+'.h5')
else:
with open(hparams['model_path']+'.pkl', 'wb') as file:
pickle.dump(model, file)
outputs=top_model(vgg16_model.output))
# 学習済みの重みをロード
model.load_weights(
os.path.join(config.result_dir, 'finetuning_weights.h5'))
# compile
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
# 画像を読み込んで4次元テンソルへ変換
img = image.load_img(args.image,
target_size=(config.img_height, config.img_width))
x = image.img_to_array(img)
x = np.expand_dims(x, axis=0)
# 学習時にImageDataGeneratorのrescaleで正規化したので同じ処理が必要
x = utils.preprocess_images(x)
# クラスを予測
# 入力は1枚の画像なので[0]のみ
pred = model.predict(x)[0]
# 予測確率が高いトップ5を出力
top = 5
top_indices = pred.argsort()[-top:][::-1]
result = [(classes[i], pred[i]) for i in top_indices]
data = []
for x in result:
data.append({"name": x[0], "percentage": '%.10f' % (x[1] * 100)})
print({"error": "", "data": data})
except (KeyboardInterrupt, SystemExit):
utils.unlock()
utils.error(config.syserr)
with open('../lists/list_queries_oxford.txt', "r") as f:
for line in f:
print i
line = line.replace('\n', '')
img = np.array(
(imread(path_images_oxford + line + '.jpg')),
dtype=np.float32)
#img = np.transpose(img, (2, 0, 1))
print img.shape
if img.shape[0] > img.shape[1]:
size = size_v
else:
size = size_h
img_p = preprocess_images(img, size[0], size[1], mean_value)
if aggregation_type == 'Offline':
features, cams, cl = extract_feat_cam(model, layer, batch_size, img_p, num_classes)
if saving_CAMs:
save_data(cams, cam_path, 'cams_' + str(ind) + '.h5')
save_data(features, feature_path, 'features_' + str(ind) + '.h5')
d_wp = weighted_cam_pooling(features, cams)
desc_wp = np.concatenate((desc_wp, d_wp))
#d_sp = sum_pooling(features)
#desc_sp = np.concatenate((desc_sp, d_sp))
#d_sp_c = sum_pooling(features, CroW=True)
#desc_sp_c = np.concatenate((desc_sp_c, d_sp_c))
elif aggregation_type == 'Online':
# lime
import lime
from lime import lime_image
# creating training and test sets
# put crater_date folder on root directory too.
# create the training_set and test_set directories inside crater_data and put west region on training_set and other regions on test_set on each running
# crater_data/training_set/crater/
# crater_data/training_set/non-crater/
# west region as training_set
#preprocess_images('tile2_24', 'training_set', img_dimensions=(64, 64))
#preprocess_images('tile2_25', 'training_set', img_dimensions=(64, 64))
# east region as test_set
preprocess_images('tile1_24', 'test_set', img_dimensions=(64, 64))
preprocess_images('tile1_25', 'test_set', img_dimensions=(64, 64))
preprocess_images('tile2_24', 'test_set', img_dimensions=(64, 64))
preprocess_images('tile2_25', 'test_set', img_dimensions=(64, 64))
preprocess_images('tile3_24', 'test_set', img_dimensions=(64, 64))
preprocess_images('tile3_25', 'test_set', img_dimensions=(64, 64))
train_datagen = ImageDataGenerator(rescale = 1./255,
shear_range = 0.2,
zoom_range = 0.2,
horizontal_flip = False)
test_datagen = ImageDataGenerator(rescale = 1./255)
training_set = train_datagen.flow_from_directory('./crater_data/training_set',
target_size = (64, 64),
i = 0
with open('../lists/list_queries_oxford.txt', "r") as f:
for line in f:
print i
line = line.replace('\n', '')
img = np.array(imread(path_images_oxford + line + '.jpg'),
dtype=np.float32)
#img = np.transpose(img, (2, 0, 1))
print img.shape
if img.shape[0] > img.shape[1]:
size = size_v
else:
size = size_h
img_tensor = preprocess_images(img, size[0], size[1], stats[0],
stats[1])
img_tensor = torch.autograd.Variable(img_tensor, volatile=True)
if aggregation_type == 'Offline':
features, cams, cl, _ = extract_feat_cam(
model, model_name, 1, img_tensor, num_classes)
if saving_CAMs:
save_data(cams, cam_path, 'cams_' + str(ind) + '.h5')
save_data(features, feature_path,
'features_' + str(ind) + '.h5')
d_wp = weighted_cam_pooling(features, cams)
desc_wp = np.concatenate((desc_wp, d_wp))
d_sp = sum_pooling(features)
desc_sp = np.concatenate((desc_sp, d_sp))
def main():
# Import data
(train_images,
train_labels), (test_images,
test_labels) = keras.datasets.mnist.load_data()
# reshape images to specify that it's a single channel
train_images = train_images.reshape(train_images.shape[0], 28, 28, 1)
test_images = test_images.reshape(test_images.shape[0], 28, 28, 1)
train_images = preprocess_images(train_images)
test_images = preprocess_images(test_images)
# Create the model
caps_net = CapsNet()
# Build up architecture
caps_net.eval_architecture('test', cfg.FIG_DIR)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# Read check point file
ckpt = tf.train.get_checkpoint_state(cfg.TRAIN_DIR)
# Test the network
with tf.Session(config=config) as sess:
caps_net.saver.restore(sess, ckpt.model_checkpoint_path)
mnist_in_path = '../videos/mnist_in.mp4'
mnist_out_path = '../videos/mnist_caps_out.mp4'
cap = cv2.VideoCapture(mnist_in_path)
vw = None
frame = -1 # counter for debugging (mostly), 0-indexed
# go through all the frames and run our classifier on the high res MNIST images as they morph from number to number
while True: # should 481 frames
frame += 1
ret, img = cap.read()
if not ret: break
img = cv2.resize(img, (720, 720))
# preprocess the image for prediction
img_proc = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
img_proc = cv2.resize(img_proc, (28, 28))
img_proc = preprocess_images(img_proc)
img_proc = 1 - img_proc # inverse since training dataset is white text with black background
net_in = np.expand_dims(
img_proc,
axis=0) # expand dimension to specify batch size of 1
net_in = np.expand_dims(
net_in,
axis=3) # expand dimension to specify number of channels
img_t = img_proc.copy()
img_t = np.reshape(img_t, [28, 28])
img_t = np.reshape(img_t, [1, 784])
preds, perc = caps_net.test_eval(sess, img_t, 1)
img = 255 - img
pad_color = 0
img = np.pad(img, ((0, 0), (0, 1280 - 720), (0, 0)),
mode='constant',
constant_values=(pad_color))
line_type = cv2.LINE_AA
font_face = cv2.FONT_HERSHEY_SIMPLEX
font_scale = 1.3
thickness = 2
x, y = 740, 60
color = (255, 255, 255)
text = "HNN Output:"
cv2.putText(img,
text=text,
org=(x, y),
fontScale=font_scale,
fontFace=font_face,
thickness=thickness,
color=color,
lineType=line_type)
text = "Input:"
cv2.putText(img,
text=text,
org=(30, y),
fontScale=font_scale,
fontFace=font_face,
thickness=thickness,
color=color,
lineType=line_type)
guess = perc[1][0][0]
# sort the lists
p1 = perc[1][0]
p2 = perc[0][0]
ziplist = sorted(zip(p1, p2))
p3, p4 = zip(*ziplist)
y = 130
#for i, p in enumerate(perc[0]):
for i in range(0, 10):
pred = p3[i]
prob = p4[i]
p = np.rint(prob * 100).astype(int)
if pred == guess:
#color = (255, 218, 158)
color = (25, 200, 25)
else:
color = (100, 100, 100)
rect_width = 0
if p > 0: rect_width = int(p * 3.3)
rect_start = 180
cv2.rectangle(img, (x + rect_start, y - 5),
(x + rect_start + rect_width, y - 20), color, -1)
text = '{}: {:>3}%'.format(i, int(p))
cv2.putText(img,
text=text,
org=(x, y),
fontScale=font_scale,
fontFace=font_face,
thickness=thickness,
color=color,
lineType=line_type)
y += 60
# if you don't want to save the output as a video, set this to False
save_video = True
if save_video:
if vw is None:
codec = cv2.VideoWriter_fourcc(*'mp4v')
vid_width_height = img.shape[1], img.shape[0]
vw = cv2.VideoWriter(mnist_out_path, codec, 30,
vid_width_height)
# 15 fps above doesn't work robustly so we write frame twice at 30 fps
vw.write(img)
vw.write(img)
cap.release()
if vw is not None:
vw.release()
def extract_cam_descriptors(model,
model_name,
batch_size,
num_classes,
size,
stats,
image_train_list_path,
desc_wp,
desc_sp,
ind=0):
images = [0] * image_batch_size
image_names = [0] * image_batch_size
counter = 0
num_images = 0
t0 = time.time()
print 'Horizontal size: ', size[0]
print 'Vertical size: ', size[1]
for line in open(image_train_list_path):
if counter >= image_batch_size:
print 'Processing image batch: ', ind
t1 = time.time()
data = preprocess_images(images, size[0], size[1], stats[0],
stats[1])
print data.size()
data = torch.autograd.Variable(data, volatile=True)
if aggregation_type == 'Offline':
features, cams, cl, _ = extract_feat_cam(
model, model_name, batch_size, data, num_classes)
if saving_CAMs:
save_data(cams, cam_path, 'cams_' + str(ind) + '.h5')
save_data(features, feature_path,
'features_' + str(ind) + '.h5')
d_wp = weighted_cam_pooling(features, cams)
desc_wp = np.concatenate((desc_wp, d_wp))
d_sp = sum_pooling(features)
desc_sp = np.concatenate((desc_sp, d_sp))
elif aggregation_type == 'Online':
features, cams = extract_feat_cam_all(model, model_name,
batch_size, data,
num_classes)
d_wp = weighted_cam_pooling(features, cams)
for img_ind in range(0, image_batch_size):
print 'Saved ' + image_names[img_ind] + '.h5'
save_data(
d_wp[img_ind * nb_classes:(img_ind + 1) * nb_classes],
path_descriptors, image_names[img_ind] + '.h5')
print 'Image batch processed, CAMs descriptors obtained!'
print 'Time elapsed: ', time.time() - t1
sys.stdout.flush()
counter = 0
ind += 1
line = line.rstrip('\n')
images[counter] = imread(line)
if dataset == 'Oxford':
line = line.replace(
'/data/jim011/datasets_retrieval/Oxford5k/images/', '')
elif dataset == 'Paris':
line = line.replace(
'/data/jim011/datasets_retrieval/Paris6k/images/', '')
image_names[counter] = (line.replace('.jpg', ''))
counter += 1
num_images += 1
#Last batch
print 'Last Batch:'
data = preprocess_images(images[0:counter], size[0], size[1], stats[0],
stats[1])
data = torch.autograd.Variable(data, volatile=True)
if aggregation_type == 'Offline':
features, cams, cl, _ = extract_feat_cam(model, model_name, batch_size,
data, num_classes)
if saving_CAMs:
save_data(cams, cam_path, 'cams_' + str(ind) + '.h5')
save_data(features, feature_path, 'features_' + str(ind) + '.h5')
d_wp = weighted_cam_pooling(features, cams)
desc_wp = np.concatenate((desc_wp, d_wp))
d_sp = sum_pooling(features)
desc_sp = np.concatenate((desc_sp, d_sp))
elif aggregation_type == 'Online':
features, cams = extract_feat_cam_all(model, model_name, batch_size,
data, num_classes)
d_wp = weighted_cam_pooling(features, cams)
for img_ind in range(0, counter):
save_data(d_wp[img_ind * nb_classes:(img_ind + 1) * nb_classes],
path_descriptors, image_names[img_ind] + '.h5')
ind += 1
print desc_wp.shape
print 'Batch processed, CAMs descriptors obtained!'
print 'Total time elapsed: ', time.time() - t0
sys.stdout.flush()
return desc_wp, desc_sp
x, y, dx, dy = f_list[1], f_list[2], f_list[3], f_list[4]
# Feature map of the query bounding box (the dimensions of the last layer are 16 times smaller)
f_x, f_y, f_dx, f_dy = int((x - (x % 16)) / 16), int((y - (y % 16)) / 16), \
int((dx - (dx % 16)) / 16), int((dy - (dy % 16)) / 16)
img_cropped = img[y:dy, x:dx]
print 'Name of the query: ', query_img_name
# Make multiple of 16
h = img_cropped.shape[0] - (img_cropped.shape[0] % 16)
w = img_cropped.shape[1] - (img_cropped.shape[1] % 16)
img_cropped = preprocess_images(img_cropped, w, h, mean_value)
# Obtain the classes from the cropped query (To-do implement it directly on model --> Now we do 2 forward)
features_c, cams_c, class_list = extract_feat_cam_fast(
model, get_output, conv_layer_features, 1, img_cropped, num_cams)
if img.shape[0] > img.shape[1]:
size = size_v
else:
size = size_h
# Query resized to be 1024x720 or 720x1024
img_p = preprocess_images(img, size[0], size[1], mean_value)
# Obtain features for all the image
time_search = time.time()
def main():
# Import dataset
(train_images,
train_labels), (test_images,
test_labels) = keras.datasets.mnist.load_data()
test_images = test_images.reshape(test_images.shape[0], 28, 28, 1)
test_images = preprocess_images(test_images)
# Generate images for test
config = tf.ConfigProto()
with tf.Session(config=config) as sess:
for i in range(0, 20):
n = np.random.randint(low=1, high=100)
x1 = test_images[n]
print(test_labels[n])
img_max, img_min = np.max(x1), np.min(x1)
x1 = 255. * (x1 - img_min) / (img_max - img_min)
filename = '../images/img'
filename += str(i)
filename += '.png'
cv2.imwrite(filename, x1)
for i in range(20, 40):
n = np.random.randint(low=1, high=100)
m = np.random.randint(low=1, high=100)
x1 = test_images[n]
x2 = test_images[m]
x3 = cv2.addWeighted(x1, 0.5, x2, 0.5, 0)
print(test_labels[n], test_labels[m])
img_max, img_min = np.max(x3), np.min(x3)
x3 = 255. * (x3 - img_min) / (img_max - img_min)
filename = '../images/img'
filename += str(i)
filename += '.png'
cv2.imwrite(filename, x3)
for i in range(40, 60):
n = np.random.randint(low=1, high=100)
m = np.random.randint(low=1, high=100)
l = np.random.randint(low=1, high=100)
x1 = test_images[n]
x2 = test_images[m]
x3 = test_images[l]
x4 = cv2.addWeighted(x1, 0.5, x2, 0.5, 0)
x5 = cv2.addWeighted(x4, 0.5, x3, 0.5, 0)
print(test_labels[n], test_labels[m], test_labels[l])
img_max, img_min = np.max(x5), np.min(x5)
x5 = 255. * (x5 - img_min) / (img_max - img_min)
filename = '../images/img'
filename += str(i)
filename += '.png'
cv2.imwrite(filename, x5)
for i in range(60, 80):
n = np.random.randint(low=1, high=100)
angle = np.random.randint(low=-180, high=180)
x6 = test_images[n]
print(test_labels[n])
x6 = np.reshape(x6, [28, 28])
x6 = ndimage.rotate(x6, angle, reshape=False)
img_max, img_min = np.max(x6), np.min(x6)
x6 = 255. * (x6 - img_min) / (img_max - img_min)
filename = '../images/img'
filename += str(i)
filename += '.png'
cv2.imwrite(filename, x6)
