def main():
if len(sys.argv) != 3 and len(sys.argv) != 4:
print("python feature_extract.py <image directory> <ascii image file>")
return
if sys.argv[1] not in ["facedata", "digitdata"]:
print("Directory not supported")
print("Must be facedata or digitdata")
return
image_set = ImageSet(sys.argv[1], sys.argv[2])
# Extract images, resize, and extract hog features
for img in image_set.extract_image():
img.generate_hog_image()
image_set.add_image(img)
if len(sys.argv) == 4 and sys.argv[3] == "visualize":
print("Beginning visualization")
utils.display_images(img.image_resize, img.hog_image)
break
# Vectorize each image and add to a new matrix M
image_set.create_transpose_of_vectorized_images()
# Transpose the matrix M
m = image_set.images_matrix
return m
def balance_dataset(X_train, y_train, size=DESIRED_DATASET_SIZE):
bins, _, bin_edges = bin_dataset(y_train)
X_balanced, y_balanced = construct_balanced_dataset_from_bins(X_train,
y_train,
bins,
size=size)
for image, angle in zip(X_balanced, y_balanced):
display_images(image, message=str(angle), delay=500)
return X_balanced, y_balanced
def main():
model_filename = "save/model.h5"
model = Model(model_filename)
# model.plot_model_to_file(model_filename)
# model.show_model_from_image(model_filename)
rows = model.read_csv(DRIVING_LOG)
n_train = int(len(rows) * TRAINING_PORTION)
if DEBUG:
print("Using Training Dataset Size: {}".format(n_train))
X_train, y_train = model.rows_to_feature_labels(n_train,
hzflip=HZ_FLIP_ENABLE)
X_train, y_train = shuffle(X_train, y_train, random_state=1)
if TRAINING_ENABLE:
X_balanced, y_balanced = balance_dataset(X_train, y_train)
display_images(X_balanced, "ROI")
model.set_optimizer()
if FIT_GENERATOR_ENABLE:
history = model.train_and_validate_with_generator(
X_balanced, y_balanced, manual=MANUAL_FIT_ENABLED)
else:
history = model.train(X_balanced, y_balanced)
model.save_model(model_filename)
if PREDICTION_ENABLE:
for i, image in enumerate(X_train):
prediction = model.model.predict(image[None, :, :, :],
batch_size=1)
print("{}; {:>6.2f}".format(
model.lines[i].split(',')[0],
prediction[0][0] / STEERING_MULTIPLIER))
display_images(image, str(prediction))
# Pickle Dump
pickle.dump([history.history, X_balanced, y_balanced, y_train],
open('save/hist_xy.p', 'wb'))
if DEBUG:
print_predictions(model.model, X_balanced, y_balanced)
import gc
gc.collect() # Suppress a Keras Tensorflow Bug
def rows_to_feature_labels(self, count, hzflip=False):
# Allocate twice the size to accomodate
# both original and horizontally flipped images
size_multiple = 1
if HZ_FLIP_ENABLE:
size_multiple = 2
x = np.empty((size_multiple * count, HEIGHT, WIDTH, DEPTH),
dtype=np.float32)
y = np.empty((size_multiple * count), dtype=np.float32)
if DEBUG:
print("Index: Steering")
for idx, line in enumerate(self.lines[:count]):
[center, left, right, steering, throttle, breaks,
speed] = line.split(',')
# Adding Random Perturbations between [-STEERING_MULTIPLIER/20, STEERING_MULTIPLIER/20] to each input
steering = float(steering) * STEERING_MULTIPLIER + random.randint(
-STEERING_MULTIPLIER / 20, STEERING_MULTIPLIER / 20)
if DEBUG:
print("{:^5d} -> {:>5}".format(idx, steering))
image_data = cv2.imread(DATA_DIR + center)
message = 'Angle: {:=+03d}'.format(int(steering))
display_images(image_data, message)
image_data = preprocess_image(image_data)
x[idx, :, :, :] = np.copy(image_data)
y[idx] = np.copy(steering)
if hzflip:
for idx in range(count):
# Fill the empty end of the array
hz_flipped_image = cv2.flip(x[idx, :, :, :],
CV_FLIPTYPE_HORIZONTAL)
x[count + idx, :, :, :] = hz_flipped_image.reshape(
HEIGHT, WIDTH, DEPTH)
y[count + idx] = -1.0 * y[idx]
return x, y
def find_points(images):
pattern_size = (9, 6)
obj_points = []
img_points = []
# Assumed object points relation
a_object_point = np.zeros((PATTERN_SIZE[1] * PATTERN_SIZE[0], 3),
np.float32)
a_object_point[:, :2] = np.mgrid[0:PATTERN_SIZE[0],
0:PATTERN_SIZE[1]].T.reshape(-1, 2)
# Termination criteria for sub pixel corners refinement
stop_criteria = (cv.TERM_CRITERIA_EPS + cv.TERM_CRITERIA_MAX_ITER,
30, 0.001)
print('Finding points ', end='')
debug_images = []
for (image, color_image) in images:
found, corners = cv.findChessboardCorners(image, PATTERN_SIZE, None)
if found:
obj_points.append(a_object_point)
cv.cornerSubPix(image, corners, (11, 11), (-1, -1), stop_criteria)
img_points.append(corners)
print('.', end='')
else:
print('-', end='')
if DEBUG:
cv.drawChessboardCorners(color_image, PATTERN_SIZE, corners, found)
debug_images.append(color_image)
sys.stdout.flush()
if DEBUG:
display_images(debug_images, DISPLAY_SCALE)
print('\nWas able to find points in %s images' % len(img_points))
return obj_points, img_points
def visualize_predictions(test_loader, G, step, writer, mask=False):
if mask:
n_samples = len(test_loader.dataset)
input_batch_numpy = []
prediction_batch_numpy = []
for j in range(10):
random_index = int(np.random.random() * n_samples)
inputs, _ = test_loader.dataset[random_index]
inputs = inputs.unsqueeze(dim=0)
predictions, mask, copied, synthesized, weighted_synthesized = G(
inputs)
images_numpy = [
255 * ((0.5 * utils.tensor2image(input_tensor.cpu())[0]) + 0.5)
for input_tensor in [
inputs, predictions, mask, copied, synthesized,
weighted_synthesized
]
]
figure = utils.display_images(images_numpy)
writer.add_figure(f'translations_{step}_{j}',
figure,
global_step=step)
else:
n_samples = len(test_loader.dataset)
input_batch_numpy = []
prediction_batch_numpy = []
for j in range(10):
random_index = int(np.random.random() * n_samples)
inputs, _ = test_loader.dataset[random_index]
inputs = inputs.unsqueeze(dim=0)
predictions = G(inputs).cpu()
inputs_numpy = utils.tensor2image(inputs.cpu())[0]
predictions_numpy = utils.tensor2image(predictions.cpu())[0]
inputs_numpy = (inputs_numpy * 0.5 + 0.5) * 255
predictions_numpy = (predictions_numpy * 0.5 + 0.5) * 255
figure = utils.display_image_side(inputs_numpy, predictions_numpy)
writer.add_figure(f'translations_{step}_{j}',
figure,
global_step=step)
custom_objects = {
'BilinearUpSampling2D': BilinearUpSampling2D,
'depth_loss_function': None
}
print('Loading model...')
# Load model into GPU / CPU
model = load_model(args.model, custom_objects=custom_objects, compile=False)
print('\nModel loaded ({0}).'.format(args.model))
# Input images
inputs = load_images(glob.glob(args.input))
print('\nLoaded ({0}) images of size {1}.'.format(inputs.shape[0],
inputs.shape[1:]))
# Compute results
outputs = predict(model, inputs)
#matplotlib problem on ubuntu terminal fix
#matplotlib.use('TkAgg')
# Display results
viz = display_images(outputs.copy(), inputs.copy(), args.output_img_size)
plt.figure(figsize=(10, 5))
plt.imshow(viz)
plt.savefig('test.png')
plt.show()
'depth_loss_function': None
}
print('Loading model...')
# Load model into GPU / CPU
model = load_model(args.model, custom_objects=custom_objects, compile=False)
print('\nModel loaded ({0}).'.format(args.model))
# Input images
inputs = load_images(glob.glob(args.input))
print('\nLoaded ({0}) images of size {1}.'.format(inputs.shape[0],
inputs.shape[1:]))
# Compute results
outputs = predict(model, inputs)
#matplotlib problem on ubuntu terminal fix
#matplotlib.use('TkAgg')
# Display results
display_images(outputs.copy(),
inputs.copy(),
start=int(args.start),
end=int(args.end))
# plt.figure(figsize=(1,1))
# plt.imshow(viz)
# plt.savefig('test.png')
# plt.show()
print("done")
def remove_distortion(images):
out = calibrate(images)
matrix = out['camera_matrix']
dist = out['distortion_coefficient']
undistorted_images = []
for (image, color_image) in images:
size = image.shape[::-1]
new_matrix, roi = cv.getOptimalNewCameraMatrix(matrix, dist, size,
1, size)
img = cv.undistort(color_image, matrix, dist, None, new_matrix)
undistorted_images.append(img)
return undistorted_images
if __name__ == '__main__':
args_parser = argparse.ArgumentParser()
args_parser.add_argument('folder', help='a folder wit himages\
to be processed')
args_parser.add_argument('-s', '--scale', type=float, default=0.45,
help='display scale to control window size')
args = args_parser.parse_args()
DISPLAY_SCALE = args.scale
images = load_images(args.folder)
undistorted_images = remove_distortion(images)
display_images(undistorted_images, DISPLAY_SCALE)
wkdir, "nyu.tflite", inputs, 'float32')
#Start converting to tensorflow-lite quantize model and inference
convert_to_tensorflow_lite(wkdir, "nyu.pb", "nyuQuan.tflite", 'import/x:0',
'import/import/Identity:0',
[tf.lite.Optimize.DEFAULT])
tensorflowLiteQuanOutputs, tensorflowLiteQuanInferenceTime = tensorflow_lite_inference(
wkdir, "nyuQuan.tflite", inputs, 'float32')
#Reverse Tensorflow and Lite depth map
tensorflowOutputs = reverse_depth_value(tensorflowOutputs)
tensorflowLiteOutputs = reverse_depth_value(tensorflowLiteOutputs)
tensorflowLiteQuanOutputs = reverse_depth_value(tensorflowLiteQuanOutputs)
#Plot Keras, Tensorflow, Tensorflow-lite depth map
vizKeras = display_images(outputs=kerasOutputs, inputs=inputs)
vizTensorflow = display_images(outputs=tensorflowOutputs, inputs=inputs)
vizTensorflowLite = display_images(outputs=tensorflowLiteOutputs,
inputs=inputs)
vizTensorflowLiteQuan = display_images(outputs=tensorflowLiteQuanOutputs,
inputs=inputs)
f, axarr = plt.subplots(4, 1)
for ax in axarr.flat:
ax.axis("off")
axarr[0].text(0.5,
0.5,
"keras inference " + str(kerasInferenceTime) + " seconds",
fontsize=10,
ha='left')
axarr[1].text(0.5,
0.5,
import torch.optim as optim
from dataloader import loader
from discriminator import Discriminator
from generator import Generator
from parameters import d_conv_dim, z_size, g_conv_dim, beta1, beta2, lr_d, lr_g
from train import train
from utils import display_images, weights_init_normal, gpu_check, load_samples, view_samples
display_images(loader)
D = Discriminator(d_conv_dim)
G = Generator(z_size=z_size, conv_dim=g_conv_dim)
D.apply(weights_init_normal)
G.apply(weights_init_normal)
train_on_gpu = gpu_check()
if not train_on_gpu:
print('No GPU found. Please use a GPU to train your neural network.')
else:
print('Training on GPU!')
d_optimizer = optim.Adam(D.parameters(), lr_d, [beta1, beta2])
g_optimizer = optim.Adam(G.parameters(), lr_g, [beta1, beta2])
n_epochs = 30
losses = train(D, d_optimizer, G, g_optimizer, n_epochs=n_epochs)
resCap = grab_frame()
fig = plt.figure(figsize=(30, 10))
columns = 2
rows = 1
fig.add_subplot(rows, columns, 1)
im1 = plt.imshow(resCap)
plt.xticks([])
plt.yticks([])
fig.add_subplot(rows, columns, 2)
im2 = plt.imshow(display_images(get_depth(resCap)))
plt.xticks([])
plt.yticks([])
plt.ion()
while True:
resCap = grab_frame()
im1.set_data(resCap)
im2.set_data(display_images(get_depth(resCap)))
plt.pause(0.2)
plt.ioff() # due to infinite loop, this gets never called.
plt.show()
type=str,
help='Trained Keras model file')
parser.add_argument('--input',
default='examples HELM',
type=str,
help='Path to example images')
parser.add_argument('--scale',
default=150,
type=float,
help='Scale of color map in meters')
args = parser.parse_args()
print('Loading model...')
# Load model into GPU / CPU
model = create_model(existing=args.model)
print('\nModel loaded ({0}).'.format(args.model))
# Input images
inputs = load_images(glob.glob(args.input + "/*.*"))
print('\nLoaded ({0}) images of size {1}.'.format(inputs.shape[0],
inputs.shape[1:]))
# Compute results
outputs = model.predict(inputs, 4)
# Display results
viz = display_images(outputs.copy(), inputs.copy(), scale=args.scale)
Image.fromarray((viz * 255).astype(np.uint8)).save("test.png")
# Custom object needed for inference and training
custom_objects = {
'BilinearUpSampling2D': BilinearUpSampling2D,
'depth_loss_function': None
}
#print('Loading model...')
# Load model into GPU / CPU
model = load_model(args.model, custom_objects=custom_objects, compile=False)
#print('\nModel loaded ({0}).'.format(args.model))
# Input images
inputs = load_images(glob.glob(args.input))
#print('\nLoaded ({0}) images of size {1}.'.format(inputs.shape[0], inputs.shape[1:]))
# Compute results
outputs = predict(model, inputs)
#matplotlib problem on ubuntu terminal fix
#matplotlib.use('TkAgg')
# Display results
viz = display_images(outputs.copy(), inputs.copy(), is_colormap=False)
#plt.figure(figsize=(10,5))
#plt.imshow(viz)
#plt.savefig('test.png')
#plt.show()
default='examples/*.png',
type=str,
help='Input filename or folder.')
args = parser.parse_args()
# Custom object needed for inference and training
custom_objects = {
'BilinearUpSampling2D': BilinearUpSampling2D,
'depth_loss_function': depth_loss_function
}
print('Loading model...')
# Load model into GPU / CPU
model = load_model(args.model, custom_objects=custom_objects, compile=False)
print('\nModel loaded ({0}).'.format(args.model))
# Input images
inputs = load_images(glob.glob(args.input))
print('\nLoaded ({0}) images of size {1}.'.format(inputs.shape[0],
inputs.shape[1:]))
# Compute results
outputs = predict(model, inputs)
# Display results
viz = display_images(outputs.copy(), inputs.copy())
plt.figure(figsize=(10, 5))
plt.imshow(viz)
plt.show()
def train(context_encoder,
context_to_dist,
decoder,
aggregator,
train_loader,
test_loader,
optimizer,
n_epochs,
device,
save_path,
summary_writer,
save_every=10,
h=28,
w=28,
log=1):
context_encoder.train()
decoder.train()
grid = make_mesh_grid(h, w).to(device) # size h,w,2
for epoch in range(n_epochs):
running_loss = 0.0
last_log_time = time.time()
# Training
train_loss = 0.0
for batch_idx, (batch, _) in enumerate(train_loader):
batch = batch.to(device)
if ((batch_idx % 100) == 0) and batch_idx > 1:
print(
"epoch {} | batch {} | mean running loss {:.2f} | {:.2f} batch/s"
.format(epoch, batch_idx, running_loss / 100,
100 / (time.time() - last_log_time)))
last_log_time = time.time()
running_loss = 0.0
mask = random_mask_uniform(bsize=batch.size(0),
h=h,
w=w,
device=batch.device)
z_params_full, z_params_masked = all_forward(
batch, grid, mask, context_encoder, aggregator,
context_to_dist)
reconstruction_loss, kl_loss, reconstructed_image = compute_loss(
batch, grid, mask, z_params_full, z_params_masked, h, w,
decoder)
loss = reconstruction_loss + kl_loss
if batch_idx % 100 == 0:
print("reconstruction {:.2f} | kl {:.2f}".format(
reconstruction_loss, kl_loss))
optimizer.zero_grad()
loss.backward()
optimizer.step()
# add loss
running_loss += loss.item()
train_loss += loss.item()
print("Epoch train loss : {}".format(train_loss / len(train_loader)))
if summary_writer is not None:
summary_writer.add_scalar("train/loss",
train_loss / len(train_loader),
global_step=epoch)
if (epoch % save_every == 0) and log and epoch > 0:
save_model(save_path, "NP_model_epoch_{}.pt".format(epoch),
context_encoder, context_to_dist, decoder, aggregator,
device)
## TEST
test_loss = 0.0
for batch_idx, (batch, _) in enumerate(test_loader):
batch = batch.to(device)
mask = random_mask_uniform(bsize=batch.size(0),
h=h,
w=w,
device=batch.device)
with torch.no_grad():
z_params_full, z_params_masked = all_forward(
batch, grid, mask, context_encoder, aggregator,
context_to_dist)
reconstruction_loss, kl_loss, reconstructed_image = compute_loss(
batch, grid, mask, z_params_full, z_params_masked, h, w,
decoder)
loss = reconstruction_loss + kl_loss
test_loss += loss.item()
if summary_writer is not None:
summary_writer.add_scalar("test/loss",
test_loss / len(test_loader),
global_step=epoch)
print("TEST loss | epoch {} | {:.2f}".format(
epoch, test_loss / len(test_loader)))
# do examples
example_batch, _ = next(iter(test_loader))
example_batch = example_batch[:10].to(device)
for n_pixels in [50, 150, 450]:
mask = random_mask(example_batch.size(0),
h,
w,
n_pixels,
device=example_batch.device)
z_params_full, z_params_masked = all_forward(
example_batch, grid, mask, context_encoder, aggregator,
context_to_dist)
z_context = torch.cat([
sample_z(z_params_masked).unsqueeze(1).expand(-1, h * w, -1)
for i in range(3)
],
dim=0)
z_context = torch.cat([
z_context,
grid.view(1, h * w, 2).expand(z_context.size(0), -1, -1)
],
dim=2)
decoded_images = decoder(z_context).view(-1, 1, h, w)
stacked_images = display_images(original_image=example_batch,
mask=mask,
reconstructed_image=decoded_images)
image = torch.tensor(stacked_images)
if summary_writer is not None:
summary_writer.add_image(
"test_image/{}_pixels".format(n_pixels),
image,
global_step=epoch)
return
model = load_model(args.model, custom_objects=custom_objects, compile=False)
print('\nModel loaded ({0}).'.format(args.model))
# Input images
folder_index = args.input.rsplit("/*", 1)[0].rsplit("/", 1)[1]
files = glob.glob(args.input)
files.sort()
inputs = load_images(files)
print('\nLoaded ({0}) images of size {1}.'.format(inputs.shape[0],
inputs.shape[1:]))
# Compute results
outputs = predict(model, inputs)
#matplotlib problem on ubuntu terminal fix
#matplotlib.use('TkAgg')
# Display results
#viz = display_images(outputs.copy(), inputs.copy())
viz = display_images(outputs.copy(),
folder_index,
inputs.copy(),
is_colormap=False)
#plt.figure(figsize=(10,5))
#plt.imshow(viz)
#plt.savefig('test.png')
#plt.show()
custom_objects = {'BilinearUpSampling2D': BilinearUpSampling2D, 'depth_loss_function': None}
print('Loading model...')
# Load model into GPU / CPU
model = load_model(args.model, custom_objects=custom_objects, compile=False)
print('\nModel loaded ({0}).'.format(args.model))
# Input images
inputs = load_images( glob.glob(args.input) )
print('\nLoaded ({0}) images of size {1}.'.format(inputs.shape[0], inputs.shape[1:]))
# Compute results
outputs = predict(model, inputs, minDepth=args.mindepth, maxDepth=args.maxdepth, batch_size=args.bs)
#matplotlib problem on ubuntu terminal fix
#matplotlib.use('TkAgg')
# Display results
if args.input_depth=='examples/eye/*.png':
inputs_depth = load_images( glob.glob(args.input_depth))
viz = display_images(outputs.copy(), inputs.copy(), inputs_depth.copy())
else:
viz = display_images(outputs.copy(), inputs.copy())
plt.figure(figsize=(10,5))
plt.imshow(viz)
plt.savefig('test.png')
plt.show()
