def test_activations_source_image(self):
important_layers = ['relu1_1', 'pool1', 'pool2', 'pool3', 'pool4']
synthesis_model = utilities.load_model(PYTORCH_MODEL_PATH)
# register hooks to extract the important activations
hook = ActivationsHook()
for name, layer in synthesis_model.named_children():
if name in important_layers:
layer.register_forward_hook(hook)
# load an image and pass it through the synthesis model
source_image = utilities.preprocess_image(
utilities.load_image(SOURCE_IMG_PATH))
synthesis_model(source_image)
# check if they are correct
with h5py.File(REF_VALS_PATH, 'r') as f:
for i, layer_name in enumerate(important_layers):
if layer_name == 'relu1_1':
layer_name = 'conv1_1'
actual_activations = hook.activations[i]
expected_activations = torch.from_numpy(
f['source_img_activations_{}'.format(layer_name)][()])
assert torch.allclose(actual_activations,
expected_activations,
atol=1e-05)
def telemetry(sid, data):
if data:
# The current steering angle of the car
steering_angle = data["steering_angle"]
# The current throttle of the car
throttle = data["throttle"]
# The current speed of the car
speed = data["speed"]
# The current image from the center camera of the car
imgString = data["image"]
image = Image.open(BytesIO(base64.b64decode(imgString)))
image_array = np.asarray(image)
processed_array = utilities.preprocess_image(image_array)
steering_angle = float(
model.predict(processed_array[None, :, :, :], batch_size=1))
throttle = controller.update(float(speed))
print(steering_angle, throttle)
send_control(steering_angle, throttle)
# save frame
if args.image_folder != '':
timestamp = datetime.utcnow().strftime('%Y_%m_%d_%H_%M_%S_%f')[:-3]
image_filename = os.path.join(args.image_folder, timestamp)
image.save('{}.jpg'.format(image_filename))
else:
# NOTE: DON'T EDIT THIS.
sio.emit('manual', data={}, skip_sid=True)
def main(args: Optional[argparse.Namespace] = None):
if args is None:
args = parse_arguments()
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# load model & data
target_image = utilities.preprocess_image(
utilities.load_image(args.img_path))
net = model.Model(args.model_path, device, target_image)
# synthesize
optimizer = optimize.Optimizer(net, args)
result = optimizer.optimize()
# save result
final_image = utilities.postprocess_image(
result, utilities.load_image(args.img_path))
final_image.save(os.path.join(args.out_dir, 'output.png'))
# plot loss
x = list(
range(args.checkpoint_every - 1,
len(optimizer.losses) * args.checkpoint_every,
args.checkpoint_every))
plt.plot(x, optimizer.losses)
plt.savefig(os.path.join(args.out_dir, 'loss_plot.png'))
plt.close()
# save intermediate images
for i, image in enumerate(optimizer.opt_images):
image.save(
os.path.join(
args.out_dir,
'intermediate_image_{}.png'.format(i * args.checkpoint_every)))
def test_activations_two_consecutive_runs(self):
important_layers = ['relu1_1', 'pool1', 'pool2', 'pool3', 'pool4']
# load an image
source_image = utilities.preprocess_image(
utilities.load_image(SOURCE_IMG_PATH))
activations = []
for j in range(2):
# load a model and pass the image through it
synthesis_model = utilities.load_model(PYTORCH_MODEL_PATH)
# register hooks to extract the important activations
hook = ActivationsHook()
for name, layer in synthesis_model.named_children():
if name in important_layers:
layer.register_forward_hook(hook)
synthesis_model(source_image)
# save activations from the last important layer
activations.append(hook.activations[-1])
assert torch.equal(activations[0], activations[1]), \
'mean error: {}'.format(
(activations[0] - activations[1]).abs().mean()
)
def load_training_data(batch):
"""
in: data frame batch with {0: 'Center', 1: 'Left', 2: 'Right', 3: 'Steering Angle'}
out: for this batch a list of all images (training data) and steering angles (labels)
function:
1. load images based on the paths in the data frame batch file
2. adjust steering angle for right or left shifted/recorded images
3. do image augmentation and preprocessing for the nn model
4. output all images and steering anles in 2 lists
"""
left_cf = utilities.left_cf # correction factor for steering angle for left image
right_cf = utilities.right_cf # correction factor for steering angle for right image
list_images = [] # empty list for output
list_steering_angle = [] # empty list for output
for index, row in batch.iterrows():
# load path for images
path_left = utilities.get_rel_path(row['Left'])
path_right = utilities.get_rel_path(row['Right'])
path_center = utilities.get_rel_path(row['Center'])
center_angle = float(row['Steering Angle'])
# load Images
left_img = utilities.load_image(path_left)
center_img = utilities.load_image(path_right)
right_img = utilities.load_image(path_center)
# For the shifted richt and left images: adjust the steering angle
lenft_angle = center_angle + left_cf
right_angle = center_angle - right_cf
# Augment the Image
left_img_aug, lenft_angle = pipeline_augment(left_img, lenft_angle)
center_img_aug, center_angle = pipeline_augment(center_img, center_angle)
right_img_aug, right_angle = pipeline_augment(right_img, right_angle)
# Preprocess (Cropping, resizing, transformation in YUV-Colorspace) the augmented images
left_img_aug_prepro = utilities.preprocess_image(left_img_aug)
center_img_aug_prepro = utilities.preprocess_image(center_img_aug)
right_img_aug_prepro = utilities.preprocess_image(right_img_aug)
# append Images and steering angles to lists for output
list_images.append(left_img_aug_prepro)
list_steering_angle.append(lenft_angle)
list_images.append(center_img_aug_prepro)
list_steering_angle.append(center_angle)
list_images.append(right_img_aug_prepro)
list_steering_angle.append(right_angle)
return list_images, list_steering_angle
def test_preprocessed_values(self):
source_img = utilities.load_image(SOURCE_IMG_PATH)
actual_preprocessed_image = utilities.preprocess_image(source_img)
with h5py.File(REF_VALS_PATH, 'r') as f:
expected_preprocessed_image = torch.from_numpy(
f['source_img_preprocessed'][()])
assert torch.equal(actual_preprocessed_image,
expected_preprocessed_image)
def get_test(data):
Xims = np.zeros((1, im_res, im_res, 3))
original_img = imread(data['image'])
img_name = os.path.basename(data['image'])
if original_img.ndim == 2:
copy = np.zeros((original_img.shape[0], original_img.shape[1], 3))
copy[:, :, 0] = original_img
copy[:, :, 1] = original_img
copy[:, :, 2] = original_img
original_img = copy
r_img = preprocess_image(original_img, im_res, im_res)
Xims[0, :] = np.copy(r_img)
return [Xims], original_img, img_name
def face_recognize(image):
image_path = get_file_path('data_test', image)
image_result = get_file_path('result_test', image_name)
objects_path = get_file_path('other', 'object_names')
model_path = get_file_path('result_train', 'model_name')
face_detector = get_face_detector_cnn()
face_embedding = get_vgg_face_embedding(
get_file_path('Pre_trained_CNN', 'vgg_model'))
image_org = cv2.imread(image_path)
image_gray = cv2.cvtColor(image_org, cv2.COLOR_RGB2GRAY)
coodrs = face_detector(image_gray, 1)
model = tf.keras.models.load_model(model_path)
objects = get_object_names(objects_path)
images, face_box = crop_rect_box_coodrs(coodrs, image_org, True)
image_org = Image.fromarray(cv2.cvtColor(image_org, cv2.COLOR_BGR2RGB))
for i in range(images.shape[0]):
box = face_box[i]
face_encode = preprocess_image(images[i], face_embedding)
face_embedded = K.eval(face_encode)
y = model.predict(face_embedded)
if np.max(y) > 0.7:
person = objects[np.argmax(y)]
else:
person = "*"
print(np.max(y), person)
# cv2.waitKey(0)
# image_org = cv2.rectangle(image_org,(box[0], box[1]), (box[0] + box[2], box[1] + box[3]), (0,255,0), 1)
fort = ImageFont.truetype(fontPath + "/" + "FreeMonoBold.ttf", size=60)
pos_name = (box[0] + box[2], box[1] - 5)
pos_box = (box[0] + box[2] - 1, box[1] - 5)
pos_rect = (box[0], box[1]), (box[0] + box[2], box[1] + box[3])
if person != "*":
tw, th = fort.getsize(person)
canvas = Image.new('RGB', (int(tw / 5) - 10, int(th / 5) + 1),
"orange")
image_org.paste(canvas, pos_box)
draw = ImageDraw.Draw(image_org)
if person != "*":
draw.text(pos_name, person, 'blue', fort=fort)
draw.rectangle(pos_rect, outline='green')
image_org.save(image_result)
def predict(sess, image_file, data):
"""
Runs the graph stored in "sess" to predict boxes for "image_file". Prints and plots the preditions.
Arguments:
sess -- your tensorflow/Keras session containing the YOLO graph
image_file -- name of an image stored in the "images" folder.
Returns:
out_scores -- tensor of shape (None, ), scores of the predicted boxes
out_boxes -- tensor of shape (None, 4), coordinates of the predicted boxes
out_classes -- tensor of shape (None, ), class index of the predicted boxes
Note: "None" actually represents the number of predicted boxes, it varies between 0 and max_boxes.
"""
startTime = time.time()
# Preprocess your image
image, image_data = preprocess_image(image_file,
model_image_size=(608, 608))
time1 = time.time()
out_scores, out_boxes, out_classes = sess.run((data),
feed_dict={
yolo_model.input:
image_data,
K.learning_phase(): 0
})
time2 = time.time()
colors = generate_colors(class_names)
time3 = time.time()
draw_boxes(image, out_scores, out_boxes, out_classes, class_names, colors)
time4 = time.time()
index = image_file.find('0')
new_image_path = image_file[:index] + 'processed/' + image_file[index:]
#new_image_path = "images/test8.jpg"
image.save(new_image_path, quality=90)
time5 = time.time()
#output_image = scipy.misc.imread(os.path.join("out", image_file))
#imshow(output_image)
print("Prep - %d" % (time1 - startTime))
print("Sess.run - %d" % (time2 - time1))
print("generate_colors - %d" % (time3 - time2))
print("draw_boxes - %d" % (time4 - time3))
print("save - %d" % (time5 - time4))
print("all time - %d" % (time5 - startTime))
return out_scores, out_boxes, out_classes
def test_noise_image(self):
noise_image = None
with h5py.File(REF_VALS_PATH, 'r') as f:
noise_image = torch.from_numpy(f['noise1234'][()])
target_image = utilities.preprocess_image(
utilities.load_image(SOURCE_IMG_PATH))
model = Model(PYTORCH_MODEL_PATH, torch.device('cpu'), target_image)
model(noise_image)
actual_loss_value = model.get_loss().detach().cpu()
with h5py.File(REF_VALS_PATH, 'r') as f:
expected_loss_value = torch.tensor(f['loss_value'][()],
dtype=torch.float32)
assert torch.allclose(actual_loss_value, expected_loss_value)
net,
dataset)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
m = SiameseNetwork(network_type=args.net).cuda()
m.load_state_dict(torch.load(args.resume))
CNN_net = m._modules['net']
dice_all1 = np.zeros((22,2))
dice_all2 = np.zeros((22,2))
for i in range(0, 22):
print("CLASS -- ", i)
vols = np.load(os.path.join(data_root, 'testtom_vol_{}.npy'.format(str(i).zfill(2))))
segs = np.load(os.path.join(data_root, 'testtom_seg_{}.npy'.format(str(i).zfill(2))))
vols = preprocess_image(vols)
dice_c1 = np.zeros((segs.shape[0], 1))
dice_c2 = np.zeros((segs.shape[0], 1))
for j in range(segs.shape[0]):
vol = vols[:,:,j,:,:,:]
seg = segs[j:j+1,:,:,:]
_, seg_pred = CNN_net(vol.cuda(), mode='test')
vol_np = vol[0,0,:,:,:].detach().numpy()
seg_np = seg[0,:,:,:]
seg_pred_np = seg_pred[0,0,:,:,:].detach().cpu().numpy()
# init wrapper object
dense_crf_param = {}
dense_crf_param['MaxIterations'] = 4.0
def generator_test_singleimage(b_s, image):
gaussian = np.zeros((b_s, nb_gaussian, shape_r_gt, shape_c_gt))
while True:
yield [preprocess_image(image, shape_r, shape_c), gaussian]
def process_frame(self, frame, config):
#frame = self.frame
cv.flip(frame, 1, frame)
height, width = frame.shape[:2]
paddingPercent = 30
trackingRangeX = [
0 + (paddingPercent / 100) * (width),
(100 - paddingPercent) / 100 * (width)
]
trackingRangeY = [
0 + (paddingPercent / 100) * (height),
(100 - paddingPercent) / 100 * (height)
]
if self.captureBGFlag:
self.bg_ref = frame
print('Background Captured!')
# self.remove_bg = True
self.captureBGFlag = False
if self.remove_bg:
if self.bg_ref is None:
print('Cannot Remove Background! Background Ref Not Set.')
self.remove_bg = False
else:
frame = utilities.remove_image_background(frame, self.bg_ref)
img = frame
cv.convertScaleAbs(frame,
img,
alpha=self.contrast / 10,
beta=self.brightness - 128)
# Lowerbound and Upperbound for color detection
l_b = np.array([config.lh, config.ls, config.lv])
u_b = np.array([config.uh, config.us, config.uv])
processedHSV = cv.cvtColor(utilities.preprocess_image(img),
cv.COLOR_BGR2HSV)
mask = cv.inRange(processedHSV, l_b, u_b)
maskFinal = utilities.process_mask(mask)
contours, h = cv.findContours(maskFinal.copy(), cv.RETR_LIST,
cv.CHAIN_APPROX_NONE)
# Draw Tracking Area Range Box
#frame = cv.rectangle(frame, (trackingRangeX[0], trackingRangeY[0]), (trackingRangeX[1], trackingRangeY[1]), (0,0,255), 3)
boundingAreas = utilities.bounding_areas_from_contours(contours)
# Sort the areas in descending order
boundingAreas = utilities.sort_dict_by_keys(boundingAreas,
reverse=True)
# Get the sorted bounding coordinates
rects = list(boundingAreas.values())
lenRects = len(rects)
# Get only the first two largest rectangles if there are more than two
if lenRects > 2:
lenRects = 2
points = []
for i in range(lenRects):
x, y, w, h, area = rects[i]
# Discard if subsequent boxes are less than 20% of the largest box
if i > 0 and area < 0.2 * rects[0][4]:
break
# Draw rectangle to show bounding box
frame = cv.rectangle(frame, (x, y), (x + w, y + h), (0, 0, 255), 3)
x_center = x + int(w / 2)
y_center = y + int(h / 2)
points.append((x_center, y_center))
# Draw circle to show center of bounding box
frame = cv.circle(frame, (x_center, y_center),
radius=5,
color=(0, 0, 255),
thickness=-1)
cursorLoc = ()
if len(points) == 1:
cursorLoc = points[0]
# If there are two bounding boxes, Draw a line through their centers
if len(points) == 2:
frame = cv.line(frame,
points[0],
points[1],
color=(0, 255, 0),
thickness=3)
x_center = int((points[0][0] + points[1][0]) / 2)
y_center = int((points[0][1] + points[1][1]) / 2)
cursorLoc = (x_center, y_center)
# Draw circle to show center of the line
frame = cv.circle(frame,
cursorLoc,
radius=5,
color=(255, 0, 0),
thickness=-1)
currentMouseX, currentMouseY = pyautogui.position()
if config.enableMouseTracking:
if cursorLoc:
if len(points) == 2 and not self.mouseDown:
pyautogui.mouseDown()
self.mouseDown = True
elif len(points) == 1 and self.mouseDown:
pyautogui.mouseUp()
self.mouseDown = False
# Mapped Tracking
if config.selectedTrackingType == 0:
newMouseX = int(
utilities.linear_interpolate(cursorLoc[0],
trackingRangeX,
[0, self.screenWidth]))
newMouseY = int(
utilities.linear_interpolate(cursorLoc[1],
trackingRangeY,
[0, self.screenHeight]))
# Ignore micromovements
if abs(currentMouseX -
newMouseX) > 5 or abs(currentMouseY -
newMouseY) > 5:
pyautogui.moveTo(newMouseX, newMouseY)
# Free Tracking (Indpendent of tracked object on the camera)
elif config.selectedTrackingType == 1 and self.prevCursorLoc:
cursor_deviation_x = (cursorLoc[0] - self.prevCursorLoc[0]
) * config.sensitivity / 100
cursor_deviation_y = (cursorLoc[1] - self.prevCursorLoc[1]
) * config.sensitivity / 100
# Ignore micromovements
if abs(cursor_deviation_x) < 4: cursor_deviation_x = 0
if abs(cursor_deviation_y) < 4: cursor_deviation_y = 0
newMouseX = currentMouseX + cursor_deviation_x
newMouseY = currentMouseY + cursor_deviation_y
# Ignore offscreen movements
if newMouseX < 0: newMouseX = 0
elif newMouseX > self.screenWidth:
newMouseX = self.screenWidth
if newMouseY < 0: newMouseY = 0
elif newMouseY > self.screenHeight:
newMouseY = self.screenHeight
if newMouseX != currentMouseX or newMouseY != currentMouseY:
pyautogui.moveTo(newMouseX, newMouseY)
elif len(points) == 0 and self.mouseDown:
pyautogui.mouseUp()
self.mouseDown = False
if cursorLoc:
self.prevCursorLoc = cursorLoc
else:
self.prevCursorLoc = ()
return [frame, maskFinal]
def __init__(self, base_img_path, style_img_path, output_img_path,
output_width, convnet, content_weight, style_weight,
tv_weight, content_layer, style_layers, iterations):
"""
Initialize and store parameters of the neural styler. Initialize the
desired convnet and compute the 3 losses and gradients with respect to the
output image.
Params
------
- input_img: tensor containing: content_img, style_img and output_img.
- convnet: [string], defines which VGG to use: vgg16 or vgg19.
- style_layers: list containing name of layers to use for style
reconstruction. Defined in Gatys et. al but can be changed.
- content_layer: string containing name of layer to use for content
reconstruction. Also defined in Gatys et. al.
- content_weight: weight for the content loss.
- style_weight: weight for the style loss.
- tv_weight: weight for the total variation loss.
- iterations: iterations for optimization algorithm
- output_img_path: path to output image.
Notes
-----
[1] If user specifies output width, then calculate the corresponding
image height. Else, output image height and width should be the
same as that of the content image. Also note that style image
should be resized to whatever was decided above.
[2] PIL returns (width, height) whereas numpy returns (height, width)
since nrows=height and ncols=width.
[3] L_BFGS requires that loss and grad be two functions so we create
a keras function that computes the gradients and loss and return
each separately using two different class methods.
"""
print('\nInitializing Neural Style model...')
# store paths
self.base_img_path = base_img_path
self.style_img_path = style_img_path
self.output_img_path = output_img_path
# configuring image sizes [1, 2]
print('\n\tResizing images...')
self.width = output_width
width, height = load_img(self.base_img_path).size
new_dims = (height, width)
# store shapes for future use
self.img_nrows = height
self.img_ncols = width
if self.width is not None:
# calculate new height
num_rows = int(np.floor(float(height * self.width / width)))
new_dims = (num_rows, self.width)
# update the stored shapes
self.img_nrows = num_rows
self.img_ncols = self.width
# resize content and style images to this desired shape
self.content_img = K.variable(
preprocess_image(self.base_img_path, new_dims))
self.style_img = K.variable(
preprocess_image(self.style_img_path, new_dims))
# and also create output placeholder with desired shape
if K.image_dim_ordering() == 'th':
self.output_img = K.placeholder((1, 3, new_dims[0], new_dims[1]))
else:
self.output_img = K.placeholder((1, new_dims[0], new_dims[1], 3))
# sanity check on dimensions
print("\tSize of content image is: {}".format(
K.int_shape(self.content_img)))
print("\tSize of style image is: {}".format(K.int_shape(
self.style_img)))
print("\tSize of output image is: {}".format(
K.int_shape(self.output_img)))
# combine the 3 images into a single Keras tensor
self.input_img = K.concatenate(
[self.content_img, self.style_img, self.output_img], axis=0)
self.convnet = convnet
self.iterations = iterations
# store weights of the loss components
self.content_weight = content_weight
self.style_weight = style_weight
self.tv_weight = tv_weight
# store convnet layers
self.content_layer = content_layer
self.style_layers = style_layers
# initialize the vgg16 model
print('\tLoading {} model'.format(self.convnet.upper()))
if self.convnet == 'vgg16':
self.model = vgg16.VGG16(input_tensor=self.input_img,
weights='imagenet',
include_top=False)
else:
self.model = vgg19.VGG19(input_tensor=self.input_img,
weights='imagenet',
include_top=False)
print('\tComputing losses...')
# get the symbolic outputs of each "key" layer (we gave them unique names).
outputs_dict = dict([(layer.name, layer.output)
for layer in self.model.layers])
# extract features only from the content layer
content_features = outputs_dict[self.content_layer]
# extract the activations of the base image and the output image
base_image_features = content_features[
0, :, :, :] # 0 corresponds to base
combination_features = content_features[
2, :, :, :] # 2 coresponds to output
# calculate the feature reconstruction loss
content_loss = self.content_weight * \
feature_content_loss(base_image_features,
combination_features)
# for each style layer compute style loss
# total style loss is then weighted sum of those losses
temp_style_loss = K.variable(0.0)
weight = 1.0 / float(len(self.style_layers))
for layer in self.style_layers:
# extract features of given layer
style_features = outputs_dict[layer]
# from those features, extract style and output activations
style_image_features = style_features[1, :, :, :]
output_style_features = style_features[2, :, :, :]
temp_style_loss += weight * style_reconstruction_loss(
style_image_features, output_style_features, self.img_nrows,
self.img_ncols)
style_loss = self.style_weight * temp_style_loss
# compute total variational loss
tv_loss = self.tv_weight * variation_loss(
self.output_img, self.img_nrows, self.img_ncols)
# composite loss
total_loss = content_loss + style_loss + tv_loss
# compute gradients of output img with respect to loss
print('\tComputing gradients...')
grads = K.gradients(total_loss, self.output_img)
outputs = [total_loss]
if type(grads) in {list, tuple}:
outputs += grads
else:
outputs.append(grads)
# [3]
self.loss_and_grads = K.function([self.output_img], outputs)
