def test_activations_source_image(self): important_layers = ['relu1_1', 'pool1', 'pool2', 'pool3', 'pool4'] synthesis_model = utilities.load_model(PYTORCH_MODEL_PATH) # register hooks to extract the important activations hook = ActivationsHook() for name, layer in synthesis_model.named_children(): if name in important_layers: layer.register_forward_hook(hook) # load an image and pass it through the synthesis model source_image = utilities.preprocess_image( utilities.load_image(SOURCE_IMG_PATH)) synthesis_model(source_image) # check if they are correct with h5py.File(REF_VALS_PATH, 'r') as f: for i, layer_name in enumerate(important_layers): if layer_name == 'relu1_1': layer_name = 'conv1_1' actual_activations = hook.activations[i] expected_activations = torch.from_numpy( f['source_img_activations_{}'.format(layer_name)][()]) assert torch.allclose(actual_activations, expected_activations, atol=1e-05)
def telemetry(sid, data): if data: # The current steering angle of the car steering_angle = data["steering_angle"] # The current throttle of the car throttle = data["throttle"] # The current speed of the car speed = data["speed"] # The current image from the center camera of the car imgString = data["image"] image = Image.open(BytesIO(base64.b64decode(imgString))) image_array = np.asarray(image) processed_array = utilities.preprocess_image(image_array) steering_angle = float( model.predict(processed_array[None, :, :, :], batch_size=1)) throttle = controller.update(float(speed)) print(steering_angle, throttle) send_control(steering_angle, throttle) # save frame if args.image_folder != '': timestamp = datetime.utcnow().strftime('%Y_%m_%d_%H_%M_%S_%f')[:-3] image_filename = os.path.join(args.image_folder, timestamp) image.save('{}.jpg'.format(image_filename)) else: # NOTE: DON'T EDIT THIS. sio.emit('manual', data={}, skip_sid=True)
def main(args: Optional[argparse.Namespace] = None): if args is None: args = parse_arguments() device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # load model & data target_image = utilities.preprocess_image( utilities.load_image(args.img_path)) net = model.Model(args.model_path, device, target_image) # synthesize optimizer = optimize.Optimizer(net, args) result = optimizer.optimize() # save result final_image = utilities.postprocess_image( result, utilities.load_image(args.img_path)) final_image.save(os.path.join(args.out_dir, 'output.png')) # plot loss x = list( range(args.checkpoint_every - 1, len(optimizer.losses) * args.checkpoint_every, args.checkpoint_every)) plt.plot(x, optimizer.losses) plt.savefig(os.path.join(args.out_dir, 'loss_plot.png')) plt.close() # save intermediate images for i, image in enumerate(optimizer.opt_images): image.save( os.path.join( args.out_dir, 'intermediate_image_{}.png'.format(i * args.checkpoint_every)))
def test_activations_two_consecutive_runs(self): important_layers = ['relu1_1', 'pool1', 'pool2', 'pool3', 'pool4'] # load an image source_image = utilities.preprocess_image( utilities.load_image(SOURCE_IMG_PATH)) activations = [] for j in range(2): # load a model and pass the image through it synthesis_model = utilities.load_model(PYTORCH_MODEL_PATH) # register hooks to extract the important activations hook = ActivationsHook() for name, layer in synthesis_model.named_children(): if name in important_layers: layer.register_forward_hook(hook) synthesis_model(source_image) # save activations from the last important layer activations.append(hook.activations[-1]) assert torch.equal(activations[0], activations[1]), \ 'mean error: {}'.format( (activations[0] - activations[1]).abs().mean() )
def load_training_data(batch): """ in: data frame batch with {0: 'Center', 1: 'Left', 2: 'Right', 3: 'Steering Angle'} out: for this batch a list of all images (training data) and steering angles (labels) function: 1. load images based on the paths in the data frame batch file 2. adjust steering angle for right or left shifted/recorded images 3. do image augmentation and preprocessing for the nn model 4. output all images and steering anles in 2 lists """ left_cf = utilities.left_cf # correction factor for steering angle for left image right_cf = utilities.right_cf # correction factor for steering angle for right image list_images = [] # empty list for output list_steering_angle = [] # empty list for output for index, row in batch.iterrows(): # load path for images path_left = utilities.get_rel_path(row['Left']) path_right = utilities.get_rel_path(row['Right']) path_center = utilities.get_rel_path(row['Center']) center_angle = float(row['Steering Angle']) # load Images left_img = utilities.load_image(path_left) center_img = utilities.load_image(path_right) right_img = utilities.load_image(path_center) # For the shifted richt and left images: adjust the steering angle lenft_angle = center_angle + left_cf right_angle = center_angle - right_cf # Augment the Image left_img_aug, lenft_angle = pipeline_augment(left_img, lenft_angle) center_img_aug, center_angle = pipeline_augment(center_img, center_angle) right_img_aug, right_angle = pipeline_augment(right_img, right_angle) # Preprocess (Cropping, resizing, transformation in YUV-Colorspace) the augmented images left_img_aug_prepro = utilities.preprocess_image(left_img_aug) center_img_aug_prepro = utilities.preprocess_image(center_img_aug) right_img_aug_prepro = utilities.preprocess_image(right_img_aug) # append Images and steering angles to lists for output list_images.append(left_img_aug_prepro) list_steering_angle.append(lenft_angle) list_images.append(center_img_aug_prepro) list_steering_angle.append(center_angle) list_images.append(right_img_aug_prepro) list_steering_angle.append(right_angle) return list_images, list_steering_angle
def test_preprocessed_values(self): source_img = utilities.load_image(SOURCE_IMG_PATH) actual_preprocessed_image = utilities.preprocess_image(source_img) with h5py.File(REF_VALS_PATH, 'r') as f: expected_preprocessed_image = torch.from_numpy( f['source_img_preprocessed'][()]) assert torch.equal(actual_preprocessed_image, expected_preprocessed_image)
def get_test(data): Xims = np.zeros((1, im_res, im_res, 3)) original_img = imread(data['image']) img_name = os.path.basename(data['image']) if original_img.ndim == 2: copy = np.zeros((original_img.shape[0], original_img.shape[1], 3)) copy[:, :, 0] = original_img copy[:, :, 1] = original_img copy[:, :, 2] = original_img original_img = copy r_img = preprocess_image(original_img, im_res, im_res) Xims[0, :] = np.copy(r_img) return [Xims], original_img, img_name
def face_recognize(image): image_path = get_file_path('data_test', image) image_result = get_file_path('result_test', image_name) objects_path = get_file_path('other', 'object_names') model_path = get_file_path('result_train', 'model_name') face_detector = get_face_detector_cnn() face_embedding = get_vgg_face_embedding( get_file_path('Pre_trained_CNN', 'vgg_model')) image_org = cv2.imread(image_path) image_gray = cv2.cvtColor(image_org, cv2.COLOR_RGB2GRAY) coodrs = face_detector(image_gray, 1) model = tf.keras.models.load_model(model_path) objects = get_object_names(objects_path) images, face_box = crop_rect_box_coodrs(coodrs, image_org, True) image_org = Image.fromarray(cv2.cvtColor(image_org, cv2.COLOR_BGR2RGB)) for i in range(images.shape[0]): box = face_box[i] face_encode = preprocess_image(images[i], face_embedding) face_embedded = K.eval(face_encode) y = model.predict(face_embedded) if np.max(y) > 0.7: person = objects[np.argmax(y)] else: person = "*" print(np.max(y), person) # cv2.waitKey(0) # image_org = cv2.rectangle(image_org,(box[0], box[1]), (box[0] + box[2], box[1] + box[3]), (0,255,0), 1) fort = ImageFont.truetype(fontPath + "/" + "FreeMonoBold.ttf", size=60) pos_name = (box[0] + box[2], box[1] - 5) pos_box = (box[0] + box[2] - 1, box[1] - 5) pos_rect = (box[0], box[1]), (box[0] + box[2], box[1] + box[3]) if person != "*": tw, th = fort.getsize(person) canvas = Image.new('RGB', (int(tw / 5) - 10, int(th / 5) + 1), "orange") image_org.paste(canvas, pos_box) draw = ImageDraw.Draw(image_org) if person != "*": draw.text(pos_name, person, 'blue', fort=fort) draw.rectangle(pos_rect, outline='green') image_org.save(image_result)
def predict(sess, image_file, data): """ Runs the graph stored in "sess" to predict boxes for "image_file". Prints and plots the preditions. Arguments: sess -- your tensorflow/Keras session containing the YOLO graph image_file -- name of an image stored in the "images" folder. Returns: out_scores -- tensor of shape (None, ), scores of the predicted boxes out_boxes -- tensor of shape (None, 4), coordinates of the predicted boxes out_classes -- tensor of shape (None, ), class index of the predicted boxes Note: "None" actually represents the number of predicted boxes, it varies between 0 and max_boxes. """ startTime = time.time() # Preprocess your image image, image_data = preprocess_image(image_file, model_image_size=(608, 608)) time1 = time.time() out_scores, out_boxes, out_classes = sess.run((data), feed_dict={ yolo_model.input: image_data, K.learning_phase(): 0 }) time2 = time.time() colors = generate_colors(class_names) time3 = time.time() draw_boxes(image, out_scores, out_boxes, out_classes, class_names, colors) time4 = time.time() index = image_file.find('0') new_image_path = image_file[:index] + 'processed/' + image_file[index:] #new_image_path = "images/test8.jpg" image.save(new_image_path, quality=90) time5 = time.time() #output_image = scipy.misc.imread(os.path.join("out", image_file)) #imshow(output_image) print("Prep - %d" % (time1 - startTime)) print("Sess.run - %d" % (time2 - time1)) print("generate_colors - %d" % (time3 - time2)) print("draw_boxes - %d" % (time4 - time3)) print("save - %d" % (time5 - time4)) print("all time - %d" % (time5 - startTime)) return out_scores, out_boxes, out_classes
def test_noise_image(self): noise_image = None with h5py.File(REF_VALS_PATH, 'r') as f: noise_image = torch.from_numpy(f['noise1234'][()]) target_image = utilities.preprocess_image( utilities.load_image(SOURCE_IMG_PATH)) model = Model(PYTORCH_MODEL_PATH, torch.device('cpu'), target_image) model(noise_image) actual_loss_value = model.get_loss().detach().cpu() with h5py.File(REF_VALS_PATH, 'r') as f: expected_loss_value = torch.tensor(f['loss_value'][()], dtype=torch.float32) assert torch.allclose(actual_loss_value, expected_loss_value)
net, dataset) if not os.path.exists(save_dir): os.makedirs(save_dir) m = SiameseNetwork(network_type=args.net).cuda() m.load_state_dict(torch.load(args.resume)) CNN_net = m._modules['net'] dice_all1 = np.zeros((22,2)) dice_all2 = np.zeros((22,2)) for i in range(0, 22): print("CLASS -- ", i) vols = np.load(os.path.join(data_root, 'testtom_vol_{}.npy'.format(str(i).zfill(2)))) segs = np.load(os.path.join(data_root, 'testtom_seg_{}.npy'.format(str(i).zfill(2)))) vols = preprocess_image(vols) dice_c1 = np.zeros((segs.shape[0], 1)) dice_c2 = np.zeros((segs.shape[0], 1)) for j in range(segs.shape[0]): vol = vols[:,:,j,:,:,:] seg = segs[j:j+1,:,:,:] _, seg_pred = CNN_net(vol.cuda(), mode='test') vol_np = vol[0,0,:,:,:].detach().numpy() seg_np = seg[0,:,:,:] seg_pred_np = seg_pred[0,0,:,:,:].detach().cpu().numpy() # init wrapper object dense_crf_param = {} dense_crf_param['MaxIterations'] = 4.0
def generator_test_singleimage(b_s, image): gaussian = np.zeros((b_s, nb_gaussian, shape_r_gt, shape_c_gt)) while True: yield [preprocess_image(image, shape_r, shape_c), gaussian]
def process_frame(self, frame, config): #frame = self.frame cv.flip(frame, 1, frame) height, width = frame.shape[:2] paddingPercent = 30 trackingRangeX = [ 0 + (paddingPercent / 100) * (width), (100 - paddingPercent) / 100 * (width) ] trackingRangeY = [ 0 + (paddingPercent / 100) * (height), (100 - paddingPercent) / 100 * (height) ] if self.captureBGFlag: self.bg_ref = frame print('Background Captured!') # self.remove_bg = True self.captureBGFlag = False if self.remove_bg: if self.bg_ref is None: print('Cannot Remove Background! Background Ref Not Set.') self.remove_bg = False else: frame = utilities.remove_image_background(frame, self.bg_ref) img = frame cv.convertScaleAbs(frame, img, alpha=self.contrast / 10, beta=self.brightness - 128) # Lowerbound and Upperbound for color detection l_b = np.array([config.lh, config.ls, config.lv]) u_b = np.array([config.uh, config.us, config.uv]) processedHSV = cv.cvtColor(utilities.preprocess_image(img), cv.COLOR_BGR2HSV) mask = cv.inRange(processedHSV, l_b, u_b) maskFinal = utilities.process_mask(mask) contours, h = cv.findContours(maskFinal.copy(), cv.RETR_LIST, cv.CHAIN_APPROX_NONE) # Draw Tracking Area Range Box #frame = cv.rectangle(frame, (trackingRangeX[0], trackingRangeY[0]), (trackingRangeX[1], trackingRangeY[1]), (0,0,255), 3) boundingAreas = utilities.bounding_areas_from_contours(contours) # Sort the areas in descending order boundingAreas = utilities.sort_dict_by_keys(boundingAreas, reverse=True) # Get the sorted bounding coordinates rects = list(boundingAreas.values()) lenRects = len(rects) # Get only the first two largest rectangles if there are more than two if lenRects > 2: lenRects = 2 points = [] for i in range(lenRects): x, y, w, h, area = rects[i] # Discard if subsequent boxes are less than 20% of the largest box if i > 0 and area < 0.2 * rects[0][4]: break # Draw rectangle to show bounding box frame = cv.rectangle(frame, (x, y), (x + w, y + h), (0, 0, 255), 3) x_center = x + int(w / 2) y_center = y + int(h / 2) points.append((x_center, y_center)) # Draw circle to show center of bounding box frame = cv.circle(frame, (x_center, y_center), radius=5, color=(0, 0, 255), thickness=-1) cursorLoc = () if len(points) == 1: cursorLoc = points[0] # If there are two bounding boxes, Draw a line through their centers if len(points) == 2: frame = cv.line(frame, points[0], points[1], color=(0, 255, 0), thickness=3) x_center = int((points[0][0] + points[1][0]) / 2) y_center = int((points[0][1] + points[1][1]) / 2) cursorLoc = (x_center, y_center) # Draw circle to show center of the line frame = cv.circle(frame, cursorLoc, radius=5, color=(255, 0, 0), thickness=-1) currentMouseX, currentMouseY = pyautogui.position() if config.enableMouseTracking: if cursorLoc: if len(points) == 2 and not self.mouseDown: pyautogui.mouseDown() self.mouseDown = True elif len(points) == 1 and self.mouseDown: pyautogui.mouseUp() self.mouseDown = False # Mapped Tracking if config.selectedTrackingType == 0: newMouseX = int( utilities.linear_interpolate(cursorLoc[0], trackingRangeX, [0, self.screenWidth])) newMouseY = int( utilities.linear_interpolate(cursorLoc[1], trackingRangeY, [0, self.screenHeight])) # Ignore micromovements if abs(currentMouseX - newMouseX) > 5 or abs(currentMouseY - newMouseY) > 5: pyautogui.moveTo(newMouseX, newMouseY) # Free Tracking (Indpendent of tracked object on the camera) elif config.selectedTrackingType == 1 and self.prevCursorLoc: cursor_deviation_x = (cursorLoc[0] - self.prevCursorLoc[0] ) * config.sensitivity / 100 cursor_deviation_y = (cursorLoc[1] - self.prevCursorLoc[1] ) * config.sensitivity / 100 # Ignore micromovements if abs(cursor_deviation_x) < 4: cursor_deviation_x = 0 if abs(cursor_deviation_y) < 4: cursor_deviation_y = 0 newMouseX = currentMouseX + cursor_deviation_x newMouseY = currentMouseY + cursor_deviation_y # Ignore offscreen movements if newMouseX < 0: newMouseX = 0 elif newMouseX > self.screenWidth: newMouseX = self.screenWidth if newMouseY < 0: newMouseY = 0 elif newMouseY > self.screenHeight: newMouseY = self.screenHeight if newMouseX != currentMouseX or newMouseY != currentMouseY: pyautogui.moveTo(newMouseX, newMouseY) elif len(points) == 0 and self.mouseDown: pyautogui.mouseUp() self.mouseDown = False if cursorLoc: self.prevCursorLoc = cursorLoc else: self.prevCursorLoc = () return [frame, maskFinal]
def __init__(self, base_img_path, style_img_path, output_img_path, output_width, convnet, content_weight, style_weight, tv_weight, content_layer, style_layers, iterations): """ Initialize and store parameters of the neural styler. Initialize the desired convnet and compute the 3 losses and gradients with respect to the output image. Params ------ - input_img: tensor containing: content_img, style_img and output_img. - convnet: [string], defines which VGG to use: vgg16 or vgg19. - style_layers: list containing name of layers to use for style reconstruction. Defined in Gatys et. al but can be changed. - content_layer: string containing name of layer to use for content reconstruction. Also defined in Gatys et. al. - content_weight: weight for the content loss. - style_weight: weight for the style loss. - tv_weight: weight for the total variation loss. - iterations: iterations for optimization algorithm - output_img_path: path to output image. Notes ----- [1] If user specifies output width, then calculate the corresponding image height. Else, output image height and width should be the same as that of the content image. Also note that style image should be resized to whatever was decided above. [2] PIL returns (width, height) whereas numpy returns (height, width) since nrows=height and ncols=width. [3] L_BFGS requires that loss and grad be two functions so we create a keras function that computes the gradients and loss and return each separately using two different class methods. """ print('\nInitializing Neural Style model...') # store paths self.base_img_path = base_img_path self.style_img_path = style_img_path self.output_img_path = output_img_path # configuring image sizes [1, 2] print('\n\tResizing images...') self.width = output_width width, height = load_img(self.base_img_path).size new_dims = (height, width) # store shapes for future use self.img_nrows = height self.img_ncols = width if self.width is not None: # calculate new height num_rows = int(np.floor(float(height * self.width / width))) new_dims = (num_rows, self.width) # update the stored shapes self.img_nrows = num_rows self.img_ncols = self.width # resize content and style images to this desired shape self.content_img = K.variable( preprocess_image(self.base_img_path, new_dims)) self.style_img = K.variable( preprocess_image(self.style_img_path, new_dims)) # and also create output placeholder with desired shape if K.image_dim_ordering() == 'th': self.output_img = K.placeholder((1, 3, new_dims[0], new_dims[1])) else: self.output_img = K.placeholder((1, new_dims[0], new_dims[1], 3)) # sanity check on dimensions print("\tSize of content image is: {}".format( K.int_shape(self.content_img))) print("\tSize of style image is: {}".format(K.int_shape( self.style_img))) print("\tSize of output image is: {}".format( K.int_shape(self.output_img))) # combine the 3 images into a single Keras tensor self.input_img = K.concatenate( [self.content_img, self.style_img, self.output_img], axis=0) self.convnet = convnet self.iterations = iterations # store weights of the loss components self.content_weight = content_weight self.style_weight = style_weight self.tv_weight = tv_weight # store convnet layers self.content_layer = content_layer self.style_layers = style_layers # initialize the vgg16 model print('\tLoading {} model'.format(self.convnet.upper())) if self.convnet == 'vgg16': self.model = vgg16.VGG16(input_tensor=self.input_img, weights='imagenet', include_top=False) else: self.model = vgg19.VGG19(input_tensor=self.input_img, weights='imagenet', include_top=False) print('\tComputing losses...') # get the symbolic outputs of each "key" layer (we gave them unique names). outputs_dict = dict([(layer.name, layer.output) for layer in self.model.layers]) # extract features only from the content layer content_features = outputs_dict[self.content_layer] # extract the activations of the base image and the output image base_image_features = content_features[ 0, :, :, :] # 0 corresponds to base combination_features = content_features[ 2, :, :, :] # 2 coresponds to output # calculate the feature reconstruction loss content_loss = self.content_weight * \ feature_content_loss(base_image_features, combination_features) # for each style layer compute style loss # total style loss is then weighted sum of those losses temp_style_loss = K.variable(0.0) weight = 1.0 / float(len(self.style_layers)) for layer in self.style_layers: # extract features of given layer style_features = outputs_dict[layer] # from those features, extract style and output activations style_image_features = style_features[1, :, :, :] output_style_features = style_features[2, :, :, :] temp_style_loss += weight * style_reconstruction_loss( style_image_features, output_style_features, self.img_nrows, self.img_ncols) style_loss = self.style_weight * temp_style_loss # compute total variational loss tv_loss = self.tv_weight * variation_loss( self.output_img, self.img_nrows, self.img_ncols) # composite loss total_loss = content_loss + style_loss + tv_loss # compute gradients of output img with respect to loss print('\tComputing gradients...') grads = K.gradients(total_loss, self.output_img) outputs = [total_loss] if type(grads) in {list, tuple}: outputs += grads else: outputs.append(grads) # [3] self.loss_and_grads = K.function([self.output_img], outputs)