def process(self, frame): if self.display.get(): input_image = cv2.resize(frame, None, fx=self.scale_factor, fy=self.scale_factor) input_image = Image.fromarray(input_image) mask = self.get_mask(input_image) mask = cv2.resize((mask * 255).astype(np.uint8), frame.shape[:2][::-1], interpolation=cv2.INTER_CUBIC) kernel = np.ones((7, 7), np.uint8) mask = cv2.erode(mask, kernel, iterations=1) kernel = kernel / kernel.sum() mask = cv2.filter2D(mask, -1, kernel) mask = Image.fromarray(mask) background = self.get_background_frame()[:, ::-1, ::-1] if background.shape != frame.shape: background = np.array( crop_center(Image.fromarray(background), *frame.shape[:2][::-1])) frame = np.array( Image.composite(Image.fromarray(frame), Image.fromarray(background), mask)) return frame
def eco_preprocessing(frames, batch_size, time_step, image_size, input_channels, is_training): res = [] selected_frames = [] chunks = np.array_split(frames, time_step) for c in chunks: selected_idx = np.random.randint(c.shape[0]) frame_sample = c[selected_idx] selected_frames.append(frame_sample) is_flip = random.randint(1, 10) % 2 == 0 for f in selected_frames: if is_training: resized = utils.random_crop(f, image_size[0], image_size[1]) if is_flip: resized = np.fliplr(resized) else: resized = utils.crop_center(f, image_size[0], image_size[1]) resized = resized - [104, 117, 123] resized = resized.astype(np.float32) res.append(resized) return np.array(res)
def load_images_and_labels(self, imgs_names, image_dir, n_class, file_names_dict, num_channel=3, do_center_crop=False): imgs = np.zeros((imgs_names.shape[0], self.input_size, self.input_size, num_channel), dtype=np.float32) labels = np.zeros((imgs_names.shape[0], n_class), dtype=np.float32) for i, img_name in tqdm(enumerate(imgs_names)): img = imread(os.path.join(image_dir, img_name)) if do_center_crop and self.input_size == 128: img = crop_center(img, 150, 150) img = np.array( Image.fromarray(img).resize( (self.input_size, self.input_size))) # img = scm.imresize(img, [self.input_size, self.input_size, num_channel]) # not supported by scipy>=1.4 img = np.reshape(img, [self.input_size, self.input_size, num_channel]) img = img / 255.0 img = img - 0.5 img = img * 2.0 imgs[i] = img try: labels[i] = file_names_dict[img_name] except: print(img_name) labels[np.where(labels == -1)] = 0 return imgs, labels
def __getitem__(self, index): if isinstance(index, ImagePatchIndex): patch_index = index.patch_index index = index.image_index else: patch_index = 0 #FIXME sort this out input_id = self.input_index[index % len(self)] input_img = self._load_input(input_id) #print(input_id, index, patch_index) h, w = input_img.shape[:2] if self.train: if self.generate_target: target_arr = self.data_by_id[input_id]['coords'] else: target_arr = self._load_target(input_id) #print(target_arr.shape) attempts = 2 for i in range(attempts): pw, ph = self.patch_size cx, cy = self._random_patch_center(input_id, w, h) input_patch, target_patch = self._crop_and_transform( cx, cy, input_img, target_arr, randomize=True) # check centre of chosen patch_index for valid pixels if np.any( utils.crop_center(input_patch, pw // 2, ph // 2, pw // 4, ph // 4)): break input_tile_tensor = self.transform(input_patch) target_tile_tensor = to_tensor(target_patch) else: target_arr = None if self.has_targets: if self.generate_target: target_arr = self.data_by_id[input_id]['coords'] else: target_arr = self._load_target(input_id) cx, cy = self._indexed_patch_center(input_id, patch_index) input_patch, target_patch = self._crop_and_transform( cx, cy, input_img, target_arr, randomize=False) input_tile_tensor = self.transform(input_patch) if target_patch is None: target_tile_tensor = torch.zeros(1) else: target_tile_tensor = to_tensor(target_patch) #print(input_tile_tensor.size(), target_tile_tensor) #cv2.imwrite('test-scaled-input-%d.png' % index, input_patch) #cv2.imwrite('test-scaled-target-%d.png' % index, 4096*target_tile[:, :, :3]) index_tensor = torch.LongTensor([input_id, index, patch_index]) return input_tile_tensor, target_tile_tensor, index_tensor
def remove_ramp(arr): # pad zeros around arr, to the size of 3 times (ups = 3) of arr size padded = ut.zero_pad(arr, arr.shape) data = np.fft.fftshift(np.fft.fftn(np.fft.fftshift(padded))) com = center_of_mass(np.power(np.absolute(data), 2)) - .5 sub_pixel_shifted = sub_pixel_shift(data, (com[1], com[0], com[2])) ramp_removed_padded = np.fft.fftshift(np.fft.ifftn(np.fft.fftshift(sub_pixel_shifted))) ramp_removed = ut.crop_center(ramp_removed_padded, arr.shape) return ramp_removed
def inception_preprocessing(img, threshold=0.875): import cv2 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) img = utils.crop_center(img, threshold=threshold) img = cv2.resize(img, dsize=(224, 224)) img = np.expand_dims(img, axis=0) img = img / 256 img = (img - 0.5) * 2 img = img.astype('float32') return img
def _centre_crop_and_transform(self, input_img, scale=1.0, trans=False, vflip=False, hflip=False): h, w = input_img.shape[:2] cx = w // 2 cy = h // 2 crop_w, crop_h = utils.calc_crop_size(self.img_size[0], self.img_size[1], scale=scale) input_img = utils.crop_center(input_img, cx, cy, crop_w, crop_h) if trans: input_img = cv2.transpose(input_img) if hflip or vflip: if hflip and vflip: c = -1 else: c = 0 if vflip else 1 input_img = cv2.flip(input_img, flipCode=c) if scale != 1.0: input_img = cv2.resize(input_img, self.img_size, interpolation=cv2.INTER_LINEAR) return input_img
def _crop_and_transform(self, cx, cy, input_img, target_arr, randomize=False): target_tile = None transform_target = False if target_arr is None else True target_is_coords = True if transform_target and target_arr.shape[ 1] == 3 else False if randomize: angle = 0. hflip = random.random() < 0.5 vflip = random.random() < 0.5 do_rotate = random.random( ) < 0.25 if not hflip and not vflip else False if do_rotate: angle = random.random() * 360 scale = random.uniform(0.5, 1.125) #print('hflip: %d, vflip: %d, angle: %f, scale: %f' % (hflip, vflip, angle, scale)) else: angle = 0. scale = 1. hflip = False vflip = False crop_w, crop_h = utils.calc_crop_size(self.patch_size[0], self.patch_size[1], angle, scale) input_tile = utils.crop_center(input_img, cx, cy, crop_w, crop_h) if transform_target: if target_is_coords: target_points = target_arr.copy() target_points = utils.crop_points_center( target_points, cx, cy, crop_w, crop_h) #print(cx, cy, crop_w, crop_h, angle, scale, hflip, vflip) #print(target_points) target_points[:, :2] = target_points[:, :2] - [cx, cy] else: target_tile = utils.crop_center(target_arr, cx, cy, crop_w, crop_h) # Perform tile geometry transforms if needed if angle or scale != 1. or hflip or vflip: Mtrans = np.identity(3) Mtrans[0, 2] = (self.patch_size[0] - crop_w) // 2 Mtrans[1, 2] = (self.patch_size[1] - crop_h) // 2 if hflip: Mtrans[0, 0] *= -1 Mtrans[0, 2] = self.patch_size[0] - Mtrans[0, 2] if vflip: Mtrans[1, 1] *= -1 Mtrans[1, 2] = self.patch_size[1] - Mtrans[1, 2] if angle or scale != 1.: Mrot = cv2.getRotationMatrix2D((crop_w // 2, crop_h // 2), angle, scale) Mfinal = np.dot(Mtrans, np.vstack([Mrot, [0, 0, 1]])) else: Mfinal = Mtrans input_tile = cv2.warpAffine(input_tile, Mfinal[:2, :], tuple(self.patch_size)) if transform_target: if target_is_coords: if len(target_points): target_cats = target_points[:, 2].copy() target_points[:, 2] = np.ones(len(target_points)) target_points = np.dot(target_points, Mfinal) #print(target_points) target_points[:, 2] = target_cats else: tt64 = target_tile.astype(np.float64) tt64 = cv2.warpAffine(tt64, Mfinal[:2, :], tuple(self.patch_size)) if scale != 1.: tt64 /= scale**2 target_tile = tt64.astype(np.float32) if target_is_coords: target_points = np.rint(target_points).astype(np.int) target_points[:, :2] = target_points[:, :2] + [ self.patch_size[0] // 2, self.patch_size[1] // 2 ] target_points = utils.crop_points(target_points, 0, 0, self.patch_size[0], self.patch_size[1]) #print(target_points) if self.target_type == 'countception': dtype = np.uint8 if self.num_logits else np.float32 max_count = self.num_logits - 1 if self.num_logits else 0 target_tile = gen_target_countception(target_points, self.patch_size, max_count=max_count, dtype=dtype) else: target_tile = gen_target_gauss(target_points, self.patch_size, factor=1024.) return input_tile, target_tile
def train(self, config): # NOTE : if train, the nx, ny are ingnored #print("config.is_train:", config.is_train) nx, ny, original_shape = input_setup(config) #print("nx, ny, original_shape:", nx, ny, original_shape) data_dir = checkpoint_dir(config) print("reading data..") input_, label_ = read_data(data_dir) print("input_", input_.shape) merged_summary_op = tf.summary.merge_all() summary_writer = tf.summary.FileWriter("./log/train_300") #, self.sess.graph) #self.summary_writer = tf.summary.FileWriter("./log/", tf.get_default_graph()) # Stochastic gradient descent with the standard backpropagation #self.train_op = tf.train.GradientDescentOptimizer(config.learning_rate).minimize(self.loss) self.optimizer = tf.train.AdamOptimizer(learning_rate=config.learning_rate) self.train_op = self.optimizer.minimize(self.loss) #self.train_op = tf.train.AdamOptimizer(learning_rate=config.learning_rate).minimize(self.loss) tf.initialize_all_variables().run() counter = 0 time_ = time.time() self.load(config.checkpoint_dir) # Train if config.is_train: print("Now Start Training...") #for ep in range(config.epoch): for ep in range(300, 1000+1, 1): #print("ep:", ep) #sys.exit() loss_summary_per_batch = [] # Run by batch images batch_idxs = len(input_) // config.batch_size for idx in range(0, batch_idxs): batch_images = input_[idx * config.batch_size : (idx + 1) * config.batch_size] batch_labels = label_[idx * config.batch_size : (idx + 1) * config.batch_size] counter += 1 _, err, summary = self.sess.run([self.train_op, self.loss, merged_summary_op], feed_dict={self.images: batch_images, self.labels: batch_labels}) summary_pb = tf.summary.Summary() summary_pb.ParseFromString(summary) summaries = {} for val in summary_pb.value: summaries[val.tag] = val.simple_value #print("summaries:", summaries) loss_summary_per_batch.append(summaries['loss']) summary_writer.add_summary(summary, (ep) * counter) #self.summary_writer.add_summary(summary, (ep+1) * counter) if counter % 1000 == 0: print("Epoch: [%2d], step: [%2d], time: [%4.4f], loss: [%.8f]" % ((ep), counter, time.time()-time_, err)) #print(label_[1] - self.pred.eval({self.images: input_})[1],'loss:]',err) #print("Epoch: [%2d], loss: [%.8f]", (ep+1), tf.reduce_mean(tf.square(label_ - self.pred.eval({self.images: input_})))) #if counter % 500 == 0: #if counter % 20 == 0: # self.save(config.checkpoint_dir, counter) if ep ==0 or ep % 10 == 0: self.save(config.checkpoint_dir, ep) ### ''' try: config.is_train = False nx_, ny_, original_shape_ = input_setup(config) data_dir_ = checkpoint_dir(config) input__, label__ = read_data(data_dir_) print("Now Start Testing...") result_ = self.pred.eval({self.images: input__}) image_ = merge(result_, [nx_, ny_], self.c_dim) print("image after merge:", image_.shape) print("[nx_, ny_]:", [nx_, ny_]) print("original_shape:", original_shape_) print(type(image__), type(original_shape_[0]), type(original_shape_[1])) cropped_img_ = crop_center(image, original_shape_[0], original_shape_[1]) print("cropped_img_:", cropped_img_.shape) imsave(image_, config.result_dir + '/result-' + ep + '.png', config) imsave(cropped_img_, config.result_dir + '/result_crop-' + ep + '.png', config) except: print("Unexpected error while evaluating image:", sys.exc_info()[0]) config.is_train = True ''' ### print("loss per epoch[%d] loss: [%.8f]" % ((ep), np.mean(loss_summary_per_batch))) summary_writer.add_summary(tf.Summary(value=[tf.Summary.Value(tag="loss per epoch", simple_value=np.mean(loss_summary_per_batch)),]), ((ep))) summary_writer.add_summary(tf.Summary(value=[tf.Summary.Value(tag="learning rate", simple_value=self.optimizer._lr),]), ((ep))) #print("learning rate:", self.optimizer._lr) # Test else: print("Now Start Testing...") #print("nx","ny",nx,ny) result = self.pred.eval({self.images: input_}) print("result:", result.shape) #print(label_[1] - result[1]) image = merge(result, [nx, ny], self.c_dim) print("image after merge:", image.shape) print("[nx, ny]:", [nx, ny]) print("original_shape:", original_shape) print(type(image), type(original_shape[0]), type(original_shape[1])) cropped_img = crop_center(image, original_shape[0], original_shape[1]) print("cropped_img:", cropped_img.shape) #image_LR = merge(input_, [nx, ny], self.c_dim) #checkimage(image_LR) imsave(image, config.result_dir+'/result.png', config) imsave(cropped_img, config.result_dir+'/result_crop.png', config)
def _random_crop_and_transform(self, input_img, scale_range=(1.0, 1.0), rot=0.0): angle = 0. hflip = random.random() < 0.5 vflip = random.random() < 0.5 trans = random.random() < 0.5 do_rotate = (rot > 0 and random.random() < 0.2) if not hflip and not vflip else False h, w = input_img.shape[:2] # Favour rotation/scale choices that involve cropping within image bounds attempts = 0 while attempts < 3: if do_rotate: angle = random.uniform(-rot, rot) scale = random.uniform(*scale_range) crop_w, crop_h = utils.calc_crop_size(self.img_size[0], self.img_size[1], angle, scale) if crop_w <= w and crop_h <= h: break attempts += 1 if crop_w > w or crop_h > h: # We can still handle crops larger than the source, add a border angle = 0.0 #scale = 1.0 border_w = crop_w - w border_h = crop_h - h input_img = cv2.copyMakeBorder( input_img, border_h//2, border_h - border_h//2, border_w//2, border_w - border_w//2, cv2.BORDER_REFLECT_101) input_img = np.ascontiguousarray(input_img) # trying to hunt a pytorch/cuda crash, is was this necessary? assert input_img.shape[:2] == (crop_h, crop_w) else: hd = max(0, h - crop_h) wd = max(0, w - crop_w) ho = random.randint(0, hd) - math.ceil(hd / 2) wo = random.randint(0, wd) - math.ceil(wd / 2) cx = w // 2 + wo cy = h // 2 + ho input_img = utils.crop_center(input_img, cx, cy, crop_w, crop_h) #print('hflip: %d, vflip: %d, angle: %f, scale: %f' % (hflip, vflip, angle, scale)) if angle: if trans: input_img = cv2.transpose(input_img) m_translate = np.identity(3) if hflip: m_translate[0, 0] *= -1 m_translate[0, 2] = (self.img_size[0] + crop_w) / 2 - 1 else: m_translate[0, 2] = (self.img_size[0] - crop_w) / 2 if vflip: m_translate[1, 1] *= -1 m_translate[1, 2] = (self.img_size[1] + crop_h) / 2 - 1 else: m_translate[1, 2] = (self.img_size[1] - crop_h) / 2 if angle or scale != 1.: m_rotate = cv2.getRotationMatrix2D((crop_w / 2, crop_h / 2), angle, scale) m_final = np.dot(m_translate, np.vstack([m_rotate, [0, 0, 1]])) else: m_final = m_translate input_img = cv2.warpAffine(input_img, m_final[:2, :], self.img_size, borderMode=cv2.BORDER_REFLECT_101) else: if trans: input_img = cv2.transpose(input_img) if hflip or vflip: if hflip and vflip: c = -1 else: c = 0 if vflip else 1 input_img = cv2.flip(input_img, flipCode=c) input_img = cv2.resize(input_img, self.img_size, interpolation=cv2.INTER_LINEAR) return input_img