def loadTargetImage(self, src, remove_bg=True): tmp = cv2.imread(src, cv2.IMREAD_GRAYSCALE) if tmp is None: raise Exception('Cant load image from file %s' % (src, )) cam = self.systemModel.cam if tmp.shape != (cam.height, cam.width): # visit fails to generate 1024 high images tmp = cv2.resize(tmp, None, fx=cam.width / tmp.shape[1], fy=cam.height / tmp.shape[0], interpolation=cv2.INTER_CUBIC) if BATCH_MODE and self.add_image_noise and self._noise_image: tmp = ImageProc.add_noise_to_image(tmp, self._noise_image) self.image_file = src if remove_bg: self.full_image, h, th = ImageProc.process_target_image(tmp) self.image_bg_threshold = th self.parent().centroid.bg_threshold = th else: self.full_image = tmp self.image_bg_threshold = None self.parent().centroid.bg_threshold = None self.setImageZoomAndResolution(im_scale=self.im_def_scale)
def _get_motion_kernel(psf_sd, line_xy): if len(psf_sd) == 3: sd1, w, sd2 = psf_sd else: sd1, w, sd2 = psf_sd[0], 0, 0 psf_hw = math.ceil(max(sd1 * 3, sd2 * 2)) psf_fw = 1 + 2 * psf_hw psf = ImageProc.gkern2d(psf_fw, sd1) + (0 if w == 0 else w * ImageProc.gkern2d(psf_fw, sd2)) line_xy = np.array(line_xy) line = np.zeros( np.ceil(np.abs(np.flip(line_xy))).astype(np.int) + psf_fw) cnt = np.flip(line.shape) / 2 start = tuple(np.round(cnt - line_xy / 2).astype(np.int)) end = tuple(np.round(cnt + line_xy / 2).astype(np.int)) cv2.line(line, start, end, color=1.0, thickness=1, lineType=cv2.LINE_AA) mb_psf = cv2.filter2D(line, cv2.CV_64F, psf) mb_psf /= np.sum(mb_psf) # normalize to one return mb_psf
def _motion_kernel_psf_saturation(self, du, psf_sd, get_px_du_sat=False): read_sd = None if len(psf_sd) in (2, 4): psf_sd, read_sd = psf_sd[:-1], psf_sd[-1] line_xy = self.frame.motion_in_px(self.ixy) mb_psf = self._get_motion_kernel(psf_sd, line_xy) px_du_sat = np.clip(mb_psf * du, 0, self.frame.max_signal) if read_sd: noise = trunc_gaussian_shift(px_du_sat, read_sd * self.frame.max_signal, self.frame.max_signal) if 1: px_du_sat = np.clip(px_du_sat - noise, 0, self.frame.max_signal) du_sat = np.sum(px_du_sat) else: noise = np.random.normal(0, read_sd * saturation_val, px_du_sat.shape) noise = cv2.filter2D(noise, cv2.CV_64F, ImageProc.gkern2d(5, 1.0)) px_du_sat = np.clip(px_du_sat + noise, 0, saturation_val) else: du_sat = np.sum(px_du_sat) return (du_sat, ) + ((px_du_sat, ) if get_px_du_sat else tuple())
def test_thumbnail_gamma_effect(): img = np.zeros((1536, 2048)) j = 1536 // 2 + 8 I = np.array(tuple(range(8, 2048, 32))[:-1], dtype='int') mag = I[-1] * 0.85 img[j - 8:j + 8, I] = np.array(I) * (mag / I[-1]) img = ImageProc.apply_point_spread_fn(img, 0.4) print('max: %d' % np.max(img)) img = np.clip(img, 0, 1023).astype('uint16') plt.imshow(img) plt.show() thb_real = cv2.resize(img, None, fx=1 / 16, fy=1 / 16, interpolation=cv2.INTER_AREA) plt.imshow(thb_real) plt.show() img_gamma = ImageProc.adjust_gamma(img, 2.2, 0.1, max_val=1023) thb_gamma = cv2.resize(img_gamma, None, fx=1 / 16, fy=1 / 16, interpolation=cv2.INTER_AREA) thb = ImageProc.adjust_gamma(thb_gamma, 2.2, 0.1, inverse=1, max_val=1023) x = thb_real[j // 16, I // 16] xf = img[j, I] xfg = img_gamma[j, I] yg = thb_gamma[j // 16, I // 16] y = thb[j // 16, I // 16] line = np.linspace(0, np.max(x)) plt.plot(x, y, 'x') plt.plot(line, line) gamma, gamma_break, max_val, scale = fit_gamma(x, y) plt.plot( line, ImageProc.adjust_gamma(line, gamma, gamma_break, max_val=max_val) * scale) plt.show() quit()
def output(img, show, maxval=1.0, gamma=1.0, outfile=None): img = ImageProc.adjust_gamma(maxval * img / np.max(img) * 255, gamma=gamma) / 255 cv2.imwrite(outfile or (sys.argv[1] if len(sys.argv) > 1 else 'test.png'), (255 * img).astype('uint8')) if show: img_sc = cv2.resize(img, (700, 700)) cv2.imshow('test.png', img_sc) return cv2.waitKey()
def adjust_iteratively(self, sce_img, outfile=None, **kwargs): self.debug_filebase = outfile self._bg_threshold = kwargs.get('bg_threshold', self._bg_threshold) sce_img = self.maybe_load_scene_image(sce_img, preproc='bg_threshold' not in kwargs) if DEBUG: sc = self.system_model.view_width / sce_img.shape[1] cv2.imshow('target img', cv2.resize(sce_img, None, fx=sc, fy=sc)) self.system_model.spacecraft_pos = ( 0, 0, -self.system_model.min_med_distance) for i in range(self.MAX_ITERATIONS): ox, oy, oz = self.system_model.spacecraft_pos od = math.sqrt(ox**2 + oy**2 + oz**2) if not DEBUG: self.adjust(sce_img, preproc=False) else: try: self.adjust(sce_img, preproc=False) except PositioningException as e: print(str(e)) break finally: cv2.imshow('rendered img', self._ref_img) cv2.waitKey() nx, ny, nz = self.system_model.spacecraft_pos ch = math.sqrt((nx - ox)**2 + (ny - oy)**2 + (nz - oz)**2) if DEBUG: print('i%d: d0=%.2f, ch=%.2f, rel_ch=%.2f%%' % (i, od, ch, ch / od * 100)) if ch / od < self.ITERATION_TOL: break if self.CHECK_RESULT_VALIDITY: result_quality = ImageProc.norm_xcorr(sce_img, self._ref_img) if result_quality < self.MIN_RESULT_XCORR: raise PositioningException( 'Result failed quality test with score: %.3f' % (result_quality, )) if BATCH_MODE and self.debug_filebase: img = self.render(shadows=self.RENDER_SHADOWS, textures=self.RENDER_TEXTURES) cv2.imwrite(self.debug_filebase + 'r.png', img) if DEBUG: cv2.waitKey() cv2.destroyAllWindows()
def show_image(self, gain=1, processed=False, compare=False, median_filter=False, zero_bg=False, save_as=None): img = self.image.astype('float') if processed: if zero_bg: img = np.clip( img - np.min(img) - (0 if zero_bg is True else zero_bg), 0, np.inf) img *= gain if median_filter: img = cv2.medianBlur(img.astype('uint16'), median_filter) img = ImageProc.color_correct(img, self.applied_bgr_mx, max_val=self.max_val) img = ImageProc.adjust_gamma(img, self.applied_gamma, self.applied_gamma_break, max_val=self.max_val) else: img = np.clip(img * gain, 0, 2**self.bits - 1) img = ImageProc.change_color_depth(img, self.bits, 8).astype('uint8') if save_as is not None: cv2.imwrite(save_as, img) s = self.image.shape if compare: img = np.hstack((self.raw_image.astype(img.dtype), np.ones((s[0], 1, s[2]), dtype=img.dtype), img)) sc = 1 plt.imshow(np.flip(img, axis=2)) plt.show() return img, sc
def generate_field_fft(shape, sd=(0.33, 0.33, 0.34), len_sc=(0.5, 0.5 / 4, 0.5 / 16)): from visnav.algo.image import ImageProc sds = sd if getattr(sd, '__len__', False) else [sd] len_scs = len_sc if getattr(len_sc, '__len__', False) else [len_sc] assert len(shape) == 2, 'only 2d shapes are valid' assert len(sds) == len(len_scs), 'len(sd) differs from len(len_sc)' n = np.prod(shape) kernel = np.sum( np.stack([1 / len_sc * sd * n * ImageProc.gkern2d(shape, 1 / len_sc) for sd, len_sc in zip(sds, len_scs)], axis=2), axis=2) f_img = np.random.normal(0, 1, shape) + np.complex(0, 1) * np.random.normal(0, 1, shape) f_img = np.real(np.fft.ifft2(np.fft.fftshift(kernel * f_img))) return f_img
def _count_du(self, x, y, size=5, bg=None): wmrg = size // 4 mmrg = 1 if bg is None else 0 mask = ImageProc.bsphkern(size + 2 * mmrg) if bg is None: mask[0, :] = 0 mask[-1, :] = 0 mask[:, 0] = 0 mask[:, -1] = 0 mask = mask.astype(np.bool) mr = size // 2 + mmrg mn = size + 2 * mmrg h, w, _ = self.image.shape x, y = int(round(x)), int(round(y)) if h - y + wmrg <= mr or w - x + wmrg <= mr or x + wmrg < mr or y + wmrg < mr: return zip([None] * 3, [None] * 3) win = self.image[max(0, y - mr):min(h, y + mr + 1), max(0, x - mr):min(w, x + mr + 1), :].reshape((-1, 3)) mx0, mx1 = -min(0, x - mr), mn - (max(w, x + mr + 1) - w) my0, my1 = -min(0, y - mr), mn - (max(h, y + mr + 1) - h) mask = mask[my0:my1, mx0:mx1].flatten() bg = np.mean(win[np.logical_not(mask), :], axis=0) if bg is None else bg if False: tot = np.sum(win[mask, :], axis=0) tot_bg = bg * np.sum(mask) tot = np.max(np.array((tot, tot_bg)), axis=0) # tried to take into account thumbnail mean resizing after gamma correction, # also assuming no saturation of original pixels because of motion blur # => better results if tune Camera->emp_coef instead resizing_gain = (1 / self.resize_scale)**2 g = self.applied_gamma # ([sum over i in n: (bg+s_i)**g] / n) ** (1/g) # => cannot compensate for gamma correction as signal components not summable anymore, # only possible if assume that only a single pixel has signal (or some known distribution of signal?) # signal in a single, non-saturating pixel (conflicting assumptions): adj_tot = (((tot - tot_bg + bg)**g * resizing_gain) - (resizing_gain - 1) * bg**g)**(1 / g) - bg signal = adj_tot else: #signal = tot - tot_bg signal = np.clip(np.sum(win[mask, :] - bg, axis=0), 0, np.inf) return zip(signal, bg)
def write_img(raw_imgs, outfile): imgs = [] for raw in raw_imgs: img = ImageProc.change_color_depth(raw.astype('float'), 8, bits) img = ImageProc.adjust_gamma(img, gamma, gamma_break=gamma_break, inverse=True, max_val=max_val) if bgr_cc_mx is not None: img = ImageProc.color_correct(img, bgr_cc_mx, inverse=True, max_val=max_val) imgs.append(np.expand_dims(img, axis=0)) if len(imgs) == 1: imgs = imgs[0] stacked = np.stack(imgs, axis=0) reduced = np.median(stacked, axis=0) if len(imgs) > 2 else np.min( stacked, axis=0) bg_img = np.round(reduced).squeeze().astype('uint16') cv2.imwrite(outfile, bg_img, (cv2.CV_16U, ))
def detect_moon(self): x, y = tuple(int(v) for v in self.moon_loc) win = self.image[y - 24:y + 25, x - 29:x + 30, :] if 0: # TODO: what transformation would make this work? win = ImageProc.adjust_gamma(win / 662 * 1023, 2.2, inverse=1, max_val=1023) h, w, s, c = *win.shape[0:2], 19, 18 mask = np.zeros((h, w), dtype='uint8') mask[(h // 2 - s):(h // 2 + s + 1), (w // 2 - s):(w // 2 + s + 1)] = ImageProc.bsphkern(s * 2 + 1).astype('uint8') mask[0:c, :] = 0 if SHOW_MEASURES: mask_img = (mask.reshape( (h, w, 1)) * np.array([255, 0, 0]).reshape( (1, 1, 3))).astype('uint8') win_img = np.clip(win, 0, 255).astype('uint8') plt.imshow(np.flip(ImageProc.merge((mask_img, win_img)), axis=2)) plt.show() mask = mask.flatten().astype('bool') n = np.sum(mask) measures = [] for i, cam in enumerate(self.cam): raw_du = np.sum(win[:, :, i].flatten()[mask]) bg_du = np.mean(win[:, :, i].flatten()[np.logical_not(mask)]) du_count = raw_du - bg_du * n measures.append(MoonMeasure(self, i, du_count)) self.measures = measures return measures
def _feature_detection_mask(self, image): _, mask = cv2.threshold(image, self.min_feature_intensity, 255, cv2.THRESH_BINARY) kernel = ImageProc.bsphkern(round(6 * image.shape[0] / 512) * 2 + 1) # exclude asteroid limb from feature detection mask = cv2.erode(mask, ImageProc.bsphkern(7), iterations=1) # remove stars mask = cv2.dilate(mask, kernel, iterations=1) # remove small shadows inside asteroid mask = cv2.erode(mask, kernel, iterations=2) # remove asteroid limb # exclude overexposed parts _, mask_oe = cv2.threshold(image, self.max_feature_intensity, 255, cv2.THRESH_BINARY) mask_oe = cv2.dilate(mask_oe, kernel, iterations=1) mask_oe = cv2.erode(mask_oe, kernel, iterations=1) mask[mask_oe > 0] = 0 if 0: cv2.imshow('mask', ImageProc.overlay_mask(image, mask)) cv2.waitKey() return mask
def get_kernel(peak, fwhm, psfd, size): var = (fwhm / np.sqrt(8 * np.log(2)))**2 amp = psfd / gaussian(0, var, 0) spectrum_fn = lambda lam: amp * gaussian(peak, var, lam) kernel = LabFrame.calc_source_kernel(bgr_cam, spectrum_fn, size, points=(peak, )) if IGNORE_INVERT_COLOR_CORRECTION: kernel = ImageProc.color_correct( kernel, np.array([ [2.083400, -0.524300, -0.389100], [-0.516800, 2.448100, -0.761300], [-0.660600, 0.149600, 1.680900], ])) return kernel
def calc_source_kernel(cams, spectrum_fn, patch_size, points=None): # detect source kernel = ImageProc.bsphkern( tuple(map(int, patch_size)) if '__iter__' in dir(patch_size) else int(patch_size)) expected_bgr = np.zeros(3) for i, cam in enumerate(cams): ef, _ = Camera.electron_flux_in_sensed_spectrum_fn(cam.qeff_coefs, spectrum_fn, cam.lambda_min, cam.lambda_max, fast=False, points=points) expected_bgr[i] = cam.gain * cam.aperture_area * cam.emp_coef * ef kernel = np.repeat(np.expand_dims(kernel, axis=2), 3, axis=2) * expected_bgr return kernel
def setImageZoomAndResolution(self, im_xoff=0, im_yoff=0, im_width=None, im_height=None, im_scale=1): self.im_xoff = im_xoff self.im_yoff = im_yoff self.im_width = im_width or self.systemModel.cam.width self.im_height = im_height or self.systemModel.cam.height self.im_scale = im_scale self.image = ImageProc.crop_and_zoom_image(self.full_image, self.im_xoff, self.im_yoff, self.im_width, self.im_height, self.im_scale) self._image_h = self.image.shape[0] self._image_w = self.image.shape[1] if self._show_target_image: # form _gl_image that is used for rendering # black => 0 alpha, non-black => white => .5 alpha im = self.image.copy() alpha = np.zeros(im.shape, im.dtype) #im[im > 0] = 255 alpha[im > 0] = 128 self._gl_image = np.flipud(cv2.merge( (im, im, im, alpha))).tobytes() self.updateFrustum() # WORK-AROUND: for some reason wont use new frustum if window not resized s = self.parent().size() self.parent().resize(s.width() + 1, s.height()) self.parent().resize(s.width(), s.height()) self.update() QCoreApplication.processEvents()
def est_refl_model(hapke=True, iters=1, init_noise=0.0, verbose=True): sm = RosettaSystemModel() imgsize = (512, 512) imgs = { 'ROS_CAM1_20140831T104353': 3.2, # 60, 3.2s 'ROS_CAM1_20140831T140853': 3.2, # 62, 3.2s 'ROS_CAM1_20140831T103933': 3.2, # 65, 3.2s 'ROS_CAM1_20140831T022253': 3.2, # 70, 3.2s 'ROS_CAM1_20140821T100719': 2.8, # 75, 2.8s 'ROS_CAM1_20140821T200718': 2.0, # 80, 2.0s 'ROS_CAM1_20140822T113854': 2.0, # 85, 2.0s 'ROS_CAM1_20140823T021833': 2.0, # 90, 2.0s 'ROS_CAM1_20140819T120719': 2.0, # 95, 2.0s 'ROS_CAM1_20140824T021833': 2.8, # 100, 2.8s 'ROS_CAM1_20140824T020853': 2.8, # 105, 2.8s 'ROS_CAM1_20140824T103934': 2.8, # 110, 2.8s 'ROS_CAM1_20140818T230718': 2.0, # 113, 2.0s 'ROS_CAM1_20140824T220434': 2.8, # 120, 2.8s 'ROS_CAM1_20140828T020434': 2.8, # 137, 2.8s 'ROS_CAM1_20140827T140434': 3.2, # 145, 3.2s 'ROS_CAM1_20140827T141834': 3.2, # 150, 3.2s 'ROS_CAM1_20140827T061834': 3.2, # 155, 3.2s 'ROS_CAM1_20140827T021834': 3.2, # 157, 3.2s 'ROS_CAM1_20140826T221834': 2.8, # 160, 2.8s } target_exposure = np.min(list(imgs.values())) for img, exposure in imgs.items(): real = cv2.imread( os.path.join(sm.asteroid.image_db_path, img + '_P.png'), cv2.IMREAD_GRAYSCALE) real = ImageProc.adjust_gamma(real, 1 / 1.8) #dark_px_lim = np.percentile(real, 0.1) #dark_px = np.mean(real[real<=dark_px_lim]) real = cv2.resize(real, imgsize) # remove dark pixel intensity and normalize based on exposure #real = real - dark_px #real *= (target_exposure / exposure) imgs[img] = real re = RenderEngine(*imgsize, antialias_samples=0) obj_idx = re.load_object(sm.asteroid.hires_target_model_file, smooth=False) ab = AlgorithmBase(sm, re, obj_idx) model = RenderEngine.REFLMOD_HAPKE if hapke else RenderEngine.REFLMOD_LUNAR_LAMBERT defs = RenderEngine.REFLMOD_PARAMS[model] if hapke: # L, th, w, b (scattering anisotropy), c (scattering direction from forward to back), B0, hs #real_ini_x = [515, 16.42, 0.3057, 0.8746] sppf_n = 2 real_ini_x = defs[:2] + defs[3:3 + sppf_n] scales = np.array((500, 20, 3e-1, 3e-1))[:2 + sppf_n] else: ll_poly = 5 #real_ini_x = np.array(defs[:7]) real_ini_x = np.array( (9.95120e-01, -6.64840e-03, 3.96267e-05, -2.16773e-06, 2.08297e-08, -5.48768e-11, 1)) # theta=20 real_ini_x = np.hstack((real_ini_x[0:ll_poly + 1], (real_ini_x[-1], ))) scales = np.array((3e-03, 2e-05, 1e-06, 1e-08, 5e-11, 1)) scales = np.hstack((scales[0:ll_poly], (scales[-1], ))) def set_params(x): if hapke: # optimize J, th, w, b, (c), B_SH0, hs xsc = list(np.array(x) * scales) vals = xsc[:2] + [defs[2]] + xsc[2:] + defs[len(xsc) + 1:] else: vals = [1] + list(np.array(x)[:-1] * scales[:-1]) + [0] * ( 5 - ll_poly) + [x[-1] * scales[-1], 0, 0, 0] RenderEngine.REFLMOD_PARAMS[model] = vals # debug 1: real vs synth, 2: err img, 3: both def costfun(x, debug=0, verbose=True): set_params(x) err = 0 for file, real in imgs.items(): lblloader.load_image_meta( os.path.join(sm.asteroid.image_db_path, file + '.LBL'), sm) sm.swap_values_with_real_vals() synth2 = ab.render(shadows=True, reflection=model, gamma=1) err_img = (synth2.astype('float') - real)**2 lim = np.percentile(err_img, 99) err_img[err_img > lim] = 0 err += np.mean(err_img) if debug: if debug % 2: cv2.imshow( 'real vs synthetic', np.concatenate((real.astype('uint8'), 255 * np.ones( (real.shape[0], 1), dtype='uint8'), synth2), axis=1)) if debug > 1: err_img = err_img**0.2 cv2.imshow('err', err_img / np.max(err_img)) cv2.waitKey() err /= len(imgs) if verbose: print('%s => %f' % (', '.join(['%.4e' % i for i in np.array(x) * scales]), err)) return err best_x = None best_err = float('inf') for i in range(iters): if hapke: ini_x = tuple(real_ini_x + init_noise * np.random.normal(0, 1, (len(scales), )) * scales) else: ini_x = tuple(real_ini_x[1:-1] / real_ini_x[0] + init_noise * np.random.normal(0, 1, (len(scales) - 1, )) * scales[:-1]) + (real_ini_x[-1] * real_ini_x[0], ) if verbose: print('\n\n\n==== i:%d ====\n' % i) res = minimize( costfun, tuple(ini_x / scales), args=(0, verbose), #method="BFGS", options={'maxiter': 10, 'eps': 1e-3, 'gtol': 1e-3}) method="Nelder-Mead", options={ 'maxiter': 120, 'xtol': 1e-4, 'ftol': 1e-4 }) #method="COBYLA", options={'rhobeg': 1.0, 'maxiter': 200, 'disp': False, 'catol': 0.0002}) if not verbose: print('%s => %f' % (', '.join(['%.5e' % i for i in np.array(res.x) * scales]), res.fun)) if res.fun < best_err: best_err = res.fun best_x = res.x if verbose: costfun(best_x, 3, verbose=True) if hapke: x = tuple(best_x * scales) else: x = (1, ) + tuple(best_x * scales) if verbose: p = np.linspace(0, 160, 100) L = get_graph_L(20, p) plt.plot(p, L, p, Lfun(x[:-1], p)) plt.show() return x
def process(self, orig_sce_img, outfile, rotate_sc=False, **kwargs): # maybe load torch model if self.model is None: self.load_model() if outfile is not None: self.debug_filebase = outfile + ('n' if isinstance( orig_sce_img, str) else '') # maybe load scene image if isinstance(orig_sce_img, str): orig_sce_img = self.load_target_image(orig_sce_img) self.timer = Stopwatch() self.timer.start() if self.DEF_ESTIMATE_THRESHOLD: threshold = ImageProc.optimal_threshold(None, orig_sce_img) else: threshold = self.DEF_LUMINOSITY_THRESHOLD # detect target, get bounds x, y, w, h = ImageProc.single_object_bounds( orig_sce_img, threshold=threshold, crop_marg=self.DEF_CROP_MARGIN, min_px=self.DEF_MIN_PIXELS, debug=DEBUG) if x is None: raise PositioningException('asteroid not detected in image') # crop image img_bw = ImageProc.crop_and_zoom_image(orig_sce_img, x, y, w, h, None, (224, 224)) # save cropped image in log archive if BATCH_MODE and self.debug_filebase: self.timer.stop() cv2.imwrite(self.debug_filebase + 'a.png', img_bw) self.timer.start() # massage input input = cv2.cvtColor(img_bw, cv2.COLOR_GRAY2BGR) input = Image.fromarray(input) input = PoseIllumiDataset.eval_transform(input)[None, :, :, :].to( self.device, non_blocking=True) # run model with torch.no_grad(): output = self.model(input) # massage output output = output[0] if isinstance(output, (list, tuple)) else output output = output.detach().cpu().numpy() # check if estimated illumination direction is close or not ill_est = self.model.illumination(output)[0] r_ini, q_ini, ill_ini = self.system_model.get_cropped_system_scf( x, y, w, h) if tools.angle_between_v( ill_est, ill_ini) > 10: # max 10 degree discrepancy accepted print( 'bad illumination direction estimated, initial=%s, estimated=%s' % (ill_ini, ill_est)) # apply result r_est = self.model.position(output)[0] q_est = np.quaternion(*self.model.rotation(output)[0]) self.system_model.set_cropped_system_scf(x, y, w, h, r_est, q_est, rotate_sc=rotate_sc) self.timer.stop() if False: r_est2, q_est2, ill_est2 = self.system_model.get_cropped_system_scf( x, y, w, h) self.system_model.swap_values_with_real_vals() r_real, q_real, ill_real = self.system_model.get_cropped_system_scf( x, y, w, h) self.system_model.swap_values_with_real_vals() print('compare q_est vs q_est2, q_real vs q_est, q_real vs q_est2') # save result image if BATCH_MODE and self.debug_filebase: # save result in log archive res_img = self.render(textures=False) sce_img = cv2.resize(orig_sce_img, tuple(np.flipud(res_img.shape))) cv2.imwrite(self.debug_filebase + 'b.png', np.concatenate((sce_img, res_img), axis=1)) if DEBUG: cv2.imshow('compare', np.concatenate((sce_img, res_img), axis=1)) cv2.waitKey()
def texture_noise(model, support=None, L=None, noise_sd=SHAPE_MODEL_NOISE_LV['lo'], len_sc=SHAPE_MODEL_NOISE_LEN_SC, max_rng=None, max_n=1e4, hf_noise=True): tex = model.load_texture() if tex is None: print('tools.texture_noise: no texture loaded') return [None] * 3 r = np.sqrt(max_n / np.prod(tex.shape[:2])) ny, nx = (np.array(tex.shape[:2]) * r).astype(np.int) n = nx * ny tx_grid_xx, tx_grid_yy = np.meshgrid(np.linspace(0, 1, nx), np.linspace(0, 1, ny)) tx_grid = np.hstack((tx_grid_xx.reshape((-1, 1)), tx_grid_yy.reshape((-1, 1)))) support = support if support else model points = np.array(support.vertices) max_rng = np.max(np.ptp(points, axis=0)) if max_rng is None else max_rng # use vertices for distances, find corresponding vertex for each pixel y_cov = None if L is None: try: from sklearn.gaussian_process.kernels import Matern, WhiteKernel except: print('Requires scikit-learn, install using "conda install scikit-learn"') sys.exit() kernel = 1.0 * noise_sd * Matern(length_scale=len_sc * max_rng, nu=1.5) \ + 0.5 * noise_sd * Matern(length_scale=0.1 * len_sc * max_rng, nu=1.5) \ + WhiteKernel( noise_level=1e-5 * noise_sd * max_rng) # white noise for positive definite covariance matrix only # texture coordinates given so that x points left and *Y POINTS UP* tex_img_coords = np.array(support.texcoords) tex_img_coords[:, 1] = 1 - tex_img_coords[:, 1] _, idxs = find_nearest_each(haystack=tex_img_coords, needles=tx_grid) tx2vx = support.texture_to_vertex_map() y_cov = kernel(points[tx2vx[idxs], :] - np.mean(points, axis=0)) if 0: # for debugging distances import matplotlib.pyplot as plt import cv2 from visnav.algo.image import ImageProc orig_tx = cv2.imread(os.path.join(DATA_DIR, '67p+tex.png'), cv2.IMREAD_GRAYSCALE) gx, gy = np.gradient(points[tx2vx[idxs], :].reshape((ny, nx, 3)), axis=(1, 0)) gxy = np.linalg.norm(gx, axis=2) + np.linalg.norm(gy, axis=2) gxy = (gxy - np.min(gxy)) / (np.max(gxy) - np.min(gxy)) grad_img = cv2.resize((gxy * 255).astype('uint8'), orig_tx.shape) overlaid = ImageProc.merge((orig_tx, grad_img)) plt.figure(1) plt.imshow(overlaid) plt.show() # sample gp e0, L = mv_normal(np.zeros(n), cov=y_cov, L=L) e0 = e0.reshape((ny, nx)) # interpolate for final texture x = np.linspace(np.min(tx_grid_xx), np.max(tx_grid_xx), tex.shape[1]) y = np.linspace(np.min(tx_grid_yy), np.max(tx_grid_yy), tex.shape[0]) interp0 = RectBivariateSpline(tx_grid_xx[0, :], tx_grid_yy[:, 0], e0, kx=1, ky=1) err0 = interp0(x, y) if 0: import matplotlib.pyplot as plt import cv2 from visnav.algo.image import ImageProc orig_tx = cv2.imread(os.path.join(DATA_DIR, '67p+tex.png'), cv2.IMREAD_GRAYSCALE) err_ = err0 if 1 else e0 eimg = (err_ - np.min(err_)) / (np.max(err_) - np.min(err_)) eimg = cv2.resize((eimg * 255).astype('uint8'), orig_tx.shape) overlaid = ImageProc.merge((orig_tx, eimg)) plt.figure(1) plt.imshow(overlaid) plt.show() err1 = 0 if hf_noise: e1, L = mv_normal(np.zeros(n), L=L) e1 = e1.reshape((ny, nx)) interp1 = RectBivariateSpline(tx_grid_xx[0, :], tx_grid_yy[:, 0], e1, kx=1, ky=1) err_coef = interp1(x, y) lo, hi = np.min(err_coef), np.max(err_coef) err_coef = (err_coef - lo) / (hi - lo) len_sc = 10 err1 = generate_field_fft(tex.shape, (6 * noise_sd, 4 * noise_sd), (len_sc / 1000, len_sc / 4500)) if hf_noise else 0 err1 *= err_coef noisy_tex = tex + err0 + err1 min_v, max_v = np.quantile(noisy_tex, (0.0001, 0.9999)) min_v = min(0, min_v) noisy_tex = (np.clip(noisy_tex, min_v, max_v) - min_v) / (max_v - min_v) if 0: import matplotlib.pyplot as plt plt.figure(1) plt.imshow(noisy_tex) plt.figure(2) plt.imshow(err0) plt.figure(3) plt.imshow(err1) plt.show() return noisy_tex, np.std(err0 + err1), L
def analyze_aurora_img(img_file, get_bgr_cam): debug = 1 n = 0 Frame.MISSING_BG_REMOVE_STRIPES = 0 bgr_cam = get_bgr_cam(thumbnail=False, estimated=1, final=1) f = Frame.from_file(bgr_cam, img_file, img_file[:-4]+'.lbl', bg_offset=False, debug=debug) if 0: f.show_image(processed=True, save_as='C:/projects/s100imgs/processed-aurora.png') img = f.image.astype('float') if 0: img = img - np.percentile(img, 5, axis=1).reshape((-1, 1, 3)) bg1 = (830, 1070), (1180, 1400) # bg1 = (0, 900), (660, 1280) # bg2 = (1560, 1050), (2048, 1350) mean_bg = np.mean(img[bg1[0][1]:bg1[1][1], bg1[0][0]:bg1[1][0], :].reshape((-1, 3)), axis=0) # mean_bg = np.mean(np.vstack((img[bg1[0][1]:bg1[1][1], bg1[0][0]:bg1[1][0], :].reshape((-1, 3)), # img[bg2[0][1]:bg2[1][1], bg2[0][0]:bg2[1][0], :].reshape((-1, 3)))), axis=0) img = img - mean_bg # img = ImageProc.apply_point_spread_fn(img - mean_bg, 0.01) # img = np.clip(img, 0, 1023).astype('uint16') # img = cv2.medianBlur(img, 31) # img = np.clip(img, 0, RAW_IMG_MAX_VALUE) / RAW_IMG_MAX_VALUE n += 1 plt.figure(n) imsh = (np.clip(img * 2 + RAW_IMG_MAX_VALUE * 0.3, 0, RAW_IMG_MAX_VALUE) / RAW_IMG_MAX_VALUE * 255).astype('uint8') rd_y = (720, 700), (890, 720) rd_r1 = (720, 830), (890, 870) rd_r2 = (1080, 820), (1250, 860) gr_r = (1280, 770), (1450, 795) cv2.rectangle(imsh, bg1[0], bg1[1], (255, 0, 0), 2) # bg1 # cv2.rectangle(imsh, bg2[0], bg2[1], (255, 0, 0), 2) # bg2 cv2.rectangle(imsh, rd_y[0], rd_y[1], (0, 200, 200), 2) # yellow cv2.rectangle(imsh, rd_r1[0], rd_r1[1], (0, 0, 255), 2) # red1 cv2.rectangle(imsh, rd_r2[0], rd_r2[1], (0, 0, 255), 2) # red2 cv2.rectangle(imsh, gr_r[0], gr_r[1], (0, 255, 0), 2) # green plt.imshow(np.flip(imsh, axis=2)) plt.show() def electrons(lam): h = 6.626e-34 # planck constant (m2kg/s) c = 3e8 # speed of light return h * c / lam # energy per photon blue = 427.8e-9 green = 557.7e-9 yellow = 589.3e-9 red = 630.0e-9 colors = (blue, green, yellow, red) dus_per_rad = dict(zip(colors, ([], [], [], []))) # DUs per 1 W/m2/sr of radiance coef = f.exposure * f.gain * RAW_IMG_MAX_VALUE for wl in dus_per_rad.keys(): for cam in bgr_cam: cgain = cam.gain * cam.emp_coef * cam.aperture_area fn, _ = Camera.qeff_fn(tuple(cam.qeff_coefs), 350e-9, 1000e-9) # W/m2/sr => phot/s/m2/sr => elec/s/m2/sr => DUs/sr dus_per_rad[wl].append(1/electrons(wl) * fn(wl) * coef * cgain) for wl in dus_per_rad.keys(): dus_per_rad[wl] = np.array(dus_per_rad[wl]) class Patch: def __init__(self, name, rect, bands, mean=None, rad=None): self.name, self.rect, self.bands, self.mean, self.rad = name, rect, bands, mean, rad nt = lambda n, r, b: Patch(name=n, rect=r, bands=b) patches = [ nt('Clean Red', rd_r1, (blue, green, red)), nt('Strong Red', rd_r2, (blue, green, red)), nt('Green', gr_r, (blue, green, red)), nt('Sodium', rd_y, (blue, green, yellow)), ] # pseudo inverse for p in patches: p.mean = np.mean(img[p.rect[0][1]:p.rect[1][1], p.rect[0][0]:p.rect[1][0], :].reshape((-1, 3)), axis=0) px_sr = cam.pixel_solid_angle((p.rect[0][0]+p.rect[1][0])//2, (p.rect[0][1]+p.rect[1][1])//2) E = np.hstack((dus_per_rad[p.bands[0]], dus_per_rad[p.bands[1]], dus_per_rad[p.bands[2]])) * px_sr invE = np.linalg.inv(E.T.dot(E)).dot(E.T) rad = invE.dot(p.mean) # radiance in W/m2/sr # e = E.dot(rad) # diff = (p.mean - e) * 100 / np.linalg.norm(p.mean) p.rad = [''] * len(colors) for i, b in enumerate(p.bands): idx = colors.index(b) p.rad[idx] = rad[i] sep = '\t' if 1 else ' & ' le = '\n' if 1 else ' \\\\\n' if 0: print(sep.join(('Patch', 'Emission at', '', '', 'Red', 'Green', 'Blue')), end=le) for name, irr, mean, model, diff in patches: print(sep.join((name, '428 nm', ('%.3e' % irr[0]) if irr[0] else 'n/a', 'Mean', *('%.1f' % m for m in np.flip(mean.flatten())))), end=le) print(sep.join(('', '557.7 nm', ('%.3e' % irr[1]) if irr[1] else 'n/a', 'Modeled', *('%.1f' % m for m in np.flip(model.flatten())))), end=le) print(sep.join(('', '589 nm', ('%.3e' % irr[2]) if irr[2] else 'n/a', 'Diff. [%]', *('%.1f' % m for m in np.flip(diff.flatten())))), end=le) print(sep.join(('', '630 nm', ('%.3e' % irr[3]) if irr[3] else 'n/a', *(['']*4))), end=le) else: print(sep.join(('Patch', 'Red', 'Green', 'Blue', '428 nm', '557.7 nm', '589 nm', '630 nm')), end=le) for p in patches: # in kilo Rayleigh (kR) == 6330*1e9*lambda * W/m2/sr or 4*pi*10^(-10)*10^(-3) * photon flux print(sep.join((p.name, *('%.1f' % m for m in np.flip(p.mean.flatten())), *(tools.fixed_precision(r*4*np.pi*1e-13/electrons(colors[i]), 3, True) if r else '' # r*1e-13*4*np.pi for i, r in enumerate(p.rad)))), end=le) quit() aurora = np.zeros_like(img) for i, color in enumerate(colors): # d/dx[(r-aw)'*(r-aw)] == 0 # => w == (r'*a)/(a'*a) a = emission[color] w = np.sum(img.reshape((-1, 3)) * a.T, axis=1) / sum(a**2) e = (w*a).T r = img.reshape((-1, 3)) - e x = w / np.linalg.norm(r, axis=1) # plt.figure(2) # plt.imshow(w.reshape(img.shape[:2])/np.max(w)) # plt.title('weight (max=%f)' % np.max(w)) # plt.figure(3) # plt.imshow(x.reshape(img.shape[:2])/np.max(x)) # plt.title('x (max=%f)' % np.max(x)) n += 1 plt.figure(n) x[x < {red: 10, green: 6, yellow: 16}[color]] = 0 x[w < 100] = 0 xf = ImageProc.apply_point_spread_fn(x.reshape(img.shape[:2]), 0.03) xf = cv2.medianBlur(xf.astype('uint16'), 11) plt.imshow(xf / np.max(xf)) plt.title('Emission detection @ %.1fnm' % (color * 1e9)) e[xf.flatten() == 0, :] = (0, 0, 0) aurora += e.reshape(img.shape) # plt.figure(6) # plt.imshow(np.flip(e.reshape(img.shape) / np.max(e), axis=2)) # plt.title('modeled aurora') # plt.figure(7) # plt.imshow(np.flip(r.reshape(img.shape)/np.max(r), axis=2)) # plt.title('residual') # plt.show() plt.figure(8) plt.imshow(np.flip(aurora / np.max(aurora), axis=2)) plt.title('modeled aurora') plt.show() # TODO: translate rgb values to aurora (ir)radiance # - following uses W/m2/sr for "in-band radiance" # - https://www.osapublishing.org/DirectPDFAccess/A2F3D832-975A-1850-088634AAFCF21258_186134/ETOP-2009-ESB4.pdf?da=1&id=186134&uri=ETOP-2009-ESB4&seq=0&mobile=no # - use pixel sr? print('done')
def render(self, obj_idxs, rel_pos_v, rel_rot_q, light_v, get_depth=False, shadows=True, textures=True, gamma=1.0, reflection=REFLMOD_LUNAR_LAMBERT, flux_density=False): obj_idxs = [obj_idxs] if isinstance(obj_idxs, int) else obj_idxs rel_pos_v = np.array(rel_pos_v).reshape((-1, 3)) rel_rot_q = np.array(rel_rot_q).reshape((-1, )) light_v = np.array(light_v) assert len(obj_idxs) == rel_pos_v.shape[0] == rel_rot_q.shape[ 0], 'obj_idxs, rel_pos_v and rel_rot_q dimensions dont match' shadow_mvps = None if shadows: shadow_mvps = self._render_shadowmap(obj_idxs, rel_pos_v, rel_rot_q, light_v) self._fbo.use() self._ctx.enable(moderngl.DEPTH_TEST) self._ctx.enable(moderngl.CULL_FACE) self._ctx.front_face = 'ccw' # cull back faces self._ctx.clear(0, 0, 0, 0, float('inf')) if shadows: self._shadow_map.use(RenderEngine._LOC_SHADOW_MAP) self._prog['shadow_map'].value = RenderEngine._LOC_SHADOW_MAP for i, obj_idx in enumerate(obj_idxs): self._set_params(obj_idx, rel_pos_v[i], rel_rot_q[i], light_v, shadow_mvps, textures, reflection, prog=self._prog, flux_density=flux_density) self._objs[obj_idx].render() if self._samples > 0: self._ctx.copy_framebuffer(self._fbo2, self._fbo) fbo = self._fbo2 dbo = self._dbo2 else: fbo = self._fbo dbo = self._dbo data = np.frombuffer(fbo.read(components=1, alignment=1, dtype='f4'), dtype='f4').reshape((self._height, self._width)) data = np.flipud(data) if get_depth: depth = np.frombuffer(dbo.read(alignment=1), dtype='f4').reshape( (self._height, self._width)) depth = np.flipud(depth) # normalize depth if self._persp_proj: # for perspective projection a = -(self._frustum_far - self._frustum_near) / ( 2.0 * self._frustum_far * self._frustum_near) b = (self._frustum_far + self._frustum_near) / ( 2.0 * self._frustum_far * self._frustum_near) if self._frustum_far / self._frustum_near < 1e7: depth = np.divide(1.0, (2.0 * a) * depth - (a - b)) # 1/((2*X-1)*a+b) else: # up to difference of 1e14 depth = np.divide(1.0, (2.0 * a) * depth.astype(np.float64) - (a - b)).astype(np.float32) else: # for orthographic projection # - depth is between 0 and 1 depth = depth * (self._frustum_far - self._frustum_near) + self._frustum_near # free memory to avoid memory leaks if shadows: self._shadow_map.release() if flux_density: data = data * flux_density else: data = np.clip(data * 255, 0, 255).astype('uint8') if gamma != 1.0: data = ImageProc.adjust_gamma(data, gamma) return (data, depth) if get_depth else data
def export(sm, dst_path, src_path=None, src_imgs=None, trg_shape=(224, 224), crop=False, debug=False, img_prefix="", title=""): trg_w, trg_h = trg_shape assert (src_path is not None) + (src_imgs is not None) == 1, 'give either src_path or src_imgs, not both' if debug: renderer = RenderEngine(sm.cam.width, sm.cam.height, antialias_samples=0) obj_idx = renderer.load_object(sm.asteroid.target_model_file, smooth=sm.asteroid.render_smooth_faces) algo = AlgorithmBase(sm, renderer, obj_idx) metadatafile = os.path.join(dst_path, 'dataset_all.txt') if not os.path.exists(metadatafile): with open(metadatafile, 'w') as f: f.write('\n'.join(['%s, camera centric coordinate frame used' % title, 'Image ID, ImageFile, Target Pose [X Y Z W P Q R], Sun Vector [X Y Z]', '', ''])) files = list(os.listdir(src_path)) if src_imgs is None else src_imgs files = sorted(files) id = 0 for i, fn in enumerate(files): if src_imgs is not None or re.search(r'(?<!far_)\d{4}\.png$', fn): c = 2 if src_imgs is None else 1 tools.show_progress(len(files)//c, i//c) id += 1 # read system state, write out as relative to s/c fname = os.path.basename(fn) if src_imgs is None: fn = os.path.join(src_path, fn) lbl_fn = re.sub(r'_%s(\d{4})' % img_prefix, r'_\1', fn[:-4]) + '.lbl' sm.load_state(lbl_fn) sm.swap_values_with_real_vals() if not crop: shutil.copy2(fn, os.path.join(dst_path, fname)) if os.path.exists(fn[:-4] + '.d.exr'): shutil.copy2(fn[:-4] + '.d.exr', os.path.join(dst_path, fname[:-4] + '.d.exr')) if os.path.exists(fn[:-4] + '.xyz.exr'): shutil.copy2(fn[:-4] + '.xyz.exr', os.path.join(dst_path, fname[:-4] + '.xyz.exr')) if os.path.exists(fn[:-4] + '.s.exr'): shutil.copy2(fn[:-4] + '.s.exr', os.path.join(dst_path, fname[:-4] + '.s.exr')) _write_metadata(metadatafile, id, fname, sm.get_system_scf()) continue from visnav.algo.absnet import AbsoluteNavigationNN # read image, detect box, resize, adjust relative pose img = cv2.imread(fn, cv2.IMREAD_GRAYSCALE) assert img is not None, 'image file %s not found' % fn # detect target, get bounds x, y, w, h = ImageProc.single_object_bounds(img, threshold=AbsoluteNavigationNN.DEF_LUMINOSITY_THRESHOLD, crop_marg=AbsoluteNavigationNN.DEF_CROP_MARGIN, min_px=AbsoluteNavigationNN.DEF_MIN_PIXELS, debug=debug) if x is None: continue # write image metadata system_scf = sm.get_cropped_system_scf(x, y, w, h) _write_metadata(metadatafile, id, fname, system_scf) others, (depth, coords, px_size), k = [], [False] * 3, 1 if os.path.exists(fn[:-4] + '.d.exr'): depth = True others.append(cv2.imread(fn[:-4] + '.d.exr', cv2.IMREAD_UNCHANGED)) if os.path.exists(fn[:-4] + '.xyz.exr'): coords = True others.append(cv2.imread(fn[:-4] + '.xyz.exr', cv2.IMREAD_UNCHANGED)) if os.path.exists(fn[:-4] + '.s.exr'): px_size = True others.append(cv2.imread(fn[:-4] + '.s.exr', cv2.IMREAD_UNCHANGED)) # crop & resize image, write it cropped = ImageProc.crop_and_zoom_image(img, x, y, w, h, None, (trg_w, trg_h), others=others) cv2.imwrite(os.path.join(dst_path, fname), cropped[0], [cv2.IMWRITE_PNG_COMPRESSION, 9]) if depth: cv2.imwrite(os.path.join(dst_path, fname[:-4] + '.d.exr'), cropped[k], (cv2.IMWRITE_EXR_TYPE, cv2.IMWRITE_EXR_TYPE_FLOAT)) k += 1 if coords: cv2.imwrite(os.path.join(dst_path, fname[:-4] + '.xyz.exr'), cropped[k], (cv2.IMWRITE_EXR_TYPE, cv2.IMWRITE_EXR_TYPE_FLOAT)) k += 1 if px_size: cv2.imwrite(os.path.join(dst_path, fname[:-4] + '.s.exr'), cropped[k], (cv2.IMWRITE_EXR_TYPE, cv2.IMWRITE_EXR_TYPE_FLOAT)) if debug: sc, dq = sm.cropped_system_tf(x, y, w, h) sm.spacecraft_pos = tools.q_times_v(SystemModel.sc2gl_q.conj(), sc_ast_lf_r) sm.rotate_spacecraft(dq) #sm.set_cropped_system_scf(x, y, w, h, sc_ast_lf_r, sc_ast_lf_q) if False: sm.load_state(lbl_fn) sm.swap_values_with_real_vals() imgd = cv2.resize(img, (trg_h, trg_w)) imge = algo.render(center=False, depth=False, shadows=True) h, w = imge.shape imge = cv2.resize(imge[:, (w - h)//2:(w - h)//2+h], cropped[0].shape) cv2.imshow('equal?', np.hstack(( cropped[0], np.ones((cropped[0].shape[0], 1), dtype=cropped[0].dtype) * 255, imge, ))) cv2.waitKey() if i > 60: quit()
def costfn(p, x, y): gamma, gamma_break, max_val, scale = tuple(map(abs, p)) diff = ImageProc.adjust_gamma(x, gamma, gamma_break, max_val=max_val) * scale - y diff = tools.pseudo_huber_loss(120, diff) return np.sum(diff) if _USE_BFGS else diff
def __init__(self, cam, gain, exposure, timestamp, raw_image, background_img, bg_offset=0, bits=8, applied_gamma=1.0, applied_gamma_break=0.0, applied_bgr_mx=None, debug=False): self.id = Frame.CURRENT_ID Frame.CURRENT_ID += 1 self.cam = [cam] if isinstance(cam, Camera) else cam self.resize_scale = raw_image.shape[1] / self.cam[0].width for c in self.cam: c.height, c.width = raw_image.shape[:2] self.bits = bits = int(bits) self.gain = gain self.exposure = exposure self.timestamp = timestamp self.raw_image = raw_image self.applied_gamma = applied_gamma self.applied_gamma_break = applied_gamma_break self.applied_bgr_mx = applied_bgr_mx self.debug = debug img_bits = int(str(raw_image.dtype)[4:]) max_val = 2**img_bits - 1 img = raw_image.astype('float') # NOTE: NanoCam has this, doesnt make sense in general! operation_order = reversed(( 'ex_gamma', 'depth', 'color', 'gamma', )) for op in operation_order: if op == 'depth' and img_bits != bits: img = ImageProc.change_color_depth(img, img_bits, bits) max_val = 2**bits - 1 if op == 'gamma' and applied_gamma != 1.0: img = ImageProc.adjust_gamma(img, applied_gamma, gamma_break=applied_gamma_break, inverse=True, max_val=max_val) if op == 'color' and applied_bgr_mx is not None: img = ImageProc.color_correct(img, applied_bgr_mx, inverse=True, max_val=max_val) # if op == 'ex_gamma' and GAMMA_ADJUSTMENT: # img = ImageProc.adjust_gamma(img, GAMMA_ADJUSTMENT, inverse=True, max_val=max_val) self.background_img = background_img if background_img is not None: self.image = ImageProc.remove_bg(img, background_img, gain=1, offset=bg_offset, max_val=max_val) elif self.MISSING_BG_REMOVE_STRIPES: for k in range(img.shape[2]): img[:, :, k] -= np.percentile(img[:, :, k], 50, axis=0).reshape((1, -1)) img[:, :, k] -= np.percentile(img[:, :, k], 50, axis=1).reshape((-1, 1)) img += bg_offset - np.min(img) self.image = np.clip(img, 0, max_val) else: self.image = img if bg_offset is not False: self.image = np.round(self.image).astype('uint16') self.measures = []
plt.show() for hd in ( 48737, 35468, 39801 ): # Lambda Orionis (HD36861) Teff too high for model (37689K) fname = r'C:\projects\s100imgs\spectra\%s.fits' % hd fdat = fits.getdata(fname) teff, logg, feh = [stars[hd][f] for f in (f_teff, f_logg, f_feh)] if teff > 30000: logg = max(logg, 4.0) testf(fdat, teff, logg, feh or 0) quit() # cam = RosettaSystemModel(focused_attenuated=False).cam cam = DidymosSystemModel(use_narrow_cam=True).cam # cam_q = tools.rand_q(math.radians(180)) cam_q = quaternion.one for i in range(100): cam_q = tools.ypr_to_q(0, np.radians(1), 0) * cam_q flux_density = Stars.flux_density(cam_q, cam) img = cam.sense(flux_density, exposure=2, gain=2) img = np.clip(img * 255, 0, 255).astype('uint8') img = ImageProc.adjust_gamma(img, 1.8) sc = min(768 / cam.width, 768 / cam.height) cv2.imshow('stars', cv2.resize(img, None, fx=sc, fy=sc)) cv2.waitKey() print('done')
def detect_source(self, kernel, total_radiation=False): assert kernel.shape[0] % 2 and kernel.shape[ 1] % 2, 'kernel width and height must be odd numbers' kernel = self.gain * self.exposure * kernel fkernel = ImageProc.fuzzy_kernel(kernel, self.impulse_spread) method = [cv2.TM_CCORR_NORMED, cv2.TM_SQDIFF][0] corr = cv2.matchTemplate(self.image.astype(np.float32), fkernel.astype(np.float32), method) _, _, minloc, maxloc = cv2.minMaxLoc( corr) # minval, maxval, minloc, maxloc loc = minloc if method in (cv2.TM_SQDIFF, cv2.TM_SQDIFF_NORMED) else maxloc loc_i = tuple(np.round(np.array(loc)).astype(np.int)) if SHOW_LAB_MEASURES: sc = 1024 / self.image.shape[1] img = cv2.resize(self.image.astype(np.float), None, fx=sc, fy=sc) / np.max(self.image) center = np.array(loc) + np.flip(fkernel.shape[:2]) / 2 img = cv2.circle( img, tuple(np.round(center * sc).astype(np.int)), round((kernel.shape[0] - self.impulse_spread) / 2 * sc), [0, 0, 1.0]) cv2.imshow('te', img) print('waiting...', end='', flush=True) cv2.waitKey() print('done') if True: # use result as a mask to calculate mean and variance kh, kw = np.array(fkernel.shape[:2]) // 2 win = self.image[loc_i[1]:loc_i[1] + kh * 2 + 1, loc_i[0]:loc_i[0] + kw * 2 + 1, :].reshape( (-1, 3)) kernel_max = np.max(np.sum(kernel, axis=2)) mask = np.sum(fkernel, axis=2) / kernel_max > 0.95 mean = np.median(win[mask.flatten(), :], axis=0) std = np.std(win[mask.flatten(), :], axis=0) n = np.sum(mask) tmp = np.zeros(self.image.shape[:2]) tmp[loc_i[1]:loc_i[1] + kh * 2 + 1, loc_i[0]:loc_i[0] + kw * 2 + 1] = mask img_m = tmp else: # calculate a correlation channel (over whole image) k = kernel.shape[0] // 2 corr = cv2.matchTemplate( self.image.astype(np.float32), kernel[k:k + 1, k:k + 1, :].astype(np.float32), method) # calculate mean & variance of kernel area using corr channel win = corr[loc_i[1] - k:loc_i[1] + k + 1, loc_i[0] - k:loc_i[0] + k + 1] corr_mean = np.mean(win) corr_std = np.std(win) # threshold using mean - sd _, mask = cv2.threshold(corr, corr_mean - corr_std, 1, cv2.THRESH_BINARY) # dilate & erode to remove inner spots krn1 = ImageProc.bsphkern(round(1.5 * corr.shape[0] / 512) * 2 + 1) krn2 = ImageProc.bsphkern(round(2 * corr.shape[0] / 512) * 2 + 1) mask = cv2.dilate(mask, krn1, iterations=1) # remove holes mask = cv2.erode(mask, krn2, iterations=1) # same size mask = mask.astype(np.bool) # use result as a mask to calculate mean and variance mean = np.mean(self.image.reshape((-1, 3))[mask.flatten()], axis=0) var = np.var(self.image.reshape((-1, 3))[mask.flatten()], axis=0) n = np.sum(mask) if self.debug: sc = 1024 / self.image.shape[1] img_m = np.repeat(np.atleast_3d(img_m.astype(np.uint8) * 127), 3, axis=2) merged = ImageProc.merge( (self.image.astype(np.float32) / np.max(self.image), img_m.astype(np.float32) / 255)) img = cv2.resize(merged, None, fx=sc, fy=sc) cv2.imshow('te', img) arr_n, lims, _ = plt.hist(self.image[:, :, 1].flatten(), bins=np.max(self.image) + 1, log=True, histtype='step') plt.hist(win[mask.flatten(), 1].flatten(), bins=np.max(self.image) + 1, log=True, histtype='step') x = np.linspace(0, np.max(win), np.max(win) + 1) i = list(np.logical_and(lims[1:] > mean[1], arr_n > 0)).index(True) plt.plot(x, arr_n[i] * np.exp(-((x - mean[1]) / std[1])**2)) plt.ylim(1e-1, 1e6) plt.figure() plt.imshow(self.image[loc_i[1]:loc_i[1] + kh * 2 + 1, loc_i[0]:loc_i[0] + kw * 2 + 1, :] / np.max(self.image)) print('waiting (%.1f, %.1f)...' % (mean[1], std[1]), end='', flush=True) cv2.waitKey(1) plt.show() print('done') return LabMeasure(self, mean, std, n)
writer = cv2.VideoWriter(target_file, cv2.VideoWriter_fourcc(*codecs[0]), framerate, (dw, dh)) imgs = [] times = [] try: for i, f in enumerate(img_files): if i % skip_mult == 0: tools.show_progress( len(img_files) // skip_mult, i // skip_mult) img = cv2.imread(os.path.join(folder, f), cv2.IMREAD_COLOR) if sw != dw or sh != dh: img = cv2.resize(img, (dw, dh), interpolation=cv2.INTER_AREA) if exposure: # blend images to simulate blur due to long exposure times timestr = f[0:17] time = datetime.datetime.strptime(timestr, '%Y-%m-%dT%H%M%S') imgs.append(img) times.append(time) idxs = np.where( np.array(times) > time - datetime.timedelta(seconds=exposure)) if len(idxs) < np.ceil(exposure): continue img = ImageProc.merge(np.array(imgs)[idxs]) writer.write(img) finally: writer.release()
real = cv2.imread(os.path.join(sm.asteroid.image_db_path, img + '_P.png'), cv2.IMREAD_GRAYSCALE) if 1: img, dist = ab.render(shadows=True, textures=textures, reflection=model, depth=True) base_loc = np.array(sm.spacecraft_pos) + np.array([-0.1, 1.1, 0]) if 1: img = ImageProc.add_jets( sm.cam, img, dist < 1000, (base_loc, base_loc, base_loc), (0.6, 0.4, 0.8), phase_angles=(math.radians(60), None, None), directions=(math.radians(120), None, None), intensities=(0.35, 0.2, 0.3), angular_radii=(np.pi / 30, np.pi / 20, np.pi / 40), down_scaling=6) if 1: img = ImageProc.add_haze(img, dist < 1000, 0.15) if 1: sc = min(1536 / img.shape[1], 1024 / img.shape[0]) cv2.imshow('jets', cv2.resize(img, None, fx=sc, fy=sc)) cv2.waitKey() else: import matplotlib.pyplot as plt plt.imshow(img) plt.show()
def remove_background(self, img): res_img, h, th = ImageProc.process_target_image(img) return res_img, th
def solve_pnp(self, orig_sce_img, outfile, feat=ORB, use_feature_db=False, adjust_sc_rot=False, add_noise=False, scale_cam_img=False, vary_scale=False, match_mask_params=None, verbose=1, **kwargs): # set max mem usable by reference features, scene features use rest of MAX_WORK_MEM ref_max_mem = KeypointAlgo.FDB_MAX_MEM if use_feature_db else KeypointAlgo.MAX_WORK_MEM / 2 sm = self.system_model self._ransac_err = KeypointAlgo.DEF_RANSAC_ERROR self._render_z = -sm.min_med_distance init_z = kwargs.get('init_z', self._render_z) ref_img_sc = min( 1, self._render_z / init_z) * (sm.view_width if scale_cam_img else self._cam.width) / sm.view_width self.extra_values = None if outfile is not None: self.debug_filebase = outfile + (self.DEBUG_IMG_POSTFIX if isinstance(orig_sce_img, str) else '') if self.est_real_ast_orient: # so that can track rotation of 67P sm.reset_to_real_vals() if use_feature_db and self._fdb_helper is None: from visnav.algo.fdbgen import FeatureDatabaseGenerator self._fdb_helper = FeatureDatabaseGenerator( self.system_model, self.render_engine, self.obj_idx) # maybe load scene image if isinstance(orig_sce_img, str): orig_sce_img = self.load_target_image(orig_sce_img) if add_noise: self._shape_model_rng = np.max( np.ptp(sm.asteroid.real_shape_model.vertices, axis=0)) self.timer = Stopwatch() self.timer.start() if use_feature_db: if KeypointAlgo.FDB_REAL: # find correct set of keypoints & descriptors from features db ref_desc, ref_kp_3d, ref_kp, ref_img = self._query_fdb(feat) else: # calculate on-the-fly exactly the same features that would be returned from a feature db ref_desc, ref_kp_3d, ref_kp, ref_img = self._fake_fdb(feat) else: # render model image ref_img, depth_result = self.render_ref_img(ref_img_sc) if False: # normalize ref_img to match sce_img ref_img = ImageProc.equalize_brightness(ref_img, orig_sce_img, percentile=99.999, image_gamma=1.8) if False: gamma = 1.0 / 1.8 ref_img = ImageProc.adjust_gamma(ref_img, gamma) orig_sce_img = ImageProc.adjust_gamma(orig_sce_img, gamma) # get keypoints and descriptors ee = sm.pixel_extent(abs(self._render_z)) ref_kp, ref_desc, self._latest_detector = KeypointAlgo.detect_features( ref_img, feat, maxmem=ref_max_mem, max_feats=KeypointAlgo.MAX_FEATURES, for_ref=True, expected_pixel_extent=ee) if BATCH_MODE and self.debug_filebase: # save start situation in log archive self.timer.stop() img1 = cv2.resize(orig_sce_img, (sm.view_width, sm.view_height)) img2 = cv2.resize(ref_img, (sm.view_width, sm.view_height)) cv2.imwrite(self.debug_filebase + 'a.png', np.concatenate((img1, img2), axis=1)) if DEBUG: cv2.imshow('compare', np.concatenate((img1, img2), axis=1)) self.timer.start() # AKAZE, SIFT, SURF are truly scale invariant, couldnt get ORB to work as good vary_scale = vary_scale if feat == self.ORB else False if len(ref_kp) < KeypointAlgo.MIN_FEATURES: raise PositioningException( 'Too few (%d) reference features found' % (len(ref_kp), )) ok = False for i in range(self.MAX_SCENE_SCALE_STEPS): try: # resize scene image if necessary sce_img_sc = (sm.view_width if scale_cam_img else self._cam.width ) / self._cam.width / self.SCENE_SCALE_STEP**i if np.isclose(sce_img_sc, 1): sce_img = orig_sce_img else: sce_img = cv2.resize(orig_sce_img, None, fx=sce_img_sc, fy=sce_img_sc, interpolation=cv2.INTER_AREA) # detect features in scene image sce_max_mem = KeypointAlgo.MAX_WORK_MEM - ( KeypointAlgo.BYTES_PER_FEATURE[feat] + 12) * len(ref_desc) ee = sm.pixel_extent(abs(match_mask_params[2]) ) if match_mask_params is not None else 0 sce_kp, sce_desc, self._latest_detector = KeypointAlgo.detect_features( sce_img, feat, maxmem=sce_max_mem, max_feats=KeypointAlgo.MAX_FEATURES, expected_pixel_extent=ee) if len(sce_kp) < KeypointAlgo.MIN_FEATURES: raise PositioningException( 'Too few (%d) scene features found' % (len(sce_kp), )) # match descriptors try: mask = None if match_mask_params is not None: mask = KeypointAlgo.calc_match_mask( sm, sce_kp, ref_kp, self._render_z, sce_img_sc, ref_img_sc, *match_mask_params) matches = KeypointAlgo.match_features( sce_desc, ref_desc, self._latest_detector.defaultNorm(), mask=mask, method='brute') error = None except PositioningException as e: matches = [] error = e # debug by drawing matches if verbose > 0 and (not BATCH_MODE or DEBUG): logger.info('matches: %s/%s' % (len(matches), min(len(sce_kp), len(ref_kp)))) if verbose > 1: self._draw_matches(sce_img, sce_img_sc, sce_kp, ref_img, ref_img_sc, ref_kp, matches, pause=False, show=DEBUG) if error is not None: raise error # select matched scene feature image coordinates sce_kp_2d = np.array([ tuple(np.divide(sce_kp[m.queryIdx].pt, sce_img_sc)) for m in matches ], dtype='float') # prepare reference feature 3d coordinates (for only matched features) if use_feature_db: ref_kp_3d = ref_kp_3d[[m.trainIdx for m in matches], :] if add_noise: # add noise to noiseless 3d ref points from fdb self.timer.stop() ref_kp_3d, self.sm_noise, _ = tools.points_with_noise( ref_kp_3d, only_z=True, noise_lv=SHAPE_MODEL_NOISE_LV[add_noise], max_rng=self._shape_model_rng) self.timer.start() else: # get feature 3d points using 3d model ref_kp_3d = KeypointAlgo.inverse_project( sm, [ref_kp[m.trainIdx].pt for m in matches], depth_result, self._render_z, ref_img_sc) if KeypointAlgo.DISCARD_OFF_OBJECT_FEATURES: I = np.where(np.logical_not(np.isnan(ref_kp_3d[:, 0])))[0] if len(I) < self.MIN_FEATURES: raise PositioningException('Too few matches found') sce_kp_2d = sce_kp_2d[I, :] ref_kp_3d = ref_kp_3d[I, :] matches = [matches[i] for i in I] # finally solve pnp with ransac rvec, tvec, inliers = KeypointAlgo.solve_pnp_ransac( sm, sce_kp_2d, ref_kp_3d, self._ransac_err) # debug by drawing inlier matches if verbose > 1: self._draw_matches(sce_img, sce_img_sc, sce_kp, ref_img, ref_img_sc, ref_kp, [matches[i[0]] for i in inliers], label='c) inliers', pause=self._pause) inlier_count = self.count_inliers(sce_kp, ref_kp, matches, inliers) if verbose > 0: logger.info('inliers: %s/%s, ' % (inlier_count, len(matches))) if inlier_count < KeypointAlgo.MIN_FEATURES: raise PositioningException( 'RANSAC algorithm was left with too few inliers') # dont try again if found enough inliers ok = True break except PositioningException as e: if not vary_scale: raise e # maybe try again using scaled down scene image if not ok: raise PositioningException( 'Not enough inliers even if tried scaling scene image down x%.1f' % (1 / sce_img_sc)) elif vary_scale: logger.info('success at x%.1f' % (1 / sce_img_sc)) self.timer.stop() # set model params to solved pose & pos self._set_sc_from_ast_rot_and_trans(rvec, tvec, self.latest_discretization_err_q, rotate_sc=adjust_sc_rot) # debugging if verbose > 0 and (not BATCH_MODE or DEBUG): rp_err = KeypointAlgo.reprojection_error(self._cam, sce_kp_2d, ref_kp_3d, inliers, rvec, tvec) sh_err = sm.calc_shift_err() logger.info( 'repr-err: %.2f, rel-rot-err: %.2f°, dist-err: %.2f%%, lat-err: %.2f%%, shift-err: %.1fm' % ( rp_err, math.degrees(sm.rel_rot_err()), sm.dist_pos_err() * 100, sm.lat_pos_err() * 100, sh_err * 1000, )) # save result image if BATCH_MODE and self.debug_filebase: # save result in log archive res_img = self.render(shadows=self.RENDER_SHADOWS) sce_img = cv2.resize(orig_sce_img, tuple(np.flipud(res_img.shape))) cv2.imwrite(self.debug_filebase + 'd.png', np.concatenate((sce_img, res_img), axis=1))
def _render(self, params): time = params[0] sun_distance = np.linalg.norm(np.array(params[1][:3])) # in meters sun_ast_v = tools.normalize_v(np.array(params[1][:3])) d1_v, d1_q, d2_v, d2_q, sc_v, sc_q = self._parse_poses(params, offset=2) d1, d2 = self.asteroids q = SystemModel.sc2gl_q.conj() * sc_q.conj() rel_rot_q = np.array([ q * d1_q * d1.ast2sc_q.conj(), q * d2_q * d2.ast2sc_q.conj(), np.quaternion(1, 0, 1, 0).normalized() ]) # last one is the for the spacecraft rel_pos_v = np.array([ tools.q_times_v(q, d1_v - sc_v), tools.q_times_v(q, d2_v - sc_v), [0, 0, 0] ]) light_v = tools.q_times_v(q, sun_ast_v) self._maybe_load_objects() # lazy load objects exp_range = (0.001, 3.5) for i in range(20): if self._autolevel and self._current_level: level = self._current_level else: level = 3 * 2.5 * 1.3e-3 if self._use_nac else 1.8 * 2.5 * 1.3e-3 exp, gain = self._sm.cam.level_to_exp_gain(level, exp_range) img = TestLoop.render_navcam_image_static(self._sm, self._renderer, self._obj_idxs, rel_pos_v, rel_rot_q, light_v, sc_q, sun_distance, exposure=exp, gain=gain, auto_gain=False, gamma=1.0, use_shadows=True, use_textures=True) if self._autolevel: v = np.percentile(img, 100 - 0.0003) level_trg = level * 170 / v print( 'autolevel (max_v=%.1f, e=%.3f, g=%.1f) current: %.3f, target: %.3f' % (v, exp, gain, level, level_trg)) self._current_level = level_trg if not self._current_level else \ (self._current_level*self._level_lambda + level_trg*(1-self._level_lambda)) if v < 85 or (v == 255 and level > exp_range[0]): level = level_trg if v < 85 else level * 70 / v self._current_level = level continue break if False: img = ImageProc.default_preprocess(img) date = datetime.fromtimestamp(time, pytz.utc) # datetime.now() fname = os.path.join(self._logpath, date.isoformat()[:-6].replace(':', '')) + '.png' cv2.imwrite(fname, img, [cv2.IMWRITE_PNG_COMPRESSION, 0]) return fname