# Modify the network so that we bypass the keypoint part and the # descriptor part. param.model.sDetector = 'bypass' # This ensures that you don't create unecessary scale space param.model.fScaleList = np.array([1.0]) param.patch.fMaxScale = np.max(param.model.fScaleList) # this ensures that you don't over eliminate features at boundaries param.model.nPatchSize = int(np.round(param.model.nDescInputSize) * np.sqrt(2)) param.patch.fRatioScale = (float(param.patch.nPatchSize) / float(param.model.nDescInputSize)) * 6.0 param.model.sDescriptor = 'bypass' # add the mean and std of the learned model to the param mean_std_file = pathconf.result + 'mean_std.h5' mean_std_dict = loadh5(mean_std_file) param.online = paramGroup() setattr(param.online, 'mean_x', mean_std_dict['mean_x']) setattr(param.online, 'std_x', mean_std_dict['std_x']) # ------------------------------------------------------------------------- # Load data in the test format test_data_in = data_module.data_obj(param, image_file_name, kp_file_name) # ------------------------------------------------------------------------- # Test using the test function start_time = time.clock() _, oris, compile_time = Test( pathconf, param, test_data_in, test_mode="ori") end_time = time.clock() compute_time = (end_time - start_time) * 1000.0 - compile_time
# Modify the network so that we bypass the keypoint part and the # orientation part param.model.sDetector = 'bypass' # This ensures that you don't create unecessary scale space param.model.fScaleList = np.array([1.0]) param.patch.fMaxScale = np.max(param.model.fScaleList) # this ensures that you don't over eliminate features at boundaries param.model.nPatchSize = int( np.round(param.model.nDescInputSize) * np.sqrt(2)) param.patch.fRatioScale = (float(param.patch.nPatchSize) / float(param.model.nDescInputSize)) * 6.0 param.model.sOrientation = 'bypass' # add the mean and std of the learned model to the param mean_std_file = pathconf.result + 'mean_std.h5' mean_std_dict = loadh5(mean_std_file) param.online = paramGroup() setattr(param.online, 'mean_x', mean_std_dict['mean_x']) setattr(param.online, 'std_x', mean_std_dict['std_x']) # ------------------------------------------------------------------------- # Load data in the test format test_data_in = data_module.data_obj(param, image_file_name, kp_file_name) # ------------------------------------------------------------------------- # Test using the test function start_time = time.clock() descs, _, compile_time = Test(pathconf, param, test_data_in, test_mode="desc")
def build(myNet, idxSiam, verbose=True): # Load model fn = '%s/%s' % (myNet.config.descriptor_export_folder, myNet.config.descriptor_model) model_dict = dt.loadh5(fn) # Load training mean/std # if we have the normalization setup if myNet.config.bNormalizeInput: kwang_mean = np.cast[floatX](myNet.config.mean_x) kwang_std = np.cast[floatX](myNet.config.std_x) # else, simply divide with 255 else: kwang_mean = np.cast[floatX](0.0) kwang_std = np.cast[floatX](255.0) if 'patch-mean' in model_dict.keys(): desc_mean_x = np.cast[floatX](model_dict['patch-mean'][0]) desc_std_x = np.cast[floatX](model_dict['patch-std'][0]) else: print('Warning: no mean/std in the model file') desc_mean_x = kwang_mean desc_std_x = kwang_std # Layer indices indices = model_dict['layers'] if verbose and idxSiam == 0: print('*** Loading descriptor "%s" ***' % fn) print('Number of elements: %d' % indices.size) # Add another layer that transforms the original input curr_name = 'desc-re-normalize' curr_input = myNet.config.descriptor_input myNet.layers[idxSiam][curr_name] = ExpressionLayer( myNet.layers[idxSiam][curr_input], lambda x: (x * kwang_std + kwang_mean - desc_mean_x) / desc_std_x, name=curr_name) curr_input = curr_name # Loop over layers for i in six.moves.xrange(indices.size): if indices[i] == 1: if verbose and idxSiam == 0: print('%d -> SpatialConvolution' % i) curr_name = 'desc-%d-conv' % (i + 1) # read actual value for siamese 0 w = model_dict['l-%d-weights' % (i + 1)].astype(floatX) b = model_dict['l-%d-bias' % (i + 1)].astype(floatX) num_filters, num_input_channels, filter_size, filter_size = w.shape # assert num_input_channels == myNet.config.nDescInputChannels # assert filter_size == myNet.config.nDescInputSize if verbose and idxSiam == 0: print(' Number of filters: %d' % num_filters) print(' Filter size: %d' % filter_size) # Manually create shared variables if idxSiam == 0: w = theano.shared(w, name=curr_name + '.W') b = theano.shared(b, name=curr_name + '.b') else: w = myNet.layers[0][curr_name].W b = myNet.layers[0][curr_name].b myNet.layers[idxSiam][curr_name] = Conv2DLayer( myNet.layers[idxSiam][curr_input], num_filters, filter_size, W=w, b=b, nonlinearity=None, # no activation flip_filters=False, name=curr_name) elif indices[i] == 2: if verbose and idxSiam == 0: print('%d -> Linear' % i) raise RuntimeError('Layer type %d TODO' % i) elif indices[i] == 3: if verbose and idxSiam == 0: print('%d -> SpatialMaxPooling' % i) curr_name = 'desc-%d-maxpool' % (i + 1) kw = model_dict['l-%d-kw' % (i + 1)].astype(np.int32)[0] kh = model_dict['l-%d-kh' % (i + 1)].astype(np.int32)[0] if verbose and idxSiam == 0: print(' Region size: %dx%d' % (kw, kh)) assert kw == kh kw = int(kw) myNet.layers[idxSiam][curr_name] = MaxPool2DLayer( myNet.layers[idxSiam][curr_input], pool_size=kw, stride=None, name=curr_name) elif indices[i] == 4: if verbose and idxSiam == 0: print('%d -> SpatialLPPooling' % i) curr_name = 'desc-%d-lppool' % (i + 1) kw = model_dict['l-%d-kw' % (i + 1)].astype(np.int32)[0] kh = model_dict['l-%d-kh' % (i + 1)].astype(np.int32)[0] if verbose and idxSiam == 0: print(' Region size: %dx%d' % (kw, kh)) assert kw == kh kw = int(kw) myNet.layers[idxSiam][curr_name] = LPPool2DLayer( myNet.layers[idxSiam][curr_input], pnorm=2, pool_size=kw, stride=None, name=curr_name) elif indices[i] == 5: if verbose and idxSiam == 0: print('%d -> Tanh' % i) curr_name = 'desc-%d-tanh' % (i + 1) myNet.layers[idxSiam][curr_name] = NonlinearityLayer( myNet.layers[idxSiam][curr_input], nonlinearity=nonlinearities.tanh, name=curr_name) elif indices[i] == 6: if verbose and idxSiam == 0: print('%d -> ReLU' % i) curr_name = 'desc-%d-relu' % (i + 1) myNet.layers[idxSiam][curr_name] = NonlinearityLayer( myNet.layers[idxSiam][curr_input], nonlinearity=nonlinearities.rectify, name=curr_name) elif indices[i] == 7: if verbose and idxSiam == 0: print('%d -> SpatialSubtractiveNormalization' % i) curr_name = 'desc-%d-subt-norm' % (i + 1) kernel = model_dict['l-%d-filter' % (i + 1)].astype(floatX) w_kernel, h_kernel = kernel.shape if verbose and idxSiam == 0: print(' Kernel size: %dx%d' % (w_kernel, h_kernel)) myNet.layers[idxSiam][curr_name] = SubtractiveNormalization2DLayer( myNet.layers[idxSiam][curr_input], kernel=kernel, name=curr_name) else: raise RuntimeError('Layer type %d: unknown' % i) # Input to the next layer curr_input = curr_name # Flatten output and rename myNet.layers[idxSiam]['desc-output'] = FlattenLayer( myNet.layers[idxSiam][curr_input], outdim=2)
def apply_learned_filter_2_image_no_theano(image, save_dir, bNormalizeInput, sKpNonlinearity, verbose=True, network_weights=None): """Apply the learned keypoint detector to image. Parameters ---------- image: np.ndarray Raw image, without any pre-processing. save_dir: str Full path to the model directory. bNormalizeInput: bool Normalization parameter used for training. sKpNonlinearity: str Nonlinearity applied to the keypoint scoremap. """ if len(image.shape) > 2: raise NotImplementedError("This function is only meant for " "grayscale images!") num_in_sum = 4 num_in_max = 4 if network_weights is None: # Load model if os.path.exists(save_dir + 'model.h5'): model = loadh5(save_dir + 'model.h5') W = model['kp-c0']['kp-c0.W'].astype(floatX) b = model['kp-c0']['kp-c0.b'].astype(floatX) elif os.path.exists(save_dir + 'model.pklz'): model = loadpklz(save_dir + 'model.pklz') W = model[0].astype(floatX) b = model[1].astype(floatX) else: raise RuntimeError('Learned model does not exist!') else: model = loadh5(network_weights) W = model['kp-c0']['kp-c0.W'].astype(floatX) b = model['kp-c0']['kp-c0.b'].astype(floatX) # add a new axis at the end if the filter is 4D if W.ndim == 4: W = W[..., np.newaxis] if bNormalizeInput: print("Using trained mean and Std for normalization") mean_std_dict = loadh5(save_dir + 'mean_std.h5') image_prep = ((image - mean_std_dict['mean_x']) / mean_std_dict['std_x']) else: print("Just dividing with 255!") image_prep = image / np.cast[floatX](255.0) # Do subtracive normalization if it is there bSubtractiveNormalization = False if 'kp-subnorm' in model.keys(): norm_kern = model['kp-subnorm']['kp-subnorm.kernel'] assert np.isclose(np.sum(norm_kern), 1.0) bSubtractiveNormalization = True # prepare image if bSubtractiveNormalization: print("Performing subtractive normalization!") # compute the coeffs to make adjust at borders coef = _subtractive_norm_make_coef(norm_kern, image.shape[:2]) # run the filter to get means conv_mean = cv2.filter2D(image_prep, -1, norm_kern, borderType=cv2.BORDER_CONSTANT) # adjust means with precomputed coef adj_mean = conv_mean / coef # subtract the mean image_prep -= adj_mean # Do LCN if it is there bLCN = False if 'kp-lcn' in model.keys(): norm_kern = model['kp-lcn']['kp-lcn.kernel'] assert np.isclose(np.sum(norm_kern), 1.0) bLCN = True if bLCN: # compute the coeffs to make adjust at borders coef = _lcn_make_coef(norm_kern, image.shape[:2]) # run the filter to get means and adjust conv_mean = cv2.filter2D(image_prep, -1, norm_kern, borderType=cv2.BORDER_CONSTANT) adj_mean = conv_mean / coef # subtract the mean sub_norm = image_prep - adj_mean # run the filter to get std and adjust conv_std = np.sqrt( cv2.filter2D(sub_norm**2.0, -1, norm_kern, borderType=cv2.BORDER_CONSTANT) ) adj_std = np.sqrt(conv_std / coef) # run the filter once more to get std mean and adjust it conv_mean_std = np.sqrt( cv2.filter2D(adj_std, -1, norm_kern, borderType=cv2.BORDER_CONSTANT) ) adj_mean_std = conv_mean_std / coef # now divide image_prep = sub_norm / np.maximum(adj_mean_std, adj_std) h, w = image_prep.shape # NEW implementation nonlinearity = sKpNonlinearity resp = apply_ghh_filter( image_prep, W, b, num_in_sum, num_in_max, nonlinearity) # The final return of this function should be a single channel image if len(resp.shape) == 3: assert resp.shape[2] == 1 resp = np.squeeze(resp) return resp
def apply_learned_filter_2_image_no_theano(image, save_dir, bNormalizeInput, sKpNonlinearity, verbose=True, network_weights=None): """Apply the learned keypoint detector to image. Parameters ---------- image: np.ndarray Raw image, without any pre-processing. save_dir: str Full path to the model directory. bNormalizeInput: bool Normalization parameter used for training. sKpNonlinearity: str Nonlinearity applied to the keypoint scoremap. """ if len(image.shape) > 2: raise NotImplementedError("This function is only meant for " "grayscale images!") num_in_sum = 4 num_in_max = 4 if network_weights is None: # Load model if os.path.exists(save_dir + 'model.h5'): model = loadh5(save_dir + 'model.h5') W = model['kp-c0']['kp-c0.W'].astype(floatX) b = model['kp-c0']['kp-c0.b'].astype(floatX) elif os.path.exists(save_dir + 'model.pklz'): model = loadpklz(save_dir + 'model.pklz') W = model[0].astype(floatX) b = model[1].astype(floatX) else: raise RuntimeError('Learned model does not exist!') else: model = loadh5(network_weights) W = model['kp-c0']['kp-c0.W'].astype(floatX) b = model['kp-c0']['kp-c0.b'].astype(floatX) # add a new axis at the end if the filter is 4D if W.ndim == 4: W = W[..., np.newaxis] if bNormalizeInput: print("Using trained mean and Std for normalization") mean_std_dict = loadh5(save_dir + 'mean_std.h5') image_prep = ((image - mean_std_dict['mean_x']) / mean_std_dict['std_x']) else: print("Just dividing with 255!") image_prep = image / np.cast[floatX](255.0) # Do subtracive normalization if it is there bSubtractiveNormalization = False if 'kp-subnorm' in model.keys(): norm_kern = model['kp-subnorm']['kp-subnorm.kernel'] assert np.isclose(np.sum(norm_kern), 1.0) bSubtractiveNormalization = True # prepare image if bSubtractiveNormalization: print("Performing subtractive normalization!") # compute the coeffs to make adjust at borders coef = _subtractive_norm_make_coef(norm_kern, image.shape[:2]) # run the filter to get means conv_mean = cv2.filter2D(image_prep, -1, norm_kern, borderType=cv2.BORDER_CONSTANT) # adjust means with precomputed coef adj_mean = conv_mean / coef # subtract the mean image_prep -= adj_mean # Do LCN if it is there bLCN = False if 'kp-lcn' in model.keys(): norm_kern = model['kp-lcn']['kp-lcn.kernel'] assert np.isclose(np.sum(norm_kern), 1.0) bLCN = True if bLCN: # compute the coeffs to make adjust at borders coef = _lcn_make_coef(norm_kern, image.shape[:2]) # run the filter to get means and adjust conv_mean = cv2.filter2D(image_prep, -1, norm_kern, borderType=cv2.BORDER_CONSTANT) adj_mean = conv_mean / coef # subtract the mean sub_norm = image_prep - adj_mean # run the filter to get std and adjust conv_std = np.sqrt( cv2.filter2D(sub_norm**2.0, -1, norm_kern, borderType=cv2.BORDER_CONSTANT)) adj_std = np.sqrt(conv_std / coef) # run the filter once more to get std mean and adjust it conv_mean_std = np.sqrt( cv2.filter2D(adj_std, -1, norm_kern, borderType=cv2.BORDER_CONSTANT)) adj_mean_std = conv_mean_std / coef # now divide image_prep = sub_norm / np.maximum(adj_mean_std, adj_std) h, w = image_prep.shape # NEW implementation nonlinearity = sKpNonlinearity resp = apply_ghh_filter(image_prep, W, b, num_in_sum, num_in_max, nonlinearity) # The final return of this function should be a single channel image if len(resp.shape) == 3: assert resp.shape[2] == 1 resp = np.squeeze(resp) return resp
def build(myNet, idxSiam, verbose=True): # Load model fn = '%s/%s' % (myNet.config.descriptor_export_folder, myNet.config.descriptor_model) model_dict = dt.loadh5(fn) # Load training mean/std # if we have the normalization setup if myNet.config.bNormalizeInput: kwang_mean = np.cast[floatX](myNet.config.mean_x) kwang_std = np.cast[floatX](myNet.config.std_x) # else, simply divide with 255 else: kwang_mean = np.cast[floatX](0.0) kwang_std = np.cast[floatX](255.0) if 'patch-mean' in model_dict.keys(): desc_mean_x = np.cast[floatX](model_dict['patch-mean'][0]) desc_std_x = np.cast[floatX](model_dict['patch-std'][0]) else: print('Warning: no mean/std in the model file') desc_mean_x = kwang_mean desc_std_x = kwang_std # Layer indices indices = model_dict['layers'] if verbose and idxSiam == 0: print('*** Loading descriptor "%s" ***' % fn) print('Number of elements: %d' % indices.size) # Add another layer that transforms the original input curr_name = 'desc-re-normalize' curr_input = myNet.config.descriptor_input myNet.layers[idxSiam][curr_name] = ExpressionLayer( myNet.layers[idxSiam][curr_input], lambda x: (x * kwang_std + kwang_mean - desc_mean_x) / desc_std_x, name=curr_name ) curr_input = curr_name # Loop over layers for i in six.moves.xrange(indices.size): if indices[i] == 1: if verbose and idxSiam == 0: print('%d -> SpatialConvolution' % i) curr_name = 'desc-%d-conv' % (i + 1) # read actual value for siamese 0 w = model_dict['l-%d-weights' % (i + 1)].astype(floatX) b = model_dict['l-%d-bias' % (i + 1)].astype(floatX) num_filters, num_input_channels, filter_size, filter_size = w.shape # assert num_input_channels == myNet.config.nDescInputChannels # assert filter_size == myNet.config.nDescInputSize if verbose and idxSiam == 0: print(' Number of filters: %d' % num_filters) print(' Filter size: %d' % filter_size) # Manually create shared variables if idxSiam == 0: w = theano.shared(w, name=curr_name + '.W') b = theano.shared(b, name=curr_name + '.b') else: w = myNet.layers[0][curr_name].W b = myNet.layers[0][curr_name].b myNet.layers[idxSiam][curr_name] = Conv2DLayer( myNet.layers[idxSiam][curr_input], num_filters, filter_size, W=w, b=b, nonlinearity=None, # no activation flip_filters=False, name=curr_name ) elif indices[i] == 2: if verbose and idxSiam == 0: print('%d -> Linear' % i) raise RuntimeError('Layer type %d TODO' % i) elif indices[i] == 3: if verbose and idxSiam == 0: print('%d -> SpatialMaxPooling' % i) curr_name = 'desc-%d-maxpool' % (i + 1) kw = model_dict['l-%d-kw' % (i + 1)].astype(np.int32)[0] kh = model_dict['l-%d-kh' % (i + 1)].astype(np.int32)[0] if verbose and idxSiam == 0: print(' Region size: %dx%d' % (kw, kh)) assert kw == kh kw = int(kw) myNet.layers[idxSiam][curr_name] = MaxPool2DLayer( myNet.layers[idxSiam][curr_input], pool_size=kw, stride=None, name=curr_name ) elif indices[i] == 4: if verbose and idxSiam == 0: print('%d -> SpatialLPPooling' % i) curr_name = 'desc-%d-lppool' % (i + 1) kw = model_dict['l-%d-kw' % (i + 1)].astype(np.int32)[0] kh = model_dict['l-%d-kh' % (i + 1)].astype(np.int32)[0] if verbose and idxSiam == 0: print(' Region size: %dx%d' % (kw, kh)) assert kw == kh kw = int(kw) myNet.layers[idxSiam][curr_name] = LPPool2DLayer( myNet.layers[idxSiam][curr_input], pnorm=2, pool_size=kw, stride=None, name=curr_name ) elif indices[i] == 5: if verbose and idxSiam == 0: print('%d -> Tanh' % i) curr_name = 'desc-%d-tanh' % (i + 1) myNet.layers[idxSiam][curr_name] = NonlinearityLayer( myNet.layers[idxSiam][curr_input], nonlinearity=nonlinearities.tanh, name=curr_name ) elif indices[i] == 6: if verbose and idxSiam == 0: print('%d -> ReLU' % i) curr_name = 'desc-%d-relu' % (i + 1) myNet.layers[idxSiam][curr_name] = NonlinearityLayer( myNet.layers[idxSiam][curr_input], nonlinearity=nonlinearities.rectify, name=curr_name ) elif indices[i] == 7: if verbose and idxSiam == 0: print('%d -> SpatialSubtractiveNormalization' % i) curr_name = 'desc-%d-subt-norm' % (i + 1) kernel = model_dict['l-%d-filter' % (i + 1)].astype(floatX) w_kernel, h_kernel = kernel.shape if verbose and idxSiam == 0: print(' Kernel size: %dx%d' % (w_kernel, h_kernel)) myNet.layers[idxSiam][curr_name] = SubtractiveNormalization2DLayer( myNet.layers[idxSiam][curr_input], kernel=kernel, name=curr_name ) else: raise RuntimeError('Layer type %d: unknown' % i) # Input to the next layer curr_input = curr_name # Flatten output and rename myNet.layers[idxSiam]['desc-output'] = FlattenLayer( myNet.layers[idxSiam][curr_input], outdim=2 )
def load_data_for_set(self, pathconf, param, image_file_name, kp_file_name): bUseColorImage = getattr(param.patch, "bUseColorImage", False) if not bUseColorImage: # If there is not gray image, load the color one and convert to # gray if os.path.exists(image_file_name.replace( "image_color", "image_gray" )): img = cv2.imread(image_file_name.replace( "image_color", "image_gray" ), 0) assert len(img.shape) == 2 else: # read the image img = cv2.cvtColor(cv2.imread(image_file_name), cv2.COLOR_BGR2GRAY) in_dim = 1 else: img = cv2.imread(image_file_name) in_dim = 3 assert(img.shape[-1] == in_dim) bTestWithTestMeanStd = getattr( param.validation, 'bTestWithTestMeanStd', False) if bTestWithTestMeanStd: img = img - np.mean(img) mean_std_dict = loadh5(pathconf.result + 'mean_std.h5') img = img + mean_std_dict['mean_x'] print("Test image has range {} to {} after " "transforming to the test domain" "".format(np.min(img), np.max(img))) # load the keypoint informations kp = np.asarray(loadKpListFromTxt(kp_file_name)) # Assign dummy values to y, ID, angle y = np.zeros((len(kp),)) ID = np.zeros((len(kp),), dtype='int64') # angle = np.zeros((len(kp),)) angle = np.pi / 180.0 * kp[:, IDX_ANGLE] # store angle in radians # load patches with id (drop out of boundary) bPerturb = False fPerturbInfo = np.zeros((3,)) dataset = load_patches(img, kp, y, ID, angle, param.patch.fRatioScale, param.patch.fMaxScale, param.patch.nPatchSize, param.model.nDescInputSize, in_dim, bPerturb, fPerturbInfo, bReturnCoords=True) x = dataset[0] y = dataset[1] ID = dataset[2] pos = dataset[3] angle = dataset[4] coords = dataset[5] return x, y, ID, pos, angle, coords