def test_barycenter_kneighbors_graph():
X = np.array([[0, 1], [1.01, 1.0], [2, 0]])
A = barycenter_kneighbors_graph(X, 1)
assert_array_almost_equal(A.toarray(), [[0.0, 1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0]])
A = barycenter_kneighbors_graph(X, 2)
# check that columns sum to one
assert_array_almost_equal(np.sum(A.toarray(), 1), np.ones(3))
pred = np.dot(A.toarray(), X)
assert_less(linalg.norm(pred - X) / X.shape[0], 1)
def test_barycenter_kneighbors_graph():
X = np.array([[0, 1], [1.01, 1.], [2, 0]])
A = barycenter_kneighbors_graph(X, 1)
assert_array_almost_equal(A.todense(),
[[0., 1., 0.], [1., 0., 0.], [0., 1., 0.]])
A = barycenter_kneighbors_graph(X, 2)
# check that columns sum to one
assert_array_almost_equal(np.sum(A.todense(), 1), np.ones((3, 1)))
pred = np.dot(A.todense(), X)
assert_less(np.linalg.norm(pred - X) / X.shape[0], 1)
def test_barycenter_kneighbors_graph():
X = np.array([[0, 1], [1.01, 1.], [2, 0]])
A = barycenter_kneighbors_graph(X, 1)
assert_array_almost_equal(
A.todense(),
[[0., 1., 0.],
[1., 0., 0.],
[0., 1., 0.]])
A = barycenter_kneighbors_graph(X, 2)
# check that columns sum to one
assert_array_almost_equal(np.sum(A.todense(), 1), np.ones((3, 1)))
pred = np.dot(A.todense(), X)
assert_true(np.linalg.norm(pred - X) / X.shape[0] < 1)
def test_lle_manifold():
rng = np.random.RandomState(0)
# similar test on a slightly more complex manifold
X = np.array(list(product(np.arange(18), repeat=2)))
X = np.c_[X, X[:, 0]**2 / 18]
X = X + 1e-10 * rng.uniform(size=X.shape)
n_components = 2
for method in ["standard", "hessian", "modified", "ltsa"]:
clf = manifold.LocallyLinearEmbedding(n_neighbors=6,
n_components=n_components,
method=method,
random_state=0)
tol = 1.5 if method == "standard" else 3
N = barycenter_kneighbors_graph(X, clf.n_neighbors).toarray()
reconstruction_error = linalg.norm(np.dot(N, X) - X)
assert_less(reconstruction_error, tol)
for solver in eigen_solvers:
clf.set_params(eigen_solver=solver)
clf.fit(X)
assert_true(clf.embedding_.shape[1] == n_components)
reconstruction_error = linalg.norm(
np.dot(N, clf.embedding_) - clf.embedding_, 'fro')**2
details = ("solver: %s, method: %s" % (solver, method))
assert_less(reconstruction_error, tol, msg=details)
assert_less(np.abs(clf.reconstruction_error_ -
reconstruction_error),
tol * reconstruction_error,
msg=details)
def test_lle_simple_grid():
# note: ARPACK is numerically unstable, so this test will fail for
# some random seeds. We choose 2 because the tests pass.
rng = np.random.RandomState(2)
tol = 0.1
# grid of equidistant points in 2D, n_components = n_dim
X = np.array(list(product(range(5), repeat=2)))
X = X + 1e-10 * rng.uniform(size=X.shape)
n_components = 2
clf = manifold.LocallyLinearEmbedding(n_neighbors=5,
n_components=n_components,
random_state=rng)
tol = 0.1
N = barycenter_kneighbors_graph(X, clf.n_neighbors).todense()
reconstruction_error = linalg.norm(np.dot(N, X) - X, 'fro')
assert_less(reconstruction_error, tol)
for solver in eigen_solvers:
clf.set_params(eigen_solver=solver)
clf.fit(X)
assert_true(clf.embedding_.shape[1] == n_components)
reconstruction_error = linalg.norm(
np.dot(N, clf.embedding_) - clf.embedding_, 'fro')**2
assert_less(reconstruction_error, tol)
assert_almost_equal(clf.reconstruction_error_,
reconstruction_error,
decimal=1)
# re-embed a noisy version of X using the transform method
noise = rng.randn(*X.shape) / 100
X_reembedded = clf.transform(X + noise)
assert_less(linalg.norm(X_reembedded - clf.embedding_), tol)
def test_lle_simple_grid():
rng = np.random.RandomState(0)
# grid of equidistant points in 2D, out_dim = n_dim
X = np.array(list(product(range(5), repeat=2)))
out_dim = 2
clf = manifold.LocallyLinearEmbedding(n_neighbors=5, out_dim=out_dim)
tol = .1
N = barycenter_kneighbors_graph(X, clf.n_neighbors).todense()
reconstruction_error = np.linalg.norm(np.dot(N, X) - X, 'fro')
assert_lower(reconstruction_error, tol)
for solver in eigen_solvers:
clf.set_params(eigen_solver=solver)
clf.fit(X)
assert_true(clf.embedding_.shape[1] == out_dim)
reconstruction_error = np.linalg.norm(
np.dot(N, clf.embedding_) - clf.embedding_, 'fro') ** 2
# FIXME: ARPACK fails this test ...
if solver != 'arpack':
assert_lower(reconstruction_error, tol)
assert_almost_equal(clf.reconstruction_error_,
reconstruction_error, decimal=4)
# re-embed a noisy version of X using the transform method
noise = rng.randn(*X.shape) / 100
X_reembedded = clf.transform(X + noise)
assert_lower(np.linalg.norm(X_reembedded - clf.embedding_), tol)
def test_lle_manifold():
rng = np.random.RandomState(0)
# similar test on a slightly more complex manifold
X = np.array(list(product(range(20), repeat=2)))
X = np.c_[X, X[:, 0] ** 2 / 20]
X = X + 1e-10 * rng.uniform(size=X.shape)
n_components = 2
for method in ["standard", "hessian", "modified", "ltsa"]:
clf = manifold.LocallyLinearEmbedding(n_neighbors=6,
n_components=n_components, method=method, random_state=0)
tol = 1.5 if method == "standard" else 3
N = barycenter_kneighbors_graph(X, clf.n_neighbors).toarray()
reconstruction_error = np.linalg.norm(np.dot(N, X) - X)
assert_less(reconstruction_error, tol)
for solver in eigen_solvers:
clf.set_params(eigen_solver=solver)
clf.fit(X)
assert_true(clf.embedding_.shape[1] == n_components)
reconstruction_error = np.linalg.norm(
np.dot(N, clf.embedding_) - clf.embedding_, 'fro') ** 2
details = ("solver: %s, method: %s"
% (solver, method))
assert_less(reconstruction_error, tol, msg=details)
assert_less(np.abs(clf.reconstruction_error_ -
reconstruction_error),
tol * reconstruction_error, msg=details)
def test_lle_simple_grid():
# note: ARPACK is numerically unstable, so this test will fail for
# some random seeds. We choose 2 because the tests pass.
rng = np.random.RandomState(2)
tol = 0.1
# grid of equidistant points in 2D, n_components = n_dim
X = np.array(list(product(range(5), repeat=2)))
X = X + 1e-10 * rng.uniform(size=X.shape)
n_components = 2
clf = manifold.LocallyLinearEmbedding(n_neighbors=5,
n_components=n_components, random_state=rng)
tol = 0.1
N = barycenter_kneighbors_graph(X, clf.n_neighbors).todense()
reconstruction_error = np.linalg.norm(np.dot(N, X) - X, 'fro')
assert_less(reconstruction_error, tol)
for solver in eigen_solvers:
clf.set_params(eigen_solver=solver)
clf.fit(X)
assert_true(clf.embedding_.shape[1] == n_components)
reconstruction_error = np.linalg.norm(
np.dot(N, clf.embedding_) - clf.embedding_, 'fro') ** 2
assert_less(reconstruction_error, tol)
assert_almost_equal(clf.reconstruction_error_,
reconstruction_error, decimal=1)
# re-embed a noisy version of X using the transform method
noise = rng.randn(*X.shape) / 100
X_reembedded = clf.transform(X + noise)
assert_less(np.linalg.norm(X_reembedded - clf.embedding_), tol)
def test_lle_manifold():
rng = np.random.RandomState(0)
# similar test on a slightly more complex manifold
X = np.array(list(product(range(20), repeat=2)))
X = np.c_[X, X[:, 0]**2 / 20]
X = X + 1e-10 * rng.uniform(size=X.shape)
n_components = 2
clf = manifold.LocallyLinearEmbedding(n_neighbors=5,
n_components=n_components,
random_state=0)
tol = 1.5
N = barycenter_kneighbors_graph(X, clf.n_neighbors).toarray()
reconstruction_error = np.linalg.norm(np.dot(N, X) - X)
assert_less(reconstruction_error, tol)
for solver in eigen_solvers:
clf.set_params(eigen_solver=solver)
clf.fit(X)
assert_true(clf.embedding_.shape[1] == n_components)
reconstruction_error = np.linalg.norm(
np.dot(N, clf.embedding_) - clf.embedding_, 'fro')**2
details = "solver: " + solver
assert_less(reconstruction_error, tol, msg=details)
assert_less(np.abs(clf.reconstruction_error_ - reconstruction_error),
tol * reconstruction_error,
msg=details)
def test_lle_simple_grid():
rng = np.random.RandomState(0)
# grid of equidistant points in 2D, n_components = n_dim
X = np.array(list(product(range(5), repeat=2)))
X = X + 1e-10 * rng.uniform(size=X.shape)
n_components = 2
clf = manifold.LocallyLinearEmbedding(n_neighbors=5,
n_components=n_components)
tol = .1
N = barycenter_kneighbors_graph(X, clf.n_neighbors).todense()
reconstruction_error = np.linalg.norm(np.dot(N, X) - X, 'fro')
assert_less(reconstruction_error, tol)
for solver in eigen_solvers:
clf.set_params(eigen_solver=solver)
clf.fit(X)
assert_true(clf.embedding_.shape[1] == n_components)
reconstruction_error = np.linalg.norm(
np.dot(N, clf.embedding_) - clf.embedding_, 'fro')**2
# FIXME: ARPACK fails this test ...
if solver != 'arpack':
assert_less(reconstruction_error, tol)
assert_almost_equal(clf.reconstruction_error_,
reconstruction_error,
decimal=4)
# re-embed a noisy version of X using the transform method
noise = rng.randn(*X.shape) / 100
X_reembedded = clf.transform(X + noise)
assert_less(np.linalg.norm(X_reembedded - clf.embedding_), tol)
def test_lle_manifold():
# similar test on a slightly more complex manifold
X = np.array(list(product(range(20), repeat=2)))
X = np.c_[X, X[:, 0]**2 / 20]
out_dim = 2
clf = manifold.LocallyLinearEmbedding(n_neighbors=5,
out_dim=out_dim,
random_state=0)
tol = 1.5
N = barycenter_kneighbors_graph(X, clf.n_neighbors).toarray()
reconstruction_error = np.linalg.norm(np.dot(N, X) - X)
assert_lower(reconstruction_error, tol)
for solver in eigen_solvers:
clf.set_params(eigen_solver=solver)
clf.fit(X)
assert_true(clf.embedding_.shape[1] == out_dim)
reconstruction_error = np.linalg.norm(
np.dot(N, clf.embedding_) - clf.embedding_, 'fro')**2
details = "solver: " + solver
assert_lower(reconstruction_error, tol, details=details)
assert_lower(np.abs(clf.reconstruction_error_ - reconstruction_error),
tol * reconstruction_error,
details=details)
def test_lle_manifold():
# similar test on a slightly more complex manifold
X = np.array(list(product(range(20), repeat=2)))
X = np.c_[X, X[:, 0] ** 2 / 20]
out_dim = 2
clf = manifold.LocallyLinearEmbedding(n_neighbors=5, out_dim=out_dim)
tol = 1.5
N = barycenter_kneighbors_graph(X, clf.n_neighbors).toarray()
reconstruction_error = np.linalg.norm(np.dot(N, X) - X)
assert_lower(reconstruction_error, tol)
for solver in eigen_solvers:
clf.set_params(eigen_solver=solver)
clf.fit(X)
assert clf.embedding_.shape[1] == out_dim
reconstruction_error = np.linalg.norm(
np.dot(N, clf.embedding_) - clf.embedding_, 'fro') ** 2
details = "solver: " + solver
assert_lower(reconstruction_error, tol, details=details)
assert_lower(np.abs(clf.reconstruction_error_ - reconstruction_error),
tol * reconstruction_error, details=details)
def main(args):
i_path = args.input_path
m_path = args.mask_path
bg_path = args.bg_path
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
torch.backends.cudnn.deterministic = True
camouflage_dir = args.output_dir
os.makedirs(camouflage_dir, exist_ok=True)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
VGG = models.vgg19(pretrained=True).features
VGG.to(device)
for parameter in VGG.parameters():
parameter.requires_grad_(False)
style_net = HRNet.HRNet()
style_net.to(device)
transform = Compose([
Normalize(
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
),
ToTensorV2(),
])
# try to give fore con_layers more weight so that can get more detail in output iamge
style_weights = args.style_weight_dic
mask = cv2.imread(m_path, 0)
mask = scaling(mask, scale=args.mask_scale)
if args.crop:
idx_y, idx_x = np.where(mask > 0)
x1_m, y1_m, x2_m, y2_m = np.min(idx_x), np.min(idx_y), np.max(
idx_x), np.max(idx_y)
else:
x1_m, y1_m = 0, 0
y2_m, x2_m = mask.shape
x2_m, y2_m = 8 * (x2_m // 8), 8 * (y2_m // 8)
x1_m = 8 * (x1_m // 8)
x2_m = 8 * (x2_m // 8)
y1_m = 8 * (y1_m // 8)
y2_m = 8 * (y2_m // 8)
fore_origin = cv2.cvtColor(cv2.imread(i_path), cv2.COLOR_BGR2RGB)
fore_origin = scaling(fore_origin, scale=args.mask_scale)
fore = fore_origin[y1_m:y2_m, x1_m:x2_m]
mask_crop = mask[y1_m:y2_m, x1_m:x2_m]
mask_crop = np.where(mask_crop > 0, 255, 0).astype(np.uint8)
kernel = np.ones((15, 15), np.uint8)
mask_dilated = cv2.dilate(mask_crop, kernel, iterations=1)
origin = cv2.cvtColor(cv2.imread(bg_path), cv2.COLOR_BGR2RGB)
h_origin, w_origin, _ = origin.shape
h, w = mask_dilated.shape
assert h < h_origin, "mask height must be smaller than bg height, and lower mask_scale parameter!!"
assert w < w_origin, "mask width must be smaller than bg width, and lower mask_scale parameter!!"
print("mask size,height:{},width:{}".format(h, w))
if args.hidden_selected is None:
y_start, x_start = recommend(origin, fore, mask_dilated)
else:
y_start, x_start = args.hidden_selected
x1, y1 = x_start + x1_m, y_start + y1_m
x2, y2 = x1 + w, y1 + h
if y2 > h_origin:
y1 -= (y2 - h_origin)
y2 = h_origin
if x2 > w_origin:
x1 -= (x2 - w_origin)
x2 = w_origin
print("hidden region...,height-{}:{},width-{}:{}".format(y1, y2, x1, x2))
mat_dilated = fore * np.expand_dims(
mask_crop / 255, axis=-1) + origin[y1:y2, x1:x2] * np.expand_dims(
(mask_dilated - mask_crop) / 255, axis=-1)
bg = origin.copy()
bg[y1:y2,
x1:x2] = fore * np.expand_dims(mask_crop / 255, axis=-1) + origin[
y1:y2, x1:x2] * np.expand_dims(1 - mask_crop / 255, axis=-1)
content_image = transform(image=mat_dilated)["image"].unsqueeze(0)
style_image = transform(image=origin[y1:y2, x1:x2])["image"].unsqueeze(0)
content_image = content_image.to(device)
style_image = style_image.to(device)
style_features = get_features(style_image, VGG, mode="style")
if args.style_all:
style_image_all = transform(
image=origin)["image"].unsqueeze(0).to(device)
style_features = get_features(style_image_all, VGG, mode="style")
style_gram_matrixs = {}
style_index = {}
for layer in style_features:
sf = style_features[layer]
_, _, h_sf, w_sf = sf.shape
mask_sf = (cv2.resize(mask_dilated, (w_sf, h_sf))).flatten()
sf_idxes = np.where(mask_sf > 0)[0]
gram_matrix = gram_matrix_slice(sf, sf_idxes)
style_gram_matrixs[layer] = gram_matrix
style_index[layer] = sf_idxes
target = content_image.clone().requires_grad_(True).to(device)
foreground_features = get_features(content_image, VGG, mode="camouflage")
target_features = foreground_features.copy()
attention_layers = [
"conv3_1",
"conv3_2",
"conv3_3",
"conv3_4",
"conv4_1",
"conv4_2",
"conv4_3",
"conv4_4",
]
for u, layer in enumerate(attention_layers):
target_feature = target_features[layer].detach().cpu().numpy(
) # output image's feature map after layer
attention = attention_map_cv(target_feature)
h, w = attention.shape
if "conv3" in layer:
attention = cv2.resize(attention, (w // 2, h // 2)) * 1 / 4
if u == 0:
all_attention = attention
else:
all_attention += attention
all_attention /= 5
max_att, min_att = np.max(all_attention), np.min(all_attention)
all_attention = (all_attention - min_att) / (max_att - min_att)
if args.erode_border:
h, w = all_attention.shape
mask_erode = cv2.erode(mask_crop, kernel, iterations=3)
mask_erode = cv2.resize(mask_erode, (w, h))
mask_erode = np.where(mask_erode > 0, 1, 0)
all_attention = all_attention * mask_erode
foreground_attention = torch.from_numpy(all_attention.astype(
np.float32)).clone().to(device).unsqueeze(0).unsqueeze(0)
b, ch, h, w = foreground_features["conv4_1"].shape
mask_f = cv2.resize(mask_dilated, (w, h)) / 255
idx = np.where(mask_f > 0)
size = len(idx[0])
mask_f = torch.from_numpy(mask_f.astype(
np.float32)).clone().to(device).unsqueeze(0).unsqueeze(0)
foreground_chi = foreground_features["conv4_1"] * foreground_attention
foreground_chi = foreground_chi.detach().cpu().numpy()[0].transpose(
1, 2, 0)
foreground_cosine = cosine_distances(foreground_chi[idx])
background_features = get_features(style_image, VGG, mode="camouflage")
idxes = np.where(mask_dilated > 0)
n_neighbors, n_jobs, reg = 7, None, 1e-3
nbrs = NearestNeighbors(n_neighbors=n_neighbors + 1, n_jobs=n_jobs)
X_origin = origin[y1:y2, x1:x2][idxes] / 255
nbrs.fit(X_origin)
X = nbrs._fit_X
Weight_Matrix = barycenter_kneighbors_graph(nbrs,
n_neighbors=n_neighbors,
reg=reg,
n_jobs=n_jobs)
idx_new = np.where(idxes[0] < (y2 - y1 - 1))
idxes_h = (idxes[0][idx_new], idxes[1][idx_new])
idx_new = np.where(idxes[1] < (x2 - x1 - 1))
idxes_w = (idxes[0][idx_new], idxes[1][idx_new])
mask_norm = mask_crop / 255.
mask_norm_torch = torch.from_numpy(
(mask_norm).astype(np.float32)).unsqueeze(0).unsqueeze(0).to(device)
boundary = (mask_dilated - mask_crop) / 255
boundary = torch.from_numpy(
(boundary).astype(np.float32)).unsqueeze(0).unsqueeze(0).to(device)
content_loss_epoch = []
style_loss_epoch = []
total_loss_epoch = []
time_start = datetime.datetime.now()
epoch = 0
show_every = args.show_every
optimizer = optim.Adam(style_net.parameters(), lr=args.lr)
steps = args.epoch
mse = nn.MSELoss()
while epoch <= steps:
#############################
### boundary conceal ########
#############################
target = style_net(content_image).to(device)
target = content_image * boundary + target * mask_norm_torch
target.requires_grad_(True)
target_features = get_features(
target, VGG) # extract output image's all feature maps
#############################
### content loss #########
#############################
target_features_content = get_features(target, VGG, mode="content")
content_loss = torch.sum((target_features_content['conv4_2'] -
foreground_features['conv4_2'])**2) / 2
content_loss *= args.lambda_weights["content"]
#############################
### style loss #########
#############################
style_loss = 0
# compute each layer's style loss and add them
for layer in style_weights:
target_feature = target_features[
layer] # output image's feature map after layer
#target_gram_matrix = get_gram_matrix(target_feature)
target_gram_matrix = gram_matrix_slice(target_feature,
style_index[layer])
style_gram_matrix = style_gram_matrixs[layer]
b, c, h, w = target_feature.shape
layer_style_loss = style_weights[layer] * torch.sum(
(target_gram_matrix - style_gram_matrix)**2) / (
(2 * c * w * h)**2)
#layer_style_loss = style_weights[layer] * torch.mean((target_gram_matrix - style_gram_matrix) ** 2)
style_loss += layer_style_loss
style_loss *= args.lambda_weights["style"]
#############################
### camouflage loss #########
#############################
target_chi = target_features["conv4_1"] * foreground_attention
target_chi = target_chi.detach().cpu().numpy()[0].transpose(1, 2, 0)
target_cosine = cosine_distances(target_chi[idx])
leave_loss = (np.mean(np.abs(target_cosine - foreground_cosine)) / 2)
leave_loss = torch.Tensor([leave_loss]).to(device)
remove_matrix = (1.0 - foreground_attention) * mask_f * (
target_features["conv4_1"] - background_features["conv4_1"])
r_min, r_max = torch.min(remove_matrix), torch.max(remove_matrix)
remove_matrix = (remove_matrix - r_min) / (r_max - r_min)
remove_loss = (torch.mean(remove_matrix**2) / 2).to(device)
camouflage_loss = leave_loss + args.mu * remove_loss
camouflage_loss *= args.lambda_weights["cam"]
#############################
### regularization loss #####
#############################
target_renormalize = target.detach().cpu().numpy()[0, :].transpose(
1, 2, 0)
target_renormalize = target_renormalize * np.array(
(0.229, 0.224, 0.225)) + np.array((0.485, 0.456, 0.406))
target_renormalize = target_renormalize.clip(0, 1)[idxes]
target_reconst = torch.from_numpy(
(Weight_Matrix * target_renormalize).astype(np.float32))
target_renormalize = torch.from_numpy(
target_renormalize.astype(np.float32))
reg_loss = mse(target_renormalize, target_reconst).to(device)
reg_loss *= args.lambda_weights["reg"]
#############################
### total variation loss ####
#############################
tv_h = torch.pow(target[:, :, 1:, :] - target[:, :, :-1, :],
2).detach().cpu().numpy()[0].transpose(1, 2, 0)
tv_w = torch.pow(target[:, :, :, 1:] - target[:, :, :, :-1],
2).detach().cpu().numpy()[0].transpose(1, 2, 0)
tv_h_mask = tv_h[:, :,
0][idxes_h] + tv_h[:, :,
1][idxes_h] + tv_h[:, :,
2][idxes_h]
tv_w_mask = tv_w[:, :,
0][idxes_w] + tv_w[:, :,
2][idxes_w] + tv_w[:, :,
2][idxes_w]
tv_loss = torch.from_numpy(
(np.array(np.mean(np.concatenate([tv_h_mask,
tv_w_mask]))))).to(device)
tv_loss *= args.lambda_weights["tv"]
total_loss = content_loss + style_loss + camouflage_loss + reg_loss + tv_loss
total_loss_epoch.append(total_loss)
style_loss_epoch.append(style_loss)
optimizer.zero_grad()
total_loss.backward()
optimizer.step()
if epoch % show_every == 0:
print("After %d criterions:" % epoch)
print('Total loss: ', total_loss.item())
print('Style loss: ', style_loss.item())
print('camouflage loss: ', camouflage_loss.item())
print('camouflage loss leave: ', leave_loss.item())
print('camouflage loss remove: ', remove_loss.item())
print('regularization loss: ', reg_loss.item())
print('total variation loss: ', tv_loss.item())
print('content loss: ', content_loss.item())
print("elapsed time:{}".format(datetime.datetime.now() -
time_start))
canvas = origin.copy()
fore_gen = im_convert(target) * 255.
sub_canvas = np.vstack(
[mat_dilated, fore_gen, origin[y1:y2, x1:x2]])
canvas[y1:y2, x1:x2] = fore_gen * np.expand_dims(
mask_norm, axis=-1) + origin[y1:y2, x1:x2] * np.expand_dims(
1.0 - mask_norm, axis=-1)
canvas = canvas.astype(np.uint8)
if args.save_process:
new_path = os.path.join(
camouflage_dir, "{}_epoch{}.png".format(args.name, epoch))
cv2.imwrite(new_path, cv2.cvtColor(canvas, cv2.COLOR_RGB2BGR))
cv2.rectangle(canvas, (x1, y1), (x2, y2), (255, 0, 0), 10)
cv2.rectangle(canvas, (x1 - x1_m, y1 - y1_m), (x2, y2),
(255, 255, 0), 10)
canvas = np.vstack([canvas, bg])
h_c, w_c, _ = canvas.shape
h_s, w_s, _ = sub_canvas.shape
sub_canvas = cv2.resize(sub_canvas, (int(w_s * (h_c / h_s)), h_c))
canvas = np.hstack([sub_canvas, canvas])
canvas = canvas.astype(np.uint8)
canvas = cv2.cvtColor(canvas, cv2.COLOR_RGB2BGR)
h_show, w_show, c = canvas.shape
cv2.imshow(
"now camouflage...",
cv2.resize(
canvas,
(w_show // args.show_comp, h_show // args.show_comp)))
epoch += 1
if cv2.waitKey(1) & 0xFF == ord('q'):
break
time_end = datetime.datetime.now()
print('totally cost:{}'.format(time_end - time_start))
new_path = os.path.join(camouflage_dir, "{}.png".format(args.name))
canvas = origin.copy()
fore_gen = im_convert(target) * 255.
canvas[y1:y2,
x1:x2] = fore_gen * np.expand_dims(mask_norm, axis=-1) + origin[
y1:y2, x1:x2] * np.expand_dims(1.0 - mask_norm, axis=-1)
canvas = canvas.astype(np.uint8)
canvas = cv2.cvtColor(canvas, cv2.COLOR_RGB2BGR)
cv2.imwrite(new_path, canvas)
def locally_linear_embedding(
X, n_neighbors, n_components, reg=1e-3, eigen_solver='auto', tol=1e-6,
max_iter=100, method='standard', hessian_tol=1E-4, modified_tol=1E-12,
random_state=None, n_jobs=None):
if eigen_solver not in ('auto', 'arpack', 'dense'):
raise ValueError("unrecognized eigen_solver '%s'" % eigen_solver)
if method not in ('standard', 'hessian', 'modified', 'ltsa'):
raise ValueError("unrecognized method '%s'" % method)
nbrs = NearestNeighbors(n_neighbors=n_neighbors + 1, n_jobs=n_jobs)
nbrs.fit(X)
X = nbrs._fit_X
N, d_in = X.shape
if n_components > d_in:
raise ValueError("output dimension must be less than or equal "
"to input dimension")
if n_neighbors >= N:
raise ValueError(
"Expected n_neighbors <= n_samples, "
" but n_samples = %d, n_neighbors = %d" %
(N, n_neighbors)
)
if n_neighbors <= 0:
raise ValueError("n_neighbors must be positive")
M_sparse = (eigen_solver != 'dense')
if method == 'standard':
W = barycenter_kneighbors_graph(nbrs, n_neighbors=n_neighbors, reg=reg, n_jobs=1)
if M_sparse:
M = eye(*W.shape, format=W.format) - W
M = (M.T * M).tocsr()
else:
M = (W.T * W - W.T - W).toarray()
M.flat[::M.shape[0] + 1] += 1 # W = W - I = W - I
elif method == 'hessian':
dp = n_components * (n_components + 1) // 2
if n_neighbors <= n_components + dp:
raise ValueError("for method='hessian', n_neighbors must be "
"greater than "
"[n_components * (n_components + 3) / 2]")
neighbors = nbrs.kneighbors(X, n_neighbors=n_neighbors + 1,
return_distance=False)
neighbors = neighbors[:, 1:]
Yi = np.empty((n_neighbors, 1 + n_components + dp), dtype=np.float64)
Yi[:, 0] = 1
M = np.zeros((N, N), dtype=np.float64)
use_svd = (n_neighbors > d_in)
for i in range(N):
Gi = X[neighbors[i]]
Gi -= Gi.mean(0)
# build Hessian estimator
if use_svd:
U = svd(Gi, full_matrices=0)[0]
else:
Ci = np.dot(Gi, Gi.T)
U = eigh(Ci)[1][:, ::-1]
Yi[:, 1:1 + n_components] = U[:, :n_components]
j = 1 + n_components
for k in range(n_components):
Yi[:, j:j + n_components - k] = (U[:, k:k + 1] *
U[:, k:n_components])
j += n_components - k
Q, R = qr(Yi)
w = Q[:, n_components + 1:]
S = w.sum(0)
S[np.where(abs(S) < hessian_tol)] = 1
w /= S
nbrs_x, nbrs_y = np.meshgrid(neighbors[i], neighbors[i])
M[nbrs_x, nbrs_y] += np.dot(w, w.T)
if M_sparse:
M = csr_matrix(M)
elif method == 'modified':
if n_neighbors < n_components:
raise ValueError("modified LLE requires "
"n_neighbors >= n_components")
neighbors = nbrs.kneighbors(X, n_neighbors=n_neighbors + 1,
return_distance=False)
neighbors = neighbors[:, 1:]
# find the eigenvectors and eigenvalues of each local covariance
# matrix. We want V[i] to be a [n_neighbors x n_neighbors] matrix,
# where the columns are eigenvectors
V = np.zeros((N, n_neighbors, n_neighbors))
nev = min(d_in, n_neighbors)
evals = np.zeros([N, nev])
# choose the most efficient way to find the eigenvectors
use_svd = (n_neighbors > d_in)
if use_svd:
for i in range(N):
X_nbrs = X[neighbors[i]] - X[i]
V[i], evals[i], _ = svd(X_nbrs,
full_matrices=True)
evals **= 2
else:
for i in range(N):
X_nbrs = X[neighbors[i]] - X[i]
C_nbrs = np.dot(X_nbrs, X_nbrs.T)
evi, vi = eigh(C_nbrs)
evals[i] = evi[::-1]
V[i] = vi[:, ::-1]
# find regularized weights: this is like normal LLE.
# because we've already computed the SVD of each covariance matrix,
# it's faster to use this rather than np.linalg.solve
reg = 1E-3 * evals.sum(1)
tmp = np.dot(V.transpose(0, 2, 1), np.ones(n_neighbors))
tmp[:, :nev] /= evals + reg[:, None]
tmp[:, nev:] /= reg[:, None]
w_reg = np.zeros((N, n_neighbors))
for i in range(N):
w_reg[i] = np.dot(V[i], tmp[i])
w_reg /= w_reg.sum(1)[:, None]
# calculate eta: the median of the ratio of small to large eigenvalues
# across the points. This is used to determine s_i, below
rho = evals[:, n_components:].sum(1) / evals[:, :n_components].sum(1)
eta = np.median(rho)
# find s_i, the size of the "almost null space" for each point:
# this is the size of the largest set of eigenvalues
# such that Sum[v; v in set]/Sum[v; v not in set] < eta
s_range = np.zeros(N, dtype=int)
evals_cumsum = stable_cumsum(evals, 1)
eta_range = evals_cumsum[:, -1:] / evals_cumsum[:, :-1] - 1
for i in range(N):
s_range[i] = np.searchsorted(eta_range[i, ::-1], eta)
s_range += n_neighbors - nev # number of zero eigenvalues
# Now calculate M.
# This is the [N x N] matrix whose null space is the desired embedding
M = np.zeros((N, N), dtype=np.float64)
for i in range(N):
s_i = s_range[i]
# select bottom s_i eigenvectors and calculate alpha
Vi = V[i, :, n_neighbors - s_i:]
alpha_i = np.linalg.norm(Vi.sum(0)) / np.sqrt(s_i)
# compute Householder matrix which satisfies
# Hi*Vi.T*ones(n_neighbors) = alpha_i*ones(s)
# using prescription from paper
h = np.full(s_i, alpha_i) - np.dot(Vi.T, np.ones(n_neighbors))
norm_h = np.linalg.norm(h)
if norm_h < modified_tol:
h *= 0
else:
h /= norm_h
# Householder matrix is
# >> Hi = np.identity(s_i) - 2*np.outer(h,h)
# Then the weight matrix is
# >> Wi = np.dot(Vi,Hi) + (1-alpha_i) * w_reg[i,:,None]
# We do this much more efficiently:
Wi = (Vi - 2 * np.outer(np.dot(Vi, h), h) +
(1 - alpha_i) * w_reg[i, :, None])
# Update M as follows:
# >> W_hat = np.zeros( (N,s_i) )
# >> W_hat[neighbors[i],:] = Wi
# >> W_hat[i] -= 1
# >> M += np.dot(W_hat,W_hat.T)
# We can do this much more efficiently:
nbrs_x, nbrs_y = np.meshgrid(neighbors[i], neighbors[i])
M[nbrs_x, nbrs_y] += np.dot(Wi, Wi.T)
Wi_sum1 = Wi.sum(1)
M[i, neighbors[i]] -= Wi_sum1
M[neighbors[i], i] -= Wi_sum1
M[i, i] += s_i
if M_sparse:
M = csr_matrix(M)
elif method == 'ltsa':
neighbors = nbrs.kneighbors(X, n_neighbors=n_neighbors + 1,
return_distance=False)
neighbors = neighbors[:, 1:]
M = np.zeros((N, N))
use_svd = (n_neighbors > d_in)
for i in range(N):
Xi = X[neighbors[i]]
Xi -= Xi.mean(0)
# compute n_components largest eigenvalues of Xi * Xi^T
if use_svd:
v = svd(Xi, full_matrices=True)[0]
else:
Ci = np.dot(Xi, Xi.T)
v = eigh(Ci)[1][:, ::-1]
Gi = np.zeros((n_neighbors, n_components + 1))
Gi[:, 1:] = v[:, :n_components]
Gi[:, 0] = 1. / np.sqrt(n_neighbors)
GiGiT = np.dot(Gi, Gi.T)
nbrs_x, nbrs_y = np.meshgrid(neighbors[i], neighbors[i])
M[nbrs_x, nbrs_y] -= GiGiT
M[neighbors[i], neighbors[i]] += 1
return W
a,b = null_space(M, n_components, k_skip=1, eigen_solver=eigen_solver,
tol=tol, max_iter=max_iter, random_state=random_state)
def ller(X,
Y,
n_neighbors,
n_components,
mu=0.5,
gamma=None,
reg=1e-3,
eigen_solver='auto',
tol=1e-6,
max_iter=100,
random_state=None):
"""
Locally Linear Embedding for Regression (LLER)
Parameters
----------
X : ndarray, 2-dimensional
The data matrix, shape (num_data_points, num_dims)
Y : ndarray, 1 or 2-dimensional
The response matrix, shape (num_response_points, num_responses).
Y[0:] is assumed to provide responses for X[:num_response_points]
n_neighbors : int
Number of neighbors for kNN graph construction.
n_components : int
Number of dimensions for embedding.
mu : float, optional
Influence of the Y-similarity penalty.
gamma : float, optional
Scaling factor for RBF kernel on Y.
Defaults to the inverse of the median distance between rows of Y.
Returns
-------
embedding : ndarray, 2-dimensional
The embedding of X, shape (num_points, n_components)
lle_error : float
The embedding error of X (for a fixed reconstruction matrix W)
ller_error : float
The embedding error of X that takes Y into account.
"""
if eigen_solver not in ('auto', 'arpack', 'dense'):
raise ValueError("unrecognized eigen_solver '%s'" % eigen_solver)
if Y.ndim == 1:
Y = Y[:, None]
if gamma is None:
dists = pairwise_distances(Y)
gamma = 1.0 / np.median(dists)
nbrs = NearestNeighbors(n_neighbors=n_neighbors + 1)
nbrs.fit(X)
X = nbrs._fit_X
Nx, d_in = X.shape
Ny = Y.shape[0]
if n_components > d_in:
raise ValueError("output dimension must be less than or equal "
"to input dimension")
if n_neighbors >= Nx:
raise ValueError("n_neighbors must be less than number of points")
if n_neighbors <= 0:
raise ValueError("n_neighbors must be positive")
if Nx < Ny:
raise ValueError("X should have at least as many points as Y")
M_sparse = (eigen_solver != 'dense')
W = barycenter_kneighbors_graph(nbrs, n_neighbors=n_neighbors, reg=reg)
if M_sparse:
M = speye(*W.shape, format=W.format) - W
M = (M.T * M).tocsr()
else:
M = (W.T * W - W.T - W).toarray()
M.flat[::M.shape[0] + 1] += 1
P = rbf_kernel(Y, gamma=gamma)
L = laplacian(P, normed=False)
M /= np.abs(M).max() # optional scaling step
L /= np.abs(L).max()
if Nx > Ny:
# zeros = csr_matrix((Nx-Ny,Nx-Ny),dtype=M.dtype)
# L = bmat([[L, None], [None, zeros]])
ones = csr_matrix(np.ones((Nx - Ny, Nx - Ny)), dtype=M.dtype)
L = bmat([[L, None], [None, ones]])
omega = M + mu * L
embedding, lle_error = null_space(omega,
n_components,
k_skip=1,
eigen_solver=eigen_solver,
tol=tol,
max_iter=max_iter,
random_state=random_state)
ller_error = np.trace(embedding.T.dot(L).dot(embedding))
return embedding, lle_error, ller_error
def ller(X, Y, n_neighbors, n_components, mu=0.5, gamma=None,
reg=1e-3,eigen_solver='auto', tol=1e-6, max_iter=100,
random_state=None):
"""
Locally Linear Embedding for Regression (LLER)
Parameters
----------
X : ndarray, 2-dimensional
The data matrix, shape (num_data_points, num_dims)
Y : ndarray, 1 or 2-dimensional
The response matrix, shape (num_response_points, num_responses).
Y[0:] is assumed to provide responses for X[:num_response_points]
n_neighbors : int
Number of neighbors for kNN graph construction.
n_components : int
Number of dimensions for embedding.
mu : float, optional
Influence of the Y-similarity penalty.
gamma : float, optional
Scaling factor for RBF kernel on Y.
Defaults to the inverse of the median distance between rows of Y.
Returns
-------
embedding : ndarray, 2-dimensional
The embedding of X, shape (num_points, n_components)
lle_error : float
The embedding error of X (for a fixed reconstruction matrix W)
ller_error : float
The embedding error of X that takes Y into account.
"""
if eigen_solver not in ('auto', 'arpack', 'dense'):
raise ValueError("unrecognized eigen_solver '%s'" % eigen_solver)
if Y.ndim == 1:
Y = Y[:, None]
if gamma is None:
dists = pairwise_distances(Y)
gamma = 1.0 / np.median(dists)
nbrs = NearestNeighbors(n_neighbors=n_neighbors + 1)
nbrs.fit(X)
X = nbrs._fit_X
Nx, d_in = X.shape
Ny = Y.shape[0]
if n_components > d_in:
raise ValueError("output dimension must be less than or equal "
"to input dimension")
if n_neighbors >= Nx:
raise ValueError("n_neighbors must be less than number of points")
if n_neighbors <= 0:
raise ValueError("n_neighbors must be positive")
if Nx < Ny:
raise ValueError("X should have at least as many points as Y")
M_sparse = (eigen_solver != 'dense')
W = barycenter_kneighbors_graph(
nbrs, n_neighbors=n_neighbors, reg=reg)
if M_sparse:
M = speye(*W.shape, format=W.format) - W
M = (M.T * M).tocsr()
else:
M = (W.T * W - W.T - W).toarray()
M.flat[::M.shape[0] + 1] += 1
P = rbf_kernel(Y, gamma=gamma)
L = laplacian(P, normed=False)
M /= np.abs(M).max() # optional scaling step
L /= np.abs(L).max()
if Nx > Ny:
# zeros = csr_matrix((Nx-Ny,Nx-Ny),dtype=M.dtype)
# L = bmat([[L, None], [None, zeros]])
ones = csr_matrix(np.ones((Nx-Ny,Nx-Ny)),dtype=M.dtype)
L = bmat([[L, None], [None, ones]])
omega = M + mu * L
embedding, lle_error = null_space(omega, n_components, k_skip=1,
eigen_solver=eigen_solver, tol=tol,
max_iter=max_iter,
random_state=random_state)
ller_error = np.trace(embedding.T.dot(L).dot(embedding))
return embedding, lle_error, ller_error
