def build_loss(self, a_sem_image, a_image, b_sem_image, b_image):
'''Create an expression for the loss as a function of the image inputs.'''
print('Building loss...')
loss = super(AnalogyModel, self).build_loss(a_sem_image, a_image,
b_sem_image, b_image)
# Precompute static features for performance
print('Precomputing static features...')
all_asem_features, all_a_image_features, all_bsem_features, all_b_features = self.precompute_static_features(
a_sem_image, a_image, b_sem_image, b_image)
print('Building DSS losses...')
if self.args.analogy_weight: #PatchMatch -image-analogies, a kind of content loss (if B is not a photograph but a [0,1] mask)
for layer_name in self.args.analogy_layers:
asem_features = all_asem_features[layer_name][0] #array
a_image_features = all_a_image_features[layer_name][0] #array
bsem_features = all_bsem_features[layer_name][0] #array
# current combined output
layer_features = self.get_layer_output(layer_name) #variable
combination_features = layer_features[0, :, :, :] #variable
al = nnf_analogy_loss(asem_features,
a_image_features,
bsem_features,
combination_features,
num_steps=self.args.analogy_nnf_steps,
patch_size=self.args.patch_size,
patch_stride=self.args.patch_stride,
jump_size=1.0)
loss += (self.args.analogy_weight /
len(self.args.analogy_layers)) * al
print('Building Content losses...')
if self.args.content_weight: #content loss(now b is seg) (default=0.0)
for layer_name in self.args.b_content_layers:
b_features = K.variable(all_b_features[layer_name][0])
# current combined output
bp_features = self.get_layer_output(layer_name)
cl = content_loss(bp_features, b_features)
loss += self.args.content_weight / len(
self.args.b_content_layers) * cl
print('Building Style losses...')
if self.args.neural_style_weight != 0.0: #Gram (default=0.0)
for layer_name in self.args.neural_style_layers:
a_image_features = K.variable(
all_a_image_features[layer_name][0])
layer_features = self.get_layer_output(layer_name)
layer_shape = self.get_layer_output_shape(layer_name)
# current combined output
combination_features = layer_features[0, :, :, :]
nsl = neural_style_loss(a_image_features, combination_features,
3, self.output_shape[-2],
self.output_shape[-1])
loss += (self.args.neural_style_weight /
len(self.args.neural_style_layers)) * nsl
return loss
def build_loss(self, a_image, ap_image, b_image):
'''Create an expression for the loss as a function of the image inputs.'''
print('Building loss...')
loss = super(AnalogyModel, self).build_loss(a_image, ap_image, b_image)
# Precompute static features for performance
print('Precomputing static features...')
all_a_features, all_ap_image_features, all_b_features = self.precompute_static_features(a_image, ap_image, b_image)
print('Building and combining losses...')
if self.args.analogy_weight != 0.0:
for layer_name in self.args.analogy_layers:
a_features = all_a_features[layer_name][0]
ap_image_features = all_ap_image_features[layer_name][0]
b_features = all_b_features[layer_name][0]
# current combined output
layer_features = self.get_layer_output(layer_name)
combination_features = layer_features[0, :, :, :]
al = analogy_loss(a_features, ap_image_features,
b_features, combination_features,
use_full_analogy=self.args.use_full_analogy,
patch_size=self.args.patch_size,
patch_stride=self.args.patch_stride)
loss += (self.args.analogy_weight / len(self.args.analogy_layers)) * al
if self.args.mrf_weight != 0.0:
for layer_name in self.args.mrf_layers:
ap_image_features = K.variable(all_ap_image_features[layer_name][0])
layer_features = self.get_layer_output(layer_name)
# current combined output
combination_features = layer_features[0, :, :, :]
sl = mrf_loss(ap_image_features, combination_features,
patch_size=self.args.patch_size,
patch_stride=self.args.patch_stride)
loss += (self.args.mrf_weight / len(self.args.mrf_layers)) * sl
if self.args.b_bp_content_weight != 0.0:
for layer_name in self.args.b_content_layers:
b_features = K.variable(all_b_features[layer_name][0])
# current combined output
bp_features = self.get_layer_output(layer_name)
cl = content_loss(bp_features, b_features)
loss += self.args.b_bp_content_weight / len(self.args.b_content_layers) * cl
if self.args.neural_style_weight != 0.0:
for layer_name in self.args.neural_style_layers:
ap_image_features = K.variable(all_ap_image_features[layer_name][0])
layer_features = self.get_layer_output(layer_name)
layer_shape = self.get_layer_output_shape(layer_name)
# current combined output
combination_features = layer_features[0, :, :, :]
nsl = neural_style_loss(ap_image_features, combination_features, 3, self.output_shape[-2], self.output_shape[-1])
loss += (self.args.neural_style_weight / len(self.args.neural_style_layers)) * nsl
return loss
def build_loss(self, a_sem_image, a_image, b_sem_image,b_image):
'''Create an expression for the loss as a function of the image inputs.'''
print('Building loss...')
loss = super(NNFModel, self).build_loss(a_sem_image, a_image, b_sem_image,b_image)#viriable
# Precompute static features for performance
print('Precomputing static features...')
all_asem_features, all_a_image_features, all_bsem_features,all_b_features = self.precompute_static_features(a_sem_image, a_image, b_sem_image,b_image)
print('Building and combining losses...')
print('Building DSS losses...')
if self.args.analogy_weight:#Deep Semantic Space-guided Loss
for layer_name in self.args.analogy_layers:
asem_features = all_asem_features[layer_name][0]#array
a_image_features = all_a_image_features[layer_name][0]#array
bsem_features = all_bsem_features[layer_name][0]#array
# current combined output
layer_features = self.get_layer_output(layer_name)#variable
combination_features = layer_features[0, :, :, :]#variable
al = nnf_analogy_loss(
asem_features, a_image_features, bsem_features, combination_features,
num_steps=self.args.analogy_nnf_steps, patch_size=self.args.patch_size,
patch_stride=self.args.patch_stride, jump_size=1.0)
loss += (self.args.analogy_weight / len(self.args.analogy_layers)) * al
print('Building Content losses...')
if self.args.content_weight:#content loss
for layer_name in self.args.b_content_layers:
b_features = K.variable(all_b_features[layer_name][0])
# current combined output
bp_features = self.get_layer_output(layer_name)
cl = content_loss(bp_features, b_features)
loss += self.args.content_weight / len(self.args.b_content_layers) * cl
print('Building Style losses...')
if self.args.neural_style_weight != 0.0:#Gram-based Style loss
for layer_name in self.args.neural_style_layers:
a_image_features = K.variable(all_a_image_features[layer_name][0])
layer_features = self.get_layer_output(layer_name)
layer_shape = self.get_layer_output_shape(layer_name)
# current combined output
combination_features = layer_features[0, :, :, :]
nsl = neural_style_loss(a_image_features, combination_features, 3, self.output_shape[-2], self.output_shape[-1])
loss += (self.args.neural_style_weight / len(self.args.neural_style_layers)) * nsl
return loss
def build_loss(self, a_image, ap_image, b_image):
'''Create an expression for the loss as a function of the image inputs.'''
print('Building loss...')
loss = super(AnalogyModel, self).build_loss(a_image, ap_image, b_image)
# Precompute static features for performance
print('Precomputing static features...')
all_a_features, all_ap_image_features, all_b_features = self.precompute_static_features(
a_image, ap_image, b_image)
print('Building and combining losses...')
if self.args.analogy_weight != 0.0:
for layer_name in self.args.analogy_layers:
a_features = all_a_features[layer_name][0]
ap_image_features = all_ap_image_features[layer_name][0]
b_features = all_b_features[layer_name][0]
# current combined output
layer_features = self.get_layer_output(layer_name)
combination_features = layer_features[0, :, :, :]
al = analogy_loss(a_features,
ap_image_features,
b_features,
combination_features,
use_full_analogy=self.args.use_full_analogy,
patch_size=self.args.patch_size,
patch_stride=self.args.patch_stride)
loss += (self.args.analogy_weight /
len(self.args.analogy_layers)) * al
if self.args.mrf_weight != 0.0:
for layer_name in self.args.mrf_layers:
ap_image_features = K.variable(
all_ap_image_features[layer_name][0])
layer_features = self.get_layer_output(layer_name)
# current combined output
combination_features = layer_features[0, :, :, :]
sl = mrf_loss(ap_image_features,
combination_features,
patch_size=self.args.patch_size,
patch_stride=self.args.patch_stride)
loss += (self.args.mrf_weight / len(self.args.mrf_layers)) * sl
if self.args.b_bp_content_weight != 0.0:
for layer_name in self.args.b_content_layers:
b_features = K.variable(all_b_features[layer_name][0])
# current combined output
bp_features = self.get_layer_output(layer_name)
cl = content_loss(bp_features, b_features)
loss += self.args.b_bp_content_weight / len(
self.args.b_content_layers) * cl
if self.args.neural_style_weight != 0.0:
for layer_name in self.args.neural_style_layers:
ap_image_features = K.variable(
all_ap_image_features[layer_name][0])
layer_features = self.get_layer_output(layer_name)
layer_shape = self.get_layer_output_shape(layer_name)
# current combined output
combination_features = layer_features[0, :, :, :]
nsl = neural_style_loss(ap_image_features,
combination_features, 3,
self.output_shape[-2],
self.output_shape[-1])
loss += (self.args.neural_style_weight /
len(self.args.neural_style_layers)) * nsl
return loss
def build_loss(self, a_image, ap_image, b_image):
'''Create an expression for the loss as a function of the image inputs.'''
print('Building loss...')
loss = super(NNFModel, self).build_loss(a_image, ap_image, b_image)
# Precompute static features for performance
print('Precomputing static features...')
all_a_features, all_ap_image_features, all_b_features = self.precompute_static_features(
a_image, ap_image, b_image)
print('Building and combining losses...')
if self.args.analogy_weight:
for layer_name in self.args.analogy_layers:
a_features = all_a_features[layer_name][0]
ap_image_features = all_ap_image_features[layer_name][0]
b_features = all_b_features[layer_name][0]
# current combined output
layer_features = self.get_layer_output(layer_name)
combination_features = layer_features[0, :, :, :]
al = nnf_analogy_loss(a_features,
ap_image_features,
b_features,
combination_features,
num_steps=self.args.analogy_nnf_steps,
patch_size=self.args.patch_size,
patch_stride=self.args.patch_stride,
jump_size=1.0)
loss += (self.args.analogy_weight /
len(self.args.analogy_layers)) * al
existing_feature_nnfs = getattr(self, 'feature_nnfs',
[None] * len(self.args.mrf_layers))
self.feature_nnfs = []
if self.args.mrf_weight:
for layer_name, existing_nnf in zip(self.args.mrf_layers,
existing_feature_nnfs):
ap_image_features = all_ap_image_features[layer_name][0]
# current combined output
layer_features = self.get_layer_output(layer_name)
combination_features = layer_features[0, :, :, :]
input_shape = self.get_layer_output_shape(layer_name)
if existing_nnf and not self.args.randomize_mnf_nnf:
matcher = existing_nnf.matcher.scale(
(input_shape[3], input_shape[2], input_shape[1]),
ap_image_features)
else:
matcher = PatchMatcher(
(input_shape[3], input_shape[2], input_shape[1]),
ap_image_features,
patch_size=self.args.patch_size,
jump_size=1.0,
patch_stride=self.args.patch_stride)
nnf = NNFState(matcher, self.get_f_layer(layer_name))
self.feature_nnfs.append(nnf)
sl = content_loss(combination_features, nnf.placeholder)
loss += (self.args.mrf_weight / len(self.args.mrf_layers)) * sl
if self.args.b_bp_content_weight:
for layer_name in self.args.b_content_layers:
b_features = K.variable(all_b_features[layer_name][0])
# current combined output
bp_features = self.get_layer_output(layer_name)
cl = content_loss(bp_features, b_features)
loss += self.args.b_bp_content_weight / len(
self.args.b_content_layers) * cl
if self.args.neural_style_weight != 0.0:
for layer_name in self.args.neural_style_layers:
ap_image_features = K.variable(
all_ap_image_features[layer_name][0])
layer_features = self.get_layer_output(layer_name)
layer_shape = self.get_layer_output_shape(layer_name)
# current combined output
combination_features = layer_features[0, :, :, :]
nsl = neural_style_loss(ap_image_features,
combination_features, 3,
self.output_shape[-2],
self.output_shape[-1])
loss += (self.args.neural_style_weight /
len(self.args.neural_style_layers)) * nsl
return loss
def build_loss(self, a_image, ap_image, b_image):
'''Create an expression for the loss as a function of the image inputs.'''
print('Building loss...')
loss = super(NNFModel, self).build_loss(a_image, ap_image, b_image)
# Precompute static features for performance
print('Precomputing static features...')
all_a_features, all_ap_image_features, all_b_features = self.precompute_static_features(a_image, ap_image, b_image)
print('Building and combining losses...')
if self.args.analogy_weight:
for layer_name in self.args.analogy_layers:
a_features = all_a_features[layer_name][0]
ap_image_features = all_ap_image_features[layer_name][0]
b_features = all_b_features[layer_name][0]
# current combined output
layer_features = self.get_layer_output(layer_name)
combination_features = layer_features[0, :, :, :]
al = nnf_analogy_loss(
a_features, ap_image_features, b_features, combination_features,
num_steps=self.args.analogy_nnf_steps, patch_size=self.args.patch_size,
patch_stride=self.args.patch_stride, jump_size=1.0)
loss += (self.args.analogy_weight / len(self.args.analogy_layers)) * al
existing_feature_nnfs = getattr(self, 'feature_nnfs', [None] * len(self.args.mrf_layers))
self.feature_nnfs = []
if self.args.mrf_weight:
for layer_name, existing_nnf in zip(self.args.mrf_layers, existing_feature_nnfs):
ap_image_features = all_ap_image_features[layer_name][0]
# current combined output
layer_features = self.get_layer_output(layer_name)
combination_features = layer_features[0, :, :, :]
input_shape = self.get_layer_output_shape(layer_name)
if existing_nnf and not self.args.randomize_mnf_nnf:
matcher = existing_nnf.matcher.scale((input_shape[3], input_shape[2], input_shape[1]), ap_image_features)
else:
matcher = PatchMatcher(
(input_shape[3], input_shape[2], input_shape[1]), ap_image_features,
patch_size=self.args.patch_size, jump_size=1.0, patch_stride=self.args.patch_stride)
nnf = NNFState(matcher, self.get_f_layer(layer_name))
self.feature_nnfs.append(nnf)
sl = content_loss(combination_features, nnf.placeholder)
loss += (self.args.mrf_weight / len(self.args.mrf_layers)) * sl
if self.args.b_bp_content_weight:
for layer_name in self.args.b_content_layers:
b_features = K.variable(all_b_features[layer_name][0])
# current combined output
bp_features = self.get_layer_output(layer_name)
cl = content_loss(bp_features, b_features)
loss += self.args.b_bp_content_weight / len(self.args.b_content_layers) * cl
if self.args.neural_style_weight != 0.0:
for layer_name in self.args.neural_style_layers:
ap_image_features = K.variable(all_ap_image_features[layer_name][0])
layer_features = self.get_layer_output(layer_name)
layer_shape = self.get_layer_output_shape(layer_name)
# current combined output
combination_features = layer_features[0, :, :, :]
nsl = neural_style_loss(ap_image_features, combination_features, 3, self.output_shape[-2], self.output_shape[-1])
loss += (self.args.neural_style_weight / len(self.args.neural_style_layers)) * nsl
return loss
