def style_loss(x, targ): return metrics.mse(gram_matrix(x), gram_matrix(targ)) # In[157]: loss = sum(style_loss(l1[0], l2[0]) for l1,l2 in zip(layers, targs))
def content_recreate(): ''' returns an image of recreated content ''' model = VGG16_Avg(include_top=False) layer = model.get_layer('block5_conv1').output layer_model = Model(model.input, layer) targ = K.variable(layer_model.predict(img_arr)) loss = metrics.mse(layer, targ) grads = K.gradients(loss, model.input) function_input = [model.input] function_output = ([loss] + grads) fn = K.function(function_input, function_output) evaluator = Evaluator(fn, img_arr.shape) x = rand_img(img_arr.shape) content_iterations = 10 x_final, content_loss_history = solve_image(evaluator, content_iterations, x, path=content_result_path) c_path = content_result_path + '/res_at_iteration_9.png' return c_path
def loss(y_true, y_pred): reconstruction_loss = mse(y_true, y_pred) reconstruction_loss *= original_dimension kl_loss = -0.5 * K.sum( 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1) return K.mean(reconstruction_loss + kl_loss)
def merge(): ''' returns an image of the neural style transfer ''' input_shape = style_arr.shape[1:] model = VGG16_Avg(include_top=False, input_shape=input_shape) outputs = {l.name: l.output for l in model.layers} style_layers = [outputs['block{}_conv1'.format(o)] for o in range(1, 6)] content_layer = outputs['block4_conv2'] style_model = Model(model.input, style_layers) style_targs = [K.variable(o) for o in style_model.predict(style_arr)] content_model = Model(model.input, content_layer) content_targ = K.variable(content_model.predict(img_arr)) alpha = 0.1 beta = 0.00001 gama = 0.000001 st_loss = sum( style_loss(l1[0], l2[0]) for l1, l2 in zip(style_layers, style_targs)) loss = alpha * metrics.mse( content_layer, content_targ ) + beta * st_loss + gama * total_variation_loss(model.input) grads = K.gradients(loss, model.input) transfer_fn = K.function([model.input], [loss] + grads) evaluator = Evaluator(transfer_fn, shp) merge_iterations = 10 x = rand_img(shp) x, merge_loss_history = solve_image(evaluator, merge_iterations, x, merge_result_path)
def vae_loss(inp, out): repr_loss = self.k * mse(inp, out) kl_loss = ( -0.5 * self.beta * K.mean(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1) ) return K.mean(repr_loss + kl_loss)
def target_MSE(y_true, y_pred, val=False): tars = y_true[:, :, data_dim] preds = y_pred[:, :, data_dim] if not val: return metrics.mse(tars, preds) else: #This only works for a single example at a time #print(tars) #print(preds) tars = tars[0, 0] preds = preds[0, 0] #return mean_squared_error(tars, preds) return np.mean(np.square(tars - preds))
def vae_loss(y_true, y_pred): ''' Negative variational lower bound used as loss function for training the variational auto-encoder. ''' # Reconstruction loss rc_loss = metrics.mse(K.flatten(image), K.flatten(t_decoded)) #binary_crossentropy #rc_loss *= img_size * img_size if wandb.config.variational: # Regularization term (KL divergence) kl_loss = -0.5 * K.sum(1 + t_log_var - K.square(t_mean) - K.exp(t_log_var), axis=-1) # Average over mini-batch return K.mean(rc_loss + kl_loss) else: return K.mean(rc_loss)
def do_style_transfer(self): # Load source image im = self.load_image(self.original_image_path) img_arr = self.preproc(np.expand_dims(im, axis=0)) shp = img_arr.shape # Load style image style = self.load_image(self.style_image_path) style = style.resize((shp[1], shp[2]), resample=Image.ANTIALIAS ) # Make style and original image the same size style_arr = self.preproc(np.expand_dims(style, 0)[:, :, :, :3]) self.content_layer = self.outputs[self.content_layer_name] self.style_layers = [ self.outputs['block{}_conv2'.format(o)] for o in self.style_loss_conv_blocks ] # Content model content_model = Model(self.model.input, self.content_layer) content_targ = K.variable(content_model.predict(img_arr)) # Style model # Single input but multiple outputs style_model = Model(self.model.input, self.style_layers) style_targs = [K.variable(o) for o in style_model.predict(style_arr)] # Compute style loss between actual and generated for all layers - use the style_wgts for each layer self.style_loss = sum( self.get_style_loss(l1[0], l2[0]) * w for l1, l2, w in zip( self.style_layers, style_targs, self.style_wgts)) # Compute content loss self.content_loss = K.mean( metrics.mse(self.content_layer, content_targ)) # Compute total loss self.total_loss = self.style_loss + self.lambda_coeff * self.content_loss grads = K.gradients(self.total_loss, self.model.input) transfer_fn = K.function([self.model.input], [self.total_loss] + grads) evaluator = Evaluator(transfer_fn, shp) x = self.rand_img(shp) x = self.solve_image(evaluator, x, shp)
outputs = {l.name: l.output for l in vgg_model.layers} style_layers = [outputs['block{}_conv2'.format(o)] for o in range(1,6)] content_name = 'block4_conv2' content_layer = outputs[content_name] style_model = Model(vgg_model.input, style_layers) style_targs = [K.variable(o) for o in style_model.predict(style_image_array)] content_model = Model(vgg_model.input, content_layer) content_targ = K.variable(content_model.predict(src)) style_wgts = [0.05, 0.2, 0.2, 0.25, 0.3] loss = sum(style_loss(l1[0], l2[0])*w for l1, l2, w in zip(style_layers, style_targs, style_wgts)) loss += metrics.mse(content_layer, content_targ)/10 grads = tf.keras.backend.gradients(loss, vgg_model.input) transfer_fn = tf.keras.backend.function([vgg_model.input], [loss]+grads) evaluator = ConvexOptimiser(transfer_fn, style_image_shape) enerated_image = generate_rand_img(style_image_shape) generated_image = optimise(optimiser, iterations, generated_image, style_image_shape) # Content plus style transfer style_width, style_height = style_image.size content_image_array = content_image_array[:, :style_height, :style_width]
def style_mse_loss(x, y): return metrics.mse(grammian_matrix(x), grammian_matrix(y))
layer = vgg.get_layer('block5_conv1').output # %% Calculate target activations. ie The output given a single image layer_model = Model(vgg.input, layer) pred = K.variable(layer_model.predict(img_arr)) # %% Define class that handles loss and gradient outputs class Evaluator(object): def __init__(self, fnc, shape): self.fnc, self.shape = fnc, shape def loss(self, img): loss_, self.grads_ = self.fnc([img.reshape(self.shape)]) return loss_.astype(np.float64) def grads(self, img): return self.grads_.flatten().astype(np.float64) # %% Define loss, gradients loss = K.mean(metrics.mse(layer, pred)) # Actual layer is y_true grads = K.gradients(loss, vgg.input) fnc = K.function([vgg.input], [loss]+grads) evaluator = Evaluator(fnc, shape) # %% Perform deterministic optimization on img def solve_img(eval_obj, n_iter, x, out_shape, path): for i in range(n_iter): x, min_val, info = fmin_l_bfgs_b(eval_obj.loss, x.flatten(), fprime=eval_obj.grads, maxfun=20) x = np.clip(x, -127, 127) print('Current loss value %s/%s:' % (i+1, n_iter), min_val) imsave(f'{path}at_iteration_{i}.png', deproc(x.copy(), out_shape)[0])
def lossFunction(y_true, y_pred) : loss = mse(y_true, y_pred) loss += K.prod(val, K.sum(K.abs(weights)))
# latent_rep = E(X)[0] # output = G(latent_rep) E_mean, E_logsigma, Z = E(X) # Z = Input(shape=(512,)) # Z2 = Input(shape=(batch_size, 512)) output = G(Z) G_dec = G(E_mean + E_logsigma) D_fake, F_fake = D(output) D_fromGen, F_fromGen = D(G_dec) D_true, F_true = D(X) VAE = Model(X, output) kl = - 0.5 * K.sum(1 + E_logsigma - K.square(E_mean) - K.exp(E_logsigma), axis=-1) crossent = 64 * metrics.mse(K.flatten(X), K.flatten(output)) VAEloss = K.mean(crossent + kl) VAE.add_loss(VAEloss) VAE.compile(optimizer=SGDop) for epoch in range(epochs): latent_vect = E.predict(dataset)[0] encImg = G.predict(latent_vect) fakeImg = G.predict(noise) DlossTrue = D_true.train_on_batch(dataset, np.ones((batch_size, 1))) DlossEnc = D_fromGen.train_on_batch(encImg, np.ones((batch_size, 1))) DlossFake = D_fake.train_on_batch(fakeImg, np.zeros((batch_size, 1))) cnt = epoch while cnt > 3:
def build_go(self, gene_list, go2genes, genes2go, vertices, edges, number_of_neurons, latent_dim, var_th_index): regex = re.compile( r"[\s, \,, \+, \:, \- ,\(,\, \), \' , \[ , \], \=, \<, \>]", re.IGNORECASE) count = 0 gene_list = [x[:x.index(".")] for x in gene_list] genes2input = {} # print "prepare input layer" # for k,v in genes2go.iteritems(): # count+=1 # print count # e2e_id = entrez2ensembl_convertor([k]) # if len(e2e_id)==0 or e2e_id[0] not in gene_list: continue # genes2input[k]= Input(shape=(1,), name="{}_{}_{}".format(e2e_id[0],str(k),"input") ) # Dense(1, )(Input(shape=(1,))) input_ensembl_ids = [] print "connect input layer to GO leafs" for k, v in vertices.iteritems(): cur_layer = [] if v["n_children"] == 0 or True: # probably genes would be connect to upper layers for cur_entrez in go2genes[k]: if not genes2input.has_key(cur_entrez): e2e_id = entrez2ensembl_convertor([cur_entrez]) if len(e2e_id) != 0 and e2e_id[0] in gene_list: input_ensembl_ids.append(e2e_id[0]) genes2input[cur_entrez] = Input( shape=(1, ), name="{}_{}_{}".format(e2e_id[0], str(cur_entrez), "input")) if genes2input.has_key(cur_entrez): cur_layer.append(cur_entrez) go_name = regex.sub(app_config["go_separator"], v["name"]) + "_converged" v["neuron_converged"] = Dense( number_of_neurons, activation=app_config['activation_function'], name=go_name) v["neuron_converged_inputs"] = cur_layer print "save input layer as list " + os.path.join( constants.LIST_DIR, "{}_{}_{}_{}".format( var_th_index, number_of_neurons, latent_dim, app_config["actual_vae_input_genes_file_name"])) file( os.path.join( constants.LIST_DIR, "{}_{}_{}_{}".format( var_th_index, number_of_neurons, latent_dim, app_config["actual_vae_input_genes_file_name"])), 'w+').write("\n".join(input_ensembl_ids)) print "connect intermediate converged GO layers" for k, v in sorted([(k, v) for k, v in vertices.iteritems()], key=lambda x: max(x[1]["depth"]), reverse=True): if not v.has_key("neuron_converged"): go_name = regex.sub(app_config["go_separator"], v["name"] + "_converged") v["neuron_converged"] = Dense( number_of_neurons, activation=app_config['activation_function'], name=go_name) v["neuron_converged_inputs"] = [] inputs = [ cur_child.id for cur_child in v["obj"].children if vertices[cur_child.id].has_key("neuron_converged") ] v["neuron_converged_inputs"] = v["neuron_converged_inputs"] + inputs for k, v in sorted([(k, v) for k, v in vertices.iteritems()], key=lambda x: max(x[1]["depth"]), reverse=True): if v.has_key("neuron_converged") and v.has_key( "neuron_converged_inputs"): inputs = [genes2input[x] for x in v["neuron_converged_inputs"] if genes2input.has_key(x)] + \ [vertices[x]["neuron_converged"] for x in v["neuron_converged_inputs"] if vertices.has_key(x)] if len(inputs) == 0: del vertices[k] if len(inputs) == 1: v["neuron_converged"] = BatchNormalization()( v["neuron_converged"](inputs[0])) if len(inputs) > 1: v["neuron_converged"] = BatchNormalization()( v["neuron_converged"](concatenate(inputs))) # print [min(x["depth"]) for x in [v for k, v in vertices.iteritems()]] roots = [ vertices[x] for x in [k for k, v in vertices.iteritems() if min(v["depth"]) == 1] ] print "num of roots: {}".format(len(roots)) if app_config["is_variational"]: print "root and sampling layers" inputs = [r["neuron_converged"] for r in roots] if len(inputs) > 1: z_mean = BatchNormalization()(Dense(latent_dim, name="z_mean")( concatenate(inputs))) z_log_var = BatchNormalization()(Dense( latent_dim, name="z_log_var")(concatenate(inputs))) else: z_mean = BatchNormalization()(Dense(latent_dim, name="z_mean")(inputs[0])) z_log_var = BatchNormalization()(Dense(latent_dim, name="z_log_var")( inputs[0])) z = BatchNormalization()(Lambda( self.sampling, output_shape=(latent_dim, ), name='z', arguments={"latent_dim": latent_dim})([z_mean, z_log_var])) for r in roots: go_name = regex.sub(app_config["go_separator"], r["name"] + "_diverged") r['neuron_diverged'] = BatchNormalization()(Dense( number_of_neurons, activation=app_config['activation_function'], name=go_name)(z)) else: for r in roots: r['neuron_diverged'] = r['neuron_converged'] print "connect intermediate diverged GO layers" neuron_count = 0 is_converged = False while not is_converged: is_converged = True for k, v in sorted([(k, v) for k, v in vertices.iteritems()], key=lambda x: max(x[1]["depth"]), reverse=False): if v.has_key("neuron_diverged"): continue inputs = [ vertices[cur_parent]["neuron_diverged"] for cur_parent in v["obj"]._parents if vertices.has_key(cur_parent) and vertices[cur_parent].has_key("neuron_diverged") ] go_name = regex.sub(app_config["go_separator"], v["name"] + "_diverged") if len(inputs) == 1: v["neuron_diverged"] = BatchNormalization()(Dense( number_of_neurons, activation=app_config['activation_function'], name=go_name)(inputs[0])) neuron_count += 1 is_converged = False if len(inputs) > 1: v["neuron_diverged"] = BatchNormalization()(Dense( number_of_neurons, activation=app_config['activation_function'], name=go_name)(concatenate(inputs))) neuron_count += 1 is_converged = False print neuron_count print "intermediate layers have {} neurons".format(neuron_count) genes2output = {} print "connect input layer to output leafs" for k, v in genes2go.iteritems(): count += 1 # print count e2e_id = entrez2ensembl_convertor([k]) if len(e2e_id) == 0 or e2e_id[0] not in gene_list: continue neuron_parents = [] for cur_go_term in genes2go[k]: if vertices.has_key( cur_go_term ): # and len(vertices[cur_go_term]["obj"].children) ==0: neuron_parents.append( vertices[cur_go_term]["neuron_diverged"]) if len(neuron_parents) == 1: genes2output[k] = Dense(1, activation='sigmoid', name="{}_{}_{}".format( e2e_id[0], str(k), "output"))(neuron_parents[0]) if len(neuron_parents) > 1: genes2output[k] = Dense(1, activation='sigmoid', name="{}_{}_{}".format( e2e_id[0], str(k), "output"))( concatenate(neuron_parents)) # self.vae = Model([v for k,v in genes2input.iteritems()], root['neuron_converged']) model_inputs = [] model_outputs = [] for k in genes2output.keys(): model_inputs.append(genes2input[k]) model_outputs.append(genes2output[k]) assert len(model_inputs) == len( model_outputs), "different # of inputs and outputs" concatenated_inputs = concatenate(model_inputs) concatenated_outputs = concatenate(model_outputs) # instantiate encoder model self.encoder = Model(model_inputs, [z_mean, z_log_var, z], name='encoder') # self.encoder.summary() plot_model(self.encoder, to_file=os.path.join(constants.OUTPUT_GLOBAL_DIR, "encoder_{}.svg".format(time.time()))) # self.decoder = Model() # Overall VAE model, for reconstruction and training self.vae = Model(model_inputs, model_outputs) # concatenated_outputs # print self.vae.summary() if app_config["loss_function"] == "mse": reconstruction_loss = len(model_inputs) * metrics.mse( concatenated_inputs, concatenated_outputs) if app_config["loss_function"] == "cross_ent": reconstruction_loss = len( model_inputs) * metrics.binary_crossentropy( concatenated_inputs, concatenated_outputs) if app_config["is_variational"]: kl = -0.5 * K.sum( 1 + z_log_var - K.exp(z_log_var) - K.square(z_mean), axis=1) self.vae.add_loss(K.mean(reconstruction_loss + kl)) # weighting? average? else: self.vae.add_loss(reconstruction_loss) # for i in range(len(model_inputs)): # self.vae.add_loss(metrics.binary_crossentropy(model_inputs[i], model_outputs[i])) # loss = metrics.binary_crossentropy(concatenated_inputs, concatenated_outputs) # loss = {} # for i in range(len(model_inputs)): # loss[str(model_outputs[i].name[:model_outputs[i].name.index("/")])] = 'binary_crossentropy' # \ # metrics.binary_crossentropy(model_inputs[i], model_outputs[i]) # self.vae.add_loss(loss) self.vae.compile(optimizer='rmsprop') # , loss=loss print "number of inputs: {}".format(len(model_inputs)) print "number of outputs: {}".format(len(model_outputs)) plot_model(self.vae, to_file=os.path.join(constants.OUTPUT_GLOBAL_DIR, "model_{}.svg".format(time.time())))
class Evaluator(object): def __init__(self, f, shp): self.f, self.shp = f, shp def loss(self, x): loss_, self.grad_values = self.f([x.reshape(self.shp)]) return loss_.astype(np.float64) def grads(self, x): return self.grad_values.flatten().astype(np.float64) # In[29]: loss = metrics.mse(layer, target) grads = K.gradients(loss, model.input) fn = K.function([model.input], [loss] + grads) evaluator = Evaluator(fn, shape_content) # In[8]: def solve_image(eval_obj, niter, x, path): for i in range(niter): x, min_val, info = fmin_l_bfgs_b(eval_obj.loss, x.flatten(), fprime=eval_obj.grads, maxfun=20) x = np.clip(x, -127, 127) print('Minimum Loss Value:', min_val)
def get_style_loss(self, x, targ): # MSE between the 2 gram matrices return K.mean(metrics.mse(self.gram_matrix(x), self.gram_matrix(targ)))
style_layers = [outputs['block{}_conv2'.format(o)] for o in range(1, 6)] content_name = 'block4_conv2' content_layer = outputs[content_name] style_model = Model(vgg_model.input, style_layers) style_targs = [K.variable(o) for o in style_model.predict(style_image_array)] content_model = Model(vgg_model.input, content_layer) content_targ = K.variable(content_model.predict(src)) style_wgts = [0.05, 0.2, 0.2, 0.25, 0.3] loss = sum( style_loss(l1[0], l2[0]) * w for l1, l2, w in zip(style_layers, style_targs, style_wgts)) loss += metrics.mse(content_layer, content_targ) / 10 grads = tf.keras.backend.gradients(loss, vgg_model.input) transfer_fn = tf.keras.backend.function([vgg_model.input], [loss] + grads) evaluator = ConvexOptimiser(transfer_fn, style_image_shape) enerated_image = generate_rand_img(style_image_shape) generated_image = optimise(optimiser, iterations, generated_image, style_image_shape) # Content plus style transfer style_width, style_height = style_image.size content_image_array = content_image_array[:, :style_height, :style_width] style_layers_2 = [ style_layers['block{}_conv2'.format(block_no)] for block_no in range(1, 6)
content_layer = outputs['block4_conv2'] #gram matrix is a matrix collect the correlation of all of the vectors #in a set. Check wiki(https://en.wikipedia.org/wiki/Gramian_matrix) for more details def gram_matrix(x): #change height,width,depth to depth, height, width, it could be 2,1,0 too #maybe 2,0,1 is more efficient due to underlying memory layout features = K.permute_dimensions(x, (2,0,1)) #batch flatten make features become 2D array features = K.batch_flatten(features) return K.dot(features, K.transpose(features)) / x.get_shape().num_elements() def style_loss(x, targ): return metrics.mse(gram_matrix(x), gram_matrix(targ)) content_loss = lambda base, gen: metrics.mse(gen, base) #l[1] is the output(activation) of style_base, l[2] is the output of gen_img loss = sum([style_loss(l[1], l[2]) for l in style_layers]) #loss of style image and gen_img #content_layer[0] is the output of content_base, content_layer[2] is the output of gen_img loss += content_loss(content_layer[0], content_layer[2]) / 10. #loss of content image and gen_img #The loss need two variables but we only pass in one, #because we only got one placeholder in the graph, #the other variable already determine by K.variable grad = K.gradients(loss, gen_img) fn = K.function([gen_img], [loss] + grad) #fn will return loss and grad, but fmin_l_bfgs need to seperate them #that is why we need a class to separate loss and gradient and store them
def ConvolutionalVAE(img_rows, img_cols, img_chans, mse=True, external_model=None): filters = 16 kernel_size = 3 latent_dim = 100 original_img_size = (img_rows, img_cols, img_chans) # ------------------------------------------------------------------------------------------------------------------ # Encoder: inputs = Input(shape=original_img_size, name='encoder_input') x = inputs for i in range(2): filters *= 2 x = Conv2D(filters=filters, kernel_size=kernel_size, activation='relu', strides=2, padding='same')(x) encoder_shape = K.int_shape(x) x = Flatten()(x) # z_mean = z_log_var = z = (None, latent_dim) z_mean = Dense(latent_dim, name='z_mean')(x) z_log_var = Dense(latent_dim, name='z_log_var')(x) ## use reparameterization trick to push the sampling out as input z = Lambda(var_sampling, output_shape=(latent_dim,), name='z')([z_mean, z_log_var]) #encoder = Model(inputs=inputs, outputs = [z_mean, z_log_var, z], name='encoder') encoder = Model(inputs=inputs, outputs=z, name='encoder') encoder.summary() # ------------------------------------------------------------------------------------------------------------------ # Decoder: latent_inputs = Input(shape=(latent_dim, ), name='z_sampling') x = Dense(encoder_shape[1]*encoder_shape[2]*encoder_shape[3], activation='relu')(latent_inputs) x = Reshape((encoder_shape[1], encoder_shape[2], encoder_shape[3]))(x) for i in range(2): x = Conv2DTranspose(filters=filters, kernel_size=kernel_size, activation='relu', strides=2, padding='same')(x) filters //= 2 outputs = Conv2DTranspose(filters=img_chans, kernel_size=kernel_size, activation='sigmoid', padding='same', name='decoder_output')(x) decoder = Model(inputs=latent_inputs, outputs=outputs, name='decoder') decoder.summary() pred_outputs = decoder(encoder(inputs)) reconstruction_loss = metrics.mse(K.flatten(pred_outputs), K.flatten(inputs)) * img_rows * img_cols # KL(p(z|x), p(z = -1/2*(e^d*u^2)/d)) kl_loss = K.sum(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1) * -0.5 #vae_loss = K.mean(sigmoid(reconstruction_loss) + sigmoid(kl_loss)) vae_loss = K.mean(reconstruction_loss + kl_loss) external_loss = 0 if external_model: external_model.summary() # entropy_loss: our goal is to maxmimize entropy for gender prediction, # so we negate its original definition # p_x = external_model(pred_outputs) # entropy_gender_loss = -K.mean(-p_x * K.log(p_x)) # vae_loss += entropy_gender_loss # penalize similarity between gender predictions for original and reconstructed images external_loss = K.mean(metrics.mse(external_model(pred_outputs), (1. - external_model(inputs)))) #reranking loss as in Adversarial Transformation Networks #external_l = external_loss(external_model) vae_loss = vae_loss + 100.*external_loss vae = Model(inputs=inputs, outputs=pred_outputs, name='vae') vae.add_loss(vae_loss) vae.compile(optimizer='adam') return vae, encoder, decoder
def vae_loss(x, x_decoded_mean): reconstruction_loss = metrics.mse(x, x_decoded_mean) * self.input_dim kl_loss = -0.5 * K.mean( 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1) return reconstruction_loss + kl_loss
model = VGG16_Avg(include_top=False, input_shape=shp[1:]) outputs = {l.name: l.output for l in model.layers} style_layers = [outputs['block{}_conv2'.format(o)] for o in range(1,6)] content_layer = outputs['block4_conv2'] style_model = Model(model.input, style_layers) style_targs = [K.variable(o) for o in style_model.predict(style_arr)] content_model = Model(model.input, content_layer) content_targ = K.variable(content_model.predict(img_arr)) style_wgts = [0.05,0.2,0.2,0.25,0.3] loss = sum(style_loss(l1[0], l2[0])*w for l1,l2,w in zip(style_layers, style_targs, style_wgts)) loss += K.mean(metrics.mse(content_layer, content_targ))/10 grads = K.gradients(loss, model.input) transfer_fn = K.function([model.input], [loss]+grads) evaluator = Evaluator(transfer_fn, shp) iterations=15 x = rand_img(shp) x = solve_image(evaluator, iterations, x)
# discriminator training model discriminator_model = Model(disc_input, disc_out, name='discriminator_model') discriminator_model.compile(loss='binary_crossentropy', optimizer=optimizer) print("DISCRIMINATOR TRAINER:") discriminator_model.summary() # decoder training model (GAN) discriminator.trainable = False outputs = discriminator(decoder(latent_inputs)) generator_model = Model(latent_inputs, outputs, name='generator_model') generator_model.compile(loss='binary_crossentropy', optimizer=optimizer) print("DECODER TRAINER (GENERATOR):") generator_model.summary() # encoder & decoder training model (VAE) outputs = decoder(encoder(inputs)[2]) vae_model = Model(inputs, outputs) xent_loss = 64 * 64 * metrics.mse(K.flatten(inputs), K.flatten(outputs)) kl_loss = -0.5 * K.sum(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1) vae_loss = K.mean(xent_loss + kl_loss) vae_model.add_loss(vae_loss) vae_model.compile(optimizer=optimizer) print("ENCODER&DECODER TRAINER (VAE):") vae_model.summary() if (len(sys.argv) == 1 or sys.argv[1] == 'continue'): if (len(sys.argv) > 1 and sys.argv[1] == 'continue'): encoder.load_weights('c_vae_gan_enc.h5') decoder.load_weights('c_vae_gan_dec.h5') discriminator.load_weights('c_vae_gan_disc.h5') X_train = []
class Evaluator(object): def __init__(self, f, shp): self.f, self.shp = f, shp def loss(self, x): loss_, self.grad_values = self.f([x.reshape(self.shp)]) return loss_.astype(np.float64) def grads(self, x): return self.grad_values.flatten().astype(np.float64) # We'll define our loss function to calculate the mean squared error between the two outputs at the specified convolutional layer. # In[32]: loss = metrics.mse(layer, targ) grads = K.gradients(loss, model.input) fn = K.function([model.input], [loss]+grads) evaluator = Evaluator(fn, shp) # Now we're going to optimize this loss function with a deterministic approach to optimization that uses a line search, which we can implement with sklearn's `fmin_l_bfgs_b` funtionc. # In[33]: def solve_image(eval_obj, niter, x): for i in range(niter): x, min_val, info = fmin_l_bfgs_b(eval_obj.loss, x.flatten(), fprime=eval_obj.grads, maxfun=20) x = np.clip(x, -127,127)
def encoder_loss(y_true, y_pred): recon_loss = 64 * 64 * metrics.mse(K.flatten(y_true), K.flatten(y_pred)) kl_loss = -0.5 * K.sum(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1) return K.mean(recon_loss + kl_loss)
def vae_loss(model_inputs, model_outputs): _, target_sample = model_inputs recon_sample, z_mean, z_log_var = model_outputs mse_loss = float(K.shape(target_sample)[1] * K.shape(target_sample)[2]) * metrics.mse(target_sample, recon_sample) kl_loss = - 0.5 * K.sum(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1) return mse_loss + kl_loss
def style_loss(x, targ): ''' returns style loss by comparing the input image and target image the factor 1/(4*(N**N)* (M*M)) is calculated in the gram_matrix simplify style_loss ''' return metrics.mse(gram_matrix(x), gram_matrix(targ))
def style_loss(x, targ): return K.mean(metrics.mse(gram_matrix(x), gram_matrix(targ))) def gram_matrix(x):
def vanila_autoencoder_loss(model_inputs, model_outputs): _, target_sample = model_inputs recon_sample, _ = model_outputs return float(K.shape(target_sample)[1] * K.shape(target_sample)[2]) * metrics.mse(target_sample, recon_sample)
def style_loss(x, targ): return metrics.mse(gram_matrix(x), gram_matrix(targ))
def ConvVAE(num_filters, kernel_sizes, num_z, input_shape=kDefaultInputShape): assert (len(num_filters) == len(kernel_sizes)) last_kernel_size = input_shape[0] for sk in kernel_sizes: last_kernel_size -= (sk - 1) assert(0 < last_kernel_size) num_layers = len(num_filters) # encoder encoder_settings = [(num_filters[i], kernel_sizes[i]) for i in range(1, num_layers)] encoder = Sequential() encoder.add(Conv2D(num_filters[0], (kernel_sizes[0], input_shape[1]), input_shape=input_shape)) encoder.add(BatchNormalization()) encoder.add(Activation('relu')) for (num_fs, sz_k) in encoder_settings: encoder.add(Conv2D(num_fs, (sz_k, 1))) encoder.add(BatchNormalization()) encoder.add(Activation('relu')) # output layer of encoder encoder.add(Conv2D(num_z, (last_kernel_size, 1))) encoder.add(BatchNormalization()) encoder.add(Activation('linear', name='latent')) encoder.summary() # latent space input_sample = Input(shape=input_shape) z_mean = encoder(input_sample) z_log_var = encoder(input_sample) z = Lambda(sampling)([z_mean, z_log_var]) # for deconvolutional layers dec_num_filters = num_filters[::-1] + [input_shape[2]] dec_kernel_sizes = [last_kernel_size] + kernel_sizes[::-1] dec_channels = [1 if i < num_layers else input_shape[1] for i in range(num_layers + 1)] decoder_settings = [(dec_num_filters[i], dec_kernel_sizes[i], dec_channels[i]) for i in range(len(dec_num_filters))] # decoder decoder = Sequential() for i, (num_fs, sz_k, ch) in enumerate(decoder_settings): if i == 0: decoder.add(Conv2DTranspose(num_fs, (sz_k, ch), input_shape=(1, 1, num_z))) else: decoder.add(Conv2DTranspose(num_fs, (sz_k, ch))) if i + 1 == num_layers: decoder.add(Activation('linear')) else: decoder.add(Activation('relu')) # reconstruction recon_sample = decoder(z) target_sample = Input(shape=input_shape) decoder.summary() # instantiate VAE model vae = Model(inputs=[input_sample, target_sample], outputs=[recon_sample, z, z_mean, z_log_var]) vae.summary() # Compute VAE loss mse_loss = float(input_shape[0]) * float(input_shape[1]) * metrics.mse(target_sample, recon_sample) kl_loss = - 0.5 * K.sum(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1) vae_loss = K.mean(mse_loss + kl_loss) vae.add_loss(vae_loss) return vae
def reconstruction_loss(x_input, x_decoded): return metrics.mse(x_input, x_decoded)