class DeepQ(object): """Constructs the desired deep q learning network""" def __init__(self, action_size, observation_size, lr=LEARNING_RATE): self.action_size = action_size self.observation_size = observation_size self.lr = lr self.model = None self.target_model = None self.qvalue_evolution = [] self.construct_q_network() def construct_q_network(self): """ Construct both the actual Q-network and the target network with three hidden layers and ReLu activation functions in between. The network uses an Adam optimizer with MSE loss.""" self.model = Sequential() input_layer = Input(shape=(self.observation_size * NUM_FRAMES, )) layer1 = Dense(self.observation_size * NUM_FRAMES)(input_layer) layer1 = Activation('relu')(layer1) layer3 = Dense(self.observation_size)(layer1) layer3 = Activation('relu')(layer3) layer4 = Dense(2 * self.action_size)(layer3) layer4 = Activation('relu')(layer4) output = Dense(self.action_size)(layer4) self.model = Model(inputs=[input_layer], outputs=[output]) self.model.compile(loss='mse', optimizer=Adam(lr=self.lr)) self.target_model = Model(inputs=[input_layer], outputs=[output]) self.target_model.compile(loss='mse', optimizer=Adam(lr=self.lr)) self.target_model.set_weights(self.model.get_weights()) def predict_movement(self, data, epsilon): """ Predict the next action from the network. Epsilon is the probability of making a random move. Returns the optimal action and the predicted reward for that action. """ rand_val = np.random.random() q_actions = self.model.predict(data.reshape( 1, self.observation_size * NUM_FRAMES), batch_size=1) if rand_val < epsilon: opt_policy = np.random.randint(0, self.action_size) else: opt_policy = np.argmax(np.abs(q_actions[0])) self.qvalue_evolution.append(q_actions[0][opt_policy]) return opt_policy, q_actions[0][opt_policy] def predict_rewards(self, data): """ Like predict_movement, only without a probability of a random move and returns only the predicted q-values.""" q_actions = self.model.predict(np.array(data).reshape( 1, self.observation_size * NUM_FRAMES), batch_size=1) return q_actions[0] def train(self, s_batch, a_batch, r_batch, d_batch, s2_batch): """ Trains the network on a batch of input. The parameters are batches of states, actions, rewards, done booleans and next states. """ batch_size = s_batch.shape[0] # Train according to the Bellman Equation targets = self.model.predict(s_batch, batch_size=batch_size) fut_action = self.target_model.predict(s2_batch, batch_size=batch_size) targets[:, a_batch.flatten()] = r_batch targets[d_batch, a_batch[d_batch]] += DISCOUNT_RATE * np.max( fut_action[d_batch], axis=-1) targets_ts = tf.convert_to_tensor(targets, dtype=tf.float32) loss = self.model.train_on_batch(s_batch, targets_ts) return loss def train_imitation(self, s_batch, t_batch): """ Trains network on generated data: Imitation Learning. """ loss = self.model.train_on_batch(s_batch, t_batch) return loss def save_network(self, path): if not os.path.exists(path): os.mkdir(path) self.model.save(os.path.join(path, 'network.h5')) print("Successfully saved network.") def load_network(self, path): self.model = load_model(os.path.join(path, 'network.h5')) print("Successfully loaded network.") def target_train(self): """ The target network is updated each step by 'merging' a small part of the actual network into it. """ model_weights = self.model.get_weights() target_model_weights = self.target_model.get_weights() for i in range(len(model_weights)): target_model_weights[i] = TAU * model_weights[i] + ( 1 - TAU) * target_model_weights[i] self.target_model.set_weights(target_model_weights)
def layer_test(layer_cls, kwargs={}, input_shape=None, input_dtype=None, input_data=None, expected_output=None, expected_output_dtype=None, fixed_batch_size=False, supports_masking=False): # generate input data if input_data is None: if not input_shape: raise AssertionError() if not input_dtype: input_dtype = K.floatx() input_data_shape = list(input_shape) for i, e in enumerate(input_data_shape): if e is None: input_data_shape[i] = np.random.randint(1, 4) input_mask = [] if all(isinstance(e, tuple) for e in input_data_shape): input_data = [] for e in input_data_shape: input_data.append( (10 * np.random.random(e)).astype(input_dtype)) if supports_masking: a = np.full(e[:2], False) a[:, :e[1] // 2] = True input_mask.append(a) else: input_data = (10 * np.random.random(input_data_shape)) input_data = input_data.astype(input_dtype) if supports_masking: a = np.full(input_data_shape[:2], False) a[:, :input_data_shape[1] // 2] = True print(a) print(a.shape) input_mask.append(a) else: if input_shape is None: input_shape = input_data.shape if input_dtype is None: input_dtype = input_data.dtype if expected_output_dtype is None: expected_output_dtype = input_dtype # instantiation layer = layer_cls(**kwargs) # test get_weights , set_weights at layer level weights = layer.get_weights() layer.set_weights(weights) try: expected_output_shape = layer.compute_output_shape(input_shape) except Exception: expected_output_shape = layer._compute_output_shape(input_shape) # test in functional API if isinstance(input_shape, list): if fixed_batch_size: x = [Input(batch_shape=e, dtype=input_dtype) for e in input_shape] if supports_masking: mask = [ Input(batch_shape=e[0:2], dtype=bool) for e in input_shape ] else: x = [Input(shape=e[1:], dtype=input_dtype) for e in input_shape] if supports_masking: mask = [Input(shape=(e[1], ), dtype=bool) for e in input_shape] else: if fixed_batch_size: x = Input(batch_shape=input_shape, dtype=input_dtype) if supports_masking: mask = Input(batch_shape=input_shape[0:2], dtype=bool) else: x = Input(shape=input_shape[1:], dtype=input_dtype) if supports_masking: mask = Input(shape=(input_shape[1], ), dtype=bool) if supports_masking: y = layer(Masking()(x), mask=mask) else: y = layer(x) if not (K.dtype(y) == expected_output_dtype): raise AssertionError() # check with the functional API if supports_masking: model = Model([x, mask], y) actual_output = model.predict([input_data, input_mask[0]]) else: model = Model(x, y) actual_output = model.predict(input_data) actual_output_shape = actual_output.shape for expected_dim, actual_dim in zip(expected_output_shape, actual_output_shape): if expected_dim is not None: if not (expected_dim == actual_dim): raise AssertionError("expected_shape", expected_output_shape, "actual_shape", actual_output_shape) if expected_output is not None: assert_allclose(actual_output, expected_output, rtol=1e-3) # test serialization, weight setting at model level model_config = model.get_config() recovered_model = model.__class__.from_config(model_config) if model.weights: weights = model.get_weights() recovered_model.set_weights(weights) _output = recovered_model.predict(input_data) assert_allclose(_output, actual_output, rtol=1e-3) # test training mode (e.g. useful when the layer has a # different behavior at training and testing time). if has_arg(layer.call, 'training'): model.compile('rmsprop', 'mse') model.train_on_batch(input_data, actual_output) # test instantiation from layer config layer_config = layer.get_config() layer_config['batch_input_shape'] = input_shape layer = layer.__class__.from_config(layer_config) # for further checks in the caller function return actual_output
def main(): data_dir = str(pathlib.Path.cwd()) + "/" batch_size = 16 img_shape = (128, 128, 3) print(data_dir) gen = ImageDataGenerator(rescale=1 / 127.5, samplewise_center=True) iters = gen.flow_from_directory( directory=data_dir, # classes=['doraemon'], class_mode=None, color_mode='rgb', target_size=img_shape[:2], batch_size=batch_size, shuffle=True) x_train_batch = next(iters) for i in range(3): plt.imshow(array_to_img(x_train_batch[i])) optimizer = Adam(lr=0.00001, beta_1=0.5) generator = generator_model(img_shape) discriminator = discriminator_model(img_shape) discriminator.trainable = True generator.compile(loss='binary_crossentropy', optimizer=optimizer) discriminator.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) z = Input(shape=(128, )) img = generator(z) discriminator.trainable = False valid = discriminator(img) generator_containing_discriminator = Model(z, valid) generator_containing_discriminator.compile(loss='binary_crossentropy', optimizer=optimizer) #saveディレクトリ model_save_dir = 'models' img_save_dir = 'output_imgs' #5×5生成画像生成 imgs_shape = (4, 4) n_img_samples = np.prod(imgs_shape) sample_seeds = np.random.uniform(-1, 1, (n_img_samples, 128)) hist = [] logs = [] #保存先フォルダ生成 os.makedirs(model_save_dir, exist_ok=True) os.makedirs(img_save_dir, exist_ok=True) Totalstep = 500000 for step, batch in enumerate(iters): if len(batch) < batch_size: continue if step > Totalstep: break half_batch = int(batch_size / 2) z_d = np.random.uniform(-1, 1, (half_batch, 128)) g_pred = generator.predict(z_d) discriminator.trainable = True real_loss = discriminator.train_on_batch(batch[:half_batch], np.ones((half_batch, 1))) fake_loss = discriminator.train_on_batch(g_pred, np.zeros((half_batch, 1))) d_loss = 0.5 * np.add(real_loss, fake_loss) z_g = np.random.uniform(-1, 1, (batch_size, 128)) discriminator.trainable = False g_loss = generator_containing_discriminator.train_on_batch( z_g, np.ones((batch_size, 1))) logs.append({ 'step': step, 'd_loss': d_loss[0], 'acc[%]': 100 * d_loss[1], 'g_loss': g_loss }) print(logs[-1]) if step % 200 == 0: img_path = '{}/generate_{}.png'.format(img_save_dir, step) save_imgs(img_path, generator.predict(sample_seeds), rows=imgs_shape[0], cols=imgs_shape[1]) if step % 5000 == 0: generator.save('{}/generator_{}.hd5'.format(model_save_dir, step)) discriminator.save('{}/discriminator_{}.hd5'.format( model_save_dir, step)) ''' Totalstep=30 generator.load_weights('D:/python/jupyter_notebook/dcgan/models/generator_25000.hd5') for step, batch in enumerate(iters): if len(batch) < batch_size: continue if step > Totalstep: break z_d = np.random.uniform(-1, 1, (batch_size, 100)) g_pred = generator.predict(z_d) img_path = '{}/pred_generate_{}.png'.format(img_save_dir, step) save_imgs(img_path, generator.predict(z_d), rows=imgs_shape[0], cols=imgs_shape[1]) ''' return
def train_frcnn(options): if options.parser == 'pascal_voc': from utils import voc_parser as get_data elif options.parser == 'simple': from utils import simple_parser as get_data else: raise ValueError( "Command line option parser must be one of 'pascal_voc' or 'simple'" ) # pass the settings from the command line, and persist them in the config object C = Config() C.use_horizontal_flips = bool(options.horizontal_flips) C.use_vertical_flips = bool(options.vertical_flips) C.rot_90 = bool(options.rot_90) C.model_path = options.output_weight_path.format(options.network) C.num_rois = int(options.num_rois) if options.network == 'resnet50': C.network = 'resnet50' from utils import rpn_res as rpn from utils import classifier_res as classifier_func from utils import get_img_output_length_res as get_img_output_length from utils import nn_base_res as nn_base elif options.network == 'vgg': C.network = 'vgg' from utils import rpn_vgg as rpn from utils import classifier_vgg as classifier_func from utils import get_img_output_length_vgg as get_img_output_length from utils import nn_base_vgg as nn_base else: print('Not a valid model') raise ValueError # check if weight path was passed via command line if options.input_weight_path: C.base_net_weights = options.input_weight_path else: # set the path to weights based on backend and model C.base_net_weights = get_weight_path(options.network) all_imgs, classes_count, class_mapping = get_data(options.path) if 'bg' not in classes_count: classes_count['bg'] = 0 class_mapping['bg'] = len(class_mapping) C.class_mapping = class_mapping inv_map = {v: k for k, v in class_mapping.items()} print('Training images per class:') pprint.pprint(classes_count) print('Num classes (including bg) = {}'.format(len(classes_count))) config_output_filename = options.config_filename with open(config_output_filename, 'wb') as config_f: pickle.dump(C, config_f) print( 'Config has been written to {}, and can be loaded when testing to ensure correct results' .format(config_output_filename)) # random.shuffle(all_imgs) train_imgs = [s for s in all_imgs if s['imageset'] == 'trainval'] val_imgs = [s for s in all_imgs if s['imageset'] == 'test'] print('Num train samples {}'.format(len(train_imgs))) print('Num val samples {}'.format(len(val_imgs))) data_gen_train = get_anchor_gt(train_imgs, classes_count, C, get_img_output_length, K.backend(), mode='train') data_gen_val = get_anchor_gt(val_imgs, classes_count, C, get_img_output_length, K.backend(), mode='val') if K.backend() == "theano": input_shape_img = (3, None, None) else: input_shape_img = (None, None, 3) img_input = Input(shape=input_shape_img) roi_input = Input(shape=(None, 4)) # define the base network (resnet here, can be VGG, Inception, etc) shared_layers = nn_base(img_input, trainable=True) # define the RPN, built on the base layers num_anchors = len(C.anchor_box_scales) * len(C.anchor_box_ratios) rpn = rpn(shared_layers, num_anchors) classifier = classifier_func(shared_layers, roi_input, C.num_rois, nb_classes=len(classes_count), trainable=True) model_rpn = Model(img_input, rpn[:2]) model_classifier = Model([img_input, roi_input], classifier) # this is a model that holds both the RPN and the classifier, used to load/save weights for the models model_all = Model([img_input, roi_input], rpn[:2] + classifier) try: print('loading weights from {}'.format(C.base_net_weights)) model_rpn.load_weights(C.base_net_weights + "rpn.h5", by_name=True) model_classifier.load_weights(C.base_net_weights + "classifier.h5", by_name=True) except Exception as e: model_rpn.load_weights(C.base_net_weights, by_name=True) model_classifier.load_weights(C.base_net_weights, by_name=True) print('Exception: {}'.format(e)) optimizer = Adam(lr=1e-5, decay=2e-7) optimizer_classifier = Adam(lr=1e-5, decay=2e-7) model_rpn.compile( optimizer=optimizer, loss=[rpn_loss_cls(num_anchors), rpn_loss_regr(num_anchors)]) model_classifier.compile( optimizer=optimizer_classifier, loss=[class_loss_cls, class_loss_regr(len(classes_count) - 1)], metrics={'dense_class_{}'.format(len(classes_count)): 'accuracy'}) model_all.compile(optimizer='sgd', loss='mae') epoch_length = options.epoch_length num_epochs = int(options.num_epochs) iter_num = 0 losses = np.zeros((epoch_length, 5)) rpn_accuracy_rpn_monitor = [] rpn_accuracy_for_epoch = [] start_time = time.time() best_loss = np.Inf print('Starting training') for epoch_num in range(num_epochs): progbar = generic_utils.Progbar(epoch_length) print('Epoch {}/{}'.format(epoch_num + 1, num_epochs)) while True: try: if len(rpn_accuracy_rpn_monitor) == epoch_length and C.verbose: mean_overlapping_bboxes = float( sum(rpn_accuracy_rpn_monitor)) / len( rpn_accuracy_rpn_monitor) rpn_accuracy_rpn_monitor = [] print( 'Average number of overlapping bounding boxes from RPN = {} for {} previous iterations' .format(mean_overlapping_bboxes, epoch_length)) if mean_overlapping_bboxes == 0: print( 'RPN is not producing bounding boxes that overlap the ground truth boxes. ' 'Check RPN settings or keep training.') X, Y, img_data = next(data_gen_train) loss_rpn = model_rpn.train_on_batch(X, Y) P_rpn = model_rpn.predict_on_batch(X) R = rpn_to_roi(P_rpn[0], P_rpn[1], C, K.backend(), use_regr=True, overlap_thresh=0.7, max_boxes=300) # note: calc_iou converts from (x1,y1,x2,y2) to (x,y,w,h) format X2, Y1, Y2, IouS = calc_iou(R, img_data, C, class_mapping) if X2 is None: rpn_accuracy_rpn_monitor.append(0) rpn_accuracy_for_epoch.append(0) continue neg_samples = np.where(Y1[0, :, -1] == 1) pos_samples = np.where(Y1[0, :, -1] == 0) if len(neg_samples) > 0: neg_samples = neg_samples[0] else: neg_samples = [] if len(pos_samples) > 0: pos_samples = pos_samples[0] else: pos_samples = [] rpn_accuracy_rpn_monitor.append(len(pos_samples)) rpn_accuracy_for_epoch.append((len(pos_samples))) if C.num_rois > 1: if len(pos_samples) < C.num_rois // 2: selected_pos_samples = pos_samples.tolist() else: selected_pos_samples = np.random.choice( pos_samples, C.num_rois // 2, replace=False).tolist() try: selected_neg_samples = np.random.choice( neg_samples, C.num_rois - len(selected_pos_samples), replace=False).tolist() except: selected_neg_samples = np.random.choice( neg_samples, C.num_rois - len(selected_pos_samples), replace=True).tolist() sel_samples = selected_pos_samples + selected_neg_samples else: # in the extreme case where num_rois = 1, we pick a random pos or neg sample selected_pos_samples = pos_samples.tolist() selected_neg_samples = neg_samples.tolist() if np.random.randint(0, 2): sel_samples = random.choice(selected_neg_samples) else: sel_samples = random.choice(selected_pos_samples) loss_class = model_classifier.train_on_batch( [X, X2[:, sel_samples, :]], [Y1[:, sel_samples, :], Y2[:, sel_samples, :]]) losses[iter_num, 0] = loss_rpn[1] losses[iter_num, 1] = loss_rpn[2] losses[iter_num, 2] = loss_class[1] losses[iter_num, 3] = loss_class[2] losses[iter_num, 4] = loss_class[3] progbar.update(iter_num + 1, [('rpn_cls', losses[iter_num, 0]), ('rpn_regr', losses[iter_num, 1]), ('detector_cls', losses[iter_num, 2]), ('detector_regr', losses[iter_num, 3])]) iter_num += 1 if iter_num == epoch_length: loss_rpn_cls = np.mean(losses[:, 0]) loss_rpn_regr = np.mean(losses[:, 1]) loss_class_cls = np.mean(losses[:, 2]) loss_class_regr = np.mean(losses[:, 3]) class_acc = np.mean(losses[:, 4]) mean_overlapping_bboxes = float(sum( rpn_accuracy_for_epoch)) / len(rpn_accuracy_for_epoch) rpn_accuracy_for_epoch = [] if C.verbose: print( 'Mean number of bounding boxes from RPN overlapping ground truth boxes: {}' .format(mean_overlapping_bboxes)) print( 'Classifier accuracy for bounding boxes from RPN: {}' .format(class_acc)) print('Loss RPN classifier: {}'.format(loss_rpn_cls)) print('Loss RPN regression: {}'.format(loss_rpn_regr)) print('Loss Detector classifier: {}'.format( loss_class_cls)) print('Loss Detector regression: {}'.format( loss_class_regr)) print('Elapsed time: {}'.format(time.time() - start_time)) curr_loss = loss_rpn_cls + loss_rpn_regr + loss_class_cls + loss_class_regr iter_num = 0 start_time = time.time() if curr_loss < best_loss: if C.verbose: print( f'Total loss decreased from {best_loss:.3f} to {curr_loss:.3f}, saving weights to ' f'{C.model_path}') best_loss = curr_loss model_classifier.save_weights(C.model_path + "classifier.h5") model_rpn.save_weights(C.model_path + "rpn.h5") break except Exception as e: print('Exception: {}'.format(e)) continue print('Training complete, exiting.')
class CartoonGAN(): def __init__(self, args): self.model_name = 'CartoonGAN' self.batch_size = args.batch_size self.epochs = args.epochs self.gpu = args.gpu_num self.image_channels = args.image_channels self.image_size = args.image_size self.init_epoch = args.init_epoch self.log_dir = args.log_dir self.lr = args.lr self.model_dir = args.model_dir self.weight = args.weight # method for generator def generator(self): input_shape = [self.image_size, self.image_size, self.image_channels] input_img = Input(shape=input_shape, name="input") # first block x = ReflectionPadding2D(3)(input_img) x = Conv2D(64, (7, 7), strides=1, use_bias=True, padding='valid', name="conv1")(x) x = InstanceNormalization(name="norm1")(x) x = Activation("relu")(x) # down-convolution channel = 128 for i in range(2): x = Conv2D(channel, (3, 3), strides=2, use_bias=True, padding='same', name="conv{}_1".format(i + 2))(x) x = Conv2D(channel, (3, 3), strides=1, use_bias=True, padding='same', name="conv{}_2".format(i + 2))(x) x = InstanceNormalization(name="norm{}".format(i + 2))(x) x = Activation("relu")(x) channel = channel * 2 # residual blocks x_res = x for i in range(8): x = ReflectionPadding2D(1)(x) x = Conv2D(256, (3, 3), strides=1, use_bias=True, padding='valid', name="conv{}_1".format(i + 4))(x) x = InstanceNormalization(name="norm{}_1".format(i + 4))(x) x = Activation("relu")(x) x = ReflectionPadding2D(1)(x) x = Conv2D(256, (3, 3), strides=1, use_bias=True, padding='valid', name="conv{}_2".format(i + 4))(x) x = InstanceNormalization(name="norm{}_2".format(i + 4))(x) x = Add()([x, x_res]) x_res = x # up-convolution for i in range(2): x = Conv2DTranspose(channel // 2, 3, 2, padding="same", output_padding=1, name="deconv{}_1".format(i + 1))(x) x = Conv2D(channel // 2, (3, 3), strides=1, use_bias=True, padding="same", name="deconv{}_2".format(i + 1))(x) x = InstanceNormalization(name="norm_deconv" + str(i + 1))(x) x = Activation("relu")(x) channel = channel // 2 # last block x = ReflectionPadding2D(3)(x) x = Conv2D(3, (7, 7), strides=1, use_bias=True, padding="valid", name="deconv3")(x) x = Activation("tanh")(x) model = Model(input_img, x, name='Cartoon_Generator') return model # method for discriminator def discriminator(self): input_shape = [self.image_size, self.image_size, self.image_channels] input_img = Input(shape=input_shape, name="input") # first block x = Conv2D(32, (3, 3), strides=1, use_bias=True, padding='same', name="conv1")(input_img) x = LeakyReLU(alpha=0.2)(x) # block loop channel = 64 for i in range(2): x = Conv2D(channel, (3, 3), strides=2, use_bias=True, padding='same', name="conv{}_1".format(i + 2))(x) x = LeakyReLU(alpha=0.2)(x) x = Conv2D(channel * 2, (3, 3), strides=1, use_bias=True, padding='same', name="conv{}_2".format(i + 2))(x) x = InstanceNormalization()(x) x = LeakyReLU(alpha=0.2)(x) channel = channel * 2 # last block x = Conv2D(256, (3, 3), strides=1, use_bias=True, padding='same', name="conv4")(x) x = InstanceNormalization()(x) x = LeakyReLU(alpha=0.2)(x) x = Conv2D(1, (3, 3), strides=1, use_bias=True, padding='same', activation='sigmoid', name="conv5")(x) model = Model(input_img, x, name='Cartoon_Discriminator') return model # vgg loss function def vgg_loss(self, y_true, y_pred): # get vgg model input_shape = [self.image_size, self.image_size, self.image_channels] img_input = Input(shape=input_shape, name="vgg_input") vgg19 = tf.keras.applications.vgg19.VGG19(weights='imagenet') vggmodel = Model(inputs=vgg19.input, outputs=vgg19.get_layer('block4_conv4').output) x = vggmodel(img_input) vgg = Model(img_input, x, name='VGG_for_Feature_Extraction') # get l1 loss for the content loss y_true = vgg(y_true) y_pred = vgg(y_pred) content_loss = tf.losses.absolute_difference(y_true, y_pred) return content_loss # compile each model def compile_model(self): # init summary writer for tensorboard self.callback1 = TensorBoard(self.log_dir + '/discriminator') self.callback2 = TensorBoard(self.log_dir + '/generator') self.callback3 = TensorBoard(self.log_dir + '/generated_images') # model stuff input_shape = [self.image_size, self.image_size, self.image_channels] adam1 = Adam(lr=self.lr) adam2 = Adam(lr=self.lr * 2) # init and add multi-gpu support try: self.discriminator = multi_gpu_model(self.discriminator(), gpus=self.gpu) except: self.discriminator = self.discriminator() try: self.generator = multi_gpu_model(self.generator(), gpus=self.gpu) except: self.generator = self.generator() # compile discriminator self.discriminator.compile(loss='binary_crossentropy', optimizer=adam1) # compile generator input_tensor = Input(shape=input_shape) generated_catroon_tensor = self.generator(input_tensor) self.discriminator.trainable = False # for here we only train the generator discriminator_output = self.discriminator(generated_catroon_tensor) self.train_generator = Model( input_tensor, outputs=[generated_catroon_tensor, discriminator_output]) # add multi-gpu support try: self.train_generator = multi_gpu_model(self.train_generator, gpus=self.gpu) except: pass self.train_generator.compile( loss=[self.vgg_loss, 'binary_crossentropy'], loss_weights=[float(self.weight), 1.0], optimizer=adam2) # set callback model self.callback1.set_model(self.discriminator) self.callback2.set_model(self.train_generator) self.callback3.set_model(self.train_generator) # method for training process def train(self): # start training flip = False variance = 1 / 127.5 start_time = time.time() for epoch in range(1, self.epochs + 1): # create batch generator at each epoch batch_generator = DataGenerator(image_size=self.image_size, batch_size=self.batch_size) batch_end = len(batch_generator) print('Epoch {}'.format(epoch)) # start training for each batch for idx, (photo, cartoon, smooth_cartoon, index) in enumerate(batch_generator): # these two tensors measure the output of generator and discriminator real = np.ones((self.batch_size, ) + (64, 64, 1)) fake = np.zeros((self.batch_size, ) + (64, 64, 1)) # check if it is the end of an epoch if index + 1 == batch_end: break # initial training or start training if epoch < self.init_epoch: g_loss = self.train_generator.train_on_batch( photo, [photo, real]) generated_img = self.generator.predict(photo) print( "Batch %d (initial training for generator), g_loss: %.5f, with time: %4.4f" % (idx, g_loss[2], time.time() - start_time)) start_time = time.time() write_log(self.callback2, 'g_loss', g_loss[2], idx + (epoch + 1) * len(batch_generator)) if idx % 20 == 0: write_images(self.callback3, generated_img, 'generated_imgs', idx + (epoch + 1) * len(batch_generator)) if epoch % 20 == 0 and K.eval( self.train_generator.optimizer.lr) > 0.0001: K.set_value( self.train_generator.optimizer.lr, K.eval(self.train_generator.optimizer.lr) * 0.99) else: # add noise to the input of discriminator if variance > 0.00001: variance = variance * 0.9999 gaussian = np.random.normal( 0, variance, (cartoon.shape[1], cartoon.shape[2])) cartoon[:, :, :, 0] = cartoon[:, :, :, 0] + gaussian cartoon[:, :, :, 1] = cartoon[:, :, :, 1] + gaussian cartoon[:, :, :, 2] = cartoon[:, :, :, 2] + gaussian gaussian = np.random.normal( 0, variance, (cartoon.shape[1], cartoon.shape[2])) smooth_cartoon[:, :, :, 0] = smooth_cartoon[:, :, :, 0] + gaussian smooth_cartoon[:, :, :, 1] = smooth_cartoon[:, :, :, 1] + gaussian smooth_cartoon[:, :, :, 2] = smooth_cartoon[:, :, :, 2] + gaussian # generate cartoonized images generated_img = self.generator.predict(photo) # to certain probability: flip the label of discriminator if idx % 9 == 0 or np.random.uniform(0, 1) < 0.05: real = fake fake = fake + 1 flip = True # train discriminator and adversarial loss real_loss = self.discriminator.train_on_batch( cartoon, real) smooth_loss = self.discriminator.train_on_batch( smooth_cartoon, fake) fake_loss = self.discriminator.train_on_batch( generated_img, fake) d_loss = (real_loss + smooth_loss + fake_loss) / 3 # train generator if flip: real = fake fake = fake - 1 flip = False g_loss = self.train_generator.train_on_batch( photo, [photo, real]) print( "Batch %d, d_loss: %.5f, g_loss: %.5f, with time: %4.4f" % (idx, d_loss, g_loss[2], time.time() - start_time)) start_time = time.time() # add losses to writer write_log(self.callback1, 'd_loss', d_loss, idx + (epoch + 1) * len(batch_generator)) write_log(self.callback2, 'g_loss', g_loss[2], idx + (epoch + 1) * len(batch_generator)) if idx % 20 == 0: write_images(self.callback3, generated_img, 'generated_imgs', idx + (epoch + 1) * len(batch_generator)) # change learning rate if epoch % 20 == 0 and K.eval( self.discriminator.optimizer.lr) > 0.0001: K.set_value( self.discriminator.optimizer.lr, K.eval(self.discriminator.optimizer.lr) * 0.95) if epoch % 20 == 0 and K.eval( self.train_generator.optimizer.lr) > 0.0001: K.set_value( self.train_generator.optimizer.lr, K.eval(self.train_generator.optimizer.lr) * 0.95) # save model if epoch % 50 == 0: self.generator.save_weights( self.model_dir + '/' + 'CartoonGan_generator_epoch_{}.h5'.format(epoch)) self.discriminator.save_weights( self.model_dir + '/' + 'CartoonGan_discriminator_epoch_{}.h5'.format(epoch)) self.train_generator.save_weights( self.model_dir + '/' + 'CartoonGan_train_generator_epoch_{}.h5'.format(epoch)) print('Done!') self.generator.save('CartoonGan_generator.h5')
def layer_test(layer_cls, kwargs={}, input_shape=None, input_dtype=None, input_data=None, expected_output=None, expected_output_dtype=None, fixed_batch_size=False): # generate input data if input_data is None: if not input_shape: raise AssertionError() if not input_dtype: input_dtype = K.floatx() input_data_shape = list(input_shape) for i, e in enumerate(input_data_shape): if e is None: input_data_shape[i] = np.random.randint(1, 4) if all(isinstance(e, tuple) for e in input_data_shape): input_data = [] for e in input_data_shape: input_data.append( (10 * np.random.random(e)).astype(input_dtype)) else: input_data = (10 * np.random.random(input_data_shape)) input_data = input_data.astype(input_dtype) else: if input_shape is None: input_shape = input_data.shape if input_dtype is None: input_dtype = input_data.dtype if expected_output_dtype is None: expected_output_dtype = input_dtype # instantiation layer = layer_cls(**kwargs) # test get_weights , set_weights at layer level weights = layer.get_weights() layer.set_weights(weights) try: expected_output_shape = layer.compute_output_shape(input_shape) except Exception: expected_output_shape = layer._compute_output_shape(input_shape) # test in functional API if isinstance(input_shape, list): if fixed_batch_size: x = [Input(batch_shape=e, dtype=input_dtype) for e in input_shape] else: x = [Input(shape=e[1:], dtype=input_dtype) for e in input_shape] else: if fixed_batch_size: x = Input(batch_shape=input_shape, dtype=input_dtype) else: x = Input(shape=input_shape[1:], dtype=input_dtype) y = layer(x) if not (K.dtype(y) == expected_output_dtype): raise AssertionError() # check with the functional API model = Model(x, y) actual_output = model.predict(input_data) actual_output_shape = actual_output.shape for expected_dim, actual_dim in zip(expected_output_shape, actual_output_shape): if expected_dim is not None: if not (expected_dim == actual_dim): raise AssertionError() if expected_output is not None: assert_allclose(actual_output, expected_output, rtol=1e-3) # test serialization, weight setting at model level model_config = model.get_config() recovered_model = model.__class__.from_config(model_config) if model.weights: weights = model.get_weights() recovered_model.set_weights(weights) _output = recovered_model.predict(input_data) assert_allclose(_output, actual_output, rtol=1e-3) # test training mode (e.g. useful when the layer has a # different behavior at training and testing time). if has_arg(layer.call, 'training'): model.compile('rmsprop', 'mse') model.train_on_batch(input_data, actual_output) # test instantiation from layer config layer_config = layer.get_config() layer_config['batch_input_shape'] = input_shape layer = layer.__class__.from_config(layer_config) # for further checks in the caller function return actual_output
class CGAN: def __init__(self, config_options): """ :param config_options: config parser object """ # Images self.img_dataset = config_options['Images']['dataset'] self.img_rows = int(config_options['Images']['rows']) self.img_cols = int(config_options['Images']['cols']) self.channels = int(config_options['Images']['channels']) self.img_shape = (self.img_rows, self.img_cols, self.channels) self.output_image_grid = [ int(el) for el in config_options['Images']['out_grid'].split(', ') ] # Word embeddings self.word_embeddings = load_embeddings( config_options['Embeddings']['file']) self.embeddings_dim = self.word_embeddings.vector_size self.embeddings_vocab = len(self.word_embeddings.vocab) self.train_labels = self.read_labels( os.path.join(os.getcwd(), 'data', config_options['Embeddings']['train_vocab'])) self.test_words = config_options['Embeddings']['test_vocab'].split( ', ') _, self.index = self.build_index() # GAN optimizer = Adam( 0.0002, 0.5 ) # the first value is the learning rate and the second is the beta1, right? # Build and compile the discriminator self.discriminator = self.build_discriminator( config_options['Discriminator']) self.discriminator.compile( loss=config_options['Discriminator']['loss'], optimizer=optimizer, metrics=config_options['Discriminator']['metrics'].split(', ')) # Build the generator self.generator = self.build_generator(config_options['Generator']) # The generator takes noise and the target label as input # and generates the corresponding digit of that label -> unclear what you mean by generate the digit noise = Input(shape=(self.embeddings_dim, )) label = Input(shape=(1, )) img = self.generator([noise, label]) # For the combined model we will only train the generator self.discriminator.trainable = False # The discr takes generated image as input and determines validity # and the label of that image valid = self.discriminator([img, label]) # The combined model (stacked generator and discriminator) # Trains generator to fool discriminator self.gan = Model([noise, label], valid) self.gan.compile(loss=config_options['GAN']['loss'], optimizer=optimizer) print('Discriminator summary:') self.discriminator.summary() print('\nGenerator summary:') self.generator.summary() print('\nGAN summary:') self.gan.summary() self.epochs = int(config_options['GAN']['epochs']) self.batch_size = int(config_options['GAN']['batch_size']) self.sample_interval = int(config_options['GAN']['sample_interval']) self.image_folder = config_options['GAN']['image_folder'] if not os.path.exists(self.image_folder): os.makedirs(self.image_folder) self.outfolder = config_options['GAN']['outfolder'] if not os.path.exists(self.outfolder): os.makedirs(self.outfolder) @staticmethod def read_labels(path): with open(path, 'r') as fin: labels = [w.strip() for w in fin if not w.startswith('#')] return labels def build_index(self, training=True): target_words = deepcopy(self.train_labels) if not training: for w in self.test_words: target_words.append(w) indexed_words = [] for word in target_words: indexed_words.append(self.word_embeddings.vocab[word].index) return target_words, indexed_words def build_generator(self, config_gener): m = float(config_gener['momentum']) a = float(config_gener['leaky_relu_alpha']) model = Sequential() model.add( Dense(int(config_gener['dense1']), input_dim=self.embeddings_dim)) model.add(LeakyReLU(alpha=a)) model.add(BatchNormalization(momentum=m)) model.add(Dense(int(config_gener['dense2']))) model.add(LeakyReLU(alpha=a)) model.add(BatchNormalization(momentum=m)) model.add(Dense(int(config_gener['dense3']))) model.add(LeakyReLU(alpha=a)) model.add(BatchNormalization(momentum=m)) model.add( Dense(np.prod(self.img_shape), activation=config_gener['activation'])) model.add(Reshape(self.img_shape)) noise = Input(shape=(self.embeddings_dim, )) label = Input(shape=(1, ), dtype='int32') label_embedding = Flatten()(Embedding( self.embeddings_vocab, self.embeddings_dim, weights=[self.word_embeddings.vectors], trainable=False)(label)) model_input = multiply([noise, label_embedding]) img = model(model_input) return Model([noise, label], img) def build_discriminator(self, config_discr): a = float(config_discr['leaky_relu_alpha']) d = float(config_discr['dropout']) model = Sequential() model.add( Dense(int(config_discr['dense1']), input_dim=np.prod(self.img_shape))) model.add(LeakyReLU(alpha=a)) model.add(Dense(int(config_discr['dense2']))) model.add(LeakyReLU(alpha=a)) model.add(Dropout(d)) model.add(Dense(int(config_discr['dense3']))) model.add(LeakyReLU(alpha=a)) model.add(Dropout(d)) model.add(Dense(1, activation=config_discr['activation'])) img = Input(shape=self.img_shape) label = Input(shape=(1, ), dtype='int32') label_embedding = Flatten()(Embedding( self.embeddings_vocab, self.embeddings_dim, weights=[self.word_embeddings.vectors], trainable=False)(label)) label_expansion = Dense(self.img_rows * self.img_cols * self.channels)(label_embedding) flat_img = Flatten()(img) model_input = multiply([flat_img, label_expansion]) validity = model(model_input) return Model([img, label], validity) def train(self): # Load the dataset if 'mnist' in self.img_dataset: (X_train, y_train), (_, _) = mnist.load_data() elif 'cifar' in self.img_dataset: (X_train, y_train), (_, _) = cifar.load_data('fine') X_train = X_train / 255.0 classes2labels = {c: l for c, l in enumerate(self.train_labels)} else: raise ValueError( "Unknown dataset {}! Please use either 'fasion_mnist' or 'cifar100'." .format(self.img_dataset)) # Configure input X_train = (X_train.astype(np.float32) - 127.5) / 127.5 # if self.channels == 1: # X_train = np.expand_dims(X_train, axis=3) for n, i in enumerate(self.index): y_train = np.where(y_train == n, i, y_train) y_train = y_train.reshape(-1, 1) # Adversarial ground truths true = np.ones((self.batch_size, 1)) fake = np.zeros((self.batch_size, 1)) for epoch in range(self.epochs): # --------------------- # Train Discriminator # --------------------- # Select a random half batch of images idx = np.random.randint(0, X_train.shape[0], self.batch_size) # with np.random.randint we can pick the same image more than once: is this a feature or a bug? imgs, labels = X_train[idx], y_train[idx] shuffled_labels = np.zeros_like(labels) for i, label in enumerate(labels): modified_index = np.delete(self.index, np.where(self.index == label)) shuffled_labels[i, 0] = np.random.choice(modified_index, 1) # Sample noise as generator input noise = np.random.normal(0, 1, (self.batch_size, self.embeddings_dim)) # Generate a half batch of new images gen_imgs = self.generator.predict([noise, labels]) """ Train the discriminator to do three tasks: 1. recognize real images for a concept as true images for that concept 2. recognize generated images for a concept as fake images for that concept 3. recognize real images for a concept as fake for a different concept this way the generator cannot just fool the discriminator by generating sensible images in the abstract, but has to generate credible images for a specific concept. """ d_loss_real = self.discriminator.train_on_batch([imgs, labels], true) d_loss_fake = self.discriminator.train_on_batch([gen_imgs, labels], fake) d_loss_unrelated = self.discriminator.train_on_batch( [imgs, shuffled_labels], fake) # this is the original loss, which could be extended to # d_loss = np.add(d_loss_real, d_loss_fake, d_loss_unrelated) / 3 # to keep its spirit. # d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) # this is the loss proposed in Reed et al 2018 d_loss = np.log(d_loss_real) + (np.log(1 - d_loss_unrelated) + np.log(1 - d_loss_fake)) / 2 # --------------------- # Train Generator # --------------------- # Condition on labels sampled_labels = np.random.choice(self.index, self.batch_size).reshape(-1, 1) # Train the generator g_loss = self.gan.train_on_batch([noise, sampled_labels], true) # If at save interval => save generated image samples if epoch % self.sample_interval == 0: print("%d [D loss: %f, acc.: %.2f%%] [G loss: %f]" % (epoch, d_loss[0], 100 * d_loss[1], g_loss)) self.sample_images(epoch) self.save() def save(self): self.discriminator.trainable = False self.gan.save( os.path.join(self.outfolder, 'gan_{}_epochs'.format(self.epochs))) self.discriminator.trainable = True self.discriminator.save( os.path.join(self.outfolder, 'discriminator_{}_epochs'.format(self.epochs))) self.generator.save(os.path.join( self.outfolder, 'generator_{}_epochs'.format(self.epochs)), include_optimizer=False) def sample_images(self, epoch): r, c = self.output_image_grid vocab, extended_index = self.build_index(training=False) noise = np.random.normal(0, 1, (r * c, self.embeddings_dim)) sampled_labels = np.asarray(extended_index).reshape(-1, 1) gen_imgs = self.generator.predict([noise, sampled_labels]) # Rescale images 0 - 1 gen_imgs = 0.5 * gen_imgs + 0.5 fig, axs = plt.subplots(r, c) cnt = 0 for i in range(r): for j in range(c): if self.channels == 1: axs[i, j].imshow(gen_imgs[cnt, :, :, 0], cmap='gray') else: axs[i, j].imshow(gen_imgs[cnt, :, :, :]) axs[i, j].set_title(vocab[cnt]) axs[i, j].axis('off') cnt += 1 fig.savefig("{}/{}.png".format(self.image_folder, epoch)) plt.close()
class GAN: def __init__(self): self.img_rows = 28 self.img_cols = 28 self.channels = 1 self.img_shape = (self.img_rows, self.img_cols, self.channels) self.latent_dim = 100 optimizer = Adam(0.0002, 0.5) # Build and compile the discriminator self.discriminator = self.build_discriminator() # self.discriminator.summary() self.discriminator.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) # Build the generator self.generator = self.build_generator() # self.generator.summary() # The generator takes noise as input and generates imgs z = Input(shape=(self.latent_dim, )) img = self.generator(z) # For the combined model we will only train the generator # self.discriminator.trainable = False # The discriminator takes generated images as input and determines validity validity = self.discriminator(img) # The combined model (stacked generator and discriminator) # Trains the generator to fool the discriminator self.combined = Model(z, validity) # self.combined.summary() self.combined.compile(loss='binary_crossentropy', optimizer=optimizer) def build_generator(self): model = Sequential() model.add(Dense(256, input_dim=self.latent_dim)) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(512)) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(1024)) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(np.prod(self.img_shape), activation='tanh')) model.add(Reshape(self.img_shape)) noise = Input(shape=(self.latent_dim, )) img = model(noise) return Model(noise, img) def build_discriminator(self): model = Sequential() model.add(Flatten(input_shape=self.img_shape)) model.add(Dense(512)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(256)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(1, activation='sigmoid')) model.summary() img = Input(shape=self.img_shape) validity = model(img) return Model(img, validity) def train(self, epochs, batch_size=128, sample_interval=50): # from tensorflow.examples.tutorials.mnist import input_data # mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) # # x_train, y_train = mnist.train.images, mnist.train.labels # x_test, y_test = mnist.test.images, mnist.test.labels # X_train = x_train.reshape(-1, 28, 28, 1).astype('float32') / 127.5 - 1 # x_test = x_test.reshape(-1, 28, 28, 1).astype('float32') # Rescale -1 to 1 from keras.datasets import mnist (X_train, _), (_, _) = mnist.load_data() X_train = X_train / 127.5 - 1. X_train = np.expand_dims(X_train, axis=3) # Adversarial ground truths valid = np.ones((batch_size, 1)) fake = np.zeros((batch_size, 1)) for epoch in range(epochs): # --------------------- # Train Discriminator # --------------------- # Select a random batch of images idx = np.random.randint(0, X_train.shape[0], batch_size) imgs = X_train[idx] noise = np.random.normal(0, 1, (batch_size, self.latent_dim)) # Generate a batch of new images gen_imgs = self.generator.predict(noise) if epoch > 80 and epoch % 40: for i in range(8): cv2.imwrite("imgs/%d_%d.jpg" % (epoch, i), ((gen_imgs[i] + 1) * 127.5).astype(np.uint8)) # Train the discriminator for layer in self.discriminator.layers: layer.trainable = True self.discriminator.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) d_loss_real = self.discriminator.train_on_batch(imgs, valid) d_loss_fake = self.discriminator.train_on_batch(gen_imgs, fake) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) for layer in self.discriminator.layers: layer.trainable = False # --------------------- # Train Generator # --------------------- noise = np.random.normal(0, 1, (batch_size, self.latent_dim)) # Train the generator (to have the discriminator label samples as valid) self.combined.compile(loss='binary_crossentropy', optimizer='adam') g_loss = self.combined.train_on_batch(noise, valid) # Plot the progress print("%d [D loss: %f, acc.: %.2f%%] [G loss: %f]" % (epoch, d_loss[0], 100 * d_loss[1], g_loss))
def test_cyclegan(): mnist_shape, mnist_images = load_mnist() cats = load_input_images() nb_epochs = 50 batch_size = 512 adam_lr = 0.0002 adam_beta_1 = 0.5 adam_decay = 0 # adam_lr/(nb_epochs*120) cyc_multiplier = 10 history_size = int(batch_size * 7 / 3) SAVEPATH = os.path.abspath( os.path.join(os.path.dirname(__file__), 'output')) generation_history_mnist = None generation_history_cats = None adam_optimizer = Adam(lr=adam_lr, beta_1=adam_beta_1, decay=adam_decay) generator_cats = mnist_generator(mnist_shape) generator_cats.compile(optimizer=adam_optimizer, loss='mean_squared_error') generator_cats.summary() discriminator_cats = mnist_discriminator(mnist_shape) discriminator_cats.compile(optimizer=adam_optimizer, loss=discriminator_loss) discriminator_cats.summary() generator_mnist = mnist_generator(mnist_shape) generator_mnist.compile(optimizer=adam_optimizer, loss='mean_squared_error') generator_mnist.summary() discriminator_mnist = mnist_discriminator(mnist_shape) discriminator_mnist.compile(optimizer=adam_optimizer, loss=discriminator_loss) discriminator_mnist.summary() mnist_input = Input(mnist_shape) cat_input = Input(mnist_shape) fake_cat = generator_cats(mnist_input) fake_mnist = generator_mnist(cat_input) # Only train discriminator during first phase # (trainable only affects new models when they are compiled) discriminator_cats.trainable = False discriminator_mnist.trainable = False mnist_gen_trainer = Model(cat_input, discriminator_mnist(fake_mnist)) mnist_gen_trainer.compile(optimizer=adam_optimizer, loss='mean_squared_error') mnist_gen_trainer.summary() cats_gen_trainer = Model(mnist_input, discriminator_cats(fake_cat)) cats_gen_trainer.compile(optimizer=adam_optimizer, loss='mean_squared_error') cats_gen_trainer.summary() mnist_cyc = Model(mnist_input, generator_mnist(fake_cat)) mnist_cyc.compile(optimizer=adam_optimizer, loss=cycle_loss, loss_weights=[cyc_multiplier]) mnist_cyc.summary() cats_cyc = Model(cat_input, generator_cats(fake_mnist)) cats_cyc.compile(optimizer=adam_optimizer, loss=cycle_loss, loss_weights=[cyc_multiplier]) cats_cyc.summary() # training time mnist_discrim_loss = [] cats_discrim_loss = [] mnist_gen_loss = [] cats_gen_loss = [] mnist_cyc_loss = [] cats_cyc_loss = [] if not os.path.exists(SAVEPATH): os.makedirs(os.path.join(SAVEPATH, 'images')) for epoch in range(nb_epochs): print("\n\n================================================") print("Epoch", epoch, '\n') if epoch == 0: # Initialize history for training the discriminator mnist_indices = np.random.choice(mnist_images.shape[0], history_size) generation_history_mnist = mnist_images[mnist_indices] cats_indices = np.random.choice(cats.shape[0], history_size) generation_history_cats = cats[cats_indices] # Make and save a test collage choice = np.random.choice(mnist_images.shape[0]) if k.image_data_format() == 'channels_first': mnist_in = mnist_images[choice].reshape((1, 1, 28, 28)) else: mnist_in = mnist_images[choice].reshape((1, 28, 28, 1)) cat_out = generator_cats.predict(mnist_in) mnist_cyc_out = generator_mnist.predict(cat_out) choice = np.random.choice(cats.shape[0]) if k.image_data_format() == 'channels_first': cat_in = cats[choice].reshape((1, 1, 28, 28)) else: cat_in = cats[choice].reshape((1, 28, 28, 1)) mnist_out = generator_mnist.predict(cat_in) cat_cyc_out = generator_cats.predict(mnist_out) mnist_test_images = np.concatenate( (prettify(mnist_in), prettify(cat_out), prettify(mnist_cyc_out)), axis=1) cat_test_images = np.concatenate( (prettify(cat_in), prettify(mnist_out), prettify(cat_cyc_out)), axis=1) test_collage = np.concatenate((mnist_test_images, cat_test_images), axis=0) test_collage = Image.fromarray(test_collage, mode='L') test_collage.save(os.path.join(SAVEPATH, 'images', str(epoch) + '.png')) for batch in range(int(mnist_images.shape[0] / batch_size)): print("\nEpoch", epoch, "| Batch", batch) # Get batch. mnist_indices = np.random.choice(mnist_images.shape[0], batch_size) mnist_batch_real = mnist_images[mnist_indices] cats_indices = np.random.choice(cats.shape[0], batch_size) cats_batch_real = cats[cats_indices] # Update history with new generated images. mnist_batch_gen = generator_mnist.predict_on_batch(cats_batch_real) cats_batch_gen = generator_cats.predict_on_batch(mnist_batch_real) generation_history_mnist = np.concatenate( (generation_history_mnist[batch_size:], mnist_batch_gen)) generation_history_cats = np.concatenate( (generation_history_cats[batch_size:], cats_batch_gen)) # Train discriminators. real_label = np.ones(batch_size) fake_label = np.zeros(batch_size) mnist_discrim_loss.append( discriminator_mnist.train_on_batch( np.concatenate((generation_history_mnist[:batch_size], mnist_batch_real)), np.concatenate((fake_label, real_label)))) print("MNIST Discriminator Loss:", mnist_discrim_loss[-1]) cats_discrim_loss.append( discriminator_cats.train_on_batch( np.concatenate((generation_history_cats[:batch_size], cats_batch_real)), np.concatenate((fake_label, real_label)))) print("Cats Discriminator Loss:", cats_discrim_loss[-1]) # Train generators. mnist_gen_loss.append( mnist_gen_trainer.train_on_batch(cats_batch_real, real_label)) print("MNIST Generator Loss:", mnist_gen_loss[-1]) cats_gen_loss.append( cats_gen_trainer.train_on_batch(mnist_batch_real, real_label)) print("Cats Generator Loss:", cats_gen_loss[-1]) mnist_cyc_loss.append( mnist_cyc.train_on_batch(mnist_batch_real, mnist_batch_real)) print("MNIST Cyclic Loss:", mnist_cyc_loss[-1]) cats_cyc_loss.append( cats_cyc.train_on_batch(cats_batch_real, cats_batch_real)) print("Cats Cyclic Loss:", cats_cyc_loss[-1]) # Save models. generator_cats.save(os.path.join(SAVEPATH, 'generator_cats.h5')) generator_mnist.save(os.path.join(SAVEPATH, 'generator_mnist.h5')) discriminator_cats.save(os.path.join(SAVEPATH, 'discriminator_cats.h5')) discriminator_mnist.save(os.path.join(SAVEPATH, 'discriminator_mnist.h5')) # Save training history. output_dict = { 'mnist_discrim_loss': [str(loss) for loss in mnist_discrim_loss], 'cats_discrim_loss': [str(loss) for loss in cats_discrim_loss], 'mnist_gen_loss': [str(loss) for loss in mnist_gen_loss], 'cats_gen_loss': [str(loss) for loss in cats_gen_loss], 'mnist_cyc_loss': [str(loss) for loss in mnist_cyc_loss], 'cats_cyc_loss': [str(loss) for loss in cats_cyc_loss] } with open(os.path.join(SAVEPATH, 'log.txt'), 'w') as f: json.dump(output_dict, f, indent=4)
class merge_models: def __init__(self, epochs, batch_size): self.epochs = epochs self.batch_size = batch_size # Generator filter_kernal = [[32, 128, 128, 128, 256], [32, 128, 128, 128, 128, 256, 512], [32, 128, 128, 128, 128, 128, 128, 128, 512]] self.g_1 = generator( 3, filter_kernal[2], in_1_size, 'Adam', 'mse', path='/home/mdo2/sid_codes/new_codes/gen_1.h5').generator_model() self.g_2 = generator( 2, filter_kernal[1], in_2_size, 'Adam', 'mse', path='/home/mdo2/sid_codes/new_codes/gen_2.h5').generator_model() self.g_3 = generator( 1, filter_kernal[0], in_3_size, 'Adam', 'mse', path='/home/mdo2/sid_codes/new_codes/gen_3.h5').generator_model() # Discriminator #self.D1 = discriminator(in_2_size).modified_vgg() #self.D1.compile(loss='mse',optimizer='Adam',metrics=['accuracy']) #self.D2 = discriminator(in_3_size).modified_vgg() #self.D2.compile(loss='mse',optimizer='Adam',metrics=['accuracy']) self.D = discriminator( out, '/home/mdo2/sid_codes/new_codes/D.h5').modified_vgg() self.D.compile(loss='mse', optimizer='Adam', metrics=['accuracy']) self.g = 0 # Images lr = Input(shape=in_1_size) hr_1 = Input(shape=in_2_size) hr_2 = Input(shape=in_3_size) hr_3 = Input(shape=out) fake_hr_image_1 = self.g_1(lr) fake_hr_image_2 = self.g_2(fake_hr_image_1) fake_hr_image_3 = self.g_3(fake_hr_image_2) self.vgg_1 = vgg(in_2_size).vgg_loss_model() self.vgg_2 = vgg(in_3_size).vgg_loss_model() self.vgg_3 = vgg(out).vgg_loss_model() fake_hr_feat_1 = self.vgg_1(fake_hr_image_1) fake_hr_feat_2 = self.vgg_2(fake_hr_image_2) fake_hr_feat_3 = self.vgg_3(fake_hr_image_3) self.D.trainable = False #self.D1.trainable = False #self.D2.trainable = False validity = self.D(fake_hr_image_3) #self.D.load_weights('/media/arjun/119GB/attachments/D_best.h5') self.merged_net = Model( inputs=[lr, hr_1, hr_2, hr_3], outputs=[validity, fake_hr_feat_1, fake_hr_feat_2, fake_hr_feat_3]) self.merged_net.compile( loss=['binary_crossentropy', 'mse', 'mse', 'mse'], loss_weights=[1e-3, .3, .3, .3], optimizer='Adam') self.merged_net.summary() #self.merged_net.load_weights('/home/mdo2/sid_codes/newew/my_model.h5') print('loaded') def save(self): #self.D.save_weights('D_best.h5') #print('Discriminator saved') #self.merged_net.save('my_model.h5') self.merged_net.save('/home/mdo2/sid_codes/new_codes/model.h5') self.g_1.save('/home/mdo2/sid_codes/new_codes/gen_1.h5') self.g_2.save('/home/mdo2/sid_codes/new_codes/gen_2.h5') self.g_3.save('/home/mdo2/sid_codes/new_codes/gen_3.h5') self.D.save('/home/mdo2/sid_codes/new_codes/D.h5') print('saved') def write(self, g_loss, d_loss, flag): with open('train.csv', mode='a') as csv_file: if flag == True: fieldnames = ['g_loss', 'd_loss'] writer = csv.DictWriter(csv_file, fieldnames=fieldnames) writer.writeheader() fieldnames = ['g_loss', 'd_loss'] writer = csv.DictWriter(csv_file, fieldnames=fieldnames) writer.writerow({'g_loss': g_loss, 'd_loss': d_loss}) """ def test(self): test_path, test_path = '' img_test_path = glob.glob(test_path) random.shuffle(img_test_path) d_sum = 0. g_sum = np.array([0.,0.,0.,0.,0.,0.,0.,0.]) step_size = int(800/self.batch_size) for steps in range(step_size): # Load data print('Loading Test Batch') x_test, y_test_1, y_test_2, y_test_3 = load_data(img_test_path, start=steps*self.batch_size, batch_size=self.batch_size) fake_hr_1 = self.g_1.predict(x_test) fake_hr_2 = self.g_2.predict(fake_hr_1) fake_hr_3 = self.g_3.predict(fake_hr_2) valid = np.ones([self.batch_size,12,12,16]) fake = np.zeros([self.batch_size,12,12,16]) d_test_loss_real = self.D.test_on_batch(y_test_3, valid) d_test_loss_fake = self.D.test_on_batch(fake_hr_3, fake) d_test_loss = 0.5*np.add(d_test_loss_fake,d_test_loss_real) 8 real_feat_1 = self.vgg_1.predict(y_test_1) real_feat_2 = self.vgg_2.predict(y_test_2) real_feat_3 = self.vgg_3.predict(y_test_3) g_test_loss = self.merged_net.test_on_batch([x_test,y_test_3], [valid,real_feat_3]) d_sum += d_test_loss[0] g_sum = np.add(g_sum, g_test_loss) save(d_sum/float(step_size),g_sum/float(step_size)) """ def train(self): try: #self.D.load_weights('/media/arjun/119GB/attachments/D_best.h5') #self.g_1.load_weights('/media/arjun/119GB/attachments/g_1_best.h5') #self.g_2.load_weights('/media/arjun/119GB/attachments/g_2_best.h5') #self.g_3.load_weights('/media/arjun/119GB/attachments/g_3_best.h5') i = 0 flag = True print("Discriminator loaded") for epoch in range(self.epochs): print("loading data") train_path = dataset_dir img_train_path = glob.glob(train_path) #i=0 #if i==0: #avg_loss=0 #i+=1 #avg_loss = avg_loss/int(25000/self.batch_size) with tqdm(total=int(25000 / self.batch_size), file=sys.stdout) as t: for steps in range(int(25000 / self.batch_size)): print('Loading Train Batch:', steps + 1) x_train, y_train_1, y_train_2, y_train_3 = load_data( train_path, start=steps * self.batch_size, batch_size=self.batch_size) print('Training Discriminator') t.update() #dISCRIMINATOR valid = np.ones([self.batch_size, 6, 6, 1]) fake = np.zeros([self.batch_size, 6, 6, 1]) fake_hr_1 = self.g_1.predict(x_train) fake_hr_feat_1 = self.vgg_1(fake_hr_1) fake_hr_2 = self.g_2.predict(fake_hr_1) fake_hr_feat_2 = self.vgg_2(fake_hr_2) fake_hr_3 = self.g_3.predict(fake_hr_2) fake_hr_feat_3 = self.vgg_3(fake_hr_3) d_train_loss_fake = self.D.train_on_batch( fake_hr_3, fake) d_train_loss_real = self.D.train_on_batch( y_train_3, valid) d_train_loss = 0.5 * np.add(d_train_loss_fake, d_train_loss_real) #GENERATOR print('Training Generator') valid = np.ones([self.batch_size, 6, 6, 1]) fake = np.zeros([self.batch_size, 6, 6, 1]) real_feat_1 = self.vgg_1.predict(y_train_1) real_feat_2 = self.vgg_2.predict(y_train_2) real_feat_3 = self.vgg_3.predict(y_train_3) g_loss = self.merged_net.train_on_batch( [x_train, y_train_1, y_train_2, y_train_3], [valid, real_feat_1, real_feat_2, real_feat_3]) t.set_postfix_str(s='Epoch:' + str(epoch) + ' Generator_cost ' + str(g_loss) + ' Discriminator_cost ' + str(d_train_loss)) if i > 0: flag = False self.write(g_loss, d_train_loss, flag) i += 1 print( '*******************************************************\n' ) if self.g == 0: try: self.old_loss = np.load( '/home/mdo2/sid_codes/newew/g_loss.npy') print('latest gen') self.g += 1 except: self.old_loss = float('inf') self.g += 1 if g_loss[0] < self.old_loss: self.old_loss = g_loss[0] print('saving best') np.save('g_loss', g_loss[0]) self.g += 1 self.save() #if (steps+1)%100==0: # self.save() #self.merged_net.inputs = [x_train,y_train_1,y_train_2,y_train_3] #self.merged_net.outputs = [valid,fake_hr_feat_1,fake_hr_feat_2,fake_hr_feat_3] #c_loss = K.sqrt(K.mean((fake_hr_feat_1 - real_feat_1)**2)) #log_loss = logcosh(fake_hr_feat_1,real_feat_1) #grads = K.gradients([log_loss,c_loss],[fake_hr_feat_1,real_feat_1])[0] #print(grads) #iterate = K.function([(y_train_1)], [c_loss, grads]) #print(iterate,'c_loss') #iterate = K.function([tf.convert_to_tensor(y_train_1)], [log_loss, grads]) #print(iterate,'log_loss_1') #self.D.fit(y_train_3,valid, epochs=epoch, batch_size=self.batch_size, callbacks=[tbCallBack]) except KeyboardInterrupt: print('Saving Weights') #self.save() def feed_forward(self): in_1 = [64, 64, 3] in_2 = [64, 64, 3] in_3 = [64 * 4, 64 * 4, 3] out_ = [128, 128, 3] img = cv2.resize( cv2.imread( '/home/mdo2/sid_codes/datasets_big/qhd_no_pad/train/1.png', 1), (64, 64)) img = img.reshape(1, 64, 64, 3) filter_kernal = [[32, 128, 128, 128, 256], [32, 128, 128, 128, 128, 256, 512], [32, 128, 128, 128, 128, 128, 128, 128, 512]] g3 = generator( 3, filter_kernal[2], in_1, 'Adam', path='/home/mdo2/sid_codes/new_codes/gen_1.h5').generator_model() #g2 = generator(2,filter_kernal[1],in_2,'Adam',path = '/home/mdo2/sid_codes/new_codes/gen_2.h5').generator_model() #g1 = generator(1,filter_kernal[0],in_1,'Adam',path = '/home/mdo2/sid_codes/new_codes/gen_3.h5').generator_model() img = g3.predict(img) #img = g2.predict(img) img = np.array(img).astype('uint8') img = img.reshape(128, 128, 3) cv2.imshow('img', img) cv2.waitKey(0) cv2.imwrite('ee.jpg', img)
class TmClassify: def __init__(self): self.input_shape = (224,224) self.filter_count = 32 self.kernel_size = (3, 3) self.leakrelu_alpha = 0.2 self.encoder = self.createEncoder() Input1 = Input(shape=(224,224,3)) Input2 = Input(shape=(224,224,3)) # target = Dot(axes=1)([self.encoder(Input1), self.encoder(Input2)]) x = concatenate(inputs = [self.encoder(Input1), self.encoder(Input2)]) target = Dense(1)(x) self.discriminator = self.createDiscriminator() y = self.discriminator(x) self.model = Model(inputs=[Input1,Input2],outputs=y) self.model.summary() self.pathlist = pathlist self.train_data_count = [len(i) for i in self.pathlist] op = Adam(lr=0.0001) self.model.compile(optimizer=op,loss='mse',metrics=['accuracy']) self.pad_param = 5 self.rotate_degree_param = 90 def createEncoder(self): base_model=InceptionV3(input_shape=(224,224,3),weights=None,include_top=False) x=base_model.output x=GlobalAveragePooling2D()(x) x = LayerNormalization()(x) model=Model(inputs=base_model.input,outputs=x) return model def createDiscriminator(self): x = Input(shape = 4096) target = Dense(1)(x) model = Model(inputs=x,outputs=target) return model def gen_data(self,path): img_pil = Image.open(path).convert('RGB') # img_pil.save("test.jpg") pad_top = int(abs(np.random.uniform(0,self.pad_param))) pad_bottom = int(abs(np.random.uniform(0,self.pad_param))) pad_left = int(abs(np.random.uniform(0,self.pad_param))) pad_right = int(abs(np.random.uniform(0,self.pad_param))) rotate_param = np.random.uniform(0,self.rotate_degree_param) flip_flag = np.random.randint(0,1) mirror_flag = np.random.randint(0,1) if(flip_flag): img_pil = ImageOps.flip(img_pil) if(mirror_flag): img_pil = ImageOps.mirror(img_pil) blur_rad = np.random.normal(loc=0.0, scale=1, size=None) img_pil = img_pil.filter(ImageFilter.GaussianBlur(blur_rad)) enhancer_contrat = ImageEnhance.Contrast(img_pil) enhancer_brightness = ImageEnhance.Brightness(img_pil) enhancer_color = ImageEnhance.Color(img_pil) contrast_factor = np.random.normal(loc=1.0, scale=0.25, size=None) color_factor = np.max([0,1-abs(np.random.normal(loc=0, scale=0.5, size=None))]) translate_factor_hor = np.random.normal(loc=0, scale=5, size=None) translate_factor_ver = np.random.normal(loc=0, scale=5, size=None) brightness_factor = np.random.normal(loc=1.0, scale=0.5, size=None) img_pil = enhancer_contrat.enhance(contrast_factor) img_pil = enhancer_brightness.enhance(brightness_factor) img_pil = enhancer_color.enhance(color_factor) img_pil = ImageChops.offset(img_pil, int(translate_factor_hor), int(translate_factor_ver)) img_pil = img_pil.rotate(rotate_param,resample = Image.BILINEAR,expand = True, fillcolor = (255)) img = np.asarray(img_pil) img = cv2.copyMakeBorder(img, pad_top, pad_bottom, pad_left, pad_right, cv2.BORDER_CONSTANT,value=(255,255,255)) img= cv2.resize(img, dsize=(self.input_shape)) img = img/127.5 - 1 return img def train(self,start_epoch, max_epoch, batch_size, viz_interval): max_step = sum([len(i) for i in self.pathlist]) // batch_size # permu_ind = list(range(len(self.pathlist))) step = 0 for epoch in range(start_epoch,max_epoch): # permu_ind = np.random.permutation(permu_ind) real_index = 0 for step_index in range(max_step): batch_img1 = np.zeros((batch_size,self.input_shape[0],self.input_shape[1],3)) batch_img2 = np.zeros((batch_size,self.input_shape[0],self.input_shape[1],3)) batch_target = np.zeros(batch_size) for batch_index in range(batch_size//2+1): random_permu = np.random.permutation(len(sdir)) filelist = glob2.glob(sdir[batch_index]+'/*') file_permu = np.random.permutation(len(filelist)) img1 = self.gen_data(filelist[file_permu[0]]) img2 = self.gen_data(filelist[file_permu[1]]) batch_target[batch_index] = 1 batch_img1[batch_index] = img1 batch_img2[batch_index] = img2 real_index = real_index+1 for batch_index in range(batch_size//2,batch_size): random_permu = np.random.permutation(len(sdir)) filelist1 = glob2.glob(sdir[batch_index]+'/*') filelist2 = glob2.glob(sdir[-1-(-1*batch_index)]+'/*') file_permu1 = np.random.permutation(len(filelist1)) file_permu2 = np.random.permutation(len(filelist2)) img1 = self.gen_data(filelist1[file_permu1[0]]) img2 = self.gen_data(filelist2[file_permu2[0]]) batch_target[batch_index] = 0 batch_img1[batch_index] = img1 batch_img2[batch_index] = img2 real_index = real_index+1 train_loss = self.model.train_on_batch([batch_img1,batch_img2],batch_target) with train_summary_writer.as_default(): tf.summary.scalar('loss', train_loss[0], step=step) tf.summary.scalar('accuracy', train_loss[1], step=step) step = step + 1 # train_summary_writer = tf.summary.create_file_writer(train_log_dir) print('\r epoch ' + str(epoch) + ' / ' + str(max_epoch) + ' ' + 'step ' + str(step_index) + ' / ' + str(max_step) + ' loss = ' + str(train_loss)) if(step_index%viz_interval==0): self.encoder.save('DIPencoder.h5') self.discriminator.save('DIPdiscriminator.h5') self.model.save('DIPMatch.h5')
class StyleBank(object): def __init__(self): ######################### Tunable Parameters ################################ # General self.img_shape = (None, 128, 128, 3 ) # (None, 512, 512, 3) ## Image Shape self.n_styles = 4 # 50 ## Number of styles in the bank self.n_content = 1000 ## Number of content images self.N_steps = 300000 ## Total number of training steps self.T = 2 ## Number of consecutive steps for training styles before training the AutoEncoder self.print_iter = 100 ## Log output self.Batch_Size = 4 ## Batch size self.Use_Batch_Norm = True ## Use batch normalization instead of instance normalization # LR self.LR_Initial = 0.01 ## Initial ADAM learning rate self.LR_Current = self.LR_Initial ## For logging self.LR_Decay = 0.8 ## LR decay self.LR_Update_Every = self.N_steps / 10 ## LR decay period # Loss self.Optimizer = optimizers.Adam( lr=self.LR_Initial) ## Optimizer for both branches self.LossAlpha = 0.025 # Content weight self.LossBeta = 1.2 # Style weight self.LossGamma = 1.0 # Total Variation weight ######################### \Tunable Parameters ################################ self.StyleNetLoss = {k: None for k in range(self.n_styles)} self.StyleNetContentLoss = {k: None for k in range(self.n_styles)} self.StyleNetStyleLoss = {k: None for k in range(self.n_styles)} # Data self.Content_DB = None self.Style_DB = None self.Content_DB_path = './DB/content/' self.Style_DB_path = './DB/style/' self.Content_DB_list = glob(self.Content_DB_path + '*') self.Style_DB_list = glob(self.Style_DB_path + '*') # VGG self.VGG16 = None # auto-encoder self.encoder = None self.decoder = None # style bank self.style_bank = {k: None for k in range(self.n_styles)} self.StyleNet = {k: None for k in range(self.n_styles)} self.AutoEncoderNet = None # inputs - content and one for style self.KinputContent = None self.KinputStyle = None self.tfStyleIndices = None self.TensorBoardStyleNet = {k: None for k in range(self.n_styles)} self.TensorBoardAutoEncoder = None def initialize_placeholders(self): # initialize the content and style image tensors self.KinputContent = Input(shape=self.img_shape[1:], name="InputContent") self.KinputDecoded = None def build_models(self): ########### # Encoder # ########### print("Building Encoder") input_layer = Input(shape=self.img_shape[1:]) t_encoder = Conv2D(32, (9, 9), strides=(1, 1), padding='same', use_bias=False)(input_layer) if self.Use_Batch_Norm: t_encoder = BatchNormalization()(t_encoder) else: t_encoder = InstanceNormalization()(t_encoder) t_encoder = Activation('relu')(t_encoder) t_encoder = Conv2D(64, (3, 3), strides=(2, 2), padding='same', use_bias=False)(t_encoder) if self.Use_Batch_Norm: t_encoder = BatchNormalization()(t_encoder) else: t_encoder = InstanceNormalization()(t_encoder) t_encoder = Activation('relu')(t_encoder) t_encoder = Conv2D(128, (3, 3), strides=(2, 2), padding='same', use_bias=False)(t_encoder) if self.Use_Batch_Norm: t_encoder = BatchNormalization()(t_encoder) else: t_encoder = InstanceNormalization()(t_encoder) t_encoder = Activation('relu')(t_encoder) self.encoder = Model(input_layer, t_encoder, name='Encoder') print(self.encoder.summary()) ########### # Decoder # ########### print("Building Decoder") input_layer = Input(shape=self.encoder.layers[-1].output_shape[1:]) t_decoder = Conv2DTranspose(64, (3, 3), strides=(2, 2), padding='same', use_bias=False)(input_layer) if self.Use_Batch_Norm: t_decoder = BatchNormalization()(t_decoder) else: t_decoder = InstanceNormalization()(t_decoder) t_decoder = Activation('relu')(t_decoder) t_decoder = Conv2DTranspose(32, (3, 3), strides=(2, 2), padding='same', use_bias=False)(t_decoder) if self.Use_Batch_Norm: t_decoder = BatchNormalization()(t_decoder) else: t_decoder = InstanceNormalization()(t_decoder) t_decoder = Activation('relu')(t_decoder) t_decoder = Conv2DTranspose(3, (9, 9), strides=(1, 1), padding='same', use_bias=False)(t_decoder) self.decoder = Model(input_layer, t_decoder, name='Decoder') print(self.decoder.summary()) ############# # StyleBank # ############# for i in self.style_bank: print("Building Style {}".format(i)) bank_name = "StyleBank{}".format(i) stylenet_name = "StyleNet{}".format(i) input_layer = Input(shape=self.encoder.layers[-1].output_shape[1:]) t_style = Conv2D(256, (3, 3), strides=(1, 1), padding='same', use_bias=False)(input_layer) if self.Use_Batch_Norm: t_style = BatchNormalization()(t_style) else: t_style = InstanceNormalization()(t_style) t_style = Activation('relu')(t_style) t_style = Conv2D(256, (3, 3), strides=(1, 1), padding='same', use_bias=False)(t_style) if self.Use_Batch_Norm: t_style = BatchNormalization()(t_style) else: t_style = InstanceNormalization()(t_style) t_style = Activation('relu')(t_style) t_style = Conv2D(128, (3, 3), strides=(1, 1), padding='same', use_bias=False)(t_style) if self.Use_Batch_Norm: t_style = BatchNormalization()(t_style) else: t_style = InstanceNormalization()(t_style) t_style = Activation('relu')(t_style) self.style_bank[i] = Model(input_layer, t_style, name=bank_name) ######################### # StyleBank Full Models # ######################### input_layer = self.encoder.layers[ 0].output # layers.Input(batch_shape=self.encoder.layers[0].input_shape) prev_layer = input_layer for layer in self.encoder.layers[1:]: prev_layer = layer(prev_layer) for layer in self.style_bank[i].layers[1:]: prev_layer = layer(prev_layer) for layer in self.decoder.layers[1:]: prev_layer = layer(prev_layer) self.StyleNet[i] = Model([input_layer], [prev_layer], name=stylenet_name) print(self.StyleNet[i].summary()) ########################## # AutoEncoder Full Model # ########################## print("Building AutoEncoder") input_layer = self.encoder.layers[ 0].output # layers.Input(batch_shape=self.encoder.layers[0].input_shape) prev_layer = input_layer for layer in self.encoder.layers[1:]: prev_layer = layer(prev_layer) for layer in self.decoder.layers[1:]: prev_layer = layer(prev_layer) self.AutoEncoderNet = Model([input_layer], [prev_layer], name='AutoEncoder') print(self.AutoEncoderNet.summary()) ### VGG print("Importing VGG") self.VGG16 = VGG16(include_top=False, weights='imagenet', input_shape=self.img_shape[1:]) print("Plotting Models") plot_model(self.AutoEncoderNet, to_file='Model_AutoEncoderNet.png', show_shapes=True) plot_model(self.VGG16, to_file='Model_VGG16.png', show_shapes=True) for i in self.style_bank: stylenet_model_file = "Model_StyleNet{}.png".format(i) plot_model(self.StyleNet[i], to_file=stylenet_model_file, show_shapes=True) def compile_models(self): print("Compiling models") def total_variation_loss(x): img_nrows = self.img_shape[1] img_ncols = self.img_shape[2] assert K.ndim(x) == 4 if K.image_data_format() == 'channels_first': a = K.square(x[:, :, :img_nrows - 1, :img_ncols - 1] - x[:, :, 1:, :img_ncols - 1]) b = K.square(x[:, :, :img_nrows - 1, :img_ncols - 1] - x[:, :, :img_nrows - 1, 1:]) else: a = K.square(x[:, :img_nrows - 1, :img_ncols - 1, :] - x[:, 1:, :img_ncols - 1, :]) b = K.square(x[:, :img_nrows - 1, :img_ncols - 1, :] - x[:, :img_nrows - 1, 1:, :]) return K.sum(K.pow(a + b, 1.25)) def gram_matrix(x): assert K.ndim(x) == 4 grams = list() for i in range(self.Batch_Size): img = x[i, :, :, :] if K.image_data_format() == 'channels_first': features = K.batch_flatten(img) else: features = K.batch_flatten( K.permute_dimensions(img, (2, 0, 1))) grams.append(K.dot(features, K.transpose(features))) gram = tf.keras.backend.stack(grams) return gram def stylenet_loss_wrapper(input_tensor): def stylenet_loss(S, O): style_loss = K.variable(0.0) content_loss = K.variable(0.0) vgg16_layers = [l for l in self.VGG16.layers] vgg16_layers = vgg16_layers[1:] FlI = input_tensor #self.encoder.layers[0].output FlS = S FlO = O for i in range(len(vgg16_layers)): FlI = vgg16_layers[i](FlI) FlS = vgg16_layers[i](FlS) FlO = vgg16_layers[i](FlO) if vgg16_layers[i] == self.VGG16.get_layer( 'block4_conv2') or self.VGG16.get_layer( 'block3_conv2') or self.VGG16.get_layer( 'block2_conv2') or self.VGG16.get_layer( 'block1_conv2'): gram_mse = K.mean( K.square(gram_matrix(FlO) - gram_matrix(FlS))) layer_channels = vgg16_layers[i].output_shape[3] layer_size = vgg16_layers[i].output_shape[ 1] * vgg16_layers[i].output_shape[2] gram_mse_norm = gram_mse / (2.0**2 * (layer_channels**2) * (layer_size**2)) style_loss = style_loss + gram_mse_norm if vgg16_layers[i] == self.VGG16.get_layer('block4_conv2'): content_loss = K.mean(K.square(FlO - FlI)) break tv_loss = total_variation_loss(O) return self.LossAlpha * content_loss + self.LossBeta * style_loss + self.LossGamma * tv_loss return stylenet_loss # Compile Models for i in self.style_bank: print("Compiling StyleBank {}".format(i)) self.StyleNet[i].compile(optimizer=self.Optimizer, loss=stylenet_loss_wrapper( self.StyleNet[i].layers[0].output)) self.AutoEncoderNet.compile(optimizer=self.Optimizer, loss=mse) print("Initial learning rates: StyleNet={}, AutoEncoder={}".format( K.eval(self.StyleNet[0].optimizer.lr), K.eval(self.AutoEncoderNet.optimizer.lr))) def get_batch_ids(self, batch_size, data_size): return np.random.choice(np.arange(0, data_size), size=batch_size, replace=False) def train_models(self): style_id = 0 new_lr = self.LR_Initial for step in range(self.N_steps): style_ids = [style_id for i in range(self.Batch_Size)] batch_ids = self.get_batch_ids(self.Batch_Size, self.n_content) # Load the DB print("Loading DB, step {}...".format(step), end='') self.Content_DB = np.array([ resize(imread(self.Content_DB_list[batch_id]), self.img_shape[1:]) for batch_id in batch_ids ]) style_im = resize(imread(self.Style_DB_list[style_id]), self.img_shape[1:]) self.Style_DB = np.array([style_im for style_id in style_ids]) print("Finished Loading DB") if step % (self.T + 1) != self.T: # Train Style loss_style = self.StyleNet[style_id].train_on_batch( self.Content_DB, self.Style_DB) self.TensorBoardStyleNet[style_id].on_epoch_end( step, self.named_logs(self.StyleNet[style_id], loss_style)) else: # Train AE loss_autoencoder = self.AutoEncoderNet.train_on_batch( self.Content_DB, self.Content_DB) self.TensorBoardAutoEncoder.on_epoch_end( step, self.named_logs(self.AutoEncoderNet, loss_autoencoder)) style_id += 1 style_id = style_id % self.n_styles if step % self.print_iter == 0 and step != 0: print( "step {0}, loss_style={1}, loss_autoencoder={2}, timestamp={3}" .format(step, loss_style, loss_autoencoder, datetime.now())) if step % self.LR_Update_Every == 0 and step != 0: new_lr = new_lr * self.LR_Decay self.LR_Current = new_lr for i in self.style_bank: K.set_value(self.StyleNet[i].optimizer.lr, new_lr) K.set_value(self.AutoEncoderNet.optimizer.lr, new_lr) print("Updating LR to: StyleNet={}, AutoEncoder={}".format( K.eval(self.StyleNet[0].optimizer.lr), K.eval(self.AutoEncoderNet.optimizer.lr))) for i in self.style_bank: self.TensorBoardStyleNet[i].on_train_end(None) self.TensorBoardAutoEncoder.on_train_end(None) def prepare_tensorboard(self): for i in self.style_bank: self.TensorBoardStyleNet[i] = keras.callbacks.TensorBoard( log_dir="tb_logs/stylenet_{}".format(i), histogram_freq=0, batch_size=self.Batch_Size, write_graph=True, write_grads=True) self.TensorBoardStyleNet[i].set_model(self.StyleNet[i]) self.TensorBoardAutoEncoder = keras.callbacks.TensorBoard( log_dir="tb_logs/autoencoder", histogram_freq=0, batch_size=self.Batch_Size, write_graph=True, write_grads=True) self.TensorBoardAutoEncoder.set_model(self.AutoEncoderNet) def named_logs(self, model, logs): result = {} for l in zip(model.metrics_names, [logs]): result[l[0]] = l[1] return result def save_models(self): # serialize model to JSON if self.Use_Batch_Norm: out_mod_dir = 'Save_BatchNorm' else: out_mod_dir = 'Save_InstNorm' os.makedirs(out_mod_dir) ae_json = self.AutoEncoderNet.to_json() with open(os.path.join(out_mod_dir, "autoencoder.json"), "w") as json_file: json_file.write(ae_json) # serialize weights to HDF5 self.AutoEncoderNet.save_weights( os.path.join(out_mod_dir, "autoencoder.h5")) for i in self.style_bank: ae_json = self.StyleNet[i].to_json() with open(os.path.join(out_mod_dir, "stylenet_{}.json".format(i)), "w") as json_file: json_file.write(ae_json) # serialize weights to HDF5 self.StyleNet[i].save_weights( os.path.join(out_mod_dir, "stylenet_{}.h5".format(i))) print("Saved model to disk") def load_models(self): # load json and create model if self.Use_Batch_Norm: out_mod_dir = 'Save_BatchNorm' else: out_mod_dir = 'Save_InstNorm' ae_json = open(os.path.join(out_mod_dir, 'autoencoder.json'), 'r') ae_model_json = ae_json.read() ae_json.close() ae_model = models.model_from_json( ae_model_json, custom_objects={'InstanceNormalization': InstanceNormalization}) # load weights into new model ae_model.load_weights(os.path.join(out_mod_dir, "autoencoder.h5")) self.AutoEncoderNet = ae_model for i in self.style_bank: ae_json = open( os.path.join(out_mod_dir, "stylenet_{}.json".format(i)), 'r') ae_model_json = ae_json.read() ae_json.close() ae_model = models.model_from_json(ae_model_json, custom_objects={ 'InstanceNormalization': InstanceNormalization }) # load weights into new model ae_model.load_weights( os.path.join(out_mod_dir, "stylenet_{}.h5".format(i))) self.StyleNet[i] = ae_model print("Loaded models from disk")
class CycleGAN: """ Do OOP for this GAN due to number of generators and discriminators """ def __init__(self): # Input shape self.img_rows = 128 self.img_cols = 128 self.channels = 3 # Configure dataloader self.dataset_name = 'apple2orange' self.data_loader = DataLoader(dataset_name=self.dataset_name, img_res=(self.img_rows, self.img_cols)) # Calculate output shape of D (PatchGAN) patch = int(self.img_rows / 2**4) self.disc_patch = (patch, patch, 1) # No. of filters in the 1st layer of G and D self.generator_filters = 32 self.discriminator_filters = 64 # Loss weights self.lambda_cycle = 10. # Cycle-consistency loss self.lambda_id = 0.9 * self.lambda_cycle # Identity loss optimizer = Adam(0.0002, 0.5) # Build and compile the discriminators self.discriminator_a = self.build_discriminator() self.discriminator_b = self.build_discriminator() self.discriminator_a.compile(loss='mse', optimizer=optimizer, metrics=['accuracy']) self.discriminator_b.compile(loss='mse', optimizer=optimizer, metrics=['accuracy']) # Construct computational graph of generators # Build the generators self.generator_ab = self.build_generator() self.generator_ba = self.build_generator() # Input images from both domains img_a = layers.Input(shape=(self.img_rows, self.img_cols, self.channels)) img_b = layers.Input(shape=(self.img_rows, self.img_cols, self.channels)) # Translate images to other domains fake_b = self.generator_ab(img_a) fake_a = self.generator_ba(img_b) # Translate images back to original domain reconstr_a = self.generator_ba(fake_b) reconstr_b = self.generator_ab(fake_a) # Identity mapping of images img_a_id = self.generator_ba(img_a) img_b_id = self.generator_ab(img_b) # For the combined model, only train the generators self.discriminator_a.trainable = False self.discriminator_b.trainable = False # Discriminators determine validity of translated images valid_a = self.discriminator_a(fake_a) valid_b = self.discriminator_b(fake_b) # Combined model trains generators to fool discriminators self.combined = Model(inputs=[img_a, img_b], outputs=[ valid_a, valid_b, reconstr_a, reconstr_b, img_a_id, img_b_id ]) self.combined.compile(loss=['mse', 'mse', 'mae', 'mae', 'mae', 'mae'], loss_weights=[ 1, 1, self.lambda_cycle, self.lambda_cycle, self.lambda_id, self.lambda_id ], optimizer=optimizer) @staticmethod def conv2d(layer_input, filters, f_size=4, normalization=True): """Discriminator layer""" output = Conv2D(filters, kernel_size=f_size, strides=2, padding='same')(layer_input) output = LeakyReLU(alpha=0.2)(output) if normalization: output = InstanceNormalization()(output) return output @staticmethod def deconv2d(layer_input, skip_input, filters, f_size=4, dropout_rate=0): """Layer used during upsampling""" output = UpSampling2D(size=2)(layer_input) output = Conv2D(filters, kernel_size=f_size, strides=1, padding='same', activation='relu')(output) if dropout_rate: output = layers.Dropout(dropout_rate)(output) output = InstanceNormalization()(output) output = layers.Concatenate()([output, skip_input]) return output def build_generator(self): """U-Net Generator""" d_0 = layers.Input(shape=(self.img_rows, self.img_cols, self.channels)) # Downsampling d_1 = self.conv2d(d_0, self.generator_filters) d_2 = self.conv2d(d_1, self.generator_filters * 2) d_3 = self.conv2d(d_2, self.generator_filters * 4) d_4 = self.conv2d(d_3, self.generator_filters * 8) # Upsampling u_1 = self.deconv2d(d_4, d_3, self.generator_filters * 4) u_2 = self.deconv2d(u_1, d_2, self.generator_filters * 2) u_3 = self.deconv2d(u_2, d_1, self.generator_filters) u_4 = UpSampling2D(size=2)(u_3) output_img = Conv2D(self.channels, kernel_size=4, strides=1, padding='same', activation='tanh')(u_4) return Model(d_0, output_img) def build_discriminator(self): """ Simple CNN discriminator but outputting patches instead of a scalar """ img = layers.Input(shape=(self.img_rows, self.img_cols, self.channels)) d_1 = self.conv2d(img, self.discriminator_filters, normalization=False) d_2 = self.conv2d(d_1, self.discriminator_filters * 2) d_3 = self.conv2d(d_2, self.discriminator_filters * 4) d_4 = self.conv2d(d_3, self.discriminator_filters * 8) validity = Conv2D(1, kernel_size=4, strides=1, padding='same')(d_4) return Model(img, validity) def sample_images(self, epoch, batch_i): """ generate 2x3 grid images where each row represents a generator e.g., row 1 is A 1st column is the original image drawn from the dataset 2nd column is the generated image in the other domain 3rd column is the reconstructed image back in the original domain """ row, column = 2, 3 imgs_a = self.data_loader.load_data(domain='A', batch_size=1, is_testing=True) imgs_b = self.data_loader.load_data(domain='B', batch_size=1, is_testing=True) # Translate images to other domain fake_b = self.generator_ab.predict(imgs_a) fake_a = self.generator_ba.predict(imgs_b) # Translate back to original domain reconstr_a = self.generator_ba.predict(fake_b) reconstr_b = self.generator_ab.predict(fake_a) gen_imgs = np.concatenate( [imgs_a, fake_b, reconstr_a, imgs_b, fake_a, reconstr_b]) gen_imgs = 0.5 * gen_imgs + 0.5 titles = ['Original', 'Translated', 'Reconstructed'] fig, axs = plt.subplots(row, column) cnt = 0 for i in range(row): for j in range(column): axs[i, j].imshow(gen_imgs[cnt]) axs[i, j].set_title(titles[j]) axs[i, j].axis('off') cnt += 1 fig.savefig('data/img{}-{}.png'.format(epoch, batch_i)) def train(self, epochs, batch_size=1, sample_interval=50): """ alternately train Discriminator and Generator for specified epochs and batch_size, also generate sample images at specified intervals """ valid = np.ones((batch_size, ) + self.disc_patch) fake = np.zeros((batch_size, ) + self.disc_patch) for epoch in range(epochs): for batch_i, (imgs_a, imgs_b) in enumerate( self.data_loader.load_batch(batch_size)): # Train Discriminators # ------------------- # Translate images to opposite domains fake_b = self.generator_ab.predict(imgs_a) fake_a = self.generator_ba.predict(imgs_b) # Train discriminators (original images = real/translated = Fake) self.discriminator_a.train_on_batch(imgs_a, valid) self.discriminator_a.train_on_batch(fake_a, fake) self.discriminator_b.train_on_batch(imgs_b, valid) self.discriminator_b.train_on_batch(fake_b, fake) # Train Generators # ------------------- self.combined.train_on_batch( [imgs_a, imgs_b], [valid, valid, imgs_a, imgs_b, imgs_a, imgs_b]) # If at save interval, plot if batch_i % sample_interval == 0: self.sample_images(epoch, batch_i)
class merge_models: def __init__(self,epochs,batch_size): self.epochs = epochs self.batch_size = batch_size # Generator filter_kernal = [[32,128,128,128,256],[32,128,128,128,128,256,512],[32,128,128,128,128,128,128,128,512]] #self.g_1 = generator(3,filter_kernal[2],in_1_size,'Adam','mse',path = '/home/mdo2/sid_codes/new_codes/gen_1.h5').generator_model() #self.g_2 = generator(2,filter_kernal[1],in_2_size,'Adam','mse',path = '/home/mdo2/sid_codes/new_codes/gen_2.h5').generator_model() self.g_3 = generator(1,filter_kernal[0],in_3_size,'Adam','mse',path = '/home/mdo2/sid_codes/new_codes/gen_3.h5').generator_model() # Discriminator #self.D1 = discriminator(in_2_size).modified_vgg() #self.D1.compile(loss='mse',optimizer='Adam',metrics=['accuracy']) #self.D2 = discriminator(in_3_size).modified_vgg() #self.D2.compile(loss='mse',optimizer='Adam',metrics=['accuracy']) self.D = discriminator(out,'/home/mdo2/sid_codes/new_codes/D.h5').modified_vgg() self.D.compile(loss='mse',optimizer='Adam',metrics=['accuracy']) self.g = 0 # Images lr = Input(shape=in_1_size) hr_1 = Input(shape=in_2_size) hr_2 = Input(shape=in_2_size) hr_3 = Input(shape=out) #fake_hr_image_1 = self.g_1(lr) #fake_hr_image_2 = self.g_2(lr) fake_hr_image_3 = self.g_3(lr) #self.vgg_1 = vgg(in_2_size).vgg_loss_model() #self.vgg_2 = vgg(in_2_size).vgg_loss_model() self.vgg_3 = vgg(in_2_size).vgg_loss_model() #fake_hr_feat_1 = self.vgg_1(fake_hr_image_1) #fake_hr_feat_2 = self.vgg_2(fake_hr_image_2) fake_hr_feat_3 = self.vgg_3(fake_hr_image_3) self.D.trainable = False #self.D1.trainable = False #self.D2.trainable = False validity = self.D(fake_hr_image_3) #self.D.load_weights('/media/arjun/119GB/attachments/D_best.h5') self.merged_net = Model(inputs=[lr,hr_2],outputs=[validity,fake_hr_feat_3]) self.merged_net.compile(loss=['binary_crossentropy','mse'],loss_weights=[1e-3,.3],optimizer='Adam') self.merged_net.summary() #self.merged_net.load_weights('/home/mdo2/sid_codes/newew/my_model.h5') print('loaded') def save(self): #self.D.save_weights('D_best.h5') #print('Discriminator saved') #self.merged_net.save('my_model.h5') self.merged_net.save('/home/mdo2/sid_codes/new_codes/model.h5') #self.g_2.save('/home/mdo2/sid_codes/new_codes/gen_2.h5') #self.g_2.save('/home/mdo2/sid_codes/new_codes/gen_2.h5') self.g_3.save('/home/mdo2/sid_codes/new_codes/gen_3.h5') print('saved') """ def test(self): test_path, test_path = '' img_test_path = glob.glob(test_path) random.shuffle(img_test_path) d_sum = 0. g_sum = np.array([0.,0.,0.,0.,0.,0.,0.,0.]) step_size = int(800/self.batch_size) for steps in range(step_size): # Load data print('Loading Test Batch') x_test, y_test_1, y_test_2, y_test_3 = load_data(img_test_path, start=steps*self.batch_size, batch_size=self.batch_size) fake_hr_1 = self.g_1.predict(x_test) fake_hr_2 = self.g_2.predict(fake_hr_1) fake_hr_3 = self.g_3.predict(fake_hr_2) valid = np.ones([self.batch_size,12,12,16]) fake = np.zeros([self.batch_size,12,12,16]) d_test_loss_real = self.D.test_on_batch(y_test_3, valid) d_test_loss_fake = self.D.test_on_batch(fake_hr_3, fake) d_test_loss = 0.5*np.add(d_test_loss_fake,d_test_loss_real) 8 real_feat_1 = self.vgg_1.predict(y_test_1) real_feat_2 = self.vgg_2.predict(y_test_2) real_feat_3 = self.vgg_3.predict(y_test_3) g_test_loss = self.merged_net.test_on_batch([x_test,y_test_3], [valid,real_feat_3]) d_sum += d_test_loss[0] g_sum = np.add(g_sum, g_test_loss) save(d_sum/float(step_size),g_sum/float(step_size)) """ def train(self): try: #self.D.load_weights('/media/arjun/119GB/attachments/D_best.h5') #self.g_1.load_weights('/media/arjun/119GB/attachments/g_1_best.h5') #self.g_2.load_weights('/media/arjun/119GB/attachments/g_2_best.h5') #self.g_3.load_weights('/media/arjun/119GB/attachments/g_3_best.h5') print("Discriminator loaded") for epoch in range(self.epochs): print("loading data") train_path = dataset_dir img_train_path = glob.glob(train_path) i=0 if i==0: avg_loss=0 i+=1 avg_loss = avg_loss/int(25000/self.batch_size) for steps in range(int(25000/self.batch_size)): # Load data print(epoch) print('Loading Train Batch:',steps+1) x_train, y_train_1, y_train_2, y_train_3 = load_data(train_path, start=steps*self.batch_size, batch_size=self.batch_size) tbCallBack = keras.callbacks.TensorBoard(log_dir='./Graph', histogram_freq=0, write_graph=True, write_images=True) # Discriminator print('Training Discriminator') ''' valid1 = np.ones([self.batch_size,1,1,1]) fake1 = np.zeros([self.batch_size,1,1,1]) valid2 = np.ones([self.batch_size,3,3,1]) fake2 = np.zeros([self.batch_size,3,3,1]) fake_hr_1 = self.g_1.predict(x_train) fake_hr_feat_1 = self.vgg_1(fake_hr_1) #d_train_loss_real = self.D1.train_on_batch(y_train_1, valid1) #d_train_loss_fake = self.D1.train_on_batch(fake_hr_1, fake1) fake_hr_2 = self.g_2.predict(fake_hr_1) fake_hr_feat_2 = self.vgg_2(fake_hr_2) #d_train_loss_real = self.D2.train_on_batch(y_train_2, valid2) #d_train_loss_fake = self.D2.train_on_batch(fake_hr_2, fake2) fake_hr_3 = self.g_3.predict(fake_hr_2) fake_hr_feat_3 = self.vgg_3(fake_hr_3) d_train_loss_fake = self.D.train_on_batch(fake_hr_3, fake) d_train_loss_real = self.D.train_on_batch(y_train_3, valid) #d_train_loss_real = self.D.train_on_batch(y_train_3, valid) #d_train_loss_fake = self.D.train_on_batch(fake_hr_3, fake) #print(d_train_loss_fake,' fake loss ') #print(d_train_loss_real,' real loss ') d_train_loss = 0.5*np.add(d_train_loss_fake,d_train_loss_real) # Generator #valid = np.ones([self.batch_size,192,192,3]) ''' print('Training Generator') valid = np.ones([self.batch_size,1,1,1]) fake = np.zeros([self.batch_size,1,1,1]) #real_feat_1 = self.vgg_1.predict(y_train_1) #real_feat_2 = self.vgg_2.predict(y_train_1) real_feat_3 = self.vgg_3.predict(y_train_1) d_train_loss=0 #cv2.imshow('image',real_feat_3) g_loss = self.merged_net.train_on_batch([x_train,y_train_1], [valid,real_feat_3]) #g_loss=0 #d_train_loss = 0 print(g_loss,d_train_loss) avg_loss+=g_loss[0] #self.low = g_loss #print("aaaaaa",K.eval(K.sum(log_loss_1))) print('*******************************************************\n') if self.g == 0: try: self.old_loss = np.load('/home/mdo2/sid_codes/newew/g_loss_3.npy') print('latest gen') self.g+=1 except: self.old_loss = float('inf') self.g+=1 if g_loss[0]< self.old_loss: self.old_loss = g_loss[0] print('saving best') np.save ('g_loss_3',g_loss[0]) self.g+=1 self.save() #if (steps+1)%100==0: # self.save() #self.merged_net.inputs = [x_train,y_train_1,y_train_2,y_train_3] #self.merged_net.outputs = [valid,fake_hr_feat_1,fake_hr_feat_2,fake_hr_feat_3] #c_loss = K.sqrt(K.mean((fake_hr_feat_1 - real_feat_1)**2)) #log_loss = logcosh(fake_hr_feat_1,real_feat_1) #grads = K.gradients([log_loss,c_loss],[fake_hr_feat_1,real_feat_1])[0] #print(grads) #iterate = K.function([(y_train_1)], [c_loss, grads]) #print(iterate,'c_loss') #iterate = K.function([tf.convert_to_tensor(y_train_1)], [log_loss, grads]) #print(iterate,'log_loss_1') #self.D.fit(y_train_3,valid, epochs=epoch, batch_size=self.batch_size, callbacks=[tbCallBack]) except KeyboardInterrupt: print('Saving Weights') #self.save() def feed_forward(self): in_1 = [256,256,3] in_2 = [512,512,3] in_3 = [1024,1024,3] out_ = [128,128,3] #/home/mdo2/sid_codes/newew/ee2.jpg #/home/mdo2/sid_codes/datasets_big/qhd_no_pad/train/1.png img = cv2.resize(cv2.imread('/home/mdo2/sid_codes/datasets_big/qhd_no_pad/train/1.png',1),(512*2,512*2)) img = img.reshape(1,512*2,512*2,3) filter_kernal = [[32,128,128,128,256],[32,128,128,128,128,256,512],[32,128,128,128,128,128,128,128,512]] g3 = generator(3,filter_kernal[2],in_3,'Adam',path = '/home/mdo2/sid_codes/new_codes/gen_1.h5').generator_model() #g2 = generator(2,filter_kernal[1],in_2,'Adam',path = '/home/mdo2/sid_codes/new_codes/new_save/gen_2.h5').generator_model() #g1 = generator(1,filter_kernal[0],in_1,'Adam',path = '/home/mdo2/sid_codes/new_codes/new_save/gen_3.h5').generator_model() img = g3.predict(img) #img = g2.predict(img) img = np.array(img).astype('uint8') img = img.reshape(1024*2,1024*2,3) cv2.imshow('img',img) cv2.waitKey(0) cv2.imwrite('ee3.png',img) im1 = tf.io.decode_image('/home/mdo2/sid_codes/newew/ee3.png') #im1 = tf.image.convert_image_dtype(im1, tf.float32) im2 = tf.io.decode_image('/home/mdo2/sid_codes/datasets_big/qhd_no_pad/train/1.png') #im2 = tf.image.convert_image_dtype(im2, tf.float32) psnr1 = tf.image.psnr(im2, im2, max_val=1.0) print(psnr1)
def fit(self, X_train, epochs=50, batch_size=50, loss=tf.keras.losses.BinaryCrossentropy(), optimizer=tf.keras.optimizers.Adam(lr=1e-5, beta_1=0.5), test=tuple(), early_stop_num=50, verbose=1): #Convert training-data to numpy format X_train = np.array(X_train) #If "input_dim" is not greater than 1, it is set to the number of features in the training-data if not self.input_dim >= 1: self.input_dim = X_train.shape[1] #Define model for Discriminator self.dis = self.get_discriminator() self.dis.compile(loss=loss, optimizer=optimizer) #Define model to train Encoder self.enc = self.get_encoder() x = Input(shape=(self.input_dim, )) z_gen = self.enc(x) valid = self.dis([x, z_gen]) enc_dis = Model(inputs=x, outputs=valid, name='enc_to_dis') enc_dis.compile(loss=loss, optimizer=optimizer) #Define model to train Generator self.gen = self.get_generator() z = Input(shape=(self.latent_dim, )) x_gen = self.gen(z) valid = self.dis([x_gen, z]) gen_dis = Model(inputs=z, outputs=valid, name='gen_to_dis') gen_dis.compile(loss=loss, optimizer=optimizer) #Training min_val_loss = float('inf') stop_count = 0 for i in range(epochs): #Unfreeze Discriminator self.dis.trainable = True #From the training data, randomly choice half of the "batch_size" as "real_data" idx = np.random.randint(0, X_train.shape[0], batch_size // 2) real_data = X_train[idx] #Generates noise and get the data generated by that noise noise = np.random.normal(0, 1, (len(idx), self.latent_dim)) gen_data = self.gen.predict(noise) #Get noise predicted from "real_data" enc_noise = self.enc.predict(real_data) #Train Discriminator d_enc_loss = self.dis.train_on_batch([real_data, enc_noise], np.ones((len(real_data), 1))) d_gen_loss = self.dis.train_on_batch([gen_data, noise], np.zeros((len(gen_data), 1))) d_loss = d_enc_loss + d_gen_loss #Freeze Discriminator to train Encoder and Generator self.dis.trainable = False #Train Encoder e_loss = enc_dis.train_on_batch(real_data, np.zeros((len(real_data), 1))) #Train Generator g_loss = gen_dis.train_on_batch(noise, np.ones((len(noise), 1))) #Calculate test loss if len(test) > 0: #Get testing-data X_test = test[0] y_true = test[1] #Predict testing-data proba = self.predict(X_test) proba = minmax_scale(proba) #As "val_loss", calculate binary cross entropy val_loss = tf.keras.losses.binary_crossentropy(y_true, proba).numpy() #If "val_loss" is less than "min_val_loss" if min_val_loss > val_loss: min_val_loss = val_loss #Update "min_val_loss" to "val_loss" stop_count = 0 #Change "stop_count" to 0 #If "stop_count" is equal or more than "early_stop_num", training is end elif stop_count >= early_stop_num: break #Else, "stop_count" is added 1 else: stop_count += 1 #Display learning progress if verbose == 1 and i % 100 == 0: if len(test) == 0: print( f'epoch{i}-> d_loss:{d_loss} e_loss:{e_loss} g_loss:{g_loss}' ) else: print( f'epoch{i}-> d_loss:{d_loss} e_loss:{e_loss} g_loss:{g_loss} val_loss:{val_loss}' )
def layer_test(layer_cls, kwargs={}, input_shape=None, input_dtype=None, input_data=None, expected_output=None, expected_output_dtype=None, fixed_batch_size=False): """Test routine for a layer with a single input tensor and single output tensor. """ # generate input data if input_data is None: assert input_shape if not input_dtype: input_dtype = K.floatx() input_data_shape = list(input_shape) for i, e in enumerate(input_data_shape): if e is None: input_data_shape[i] = np.random.randint(1, 4) input_data = (10 * np.random.random(input_data_shape)) input_data = input_data.astype(input_dtype) else: if input_shape is None: input_shape = input_data.shape if input_dtype is None: input_dtype = input_data.dtype if expected_output_dtype is None: expected_output_dtype = input_dtype # instantiation layer = layer_cls(**kwargs) # test get_weights , set_weights at layer level weights = layer.get_weights() layer.set_weights(weights) try: expected_output_shape = layer.compute_output_shape(input_shape) except: expected_output_shape = layer._compute_output_shape(input_shape) # test in functional API if fixed_batch_size: x = Input(batch_shape=input_shape, dtype=input_dtype) else: x = Input(shape=input_shape[1:], dtype=input_dtype) y = layer(x) assert K.dtype(y) == expected_output_dtype # check with the functional API model = Model(x, y) actual_output = model.predict(input_data) actual_output_shape = actual_output.shape for expected_dim, actual_dim in zip(expected_output_shape, actual_output_shape): if expected_dim is not None: assert expected_dim == actual_dim if expected_output is not None: assert_allclose(actual_output, expected_output, rtol=1e-3) # test serialization, weight setting at model level model_config = model.get_config() recovered_model = model.__class__.from_config(model_config) if model.weights: weights = model.get_weights() recovered_model.set_weights(weights) _output = recovered_model.predict(input_data) assert_allclose(_output, actual_output, rtol=1e-3) # test training mode (e.g. useful when the layer has a # different behavior at training and testing time). if has_arg(layer.call, 'training'): model.compile('rmsprop', 'mse') model.train_on_batch(input_data, actual_output) # test instantiation from layer config layer_config = layer.get_config() layer_config['batch_input_shape'] = input_shape layer = layer.__class__.from_config(layer_config) # for further checks in the caller function return actual_output
size=batch_size)] #Construct different batches of real and fake data X = np.concatenate([image_batch, generated_images]) # Labels for generated and real data y_dis = np.zeros(2 * batch_size) y_dis[:batch_size] = 0.9 #Pre train discriminator on fake and real data before starting the gan. discriminator.trainable = True discriminator.train_on_batch(X, y_dis) #Tricking the noised input of the Generator as real data noise = np.random.normal(0, 1, [batch_size, 100]) y_gen = np.ones(batch_size) # During the training of gan, # the weights of discriminator should be fixed. #We can enforce that by setting the trainable flag discriminator.trainable = False #training the GAN by alternating the training of the Discriminator #and training the chained GAN model with Discriminator’s weights freezed. gan.train_on_batch(noise, y_gen) if e == 1 or e % 20 == 0: plot_generated_images(e, generator) #%%
class merge_models: def __init__(self, epochs, batch_size): self.epochs = epochs self.batch_size = batch_size # Generator filter_kernal = [[32, 128, 128, 128, 256], [32, 128, 128, 128, 128, 256, 512], [32, 128, 128, 128, 128, 128, 128, 512, 512]] self.g_1 = generator(3, filter_kernal[2], in_1_size, 'Adam', 'mse').generator_model() self.g_2 = generator(2, filter_kernal[1], in_2_size, 'Adam', 'mse').generator_model() self.g_3 = generator(1, filter_kernal[0], in_3_size, 'Adam', 'mse').generator_model() # Discriminator self.D = discriminator(out).modified_vgg() self.D.compile(loss='mse', optimizer='Adam', metrics=['accuracy']) # Images lr = Input(shape=in_1_size) hr_1 = Input(shape=in_2_size) hr_2 = Input(shape=in_3_size) hr_3 = Input(shape=out) fake_hr_image_1 = self.g_1(lr) fake_hr_image_2 = self.g_2(fake_hr_image_1) fake_hr_image_3 = self.g_3(fake_hr_image_2) self.vgg_1 = vgg(in_2_size).vgg_loss_model() self.vgg_2 = vgg(in_3_size).vgg_loss_model() self.vgg_3 = vgg(out).vgg_loss_model() fake_hr_feat_1 = self.vgg_1(fake_hr_image_1) fake_hr_feat_2 = self.vgg_2(fake_hr_image_2) fake_hr_feat_3 = self.vgg_3(fake_hr_image_3) self.D.trainable = False valid = self.D(fake_hr_image_3) self.merged_net = Model( inputs=[lr, hr_1, hr_2, hr_3], outputs=[valid, fake_hr_feat_1, fake_hr_feat_2, fake_hr_feat_3]) self.merged_net.compile( loss=['binary_crossentropy', 'mse', 'mse', 'mse'], loss_weights=[1e-3, .3, .3, .3], optimizer='Adam') self.merged_net.summary() def save(self): self.D.save_weights('D_best.h5') print('Discriminator saved') self.g_1.save_weights('g_1_best.h5') self.g_2.save_weights('g_2_best.h5') self.g_3.save_weights('g_3_best.h5') print('All generators saved') """ def test(self): test_path, test_path = '' img_test_path = glob.glob(test_path) random.shuffle(img_test_path) d_sum = 0. g_sum = np.array([0.,0.,0.,0.,0.,0.,0.,0.]) step_size = int(800/self.batch_size) for steps in range(step_size): # Load data print('Loading Test Batch') x_test, y_test_1, y_test_2, y_test_3 = load_data(img_test_path, start=steps*self.batch_size, batch_size=self.batch_size) fake_hr_1 = self.g_1.predict(x_test) fake_hr_2 = self.g_2.predict(fake_hr_1) fake_hr_3 = self.g_3.predict(fake_hr_2) valid = np.ones([self.batch_size,12,12,16]) fake = np.zeros([self.batch_size,12,12,16]) d_test_loss_real = self.D.test_on_batch(y_test_3, valid) d_test_loss_fake = self.D.test_on_batch(fake_hr_3, fake) d_test_loss = 0.5*np.add(d_test_loss_fake,d_test_loss_real) 8 real_feat_1 = self.vgg_1.predict(y_test_1) real_feat_2 = self.vgg_2.predict(y_test_2) real_feat_3 = self.vgg_3.predict(y_test_3) g_test_loss = self.merged_net.test_on_batch([x_test,y_test_3], [valid,real_feat_3]) d_sum += d_test_loss[0] g_sum = np.add(g_sum, g_test_loss) save(d_sum/float(step_size),g_sum/float(step_size)) """ def train(self): try: for epoch in range(self.epochs): train_path = dataset_dir + '/*.jpg' img_train_path = glob.glob(train_path) random.shuffle(img_train_path) for steps in range(int(25000 / self.batch_size)): # Load data print('Loading Train Batch:', steps + 1) x_train, y_train_1, y_train_2, y_train_3 = load_data( train_path, start=steps * self.batch_size, batch_size=self.batch_size) # Discriminator print('Training Discriminator') fake_hr_1 = self.g_1.predict(x_train) fake_hr_2 = self.g_2.predict(fake_hr_1) fake_hr_3 = self.g_3.predict(fake_hr_2) valid = np.ones([self.batch_size, 6, 6, 1]) fake = np.zeros([self.batch_size, 6, 6, 1]) #d_train_loss_real = self.D.train_on_batch(y_train_3, valid) #d_train_loss_fake = self.D.train_on_batch(fake_hr_3, fake) #d_train_loss = 0.5*np.add(d_train_loss_fake,d_train_loss_real) d_train_loss = 0 # Generator print('Training Generator') real_feat_1 = self.vgg_1.predict(y_train_1) real_feat_2 = self.vgg_2.predict(y_train_2) real_feat_3 = self.vgg_3.predict(y_train_3) g_loss = self.merged_net.train_on_batch( [x_train, y_train_1, y_train_2, y_train_3], [valid, real_feat_1, real_feat_2, real_feat_3]) print(g_loss, d_train_loss) self.low = g_loss print("aaaaaa", K.eval(K.sum(log_loss_1))) print( '*******************************************************\n' ) if (steps + 1) % 100 == 0: self.save() except KeyboardInterrupt: print('Saving Weights') #self.save() def loss(self): print("0000......................................")
class A2C(Agent): """Advantage Actor-Critic (A2C) A2C is a synchronous version of A3C which gives equal or better performance. For more information on A2C refer to the OpenAI blog post: https://blog.openai.com/baselines-acktr-a2c/. The A3C algorithm is described in "Asynchronous Methods for Deep Reinforcement Learning" (Mnih et al., 2016) Since this algorithm is on-policy, it can and should be trained with multiple simultaneous environment instances. The parallelism decorrelates the agents' data into a more stationary process which aids learning. """ def __init__(self, model, actions, optimizer=None, policy=None, test_policy=None, gamma=0.99, instances=8, nsteps=1, value_loss=0.5, entropy_loss=0.01): """ TODO: Describe parameters """ self.actions = actions self.optimizer = Adam(lr=3e-3) if optimizer is None else optimizer self.memory = memory.OnPolicy(steps=nsteps, instances=instances) if policy is None: # Create one policy per instance, with varying exploration parameters self.policy = [Greedy()] + [ GaussianEpsGreedy(eps, 0.1) for eps in np.arange(0, 1, 1 / (instances - 1)) ] else: self.policy = policy self.test_policy = Greedy() if test_policy is None else test_policy self.gamma = gamma self.instances = instances self.nsteps = nsteps self.value_loss = value_loss self.entropy_loss = entropy_loss self.training = True # Create output model layers based on number of actions raw_output = model.layers[-1].output actor = Dense(actions, activation='softmax')( raw_output) # Actor (Policy Network) critic = Dense(1, activation='linear')( raw_output) # Critic (Value Network) output_layer = Concatenate()([actor, critic]) self.model = Model(inputs=model.input, outputs=output_layer) def a2c_loss(targets_actions, y_pred): # Unpack input targets, actions = targets_actions[:, 0], targets_actions[:, 1:] # Unpack probs, values = y_pred[:, :-1], y_pred[:, -1] # Compute advantages and logprobabilities adv = targets - values logprob = tf.math.log( tf.reduce_sum(probs * actions, axis=1, keepdims=False) + 1e-10) # Compute composite loss loss_policy = -adv * logprob loss_value = self.value_loss * tf.square(adv) entropy = self.entropy_loss * tf.reduce_sum( probs * tf.math.log(probs + 1e-10), axis=1, keepdims=False) return tf.reduce_mean(loss_policy + loss_value + entropy) self.model.compile(optimizer=self.optimizer, loss=a2c_loss) def save(self, filename, overwrite=False): """Saves the model parameters to the specified file.""" self.model.save_weights(filename, overwrite=overwrite) def act(self, state, instance=0): """Returns the action to be taken given a state.""" qvals = self.model.predict(np.array([state]))[0][:-1] if self.training: return self.policy[instance].act(qvals) if isinstance( self.policy, list) else self.policy.act(qvals) else: return self.test_policy[instance].act(qvals) if isinstance( self.test_policy, list) else self.test_policy.act(qvals) def push(self, transition, instance=0): """Stores the transition in memory.""" self.memory.put(transition, instance) def train(self, step): """Trains the agent for one step.""" if len(self.memory) < self.instances: return state_batch, action_batch, reward_batches, end_state_batch, not_done_mask = self.memory.get( ) # Compute the value of the last next states target_qvals = np.zeros(self.instances) non_final_last_next_states = [ es for es in end_state_batch if es is not None ] if len(non_final_last_next_states) > 0: non_final_mask = list(map(lambda s: s is not None, end_state_batch)) target_qvals[non_final_mask] = self.model.predict_on_batch( np.array(non_final_last_next_states))[:, -1].squeeze() # Compute n-step discounted return # If episode ended within any sampled nstep trace - zero out remaining rewards for n in reversed(range(self.nsteps)): rewards = np.array([b[n] for b in reward_batches]) target_qvals *= np.array([t[n] for t in not_done_mask]) target_qvals = rewards + (self.gamma * target_qvals) # Prepare loss data: target Q-values and actions taken (as a mask) ran = np.arange(self.instances) targets_actions = np.zeros((self.instances, self.actions + 1)) targets_actions[ran, 0] = target_qvals targets_actions[ran, np.array(action_batch) + 1] = 1 self.model.train_on_batch(np.array(state_batch), targets_actions)
class CycleGAN(): def __init__(self): # Input shape self.class_num = config.class_num self.img_shape = config.img_shape self.img_width = config.img_width self.img_height = config.img_height self.img_channel = config.img_channel self.gf = 32 self.df = 64 self.patch = int(config.img_height // (2**4)) self.patch_size = (self.patch, self.patch, 1) self.s1 = Dataset('./t1ce', './tice_label') self.s2 = Dataset('./t2', './t2_label') self.target = Dataset('./OpenBayes', './Openbayes_label', need_resize=1) optimizer = RMSprop(0.0002) self.D = self.build_discriminator() self.G = self.build_generator() self.G.trainable = False real_img = Input(shape=config.img_shape) real_src, real_cls = self.D(real_img) fake_cls = Input(shape=(self.class_num, )) fake_img = self.G([real_img, fake_cls]) fake_src, fake_output = self.D(fake_img) self.Train_D = Model([real_img, fake_cls], [real_src, real_cls, fake_src, fake_output]) self.Train_D.compile(loss=[ 'mse', self.classification_loss, 'mse', self.classification_loss ], optimizer=optimizer, loss_weights=[1.0, 1.0, 1.0, 1.0]) self.G.trainable = True self.D.trainable = False real_x = Input(shape=self.img_shape) now_label = Input(shape=(self.class_num, )) target_label = Input(shape=(self.class_num, )) fake_x = self.G([real_x, target_label]) fake_out_src, fake_out_cls = self.D(fake_x) x_rec = self.G([fake_x, now_label]) self.train_G = Model([real_x, now_label, target_label], [fake_out_src, fake_out_cls, x_rec]) self.train_G.compile(loss=['mse', self.classification_loss, 'mae'], optimizer=optimizer, loss_weights=[1.0, 1.0, 1.0]) ''' self.AdataSet=Dataset('./done', './done_label',need_resize=1) to get different model change image path for example t1ce to t1 self.BdataSet=Dataset('./Data/2/t1','./Data/2/label') # Number of filters in the first layer of G and D self.patch = int(config.img_height //(2**4)) self.patch_size = (self.patch,self.patch,1) # Loss weights self.lambda_cycle = 10.0 # Cycle-consistency loss self.lambda_id = 0.1 * self.lambda_cycle # Identity loss optimizer = RMSprop(0.0002) # Build and compile the discriminators self.d_B = self.build_discriminator() self.d_A = self.build_discriminator() self.conv_init = RandomNormal(0, 0.02) # for convolution kernel self.gamma_init = RandomNormal(1., 0.02) self.d_A.compile(loss='mse', optimizer=optimizer, metrics=['accuracy']) self.d_B.compile(loss='mse', optimizer=optimizer, metrics=['accuracy']) #------------------------- # Construct Computational # Graph of Generators #------------------------- # Build the generators self.g_AB = self.resnet_6blocks() self.g_BA = self.resnet_6blocks() # Input images from both domains img_A = Input(shape=self.img_shape) img_B = Input(shape=self.img_shape) # Translate images to the other domain fake_B = self.g_AB(img_A) fake_A = self.g_BA(img_B) # Translate images back to original domain reconstr_A = self.g_BA(fake_B) reconstr_B = self.g_AB(fake_A) # Identity mapping of images img_A_id = self.g_BA(img_A) img_B_id = self.g_AB(img_B) # For the combined model we will only train the generators self.d_A.trainable = False self.d_B.trainable = False # Discriminators determines validity of translated images valid_A = self.d_A(fake_A) valid_B = self.d_B(fake_B) # Combined model trains generators to fool discriminators self.combined = Model(inputs=[img_A, img_B], outputs=[ valid_A, valid_B, reconstr_A, reconstr_B, img_A_id, img_B_id ]) self.combined.compile(loss=['mse', 'mse', 'mae', 'mae', 'mae', 'mae'], loss_weights=[ 1, 1, self.lambda_cycle, self.lambda_cycle, self.lambda_id, self.lambda_id ], optimizer=optimizer) ''' def classification_loss(self, Y_true, Y_pred): return tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(labels=Y_true, logits=Y_pred)) def normalize(self, **kwargs): # return batchnorm()#axis=get_filter_dim() return InstanceNormalization() def get_onehot_label(self, label): now_label = [0.] * self.class_num now_label[label] = 1. now_label = np.array(now_label).reshape([1, self.class_num]) return now_label def resnet_block(self, input, dim, ks=(3, 3), strides=(1, 1)): x = ZeroPadding2D((1, 1))(input) x = Conv2D(dim, ks, strides=strides, kernel_initializer=self.conv_init)(x) x = InstanceNormalization()(x) x = Activation('relu')(x) x = ZeroPadding2D((1, 1))(x) x = Conv2D(dim, ks, strides=strides, kernel_initializer=self.conv_init)(x) x = InstanceNormalization()(x) res = Add()([input, x]) return res def resnet_6blocks(self, ngf=64, **kwargs): input = Input(config.img_shape) x = ZeroPadding2D((3, 3))(input) x = Conv2D(ngf, (7, 7), kernel_initializer=self.conv_init)(x) x = InstanceNormalization()(x) x = Activation('relu')(x) n_downsampling = 2 for i in range(n_downsampling): mult = 2**i x = Conv2D(ngf * mult * 2, (3, 3), padding='same', strides=(2, 2), kernel_initializer=self.conv_init)(x) x = InstanceNormalization()(x) x = Activation('relu')(x) mult = 2**n_downsampling for i in range(6): x = self.resnet_block(x, ngf * mult) for i in range(n_downsampling): mult = 2**(n_downsampling - i) f = int(ngf * mult / 2) x = UpSampling2D()(x) x = InstanceNormalization()(x) x = Activation('relu')(x) x = ZeroPadding2D((3, 3))(x) x = Conv2D(config.img_channel, (7, 7), kernel_initializer=self.conv_init)(x) x = Activation('tanh')(x) model = Model(inputs=input, outputs=[x]) print('Model resnet 6blocks:') model.summary() return model def build_generator(self): """U-Net Generator""" def conv2d(layer_input, filters, f_size=4): """Layers used during downsampling""" d = Conv2D(filters, kernel_size=f_size, strides=2, padding='same')(layer_input) d = LeakyReLU(alpha=0.2)(d) d = InstanceNormalization()(d) return d def deconv2d(layer_input, skip_input, filters, f_size=4, dropout_rate=0): """Layers used during upsampling""" u = UpSampling2D(size=2)(layer_input) u = Conv2D(filters, kernel_size=f_size, strides=1, padding='same', activation='relu')(u) if dropout_rate: u = Dropout(dropout_rate)(u) u = InstanceNormalization()(u) u = Concatenate()([u, skip_input]) return u # Image input input_img = Input(shape=self.img_shape) inp_c = Input(shape=(self.class_num, )) c = Lambda(lambda x: K.repeat(x, self.img_width * self.img_height))( inp_c) c = Reshape((self.img_width, self.img_height, self.class_num))(c) d0 = Concatenate()([input_img, c]) # Downsampling d1 = conv2d(d0, self.gf) d2 = conv2d(d1, self.gf * 2) d3 = conv2d(d2, self.gf * 4) d4 = conv2d(d3, self.gf * 8) # Upsampling u1 = deconv2d(d4, d3, self.gf * 4) u2 = deconv2d(u1, d2, self.gf * 2) u3 = deconv2d(u2, d1, self.gf) u4 = UpSampling2D(size=2)(u3) output_img = Conv2D(self.img_channel, kernel_size=4, strides=1, padding='same', activation='tanh')(u4) return Model([input_img, inp_c], output_img) def build_discriminator(self): def d_layer(layer_input, filters, f_size=4, normalization=True): """Discriminator layer""" d = Conv2D(filters, kernel_size=f_size, strides=2, padding='same')(layer_input) d = LeakyReLU(alpha=0.2)(d) if normalization: d = InstanceNormalization()(d) return d img = Input(shape=self.img_shape) d1 = d_layer(img, self.df, normalization=False) d2 = d_layer(d1, self.df * 2) d3 = d_layer(d2, self.df * 4) d4 = d_layer(d3, self.df * 8) validity = Conv2D(1, kernel_size=4, strides=1, padding='same')(d4) class_cls = Conv2D(self.class_num, kernel_size=15, strides=1, padding='valid')(d4) class_cls = Reshape((self.class_num, ))(class_cls) model = Model(img, [validity, class_cls]) return model def train(self, epochs, batch_size=1, sample_interval=50): #start_time = datetime.datetime.now() # Adversarial loss ground truths valid = np.ones((batch_size, ) + self.patch_size) fake = np.zeros((batch_size, ) + self.patch_size) self.train_G.summary() self.Train_D.summary() for epoch in range(epochs): for batch_i in range(0, 1000): rand_number = random.randint(0, 1) target_label = 2 target_label = self.get_onehot_label(target_label) now_img = None now_label = None if (rand_number == 0): now_img, _ = self.s1.get_one_data() now_label = self.get_onehot_label(0) if (rand_number == 1): now_img, _ = self.s2.get_one_data() now_label = self.get_onehot_label(1) if (rand_number == 2): now_img, _ = self.target.get_one_data() now_label = self.get_onehot_label(2) #print(now_label.shape) #print(target_label.shape) #print(self.D.output) d_loss = self.Train_D.train_on_batch( [now_img, target_label], [valid, now_label, fake, target_label], class_weight=[1.0, 1.0, 1.0, 1.0]) g_loss = self.train_G.train_on_batch( [now_img, now_label, target_label], [valid, target_label, now_img], class_weight=[1.0, 1.0, 1.0]) print("[Epoch %d/%d][Batch %d/%d]" % (epoch, epochs, batch_i, 1000)) print( "[d loss: %.3f][real dloss: %.3f][real cl: %.6f][fake dloss: %.3f][fake cl:%.6f]" % (d_loss[0], d_loss[1], d_loss[2], d_loss[3], d_loss[4])) print( "[g loss: %.3f][g dloss: %.3f][g cl loss: %.6f][rec loss: %.3f]" % (g_loss[0], g_loss[1], g_loss[2], g_loss[3])) # If at save interval => save generated image samples if batch_i % sample_interval == 0: ''' change parameter path to save image to different dir ''' self.sample_images(epoch, batch_i, path='sample_t1ce') ''' remember to save model with different name ''' self.G.save_weights('g_300.h5') def sample_images(self, epoch, batch_i, path='sample'): os.makedirs(path + '/', exist_ok=True) r, c = 2, 2 imgs_A, label_A = self.s1.get_one_data() imgs_B, label_B = self.s2.get_one_data() target_label = self.get_onehot_label(2) fake_A = self.G.predict([imgs_A, target_label]) fake_B = self.G.predict([imgs_B, target_label]) gen_imgs = np.concatenate([imgs_A, fake_A, imgs_B, fake_B]) gen_imgs = np.reshape(gen_imgs, (r * c, config.img_width, config.img_height)) # Rescale images 0 - 1 gen_imgs = 0.5 * gen_imgs + 0.5 for x in range(r * c): now_image = gen_imgs[x, :, :] now_image = now_image[:, :, np.newaxis] now_image = image.array_to_img(now_image, scale=True) now_image.save(os.path.join(path, 'generated_' \ + str(epoch) + '_' + str(batch_i)+'_'+str(x) + '_.png')) '''
class Pix2Pix(): def __init__(self): # Input shape self.img_rows = 256 self.img_cols = 256 self.channels = 3 self.img_shape = (self.img_rows, self.img_cols, self.channels) # Configure data loader self.dataset_name = 'facades' self.data_loader = DataLoader(dataset_name=self.dataset_name, img_res=(self.img_rows, self.img_cols)) # Calculate output shape of D (PatchGAN) patch = int(self.img_rows / 2**4) self.disc_patch = (patch, patch, 1) # Number of filters in the first layer of G and D self.gf = 64 self.df = 64 optimizer = Adam(0.0002, 0.5) # Build and compile the discriminator self.discriminator = self.build_discriminator() self.discriminator.compile(loss='mse', optimizer=optimizer, metrics=['accuracy']) #------------------------- # Construct Computational # Graph of Generator #------------------------- # Build the generator self.generator = self.build_generator() # Input images and their conditioning images img_A = Input(shape=self.img_shape) img_B = Input(shape=self.img_shape) # By conditioning on B generate a fake version of A fake_A = self.generator(img_B) # For the combined model we will only train the generator #self.discriminator.trainable = False # Discriminators determines validity of translated images / condition pairs valid = self.discriminator([fake_A, img_B]) self.combined = Model(inputs=[img_A, img_B], outputs=[valid, fake_A]) self.combined.compile(loss=['mse', 'mae'], loss_weights=[1, 100], optimizer=optimizer) valid.trainable = False #self.combined.load_weights("Weights/199.h5") def build_generator(self): layer_per_block = [4, 4, 4, 4, 4, 15, 4, 4, 4, 4, 4] tiramisu = Tiramisu(layer_per_block) tiramisu.summary() #d0 = Input(shape=self.img_shape) return tiramisu def build_discriminator(self): def d_layer(layer_input, filters, f_size=4, bn=True): """Discriminator layer""" d = Conv2D(filters, kernel_size=f_size, strides=2, padding='same')(layer_input) d = LeakyReLU(alpha=0.2)(d) if bn: d = BatchNormalization(momentum=0.8)(d) return d img_A = Input(shape=self.img_shape) img_B = Input(shape=self.img_shape) # Concatenate image and conditioning image by channels to produce input combined_imgs = Concatenate(axis=-1)([img_A, img_B]) d1 = d_layer(combined_imgs, self.df, bn=False) d2 = d_layer(d1, self.df*2) d3 = d_layer(d2, self.df*4) d4 = d_layer(d3, self.df*8) validity = Conv2D(1, kernel_size=4, strides=1, padding='same')(d4) return Model([img_A, img_B], validity) def train(self, epochs, batch_size=1, sample_interval=50): start_time = datetime.datetime.now() # Adversarial loss ground truths valid = np.ones((batch_size,) + self.disc_patch) fake = np.zeros((batch_size,) + self.disc_patch) for epoch in range(epochs): for batch_i, (imgs_A, imgs_B) in enumerate(self.data_loader.load_batch(batch_size)): # --------------------- # Train Discriminator # --------------------- print(imgs_A.shape) # Condition on B and generate a translated version fake_A = self.generator.predict(imgs_B) # Train the discriminators (original images = real / generated = Fake) d_loss_real = self.discriminator.train_on_batch([imgs_A, imgs_B], valid) d_loss_fake = self.discriminator.train_on_batch([fake_A, imgs_B], fake) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) # ----------------- # Train Generator # ----------------- # Train the generators g_loss = self.combined.train_on_batch([imgs_A, imgs_B], [valid, imgs_A]) elapsed_time = datetime.datetime.now() - start_time # Plot the progress print ("[Epoch %d/%d] [Batch %d/%d] [D loss: %f, acc: %3d%%] [G loss: %f] time: %s" % (epoch, epochs, batch_i, self.data_loader.n_batches, d_loss[0], 100*d_loss[1], g_loss[0], elapsed_time)) # If at save interval => save generated image samples if batch_i % sample_interval == 0: self.sample_images(epoch, batch_i) self.combined.save_weights("Weights/"+str(epoch)+".h5") def img_to_frame(self,imgA,imgB,fakeA): no_images = imgA.shape[0] img_height = imgA.shape[1] img_width = imgA.shape[2] pad = 20 title_pad=20 pad_top = pad+title_pad frame=np.zeros((no_images*(img_height+pad_top),no_images*(img_width+pad),3)) count=0 gen_imgs = np.concatenate([imgB, fakeA, imgA]) gen_imgs = 0.5 * gen_imgs + 0.5 titles = ['Condition', 'Generated', 'Original'] for r in range(no_images): for c in range(no_images): im = gen_imgs[count] count=count+1 y0 = r*(img_height+pad_top) + pad//2 x0 = c*(img_width+pad) + pad//2 # print(frame[y0:y0+img_height,x0:x0+img_width,:].shape) frame[y0:y0+img_height,x0:x0+img_width,:] = im*255 frame = cv2.putText(frame, titles[r], (x0, y0-title_pad//4), cv2.FONT_HERSHEY_COMPLEX, .5, (255,255,255)) return frame def sample_images(self, epoch, batch_i): os.makedirs('images/%s' % self.dataset_name, exist_ok=True) os.makedirs('images/dehazed', exist_ok=True) os.makedirs('images/haze', exist_ok=True) os.makedirs('images/original',exist_ok=True) r, c = 3, 3 imgs_A, imgs_B, or_A, or_B = self.data_loader.load_data(batch_size=3, is_testing=True) fake_A = self.generator.predict(imgs_B) cv2.imwrite("images/dehazed"+"/"+"Img:"+str(epoch)+"_"+str(batch_i)+".jpg",(fake_A[0]*0.5+0.5)*255) cv2.imwrite("images/haze"+"/"+"Img:"+str(epoch)+"_"+str(batch_i)+".jpg",(or_B[0]*0.5+0.5)*255) cv2.imwrite("images/original"+"/"+"Img:"+str(epoch)+"_"+str(batch_i)+".jpg",(or_A[0]*0.5+0.5)*255) frame=self.img_to_frame(imgs_A,imgs_B,fake_A) cv2.imwrite("images/"+self.dataset_name+"/"+"Img:"+str(epoch)+"_"+str(batch_i)+".png",frame)
class NS_MODEL: def __init__(self): input_shape = (224, 224, 3) # 変換ネットワークの構築 self.model_gen = self.build_encoder_decoder( input_shape=input_shape ) # 学習済みモデルVGG16の呼び出し self.vgg16 = VGG16() # 重みパラメータを学習させない設定をする for layer in self.vgg16.layers: layer.trainable = False # 特徴量を抽出する層の名前を定義 self.style_layer_names = ( 'block1_conv2', 'block2_conv2', 'block3_conv3', 'block4_conv3' ) self.contents_layer_names = ('block3_conv3',) # 中間層の出力を保持するためのリスト self.style_outputs_gen = [] self.contents_outputs_gen = [] self.input_gen = self.model_gen.output # 変換ネットワークの出力を入力とする z = Lambda(self.norm_vgg16)(self.input_gen) # 入力値の正規化 for layer in self.vgg16.layers: z = layer(z) # VGG16の層を積み上げてネットワークを再構築 if layer.name in self.style_layer_names: # スタイル特徴量抽出用の中間層の出力を追加 self.style_outputs_gen.append(z) if layer.name in self.contents_layer_names: # コンテンツ特徴量抽出用の中間層の出力を追加 self.contents_outputs_gen.append(z) # モデルを定義 self.model = Model( inputs=self.model_gen.input, outputs=self.style_outputs_gen + self.contents_outputs_gen ) input_sty = Input(shape=input_shape, name='input_sty') style_outputs = [] x = Lambda(self.norm_vgg16)(input_sty) for layer in self.vgg16.layers: x = layer(x) if layer.name in self.style_layer_names: style_outputs.append(x) self.model_sty = Model( inputs=input_sty, outputs=style_outputs ) input_con = Input(shape=input_shape, name='input_con') contents_outputs = [] y = Lambda(self.norm_vgg16)(input_con) for layer in self.vgg16.layers: y = layer(y) if layer.name in self.contents_layer_names: contents_outputs.append(y) self.model_con = Model( inputs = input_con, outputs = contents_outputs ) def residual_block(self,input_ts): """ResidualBlockの構築する関数""" x = Conv2D( 128, (3, 3), strides=1, padding='same' )(input_ts) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv2D(128, (3, 3), strides=1, padding='same')(x) x = BatchNormalization()(x) return Add()([x, input_ts]) def build_encoder_decoder(self,input_shape=(224, 224, 3)): """変換用ネットワークの構築""" # Encoder部分 input_ts = Input(shape=input_shape, name='input') # 入力を[0, 1]の範囲に正規化 x = Lambda(lambda a: a/255.)(input_ts) x = Conv2D(32, (9, 9), strides=1, padding='same')(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv2D(64, (3, 3), strides=2, padding='same')(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv2D(128, (3, 3), strides=2, padding='same')(x) x = BatchNormalization()(x) x = Activation('relu')(x) # ResidualBlockを5ブロック追加 for _ in range(5): x = self.residual_block(x) # Decoder部分 x = Conv2DTranspose( 64, (3, 3), strides=2, padding='same' )(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv2DTranspose(32, (3, 3), strides=2, padding='same')(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Conv2DTranspose(3, (9, 9), strides=1, padding='same')(x) x = BatchNormalization()(x) x = Activation('tanh')(x) # 出力値が[0, 255]になるようにスケール変換 gen_out = Lambda(lambda a: (a + 1)*127.5)(x) model_gen = Model( inputs=[input_ts], outputs=[gen_out] ) return model_gen # VGG16のための入力値を正規化する関数 def norm_vgg16(self,x): """RGB->BGR変換と近似的に中心化をおこなう関数""" return (x[:, :, :, ::-1] - 120) / 255. # 画像ファイル読み込み用のラッパー関数定義 def load_imgs(self,img_paths, target_size=(224, 224)): """画像ファイルのパスのリストから、配列のバッチを返す""" _load_img = lambda x: img_to_array( load_img(x, target_size=target_size) ) img_list = [ np.expand_dims(_load_img(img_path), axis=0) for img_path in img_paths ] return np.concatenate(img_list, axis=0) def train_generator(self,img_paths, batch_size, model, y_true_sty, shuffle=True, epochs=None): """学習データを生成するジェネレータ""" n_samples = len(img_paths) indices = list(range(n_samples)) steps_per_epoch = math.ceil(n_samples / batch_size) img_paths = np.array(img_paths) cnt_epoch = 0 while True: cnt_epoch += 1 if shuffle: np.random.shuffle(indices) for i in range(steps_per_epoch): start = batch_size*i end = batch_size*(i + 1) X = self.load_imgs(img_paths[indices[start:end]]) batch_size_act = X.shape[0] y_true_sty_t = [ np.repeat(feat, batch_size_act, axis=0) for feat in y_true_sty ] # コンテンツ特徴量の抽出 y_true_con = model.predict(X) yield (X, y_true_sty_t + [y_true_con]) if epochs is not None: if cnt_epoch >= epochs: raise StopIteration def feature_loss(self, y_true, y_pred): """コンテンツ特徴量の損失関数""" norm = K.prod(K.cast(K.shape(y_true)[1:], 'float32')) return K.sum( K.square(y_pred - y_true), axis=(1, 2, 3) )/norm def gram_matrix(self, X): """グラム行列の算出""" X_sw = K.permute_dimensions( X, (0, 3, 2, 1) ) # 軸の入れ替え s = K.shape(X_sw) new_shape = (s[0], s[1], s[2]*s[3]) X_rs = K.reshape(X_sw, new_shape) X_rs_t = K.permute_dimensions( X_rs, (0, 2, 1) ) # 行列の転置 dot = K.batch_dot(X_rs, X_rs_t) # 内積の計算 norm = K.prod(K.cast(s[1:], 'float32')) return dot/norm def style_loss(self, y_true, y_pred): """スタイル用の損失関数定義""" return K.sum( K.square( self.gram_matrix(y_pred) - self.gram_matrix(y_true) ), axis=(1, 2) ) # Total Variation Regularizerの定義 def TVRegularizer(self, x, weight=1e-6, beta=1.0, input_size=(224, 224)): delta = 1e-8 h, w = input_size d_h = K.square(x[:, :h - 1, :w - 1, :] - x[:, 1:, :w - 1, :]) d_w = K.square(x[:, :h - 1, :w - 1, :] - x[:, :h - 1, 1:, :]) return weight * K.mean(K.sum(K.pow(d_h + d_w + delta, beta/2.))) def learn(self, style_name): img_paths = glob.glob("transfer/ns_model/train_img/*") batch_size = 2 epochs = 5 input_shape = (224, 224, 3) input_size = input_shape[:2] style = glob.glob("media/style/*")[0].split("\\")[-1] img_sty = load_img( 'media/style/'+style, target_size=input_size ) img_arr_sty = np.expand_dims(img_to_array(img_sty), axis=0) self.y_true_sty = self.model_sty.predict(img_arr_sty) shutil.rmtree("./media/style") os.mkdir("./media/style") self.gen = self.train_generator( img_paths, batch_size, self.model_con, self.y_true_sty, epochs=epochs ) gen_output_layer = self.model_gen.layers[-1] tv_loss = self.TVRegularizer(gen_output_layer.output) gen_output_layer.add_loss(tv_loss) self.model.compile( optimizer = Adadelta(), loss = [ self.style_loss, self.style_loss, self.style_loss, self.style_loss, self.feature_loss ], loss_weights = [1.0, 1.0, 1.0, 1.0, 3.0] ) now_epoch = 0 min_loss = np.inf steps_per_epoch = math.ceil(len(img_paths)/batch_size) # 学習 for i , (X_train, y_train) in enumerate(self.gen): if i % steps_per_epoch == 0: now_epoch += 1 loss = self.model.train_on_batch(X_train, y_train) if loss[0]<min_loss: min_loss = loss[0] self.model.save("transfer/ns_model/pretrained_model/" + style_name + ".h5") print("epoch: {}, iters: {}, loss: {:.3f}".format(now_epoch, i, loss[0]))
class GAN(): def __init__(self, rows): self.seq_length = rows self.seq_shape = (self.seq_length, 1) self.latent_dim = 1000 self.disc_loss = [] self.gen_loss = [] optimizer = Adam(0.0002, 0.5) # Build and compile the discriminator self.discriminator = self.build_discriminator() self.discriminator.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) # Build the generator self.generator = self.build_generator() # The generator takes noise as input and generates note sequences z = Input(shape=(self.latent_dim, )) generated_seq = self.generator(z) # For the combined model we will only train the generator self.discriminator.trainable = False # The discriminator takes generated images as input and determines validity validity = self.discriminator(generated_seq) # The combined model (stacked generator and discriminator) # Trains the generator to fool the discriminator self.combined = Model(z, validity) self.combined.compile(loss='binary_crossentropy', optimizer=optimizer) def build_discriminator(self): model = Sequential() model.add( CuDNNLSTM(512, input_shape=self.seq_shape, return_sequences=True)) model.add(Bidirectional(CuDNNLSTM(512))) model.add(Dense(512)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(256)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(1, activation='sigmoid')) model.summary() seq = Input(shape=self.seq_shape) validity = model(seq) return Model(seq, validity) def build_generator(self): model = Sequential() model.add(Dense(256, input_dim=self.latent_dim)) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(512)) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(1024)) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(np.prod(self.seq_shape), activation='tanh')) model.add(Reshape(self.seq_shape)) model.summary() noise = Input(shape=(self.latent_dim, )) seq = model(noise) return Model(noise, seq) def train(self, epochs, batch_size=128, sample_interval=50): # Load and convert the data notes = get_notes() n_vocab = len(set(notes)) X_train, y_train = prepare_sequences(notes, n_vocab) # Adversarial ground truths real = np.ones((batch_size, 1)) fake = np.zeros((batch_size, 1)) # Training the model for epoch in range(epochs): # Training the discriminator # Select a random batch of note sequences idx = np.random.randint(0, X_train.shape[0], batch_size) real_seqs = X_train[idx] #noise = np.random.choice(range(484), (batch_size, self.latent_dim)) #noise = (noise-242)/242 noise = np.random.normal(0, 1, (batch_size, self.latent_dim)) # Generate a batch of new note sequences gen_seqs = self.generator.predict(noise) # Train the discriminator d_loss_real = self.discriminator.train_on_batch(real_seqs, real) d_loss_fake = self.discriminator.train_on_batch(gen_seqs, fake) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) # Training the Generator noise = np.random.normal(0, 1, (batch_size, self.latent_dim)) # Train the generator (to have the discriminator label samples as real) g_loss = self.combined.train_on_batch(noise, real) # Print the progress and save into loss lists if epoch % sample_interval == 0: print("%d [D loss: %f, acc.: %.2f%%] [G loss: %f]" % (epoch, d_loss[0], 100 * d_loss[1], g_loss)) self.disc_loss.append(d_loss[0]) self.gen_loss.append(g_loss) self.generate(notes) self.plot_loss() def generate(self, input_notes): # Get pitch names and store in a dictionary notes = input_notes pitchnames = sorted(set(item for item in notes)) int_to_note = dict( (number, note) for number, note in enumerate(pitchnames)) # Use random noise to generate sequences noise = np.random.normal(0, 1, (1, self.latent_dim)) predictions = self.generator.predict(noise) pred_notes = [((x + 1) * (len(set(notes))) / 2) for x in predictions[0]] pred_notes = [int_to_note[int(x)] for x in pred_notes] create_midi(pred_notes, 'gan_final') def plot_loss(self): plt.plot(self.disc_loss, c='red') plt.plot(self.gen_loss, c='blue') plt.title("GAN Loss per Epoch") plt.legend(['Discriminator', 'Generator']) plt.xlabel('Epoch') plt.ylabel('Loss') plt.savefig('GAN_Loss_per_Epoch_final.png', transparent=True) plt.close()
class WGANGP(): def __init__(self): self.img_rows = 2 self.img_cols = 235 self.channels = 1 self.img_shape = (self.img_rows, self.img_cols, self.channels) self.latent_dim = 470 # Following parameter and optimizer set as recommended in paper self.n_critic = 5 optimizer = RMSprop(lr=0.00002) # Build the generator and critic self.generator = self.build_generator() self.critic = self.build_critic() #------------------------------- # Construct Computational Graph # for the Critic #------------------------------- # Freeze generator's layers while training critic self.generator.trainable = False # Image input (real sample) real_img = Input(shape=self.img_shape) # Noise input z_disc = Input(shape=(self.latent_dim, )) # Generate image based of noise (fake sample) fake_img = self.generator(z_disc) # Discriminator determines validity of the real and fake images fake = self.critic(fake_img) valid = self.critic(real_img) # Construct weighted average between real and fake images interpolated_img = RandomWeightedAverage()([real_img, fake_img]) # Determine validity of weighted sample validity_interpolated = self.critic(interpolated_img) # Use Python partial to provide loss function with additional # 'averaged_samples' argument partial_gp_loss = partial(self.gradient_penalty_loss, averaged_samples=interpolated_img) partial_gp_loss.__name__ = 'gradient_penalty' # Keras requires function names self.critic_model = Model(inputs=[real_img, z_disc], outputs=[valid, fake, validity_interpolated]) self.critic_model.compile(loss=[ self.wasserstein_loss, self.wasserstein_loss, partial_gp_loss ], optimizer=optimizer, loss_weights=[2, 2, 10]) #------------------------------- # Construct Computational Graph # for Generator #------------------------------- # For the generator we freeze the critic's layers self.critic.trainable = False self.generator.trainable = True # Sampled noise for input to generator z_gen = Input(shape=(470, )) # Generate images based of noise img = self.generator(z_gen) # Discriminator determines validity valid = self.critic(img) # Defines generator model self.generator_model = Model(z_gen, valid) self.generator_model.compile(loss=self.wasserstein_loss, optimizer=optimizer) def gradient_penalty_loss(self, y_true, y_pred, averaged_samples): """ Computes gradient penalty based on prediction and weighted real / fake samples """ gradients = K.gradients(y_pred, averaged_samples)[0] # compute the euclidean norm by squaring ... gradients_sqr = K.square(gradients) # ... summing over the rows ... gradients_sqr_sum = K.sum(gradients_sqr, axis=np.arange(1, len(gradients_sqr.shape))) # ... and sqrt gradient_l2_norm = K.sqrt(gradients_sqr_sum) # compute lambda * (1 - ||grad||)^2 still for each single sample gradient_penalty = K.square(1 - gradient_l2_norm) # return the mean as loss over all the batch samples return K.mean(gradient_penalty) def wasserstein_loss(self, y_true, y_pred): return K.mean(y_true * y_pred) def build_generator(self): model = Sequential() model.add( Dense(235 * 2 * 1, activation="relu", input_dim=self.latent_dim)) model.add(Reshape((2, 235, 1))) model.add(UpSampling2D()) model.add(Conv2D(128, kernel_size=4, padding="same")) model.add(BatchNormalization(momentum=0.8)) model.add(Activation("relu")) model.add(UpSampling2D()) model.add(Conv2D(64, kernel_size=4, padding="same")) model.add(BatchNormalization(momentum=0.8)) model.add(Activation("relu")) model.add(Conv2D(self.channels, 2, strides=4)) model.add(Activation("relu")) model.summary() noise = Input(shape=(self.latent_dim, )) img = model(noise) return Model(noise, img) def build_critic(self): model = Sequential() model.add( Conv2D(16, kernel_size=3, strides=2, input_shape=self.img_shape, padding="same")) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.25)) model.add(Conv2D(32, kernel_size=3, strides=2, padding="same")) model.add(ZeroPadding2D(padding=((0, 1), (0, 1)))) model.add(BatchNormalization(momentum=0.8)) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.25)) model.add(Conv2D(64, kernel_size=3, strides=2, padding="same")) model.add(BatchNormalization(momentum=0.8)) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.25)) model.add(Conv2D(128, kernel_size=3, strides=1, padding="same")) model.add(BatchNormalization(momentum=0.8)) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(1)) model.summary() img = Input(shape=self.img_shape) validity = model(img) return Model(img, validity) def train(self, epochs, batch_size, sample_interval=50): # Load the dataset X_train = trains # Adversarial ground truths valid = -np.ones((batch_size, 1)) fake = np.ones((batch_size, 1)) dummy = np.zeros((batch_size, 1)) # Dummy gt for gradient penalty for epoch in range(epochs): for _ in range(self.n_critic): # --------------------- # Train Discriminator # --------------------- # Select a random batch of images idx = np.random.randint(0, X_train.shape[0], batch_size) imgs = X_train[idx] # Sample generator input noise = np.random.normal(0, 1, (batch_size, self.latent_dim)) # Train the critic d_loss = self.critic_model.train_on_batch([imgs, noise], [valid, fake, dummy]) # --------------------- # Train Generator # --------------------- g_loss = self.generator_model.train_on_batch(noise, valid) # Plot the progress # If at save interval => save generated image samples if epoch % 200 == 0: self.sample_images(epoch) print("%d [D loss: %f] [G loss: %f]" % (epoch, d_loss[0], g_loss)) def sample_images(self, epoch): r, c = 1, 1 noise = np.random.normal(0, 1, (r * c, self.latent_dim)) gen_imgs = self.generator.predict(noise) generated_images = gen_imgs[0].transpose(2, 0, 1) plt.plot(generated_images[0][0], generated_images[0][1], "-") plt.pause(4)
class GAN(): def __init__(self): self.img_rows = 128 self.img_cols = 128 self.channels = 4 self.img_shape = (self.img_rows, self.img_cols, self.channels) self.latent_dim = 100 optimizer = Adam(0.0002, 0.5) # Build and compile the discriminator self.discriminator = self.build_discriminator() self.discriminator.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) # Build the generator self.generator = self.build_generator() # The generator takes noise as input and generates imgs z = Input(shape=(self.latent_dim, )) img = self.generator(z) # For the combined model we will only train the generator self.discriminator.trainable = False # The discriminator takes generated images as input and determines validity validity = self.discriminator(img) # The combined model (stacked generator and discriminator) # Trains the generator to fool the discriminator self.combined = Model(z, validity) self.combined.compile(loss='binary_crossentropy', optimizer=optimizer) def build_generator(self): model = Sequential() model.add(Dense(256, input_dim=self.latent_dim)) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(512)) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(1024)) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(np.prod(self.img_shape), activation='tanh')) model.add(Reshape(self.img_shape)) model.summary() noise = Input(shape=(self.latent_dim, )) img = model(noise) return Model(noise, img) def build_discriminator(self): model = Sequential() model.add(Flatten(input_shape=self.img_shape)) model.add(Dense(512)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(256)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(1, activation='sigmoid')) model.summary() img = Input(shape=self.img_shape) validity = model(img) return Model(img, validity) def train(self, epochs, batch_size=128, sample_interval=50): mystr = "" # Load the dataset X_train = np.load(os.getcwd() + "/pickles/train_data.npy") # Rescale -1 to 1 X_train = X_train / 127.5 - 1. #X_train = np.expand_dims(X_train, axis=4) # Adversarial ground truths valid = np.ones((batch_size, 1)) fake = np.zeros((batch_size, 1)) for epoch in range(epochs): # --------------------- # Train Discriminator # --------------------- # Select a random batch of images idx = np.random.randint(0, X_train.shape[0], batch_size) imgs = X_train[idx] noise = np.random.normal(0, 1, (batch_size, self.latent_dim)) # Generate a batch of new images gen_imgs = self.generator.predict(noise) # Train the discriminator d_loss_real = self.discriminator.train_on_batch(imgs, valid) d_loss_fake = self.discriminator.train_on_batch(gen_imgs, fake) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) # --------------------- # Train Generator # --------------------- noise = np.random.normal(0, 1, (batch_size, self.latent_dim)) # Train the generator (to have the discriminator label samples as valid) g_loss = self.combined.train_on_batch(noise, valid) # Plot the progress print("%d [D loss: %f, acc.: %.2f%%] [G loss: %f]" % (epoch, d_loss[0], 100 * d_loss[1], g_loss)) mystr += ("%d [D loss: %f, acc.: %.2f%%] [G loss: %f]\n" % (epoch, d_loss[0], 100 * d_loss[1], g_loss)) # If at save interval => save generated image samples if epoch % sample_interval == 0: self.sample_images(epoch) with open("analytics.txt", "w") as f: f.write(mystr) def sample_images(self, epoch): r, c = 5, 5 noise = np.random.normal(0, 1, (r * c, self.latent_dim)) gen_imgs = self.generator.predict(noise) # Rescale images 0 - 1 gen_imgs = 0.5 * gen_imgs + 0.5 fig, axs = plt.subplots(r, c) cnt = 0 for i in range(r): for j in range(c): axs[i, j].imshow(gen_imgs[cnt, :, :, :]) axs[i, j].axis('off') cnt += 1 fig.savefig("images/%d.png" % epoch) plt.close()
class GAN(): def __init__(self, model_yaml, train_yaml): """ Args: model_yaml: dictionnary with the model parameters train_yaml: dictionnary the tran parameters """ self.sigma_val = 0 self.model_yaml = model_yaml self.img_rows = 28 self.img_cols = 28 self.channels = 1 self.img_shape = (self.img_rows, self.img_cols, self.channels) if "dict_band_x" not in train_yaml: self.dict_band_X = None self.dict_band_label = None self.dict_rescale_type = None else: self.dict_band_X = train_yaml["dict_band_x"] self.dict_band_label = train_yaml["dict_band_label"] self.dict_rescale_type = train_yaml["dict_rescale_type"] self.s1bands = train_yaml["s1bands"] self.s2bands = train_yaml["s2bands"] # self.latent_dim = 100 # PATH self.model_name = model_yaml["model_name"] self.model_dir = train_yaml["training_dir"] + self.model_name + "/" self.this_training_dir = self.model_dir + "training_{}/".format( train_yaml["training_number"]) self.saving_image_path = self.this_training_dir + "saved_training_images/" self.saving_logs_path = self.this_training_dir + "logs/" self.checkpoint_dir = self.this_training_dir + "checkpoints/" self.previous_checkpoint = train_yaml["load_model"] # TRAIN PARAMETER self.normalization = train_yaml["normalization"] self.epoch = train_yaml["epoch"] self.batch_size = train_yaml["batch_size"] # self.sess = sess self.learning_rate = train_yaml["lr"] self.fact_g_lr = train_yaml["fact_g_lr"] self.beta1 = train_yaml["beta1"] self.val_directory = train_yaml["val_directory"] self.fact_s2 = train_yaml["s2_scale"] self.fact_s1 = train_yaml["s1_scale"] self.data_X, self.data_y, self.scale_dict_train = load_data( train_yaml["train_directory"], x_shape=model_yaml["input_shape"], label_shape=model_yaml["dim_gt_image"], normalization=self.normalization, dict_band_X=self.dict_band_X, dict_band_label=self.dict_band_label, dict_rescale_type=self.dict_rescale_type, fact_s2=self.fact_s2, fact_s1=self.fact_s1, s2_bands=self.s2bands, s1_bands=self.s1bands, lim=train_yaml["lim_train_tile"]) self.val_X, self.val_Y, scale_dict_val = load_data( self.val_directory, x_shape=model_yaml["input_shape"], label_shape=model_yaml["dim_gt_image"], normalization=self.normalization, dict_band_X=self.dict_band_X, dict_band_label=self.dict_band_label, dict_rescale_type=self.dict_rescale_type, dict_scale=self.scale_dict_train, fact_s2=self.fact_s2, fact_s1=self.fact_s1, s2_bands=self.s2bands, s1_bands=self.s1bands, lim=train_yaml["lim_val_tile"]) print("Loading the data done dataX {} dataY {}".format( self.data_X.shape, self.data_y.shape)) self.gpu = train_yaml["n_gpu"] self.num_batches = self.data_X.shape[0] // self.batch_size self.model_yaml = model_yaml self.im_saving_step = train_yaml["im_saving_step"] self.w_saving_step = train_yaml["weights_saving_step"] self.val_metric_step = train_yaml["metric_step"] # REDUCE THE DISCRIMINATOR PERFORMANCE self.val_lambda = train_yaml["lambda"] self.real_label_smoothing = tuple(train_yaml["real_label_smoothing"]) self.fake_label_smoothing = tuple(train_yaml["fake_label_smoothing"]) self.sigma_init = train_yaml["sigma_init"] self.sigma_step = train_yaml['sigma_step'] self.sigma_decay = train_yaml["sigma_decay"] self.ite_train_g = train_yaml["train_g_multiple_time"] self.max_im = 10 self.strategy = tf.distribute.MirroredStrategy() print('Number of devices: {}'.format( self.strategy.num_replicas_in_sync)) self.buffer_size = self.data_X.shape[0] self.global_batch_size = self.batch_size * self.strategy.num_replicas_in_sync with self.strategy.scope(): self.d_optimizer = Adam(self.learning_rate, self.beta1) self.g_optimizer = Adam(self.learning_rate * self.fact_g_lr, self.beta1) self.build_model() self.model_writer = tf.summary.create_file_writer( self.saving_logs_path) #self.strategy = tf.distribute.MirroredStrategy() def build_model(self): # strategy = tf.distribute.MirroredStrategy() # print('Number of devices: {}'.format(strategy.num_replicas_in_sync)) # We use the discriminator self.discriminator = self.build_discriminator(self.model_yaml) self.discriminator.compile(loss='binary_crossentropy', optimizer=self.d_optimizer, metrics=['accuracy']) self.generator = self.build_generator(self.model_yaml, is_training=True) print("Input G") g_input = Input(shape=(self.data_X.shape[1], self.data_X.shape[2], self.data_X.shape[3]), name="g_build_model_input_data") G = self.generator(g_input) print("G", G) # For the combined model we will only train the generator self.discriminator.trainable = False D_input = tf.concat([G, g_input], axis=-1) print("INPUT DISCRI ", D_input) # The discriminator takes generated images as input and determines validity D_output_fake = self.discriminator(D_input) # print(D_output) # The combined model (stacked generator and discriminator) # TO TRAIN WITH MULTIPLE GPU self.combined = Model(g_input, [D_output_fake, G], name="Combined_model") self.combined.compile(loss=['binary_crossentropy', L1_loss], loss_weights=[1, self.val_lambda], optimizer=self.g_optimizer) print("[INFO] combined model loss are : ".format( self.combined.metrics_names)) def build_generator(self, model_yaml, is_training=True): def build_resnet_block(input, id=0): """Define the ResNet block""" x = Conv2D(model_yaml["dim_resnet"], model_yaml["k_resnet"], padding=model_yaml["padding"], strides=tuple(model_yaml["stride"]), name="g_block_{}_conv1".format(id))(input) x = BatchNormalization(momentum=model_yaml["bn_momentum"], trainable=is_training, name="g_block_{}_bn1".format(id))(x) x = ReLU(name="g_block_{}_relu1".format(id))(x) x = Dropout(rate=model_yaml["do_rate"], name="g_block_{}_do".format(id))(x) x = Conv2D(model_yaml["dim_resnet"], model_yaml["k_resnet"], padding=model_yaml["padding"], strides=tuple(model_yaml["stride"]), name="g_block_{}_conv2".format(id))(x) x = BatchNormalization(momentum=model_yaml["bn_momentum"], trainable=is_training, name="g_block_{}_bn2".format(id))(x) x = Add(name="g_block_{}_add".format(id))([x, input]) x = ReLU(name="g_block_{}_relu2".format(id))(x) return x img_input = Input(shape=(self.data_X.shape[1], self.data_X.shape[2], self.data_X.shape[3]), name="g_input_data") if model_yaml["last_activation"] == "tanh": print("use tanh keras") last_activ = lambda x: tf.keras.activations.tanh(x) else: last_activ = model_yaml["last_activation"] x = img_input for i, param_lay in enumerate( model_yaml["param_before_resnet"] ): # build the blocks before the Resnet Blocks x = Conv2D(param_lay[0], param_lay[1], strides=tuple(model_yaml["stride"]), padding=model_yaml["padding"], name="g_conv{}".format(i))(x) x = BatchNormalization(momentum=model_yaml["bn_momentum"], trainable=is_training, name="g_{}_bn".format(i))(x) x = ReLU(name="g_{}_lay_relu".format(i))(x) for j in range(model_yaml["nb_resnet_blocs"]): # add the Resnet blocks x = build_resnet_block(x, id=j) for i, param_lay in enumerate(model_yaml["param_after_resnet"]): x = Conv2D(param_lay[0], param_lay[1], strides=tuple(model_yaml["stride"]), padding=model_yaml["padding"], name="g_conv_after_resnetblock{}".format(i))(x) x = BatchNormalization( momentum=model_yaml["bn_momentum"], trainable=is_training, name="g_after_resnetblock{}_bn2".format(i))(x) x = ReLU(name="g_after_resnetblock_relu_{}".format(i))(x) # The last layer x = Conv2D(model_yaml["last_layer"][0], model_yaml["last_layer"][1], strides=tuple(model_yaml["stride"]), padding=model_yaml["padding"], name="g_final_conv", activation=last_activ)(x) model_gene = Model(img_input, x, name="Generator") model_gene.summary() return model_gene def build_discriminator(self, model_yaml, is_training=True): discri_input = Input(shape=tuple([256, 256, 12]), name="d_input") if model_yaml["d_activation"] == "lrelu": d_activation = lambda x: tf.nn.leaky_relu( x, alpha=model_yaml["lrelu_alpha"]) else: d_activation = model_yaml["d_activation"] if model_yaml["add_discri_noise"]: x = GaussianNoise(self.sigma_val, input_shape=self.model_yaml["dim_gt_image"], name="d_GaussianNoise")(discri_input) else: x = discri_input for i, layer_index in enumerate(model_yaml["dict_discri_archi"]): layer_val = model_yaml["dict_discri_archi"][layer_index] layer_key = model_yaml["layer_key"] layer_param = dict(zip(layer_key, layer_val)) pad = layer_param["padding"] vpadding = tf.constant([[0, 0], [pad, pad], [pad, pad], [0, 0]]) # the last dimension is 12 x = tf.pad( x, vpadding, model_yaml["discri_opt_padding"], name="{}_padding_{}".format( model_yaml["discri_opt_padding"], layer_index)) # the type of padding is defined the yaml, # more infomration in https://www.tensorflow.org/api_docs/python/tf/pad # # x = ZeroPadding2D( # padding=(layer_param["padding"], layer_param["padding"]), name="d_pad_{}".format(layer_index))(x) x = Conv2D(layer_param["nfilter"], layer_param["kernel"], padding="valid", activation=d_activation, strides=(layer_param["stride"], layer_param["stride"]), name="d_conv{}".format(layer_index))(x) if i > 0: x = BatchNormalization(momentum=model_yaml["bn_momentum"], trainable=is_training, name="d_bn{}".format(layer_index))(x) # x = Flatten(name="flatten")(x) # for i, dlayer_idx in enumerate(model_yaml["discri_dense_archi"]): # dense_layer = model_yaml["discri_dense_archi"][dlayer_idx] # x = Dense(dense_layer, activation=d_activation, name="dense_{}".format(dlayer_idx))(x) if model_yaml["d_last_activ"] == "sigmoid": x_final = tf.keras.layers.Activation('sigmoid', name="d_last_activ")(x) else: x_final = x model_discri = Model(discri_input, x_final, name="discriminator") model_discri.summary() return model_discri def produce_noisy_input(self, input, sigma_val): if self.model_yaml["add_discri_white_noise"]: # print("[INFO] On each batch GT label we add Gaussian Noise before training discri on labelled image") new_gt = GaussianNoise(sigma_val, input_shape=self.model_yaml["dim_gt_image"], name="d_inputGN")(input) if self.model_yaml["add_relu_after_noise"]: new_gt = tf.keras.layers.Activation( lambda x: tf.keras.activations.tanh(x), name="d_before_activ")(new_gt) else: new_gt = input return new_gt def define_callback(self): # Define Tensorboard callbacks self.g_tensorboard_callback = TensorBoard( log_dir=self.saving_logs_path, histogram_freq=0, batch_size=self.batch_size, write_graph=True, write_grads=True) self.g_tensorboard_callback.set_model(self.combined) def train_gpu(self): valid = np.ones( (self.batch_size, 30, 30, 1)) # because of the shape of the discri fake = np.zeros((self.batch_size, 30, 30, 1)) print("valid shape {}".format(valid.shape)) if self.previous_checkpoint is not None: print("LOADING the model from step {}".format( self.previous_checkpoint)) start_epoch = int(self.previous_checkpoint) + 1 self.load_from_checkpoint(self.previous_checkpoint) else: # create_safe_directory(self.saving_logs_path) create_safe_directory(self.saving_image_path) train_dataset = tf.data.Dataset.from_tensor_slices( (self.data_X, self.data_y)).shuffle(self.batch_size).batch( self.global_batch_size) train_dist_dataset = self.strategy.experimental_distribute_dataset( train_dataset) def train(self): # Adversarial ground truths valid = np.ones((self.global_batch_size, 30, 30, 1)) # because of the shape of the discri fake = np.zeros((self.global_batch_size, 30, 30, 1)) #print("valid shape {}".format(valid.shape)) if self.previous_checkpoint is not None: print("LOADING the model from step {}".format( self.previous_checkpoint)) start_epoch = int(self.previous_checkpoint) + 1 self.load_from_checkpoint(self.previous_checkpoint) else: # create_safe_directory(self.saving_logs_path) create_safe_directory(self.saving_image_path) start_epoch = 0 train_dataset = tf.data.Dataset.from_tensor_slices( (self.data_X, self.data_y)).shuffle(self.batch_size).batch( self.global_batch_size) # loop for epoch sigma_val = self.sigma_init # dict_metric={"epoch":[],"d_loss_real":[],"d_loss_fake":[],"d_loss":[],"g_loss":[]} d_loss_real = [100, 100] # init losses d_loss_fake = [100, 100] d_loss = [100, 100] l_val_name_metrics, l_val_value_metrics = [], [] start_time = time.time() for epoch in range(start_epoch, self.epoch): # print("starting epoch {}".format(epoch)) for idx, (batch_input, batch_gt) in enumerate(train_dataset): #print(batch_input) ## TRAIN THE DISCRIMINATOR d_noise_real = random.uniform( self.real_label_smoothing[0], self.real_label_smoothing[1]) # Add noise on the loss d_noise_fake = random.uniform( self.fake_label_smoothing[0], self.fake_label_smoothing[1]) # Add noise on the loss # Create a noisy gt images batch_new_gt = self.produce_noisy_input(batch_gt, sigma_val) # Generate a batch of new images # print("Make a prediction") gen_imgs = self.generator.predict( batch_input) # .astype(np.float32) D_input_real = tf.concat([batch_new_gt, batch_input], axis=-1) D_input_fake = tf.concat([gen_imgs, batch_input], axis=-1) print("shape d train") print(valid.shape, D_input_fake.shape) d_loss_real = self.discriminator.train_on_batch( D_input_real, d_noise_real * valid) d_loss_fake = self.discriminator.train_on_batch( D_input_fake, d_noise_fake * fake) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) g_loss = self.combined.train_on_batch(batch_input, [valid, batch_gt]) # Plot the progress print("%d iter %d [D loss: %f, acc.: %.2f%%] [G loss: %f %f]" % (epoch, self.num_batches * epoch + idx, d_loss[0], 100 * d_loss[1], g_loss[0], g_loss[1])) if epoch % self.im_saving_step == 0 and idx < self.max_im: # to save some generated_images gen_imgs = self.generator.predict(batch_input) save_images(gen_imgs, self.saving_image_path, ite=idx) # LOGS to print in Tensorboard if epoch % self.val_metric_step == 0: l_val_name_metrics, l_val_value_metrics = self.val_metric() name_val_metric = [ "val_{}".format(name) for name in l_val_name_metrics ] name_logs = self.combined.metrics_names + [ "g_loss_tot", "d_loss_real", "d_loss_fake", "d_loss_tot", "d_acc_real", "d_acc_fake", "d_acc_tot" ] val_logs = g_loss + [ g_loss[0] + 100 * g_loss[1], d_loss_real[0], d_loss_fake[0], d_loss[0], d_loss_real[1], d_loss_fake[1], d_loss[1] ] # The metrics #print(type(batch_gt),type(gen_imgs)) l_name_metrics, l_value_metrics = compute_metric( batch_gt.numpy(), gen_imgs) assert len(val_logs) == len( name_logs ), "The name and value list of logs does not have the same lenght {} vs {}".format( name_logs, val_logs) write_log_tf2( self.model_writer, name_logs + l_name_metrics + name_val_metric + ["time_in_sec"], val_logs + l_value_metrics + l_val_value_metrics + [start_time - time.time()], epoch) if epoch % self.sigma_step == 0: # update simga sigma_val = sigma_val * self.sigma_decay # save the models if epoch % self.w_saving_step == 0: self.save_model(epoch) def save_model(self, step): print("Saving model at {} step {}".format(self.checkpoint_dir, step)) checkpoint_dir = self.checkpoint_dir if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) if not os.path.isfile("{}model_generator.yaml".format( self.checkpoint_dir)): gene_yaml = self.generator.to_yaml() with open("{}model_generator.yaml".format(self.checkpoint_dir), "w") as yaml_file: yaml_file.write(gene_yaml) if not os.path.isfile("{}model_combined.yaml".format( self.checkpoint_dir)): comb_yaml = self.combined.to_yaml() with open("{}model_combined.yaml".format(self.checkpoint_dir), "w") as yaml_file: yaml_file.write(comb_yaml) if not os.path.isfile("{}model_discri.yaml".format( self.checkpoint_dir)): discri_yaml = self.discriminator.to_yaml() with open("{}model_discri.yaml".format(self.checkpoint_dir), "w") as yaml_file: yaml_file.write(discri_yaml) self.generator.save_weights("{}model_gene_i{}.h5".format( self.checkpoint_dir, step)) self.discriminator.save_weights("{}model_discri_i{}.h5".format( self.checkpoint_dir, step)) self.combined.save_weights("{}model_combined_i{}.h5".format( self.checkpoint_dir, step)) def load_from_checkpoint(self, step): assert os.path.isfile("{}model_discri_i{}.h5".format( self.checkpoint_dir, step)), "No file at {}".format("{}model_discri_i{}.h5".format( self.checkpoint_dir, step)) self.discriminator.load_weights("{}model_discri_i{}.h5".format( self.checkpoint_dir, step)) self.generator.load_weights("{}model_gene_i{}.h5".format( self.checkpoint_dir, step)) self.combined.load_weights("{}model_combined_i{}.h5".format( self.checkpoint_dir, step)) def load_generator(self, path_yaml, path_weight): # load YAML and create model yaml_file = open(path_yaml, 'r') loaded_model_yaml = yaml_file.read() yaml_file.close() loaded_model = model_from_yaml(loaded_model_yaml) # load weights into new model loaded_model.load_weights(path_weight) print("Loaded model from disk") return loaded_model def val_metric(self): test_dataset = tf.data.Dataset.from_tensor_slices( (self.val_X, self.val_Y)).batch(self.val_X.shape[0]) #test_dist_dataset = self.strategy.experimental_distribute_dataset(test_dataset) for i, (x, y) in enumerate(test_dataset): #print("eval on {}".format(i)) val_pred = self.generator.predict(x) #print("type {} {}".format(type(y),type(val_pred))) label = y return compute_metric(label.numpy(), val_pred) def predict_on_iter(self, batch, path_save, l_image_id=None, un_rescale=True): """given an iter load the model at this iteration, returns the a predicted_batch but check if image have been saved at this directory :param dataset: :param batch could be a string : path to the dataset or an array corresponding to the batch we are going to predict on """ if type(batch) == type( "u" ): # the param is an string we load the bathc from this directory #print("We load our data from {}".format(batch)) l_image_id = find_image_indir(batch + XDIR, "npy") batch, _ = load_data(batch, x_shape=self.model_yaml["input_shape"], label_shape=self.model_yaml["dim_gt_image"], normalization=self.normalization, dict_band_X=self.dict_band_X, dict_band_label=self.dict_band_label, dict_rescale_type=self.dict_rescale_type, dict_scale=self.scale_dict_train, fact_s2=self.fact_s2, fact_s1=self.fact_s1, s2_bands=self.s2bands, s1_bands=self.s1bands, clip_s2=False) else: if l_image_id is None: print("We defined our own index for image name") l_image_id = [i for i in range(batch.shape[0])] assert len(l_image_id) == batch.shape[ 0], "Wrong size of the name of the images is {} should be {} ".format( len(l_image_id), batch.shape[0]) if os.path.isdir(path_save): print( "[INFO] the directory where to store the image already exists") data_array, path_tile, _ = load_from_dir( path_save, self.model_yaml["dim_gt_image"]) return data_array else: create_safe_directory(path_save) batch_res = self.generator.predict(batch) # if un_rescale: # remove the normalization made on the data # _, batch_res, _ = rescale_array(batch, batch_res, dict_group_band_X=self.dict_band_X, # dict_group_band_label=self.dict_band_label, # dict_rescale_type=self.dict_rescale_type, # dict_scale=self.scale_dict_train, invert=True, fact_scale2=self.fact_s2, # fact_scale1=self.fact_s1,clip_s2=False) assert batch_res.shape[0] == batch.shape[ 0], "Wrong prediction should have shape {} but has shape {}".format( batch_res.shape, batch.shape) if path_save is not None: # we store the data at path_save for i in range(batch_res.shape[0]): np.save( "{}_image_{}".format(path_save, l_image_id[i].split("/")[-1]), batch_res[i, :, :, :]) return batch_res
continue if step > Totalstep: break half_batch = int(batch_size / 2) z_d = np.random.uniform(-1, 1, (half_batch, 100)) g_pred = generator.predict(z_d) discriminator.trainable = True real_loss = discriminator.train_on_batch(batch[:half_batch], np.ones((half_batch, 1))) fake_loss = discriminator.train_on_batch(g_pred, np.zeros((half_batch, 1))) d_loss = 0.5 * np.add(real_loss, fake_loss) z_g = np.random.uniform(-1, 1, (batch_size, 100)) discriminator.trainable = False g_loss = generator_containing_discriminator.train_on_batch( z_g, np.ones((batch_size, 1))) logs.append({ 'step': step, 'd_loss': d_loss[0], 'acc[%]': 100 * d_loss[1], 'g_loss': g_loss }) print(logs[-1]) if step % 200 == 0: img_path = '{}/generate_{}.png'.format(img_save_dir, step) save_imgs(img_path, generator.predict(sample_seeds), rows=imgs_shape[0], cols=imgs_shape[1])
x = y = np.random.randn(32, 12) # dummy data ipt = Input((12, )) out = Dense(12)(ipt) model = Model(ipt, out) # starter_learning_rate = 0.1 # end_learning_rate = 0.01 # decay_steps = 10000 # learning_rate_fn = tf.keras.optimizers.schedules.PolynomialDecay( # starter_learning_rate, # decay_steps, # end_learning_rate, # power=0.5) # model.compile(SGD(learning_rate = 1e-4, decay=1e-2), loss='mse') model.compile(SGD(learning_rate=0.1), loss='mse') lr_list = [] for iteration in range(10): model.train_on_batch(x, y) model.optimizer.lr = model.optimizer.lr - 0.001 # print(model.optimizer.get_update()) print("lr at iteration {}: {}".format( iteration + 1, model.optimizer._decayed_lr('float32').numpy())) lr_list.append(model.optimizer._decayed_lr("float32").numpy()) print("model.optimizer.lr", model.optimizer.lr) plt.plot(range(10), lr_list) plt.show()
class DEC(object): def __init__(self, dims, n_clusters=10, alpha=1.0, init='glorot_uniform'): super(DEC, self).__init__() self.dims = dims self.input_dim = dims[0] self.n_stacks = len(self.dims) - 1 self.n_clusters = n_clusters self.alpha = alpha self.encoder = autoencoder(self.dims, init=init) # prepare DEC model clustering_layer = ClusteringLayer(self.n_clusters, name='clustering')( self.encoder.output) self.model = Model(inputs=self.encoder.input, outputs=clustering_layer) def load_weights(self, weights): # load weights of DEC model self.model.load_weights(weights) def extract_features(self, x): return self.encoder.predict(x) def predict( self, x): # predict cluster labels using the output of clustering layer q = self.model.predict(x, verbose=0) return q.argmax(1) @staticmethod def target_distribution(q): weight = q**2 / q.sum(0) return (weight.T / weight.sum(1)).T def compile(self, optimizer='sgd', loss='kld'): self.model.compile(optimizer=optimizer, loss=loss) def fit(self, x, y=None, maxiter=2e4, batch_size=256, tol=1e-3, update_interval=140, save_dir='./results/temp'): print('Update interval', update_interval) save_interval = int(x.shape[0] / batch_size) * 5 # 5 epochs print('Save interval', save_interval) # Step 1: initialize cluster centers using k-means t1 = time() print('Initializing cluster centers with k-means.') kmeans = KMeans(n_clusters=self.n_clusters, n_init=20) y_pred = kmeans.fit_predict(self.encoder.predict(x)) y_pred_last = np.copy(y_pred) self.model.get_layer(name='clustering').set_weights( [kmeans.cluster_centers_]) # Step 2: deep clustering # logging file import csv logfile = open(save_dir + '/dec_log.csv', 'w') logwriter = csv.DictWriter( logfile, fieldnames=['iter', 'acc', 'nmi', 'ari', 'loss']) logwriter.writeheader() loss = 0 index = 0 index_array = np.arange(x.shape[0]) for ite in range(int(maxiter)): if ite % update_interval == 0: q = self.model.predict(x, verbose=0) p = self.target_distribution( q) # update the auxiliary target distribution p # evaluate the clustering performance y_pred = q.argmax(1) if y is not None: acc = np.round(metrics.acc(y, y_pred), 5) nmi = np.round(metrics.nmi(y, y_pred), 5) ari = np.round(metrics.ari(y, y_pred), 5) loss = np.round(loss, 5) logdict = dict(iter=ite, acc=acc, nmi=nmi, ari=ari, loss=loss) logwriter.writerow(logdict) print( 'Iter %d: acc = %.5f, nmi = %.5f, ari = %.5f' % (ite, acc, nmi, ari), ' ; loss=', loss) # check stop criterion delta_label = np.sum(y_pred != y_pred_last).astype( np.float32) / y_pred.shape[0] y_pred_last = np.copy(y_pred) if ite > 0 and delta_label < tol: print('delta_label ', delta_label, '< tol ', tol) print('Reached tolerance threshold. Stopping training.') logfile.close() break # train on batch # if index == 0: # np.random.shuffle(index_array) idx = index_array[index * batch_size:min((index + 1) * batch_size, x.shape[0])] loss = self.model.train_on_batch(x=x[idx], y=p[idx]) index = index + 1 if (index + 1) * batch_size <= x.shape[0] else 0 # save intermediate model if ite % save_interval == 0: print('saving model to:', save_dir + '/DEC_model_' + str(ite) + '.h5') self.model.save_weights(save_dir + '/DEC_model_' + str(ite) + '.h5') ite += 1 # save the trained model logfile.close() print('saving model to:', save_dir + '/DEC_model_final.h5') self.model.save_weights(save_dir + '/DEC_model_final.h5') return y_pred
class GAN(): def __init__(self): self.img_rows = 28 self.img_cols = 28 self.channels = 1 self.img_shape = (self.img_rows, self.img_cols, self.channels) self.latent_dim = 100 # latent_dim is dimension inputted to generator, (rows, column, 1) is resolution returned by generator and inputted to # discriminator, then discriminator returns 0 or 1 - wether image is real or fake optimizer = Adam(0.0002, 0.5) self.discriminator = self.build_discriminator() self.discriminator.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) # for training discriminator self.generator = self.build_generator() z = Input(shape=(self.latent_dim,)) img = self.generator(z) self.discriminator.trainable = False # so only generator weights are trained while training entire GAN validity = self.discriminator(img) self.combined = Model(z, validity) self.combined.compile(loss='binary_crossentropy', optimizer=optimizer) # combined is the GAN - generator + discriminator(but only gen weights are trained) def build_generator(self): model = Sequential() model.add(Dense(256, input_dim=self.latent_dim)) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(512)) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(1024)) model.add(LeakyReLU(alpha=0.2)) model.add(BatchNormalization(momentum=0.8)) model.add(Dense(np.prod(self.img_shape), activation='tanh')) model.add(Reshape(self.img_shape)) model.summary() noise = Input(shape=(self.latent_dim,)) img = model(noise) return Model(noise, img) def build_discriminator(self): model = Sequential() model.add(Flatten(input_shape=self.img_shape)) model.add(Dense(512)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(256)) model.add(LeakyReLU(alpha=0.2)) model.add(Dense(1, activation='sigmoid')) model.summary() img = Input(shape=self.img_shape) validity = model(img) return Model(img, validity) def train(self, epochs, batch_size=128, sample_interval=50): (X_train, _), (_, _) = mnist.load_data() X_train = X_train / 127.5 - 1. X_train = np.expand_dims(X_train, axis=3) valid = np.ones((batch_size, 1)) fake = np.zeros((batch_size, 1)) # we will first train GAN(generator + discriminator) by inputting noise and give output as valid(1) and only generator weights # will be trained(the purpose is therefore to fool the discriminator into thinking these generated images are valid). # next we will train discriminator by inputting images obtained from generator with output - fake(0) and also by images from # mnist with output - valid(1)(basically train discriminator to label images generated by generator as fake). # therefore we set up a rivalry between generator and discriminator where purpose of generator is to fool the discriminator # and purpose of discriminator is to recognise images by generator and not get fooled. for epoch in range(epochs+1): idx = np.random.randint(0, X_train.shape[0], batch_size) imgs = X_train[idx] # real images - inputted to discriminator for training noise = np.random.normal(0, 1, (batch_size, self.latent_dim)) # random noise - inputted to gan(generator part) g_loss = self.combined.train_on_batch(noise, valid) # training gan gen_imgs = self.generator.predict(noise) # output from generator d_loss_real = self.discriminator.train_on_batch(imgs, valid) # discriminator trained on real images from mnist d_loss_fake = self.discriminator.train_on_batch(gen_imgs, fake) # discriminator trained on fake images from generator d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) print("%d [Discriminator loss: %f, acc.: %.2f%%] [GAN loss: %f]" % (epoch, d_loss[0], 100 * d_loss[1], g_loss)) if epoch % sample_interval == 0: self.sample_images(epoch) def sample_images(self, epoch): r, c = 5, 5 noise = np.random.normal(0, 1, (r * c, self.latent_dim)) gen_imgs = self.generator.predict(noise) gen_imgs = 0.5 * gen_imgs + 0.5 fig, axs = plt.subplots(r, c) cnt = 0 for i in range(r): for j in range(c): axs[i, j].imshow(gen_imgs[cnt, :, :, 0], cmap='gray') axs[i, j].axis('off') cnt += 1 plt.show()