def main(): batch_size = _BATCH_SIZE noise_dim = _NOISE_DIM lamb = 10.0 train = get_data() train_images, train_labels = make_batch(train) gen = generator() dis = discriminator() gen.summary() dis.summary() dis_opt = Adam(lr=1.0e-4, beta_1=0.0, beta_2=0.9) gen_opt = Adam(lr=1.0e-4, beta_1=0.0, beta_2=0.9) gen.trainable = True dis.trainable = False gen_inputs = Input(shape=(noise_dim, )) gen_outputs = gen(gen_inputs) dis_outputs = dis(gen_outputs) gen_model = Model(inputs=[gen_inputs], outputs=[dis_outputs]) gen_model.compile(loss=wasserstein_loss, optimizer=gen_opt) gen_model.summary() gen.trainable = False dis.trainable = True real_inputs = Input(shape=train_images.shape[1:]) dis_real_outputs = dis(real_inputs) fake_inputs = Input(shape=(noise_dim, )) gen_fake_outputs = gen(fake_inputs) dis_fake_outputs = dis(gen_fake_outputs) interpolate = RandomWeightedAverage()([real_inputs, gen_fake_outputs]) dis_interpolate_outputs = dis(interpolate) gp_reg = partial(gradient_penalty, interpolate=interpolate, lamb=lamb) #gp_reg.__name__ = 'gradient_penalty' dis_model = Model(inputs=[real_inputs, fake_inputs],\ outputs=[dis_real_outputs, dis_fake_outputs,dis_interpolate_outputs]) dis_model.compile(loss=[wasserstein_loss, wasserstein_loss, gp_reg], optimizer=dis_opt) dis_model.summary() max_epoch = 10001 max_train_only_dis = 5 minibatch_size = batch_size * max_train_only_dis max_loop = int(train_images.shape[0] / minibatch_size) real = np.zeros((batch_size, train_images.shape[1], train_images.shape[2], train_images.shape[3]), dtype=np.float32) minibatch_train_images = np.zeros( (minibatch_size, train_images.shape[1], train_images.shape[2], train_images.shape[3]), dtype=np.float32) progbar = Progbar(target=max_epoch) real_label = [-1] * batch_size fake_label = [1] * batch_size dummy_label = [0] * batch_size for epoch in range(max_epoch): np.random.shuffle(train_images) for loop in range(max_loop): minibatch_train_images = train_images[loop * minibatch_size:(loop + 1) * minibatch_size] for train_only_dis in range(max_train_only_dis): real = minibatch_train_images[train_only_dis * batch_size:(train_only_dis + 1) * batch_size] noise = np.random.uniform( -1, 1, (batch_size, noise_dim)).astype(np.float32) dis_loss = dis_model.train_on_batch( [real, noise], [real_label, fake_label, dummy_label]) noise = np.random.uniform(-1, 1, (batch_size, noise_dim)).astype( np.float32) gen_loss = gen_model.train_on_batch(noise, real_label) progbar.add(1, values=[("dis_loss", dis_loss[0]), ("gen_loss", gen_loss)]) if epoch % 100 == 0: noise = np.random.uniform(-1, 1, (batch_size, 10)).astype(np.float32) fake = gen.predict(noise) tmp = [r.reshape(-1, 32) for r in fake] tmp = np.concatenate(tmp, axis=1) img = ((tmp / 2.0 + 0.5) * 255.0).astype(np.uint8) Image.fromarray(img).save("generate/%d.png" % (epoch)) backend.clear_session()
def main(): if os.path.isfile(macro._LOCAL_SAVE_DATA) == 0: # download data and compute featuers (see "download_data.py") # atomic_numbers use to compute composition vector # labels is target properties (formation energy) train_labels, compositions, features, atomic_numbers = dl.get_data() # compute bag-of-atom vector that trains GAN (see "preprocess.py") boa_vectors = pre.compute_bag_of_atom_vector(compositions, atomic_numbers) train_data = np.concatenate([boa_vectors, features], axis=1) save_data = pd.DataFrame( np.concatenate([train_labels, train_data], axis=1)) save_data.to_csv(macro._LOCAL_SAVE_DATA, index=False, header=False) else: data = pd.read_csv(macro._LOCAL_SAVE_DATA, delimiter=',', engine="python", header=None) data = np.array(data) train_labels, train_data = np.split(data, [1], axis=1) # normalization of training data such that min is 0 and max is 1 (see "preprocess.py") normalized_train_data, data_max, data_min = pre.normalize_for_train( train_data) normalized_train_labels, max_train_prop, min_train_prop = pre.normalize_for_train( train_labels) # Save normalization parameter to .csv to use generation save_data = pd.DataFrame( np.concatenate([max_train_prop, min_train_prop, data_max, data_min], axis=0)) save_data.to_csv(macro._SAVE_NORMALIZATION_PARAM, index=False, header=False) ### start initialization of training GAN ### # set hyperparameters batch_size = macro._BATCH_SIZE # batch size noise_dim = macro._NOISE_DIM # dimension of noise to input generator property_dim = macro._PROP_DIM # the number of properties lamb = macro._LAMB # hyperparameter for W-GAN-GP max_epoch = macro._MAX_EPOCH # maximum iteration of outer loop max_train_only_dis = macro._MAX_EPOCH_TRAIN_DISCRIMINATOR # maximum iteration of inner loop defined by W-GAN-GP paper (https://arxiv.org/pdf/1704.00028.pdf) max_loop = int(train_data.shape[0] / batch_size) # set model (see "model.py") # in this code, we apply AC-GAN based network architecture (https://arxiv.org/abs/1610.09585) # difference between AC-GAN is that our model is the regression, not classification gen = model.generator(normalized_train_data.shape[1]) dis = model.discriminator(normalized_train_data.shape[1]) # rf is the output layer of discriminator that discriminates real or fake rf = model.real_fake() # pred is the output layer of discriminator that predicts target property pred = model.prediction() # set optimization method dis_opt = Adam(lr=1.0e-4, beta_1=0.0, beta_2=0.9) gen_opt = Adam(lr=1.0e-4, beta_1=0.0, beta_2=0.9) # first set discriminator's parameters for training gen.trainable = False # generator's parameter does not update dis.trainable = True rf.trainable = True pred.trainable = True # set variables when inputting real data real_inputs = Input(shape=normalized_train_data.shape[1:]) dis_real_outputs = dis(real_inputs) real_fake_from_real = rf(dis_real_outputs) predictions_from_real = pred(dis_real_outputs) # set variables when inputting fake data fake_inputs = Input(shape=(noise_dim + property_dim, )) gen_fake_outputs = gen(fake_inputs) dis_fake_outputs = dis(gen_fake_outputs) real_fake_from_fake = rf(dis_fake_outputs) # set loss function for discriminator # in this case, we apply W-GAN-GP based loss function because of improving stability # W-GAN-GP (https://arxiv.org/pdf/1704.00028.pdf) # W-GAN-GP is unsupervised training, on the other hand, our model is supervised (conditional). # So, we apply wasserstein_loss to real_fake part and apply mean_squared_error to prediction part interpolate = model.RandomWeightedAverage()( [real_inputs, gen_fake_outputs]) dis_interpolate_outputs = dis(interpolate) real_fake_interpolate = rf(dis_interpolate_outputs) # gradient penalty of W-GAN-GP gp_reg = partial(model.gradient_penalty, interpolate=interpolate, lamb=lamb) gp_reg.__name__ = 'gradient_penalty' # connect inputs and outputs of the discriminator # prediction part is trained by only using training dataset (i.e., predict part is not trained by generated samples) dis_model = Model(inputs=[real_inputs, fake_inputs],\ outputs=[real_fake_from_real, real_fake_from_fake, real_fake_interpolate, predictions_from_real]) # compile dis_model.compile(loss=[model.wasserstein_loss,model.wasserstein_loss,\ gp_reg,'mean_squared_error'],optimizer=dis_opt) # second set generator's parameters for training gen.trainable = True # generator's parameters only update dis.trainable = False rf.trainable = False pred.trainable = False # set variables when inputting noise and target property gen_inputs = Input(shape=(noise_dim + property_dim, )) gen_outputs = gen(gen_inputs) # set variables for discriminator when inputting fake data dis_outputs = dis(gen_outputs) real_fake = rf(dis_outputs) predictions = pred(dis_outputs) # connect inputs and outputs of the discriminator gen_model = Model(inputs=[gen_inputs], outputs=[real_fake, predictions]) # compile # generator is trained by real_fake classification and prediction of target property gen_model.compile(loss=[model.wasserstein_loss, 'mean_squared_error'], optimizer=gen_opt) # if you need progress bar progbar = Progbar(target=max_epoch) # set the answer to train each model real_label = [-1] * batch_size fake_label = [1] * batch_size dummy_label = [0] * batch_size #real = np.zeros((batch_size,train_data.shape[1]), dtype=np.float32) inputs = np.zeros((batch_size, noise_dim + property_dim), dtype=np.float32) # epoch for epoch in range(max_epoch): # iteration for loop in range(max_loop): # shuffle to change the trainng order and select data sdata, slabels, bak = pre.paired_shuffle(normalized_train_data, normalized_train_labels) real = sdata[loop * batch_size:(loop + 1) * batch_size] properties = slabels[loop * batch_size:(loop + 1) * batch_size] # generator's parameters does not update gen.trainable = False dis.trainable = True rf.trainable = True pred.trainable = True # train discriminator for train_only_dis in range(max_train_only_dis): noise = np.random.uniform( -1, 1, (batch_size, noise_dim)).astype(np.float32) for i in range(len(noise)): inputs[i] = np.hstack((noise[i], properties[i])) dis_loss = dis_model.train_on_batch( [real, inputs], [real_label, fake_label, dummy_label, properties]) # second train only generator gen.trainable = True dis.trainable = False rf.trainable = False pred.trainable = False noise = np.random.uniform(-1, 1, (batch_size, noise_dim)).astype( np.float32) for i in range(len(noise)): inputs[i] = np.hstack((noise[i], properties[i])) gen_loss = gen_model.train_on_batch([inputs], [real_label, properties]) # if you need progress bar progbar.add(1, values=[("dis_loss", dis_loss[0]), ("gen_loss", gen_loss[0])]) # save generated samples and models eval.save(normalized_train_data, gen, dis, pred, rf) backend.clear_session()
class DCGAN(): def __init__(self): # Input shape self.img_rows = 28 self.img_cols = 28 self.channels = 1 self.img_shape = (self.img_rows, self.img_cols, self.channels) self.latent_dim = 100 optimizer = Adam(0.0002, 0.5) # Build and compile the discriminator self.discriminator = self.build_discriminator() # Build the generator self.generator = self.build_generator() # load weight self.load_weights_from_file() # Compile the discriminator self.discriminator.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) # The generator takes noise as input and generates imgs z = Input(shape=(self.latent_dim, )) img = self.generator(z) # For the combined model we will only train the generator self.discriminator.trainable = False # The discriminator takes generated images as input and determines validity valid = self.discriminator(img) # The combined model (stacked generator and discriminator) # Trains the generator to fool the discriminator self.combined = Model(z, valid) self.combined.compile(loss='binary_crossentropy', optimizer=optimizer) def build_generator(self): model = Sequential() model.add( Dense(128 * 7 * 7, activation="relu", input_dim=self.latent_dim)) model.add(Reshape((7, 7, 128))) model.add(UpSampling2D()) model.add(Conv2D(128, kernel_size=3, padding="same")) model.add(BatchNormalization(momentum=0.8)) model.add(Activation("relu")) model.add(UpSampling2D()) model.add(Conv2D(64, kernel_size=3, padding="same")) model.add(BatchNormalization(momentum=0.8)) model.add(Activation("relu")) model.add(Conv2D(self.channels, kernel_size=3, padding="same")) model.add(Activation("tanh")) # model.summary() noise = Input(shape=(self.latent_dim, )) img = model(noise) return Model(noise, img) def build_discriminator(self): model = Sequential() model.add( Conv2D(32, kernel_size=3, strides=2, input_shape=self.img_shape, padding="same")) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.25)) model.add(Conv2D(64, kernel_size=3, strides=2, padding="same")) model.add(ZeroPadding2D(padding=((0, 1), (0, 1)))) model.add(BatchNormalization(momentum=0.8)) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.25)) model.add(Conv2D(128, kernel_size=3, strides=2, padding="same")) model.add(BatchNormalization(momentum=0.8)) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.25)) model.add(Conv2D(256, kernel_size=3, strides=1, padding="same")) model.add(BatchNormalization(momentum=0.8)) model.add(LeakyReLU(alpha=0.2)) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(1, activation='sigmoid')) # model.summary() img = Input(shape=self.img_shape) validity = model(img) return Model(img, validity) def train(self, epochs, batch_size=128, save_interval=50): # Load the dataset # (X_train, _), (_, _) = mnist.load_data() (X_train, y_train) = mnist.train.images, mnist.train.labels # Rescale -1 to 1 X_train = X_train / 127.5 - 1. print(X_train.shape) X_train = np.reshape(X_train, (-1, 28, 28, 1)) # Adversarial ground truths valid = np.ones((batch_size, 1)) fake = np.zeros((batch_size, 1)) for epoch in range(epochs): # --------------------- # Train Discriminator # --------------------- # Select a random half of images idx = np.random.randint(0, X_train.shape[0], batch_size) imgs = X_train[idx] # Sample noise and generate a batch of new images noise = np.random.normal(0, 1, (batch_size, self.latent_dim)) gen_imgs = self.generator.predict(noise) # Train the discriminator (real classified as ones and generated as zeros) d_loss_real = self.discriminator.train_on_batch(imgs, valid) d_loss_fake = self.discriminator.train_on_batch(gen_imgs, fake) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) # --------------------- # Train Generator # --------------------- # Train the generator (wants discriminator to mistake images as real) g_loss = self.combined.train_on_batch(noise, valid) # Plot the progress print("%d [D loss: %f, acc.: %.2f%%] [G loss: %f]" % (epoch, d_loss[0], 100 * d_loss[1], g_loss)) # If at save interval => save generated image samples if epoch % save_interval == 0: self.save_imgs(epoch) self.save_model() def save_imgs(self, epoch): r, c = 5, 5 noise = np.random.normal(0, 1, (r * c, self.latent_dim)) gen_imgs = self.generator.predict(noise) # Rescale images 0 - 1 gen_imgs = 0.5 * gen_imgs + 0.5 fig, axs = plt.subplots(r, c) cnt = 0 for i in range(r): for j in range(c): axs[i, j].imshow(gen_imgs[cnt, :, :, 0], cmap='gray') axs[i, j].axis('off') cnt += 1 fig.savefig("images/mnist_%d.png" % epoch) plt.close() return # Save model def save_model(self): self.discriminator.save_weights(weight_path + 'dcgan_discriminator_weight.h5') self.generator.save_weights(weight_path + 'dcgan_generator_weight.h5') print("save model") # load weights def load_weights_from_file(self): # discriminator weight if self.discriminator != None and os.path.isfile( weight_path + "dcgan_discriminator_weight.h5"): print("discriminator_weights exists") self.discriminator.load_weights(weight_path + "dcgan_discriminator_weight.h5") # generator weight if self.generator != None and os.path.isfile( weight_path + "dcgan_generator_weight.h5"): print("generator_weights exists") self.generator.load_weights(weight_path + "dcgan_generator_weight.h5")
def main(): # Define Functions build_generator = build_generator_dense build_discriminator = build_discriminator_dense # Build dictionary dictionary = {} reverse_dictionary = {} for i, c in enumerate(alphabet): dictionary[c]=i+1 reverse_dictionary[i+1]=c # Build Oprimizer optimizer = Adam(learning_rate, 0.5) # Build and compile the discriminator print ("*** BUILDING DISCRIMINATOR ***") discriminator = build_discriminator() discriminator.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) # Build and compile the generator print ("*** BUILDING GENERATOR ***") generator = build_generator() generator.compile(loss='binary_crossentropy', optimizer=optimizer) # The generator takes noise as input and generated samples z = Input(shape=noise_shape) gen = generator(z) # For the combined model we will only train the generator discriminator.trainable = False # The valid takes generated samples as input and determines validity valid = discriminator(gen) # The combined model (stacked generator and discriminator) takes # noise as input => generates samples => determines validity combined = Model(z, valid) combined.compile(loss='binary_crossentropy', optimizer=optimizer) # Load the dataset data = [] for line in open(input_data,"r").read().splitlines(): this_sample=np.zeros(url_shape) line = line.lower() if len ( set(line) - set(alphabet)) == 0 and len(line) < url_len: for i, position in enumerate(this_sample): this_sample[i][0]=1.0 for i, char in enumerate(line): this_sample[i][0]=0.0 this_sample[i][dictionary[char]]=1.0 data.append(this_sample) else: print("Uncompatible line:", line) print("Data ready. Lines:", len(data)) X_train = np.array(data) print ("Array Shape:", X_train.shape) half_batch = int(batch_size / 2) # Start Training for epoch in range(epochs): # --------------------- # Train Discriminator # --------------------- # Select a random half batch of data idx = np.random.randint(0, X_train.shape[0], half_batch) samples = X_train[idx] noise_batch_shape = (half_batch,) + noise_shape noise = np.random.normal(0, 1, noise_batch_shape) # Generate a half batch of new data gens = generator.predict(noise) # Train the discriminator d_loss_real = discriminator.train_on_batch(samples, np.ones((half_batch, 1))) d_loss_fake = discriminator.train_on_batch(gens, np.zeros((half_batch, 1))) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) # --------------------- # Train Generator # --------------------- noise_batch_shape = (batch_size,) + noise_shape noise = np.random.normal(0, 1, noise_batch_shape) # The generator wants the discriminator to label the generated samples as valid (ones) valid_y = np.array([1] * batch_size) # Train the generator g_loss = combined.train_on_batch(noise, valid_y) # Plot the progress print ("%d [D loss: %0.3f, acc.: %0.3f%%] [G loss: %0.3f]" % (epoch, d_loss[0], 100*d_loss[1], g_loss)) # If at save interval, print some examples if epoch % save_interval == 0: generated_samples=[] r, c = 5, 5 noise_batch_shape = (print_size,) + noise_shape noise = np.random.normal(0, 1, noise_batch_shape) gens = generator.predict(noise) for url in gens: this_url_gen = "" for position in url: this_index = np.argmax(position) if this_index != 0: this_url_gen += reverse_dictionary[this_index] print(this_url_gen) generated_samples.append(this_url_gen) # Save networks discriminator.save(discriminator_savefile) generator.save(generator_savefile) # Save Samples fo = open(generated_savefile, "w") for url in generated_samples: print (url, file=fo) fo.close()
def transfer_model(source_df, target_df, test_df, method_flag, fold_num): source_labels, source_data = np.split(np.array(source_df),[1],axis=1) target_labels, target_data = np.split(np.array(target_df),[1],axis=1) test_labels, test_data = np.split(np.array(test_df),[1],axis=1) # normalization #normalized_source_data = pre.normalize(source_data) #normalized_target_data = pre.normalize(target_data) #normalized_test_data = pre.normalize(test_data) normalized_source_data = source_data normalized_target_data = target_data normalized_test_data = test_data ### constuct model for source domain task ### # optimization opt = Adam() # network setting latent = models.latent(normalized_source_data.shape[1]) sll = models.source_last_layer() tll = models.target_last_layer() source_inputs = Input(shape=normalized_source_data.shape[1:]) latent_features = latent(source_inputs) source_predictors = sll(latent_features) latent.trainable = mc._SORUCE_LATENT_TRAIN source_predictors.trainable = True source_nn = Model(inputs=[source_inputs], outputs=[source_predictors]) source_nn.compile(loss=['mean_squared_error'],optimizer=opt) #source_nn.summary() # training using source domain data if method_flag != mc._SCRATCH: source_max_loop = int(normalized_source_data.shape[0]/mc._BATCH_SIZE) source_progbar = Progbar(target=mc._SOURCE_EPOCH_NUM) for epoch in range(mc._SOURCE_EPOCH_NUM): shuffle_data, shuffle_labels, _ = pre.paired_shuffle(normalized_source_data,source_labels,1) for loop in range(source_max_loop): batch_train_data = shuffle_data[loop*mc._BATCH_SIZE:(loop+1)*mc._BATCH_SIZE] batch_train_labels = shuffle_labels[loop*mc._BATCH_SIZE:(loop+1)*mc._BATCH_SIZE] batch_train_labels = np.reshape(batch_train_labels, [len(batch_train_labels)]) one_hots = np.identity(mc._SOURCE_DIM_NUM)[np.array(batch_train_labels, dtype=np.int32)] loss = source_nn.train_on_batch([batch_train_data],[one_hots]) #source_progbar.add(1, values=[("source loss",loss)]) # save #latent.save('../results/source_latent.h5') #sll.save('../results/source_last_layer.h5') # compute relation vectors if method_flag == mc._SCRATCH or method_flag == mc._CONV_TRANSFER: target_vectors = np.identity(mc._TARGET_DIM_NUM)[np.array(target_labels, dtype=np.int32)] target_vectors = np.reshape(target_vectors, [target_vectors.shape[0], target_vectors.shape[2]]) elif method_flag == mc._COUNT_ATDL: target_labels, relations = rv.compute_relation_labels(source_nn, normalized_target_data, target_labels, fold_num) target_vectors = np.identity(mc._SOURCE_DIM_NUM)[np.array(target_labels, dtype=np.int32)] target_vectors = np.reshape(target_vectors, [target_vectors.shape[0], target_vectors.shape[2]]) else: relation_vectors = rv.compute_relation_vectors(source_nn, normalized_target_data, target_labels, fold_num, method_flag) target_vectors = np.zeros((len(target_labels),mc._SOURCE_DIM_NUM), dtype=np.float32) for i in range(len(target_labels)): target_vectors[i] = relation_vectors[int(target_labels[i])] ### tuning model for target domain task ### latent.trainable = mc._TARGET_LATENT_TRAIN target_inputs = Input(shape=normalized_target_data.shape[1:]) latent_features = latent(target_inputs) if method_flag == mc._SCRATCH or method_flag == mc._CONV_TRANSFER: predictors = tll(latent_features) label_num = mc._TARGET_DIM_NUM else: predictors= sll(latent_features) label_num = mc._SOURCE_DIM_NUM target_nn = Model(inputs=[target_inputs], outputs=[predictors]) target_nn.compile(loss=['mean_squared_error'],optimizer=opt) #target_nn.summary() # training using target domain data target_max_loop = int(normalized_target_data.shape[0]/mc._BATCH_SIZE) target_progbar = Progbar(target=mc._TARGET_EPOCH_NUM) for epoch in range(mc._TARGET_EPOCH_NUM): shuffle_data, shuffle_labels, _ = \ pre.paired_shuffle(normalized_target_data, target_vectors, label_num) for loop in range(target_max_loop): batch_train_data = shuffle_data[loop*mc._BATCH_SIZE:(loop+1)*mc._BATCH_SIZE] batch_train_labels = shuffle_labels[loop*mc._BATCH_SIZE:(loop+1)*mc._BATCH_SIZE] loss = target_nn.train_on_batch([batch_train_data],[batch_train_labels]) #target_progbar.add(1, values=[("target loss",loss)]) # compute outputs of test data of target domain x = target_nn.predict([normalized_test_data]) if method_flag == mc._SCRATCH or method_flag == mc._CONV_TRANSFER: idx = np.argmax(x, axis=1) elif method_flag == mc._COUNT_ATDL: idx = np.argmax(x,axis=1) for j in range(len(test_labels)): for i in range(mc._TARGET_DIM_NUM): if test_labels[j] == i: test_labels[j] = relations[i] break else: distance, idx = Neighbors(x, relation_vectors, 1) idx = idx[:,0] backend.clear_session() return idx.T, test_labels.T