def get_all_responses_with_games(self, message, games): for_time = extract_time(message.content) game_and_players_strings = [] for game in games: db.record_would_play(message.author, game, for_time) ready_would_plays_for_game = db.get_ready_would_plays_for_game( game) unready_would_plays_for_game = db.get_unready_would_plays_for_game( game) if len(unready_would_plays_for_game) == 0: game_and_players_strings += [ "%s (%s)" % (game.name, len(game.get_ready_players())) ] else: game_and_players_strings += [ "%s %s" % (game.name, generate_ready_at_time_messages( ready_would_plays_for_game, unready_would_plays_for_game)) ] messages = [ "%s would play %s" % (message.author.display_name, make_sentence_from_strings(game_and_players_strings)) ] for game in games: messages += get_any_ready_messages(game) return messages for game in games: messages += get_any_ready_messages(game) return messages
def predictive_score_metrics(ori_data, generated_data): """Report the performance of Post-hoc RNN one-step ahead prediction. Args: - ori_data: original data - generated_data: generated synthetic data Returns: - predictive_score: MAE of the predictions on the original data """ # Initialization on the Graph tf.compat.v1.reset_default_graph() # Basic Parameters no, seq_len, dim = np.asarray(ori_data).shape # Set maximum sequence length and each sequence length ori_time, ori_max_seq_len = extract_time(ori_data) generated_time, generated_max_seq_len = extract_time(ori_data) max_seq_len = max([ori_max_seq_len, generated_max_seq_len]) ## Builde a post-hoc RNN predictive network # Network parameters hidden_dim = int(dim / 2) iterations = 5000 batch_size = 128 # Input place holders X = tf.compat.v1.placeholder(tf.float32, [None, max_seq_len - 1, dim - 1], name="myinput_x") T = tf.compat.v1.placeholder(tf.int32, [None], name="myinput_t") Y = tf.compat.v1.placeholder(tf.float32, [None, max_seq_len - 1, 1], name="myinput_y") # Predictor function def predictor(x, t): """Simple predictor function. Args: - x: time-series data - t: time information Returns: - y_hat: prediction - p_vars: predictor variables """ with tf.compat.v1.variable_scope("predictor", reuse=tf.compat.v1.AUTO_REUSE) as vs: p_cell = tf.keras.layers.GRUCell(hidden_dim, activation='tanh', name='p_cell') p_outputs = tf.keras.layers.RNN(p_cell, return_sequences=True)(x) y_hat_logit = tf.keras.layers.Dense(1, activation=None)(p_outputs) y_hat = tf.nn.sigmoid(y_hat_logit) p_vars = [ v for v in tf.compat.v1.global_variables() if v.name.startswith(vs.name) ] return y_hat, p_vars y_pred, p_vars = predictor(X, T) # Loss for the predictor p_loss = tf.compat.v1.losses.absolute_difference(Y, y_pred) # optimizer p_solver = tf.compat.v1.train.AdamOptimizer().minimize(p_loss, var_list=p_vars) ## Training # Session start sess = tf.compat.v1.Session() sess.run(tf.compat.v1.global_variables_initializer()) # Training using Synthetic dataset for itt in range(iterations): # Set mini-batch idx = np.random.permutation(len(generated_data)) train_idx = idx[:batch_size] X_mb = list(generated_data[i][:-1, :(dim - 1)] for i in train_idx) T_mb = list(generated_time[i] - 1 for i in train_idx) Y_mb = list( np.reshape(generated_data[i][1:, ( dim - 1)], [len(generated_data[i][1:, (dim - 1)]), 1]) for i in train_idx) # Train predictor _, step_p_loss = sess.run([p_solver, p_loss], feed_dict={ X: X_mb, T: T_mb, Y: Y_mb }) ## Test the trained model on the original data idx = np.random.permutation(len(ori_data)) train_idx = idx[:no] X_mb = list(ori_data[i][:-1, :(dim - 1)] for i in train_idx) T_mb = list(ori_time[i] - 1 for i in train_idx) Y_mb = list( np.reshape(ori_data[i][1:, (dim - 1)], [len(ori_data[i][1:, (dim - 1)]), 1]) for i in train_idx) # Prediction pred_Y_curr = sess.run(y_pred, feed_dict={X: X_mb, T: T_mb}) # Compute the performance in terms of MAE MAE_temp = 0 for i in range(no): MAE_temp = MAE_temp + mean_absolute_error(Y_mb[i], pred_Y_curr[i, :, :]) predictive_score = MAE_temp / no return predictive_score
def discriminative_score_metrics(ori_data, generated_data, rnn_iterations=2000): """Use post-hoc RNN to classify original data and synthetic data Args: - ori_data: original data - generated_data: generated synthetic data Returns: - discriminative_score: np.abs(classification accuracy - 0.5) """ # Initialization on the Graph tf.reset_default_graph() # Basic Parameters no, seq_len, dim = np.asarray(ori_data).shape # Set maximum sequence length and each sequence length ori_time, ori_max_seq_len = extract_time(ori_data) generated_time, generated_max_seq_len = extract_time(ori_data) max_seq_len = max([ori_max_seq_len, generated_max_seq_len]) ## Builde a post-hoc RNN discriminator network # Network parameters hidden_dim = int(dim / 2) iterations = rnn_iterations batch_size = 128 # Input place holders # Feature X = tf.placeholder(tf.float32, [None, max_seq_len, dim], name="myinput_x") X_hat = tf.placeholder(tf.float32, [None, max_seq_len, dim], name="myinput_x_hat") T = tf.placeholder(tf.int32, [None], name="myinput_t") T_hat = tf.placeholder(tf.int32, [None], name="myinput_t_hat") # discriminator function def discriminator(x, t): """Simple discriminator function. Args: - x: time-series data - t: time information Returns: - y_hat_logit: logits of the discriminator output - y_hat: discriminator output - d_vars: discriminator variables """ with tf.variable_scope("discriminator", reuse=tf.AUTO_REUSE) as vs: d_cell = tf.nn.rnn_cell.GRUCell(num_units=hidden_dim, activation=tf.nn.tanh, name='d_cell') d_outputs, d_last_states = tf.nn.dynamic_rnn(d_cell, x, dtype=tf.float32, sequence_length=t) y_hat_logit = tf.contrib.layers.fully_connected(d_last_states, 1, activation_fn=None) y_hat = tf.nn.sigmoid(y_hat_logit) d_vars = [ v for v in tf.all_variables() if v.name.startswith(vs.name) ] return y_hat_logit, y_hat, d_vars y_logit_real, y_pred_real, d_vars = discriminator(X, T) y_logit_fake, y_pred_fake, _ = discriminator(X_hat, T_hat) # Loss for the discriminator d_loss_real = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits( logits=y_logit_real, labels=tf.ones_like(y_logit_real))) d_loss_fake = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits( logits=y_logit_fake, labels=tf.zeros_like(y_logit_fake))) d_loss = d_loss_real + d_loss_fake # optimizer d_solver = tf.train.AdamOptimizer().minimize(d_loss, var_list=d_vars) ## Train the discriminator # Start session and initialize sess = tf.Session() sess.run(tf.global_variables_initializer()) # Train/test division for both original and generated data train_x, train_x_hat, test_x, test_x_hat, train_t, train_t_hat, test_t, test_t_hat = \ train_test_divide(ori_data, generated_data, ori_time, generated_time) # Training step for itt in range(iterations): # Batch setting X_mb, T_mb = batch_generator(train_x, train_t, batch_size) X_hat_mb, T_hat_mb = batch_generator(train_x_hat, train_t_hat, batch_size) # Train discriminator _, step_d_loss = sess.run([d_solver, d_loss], feed_dict={ X: X_mb, T: T_mb, X_hat: X_hat_mb, T_hat: T_hat_mb }) ## Test the performance on the testing set y_pred_real_curr, y_pred_fake_curr = sess.run([y_pred_real, y_pred_fake], feed_dict={ X: test_x, T: test_t, X_hat: test_x_hat, T_hat: test_t_hat }) y_pred_final = np.squeeze( np.concatenate((y_pred_real_curr, y_pred_fake_curr), axis=0)) y_label_final = np.concatenate((np.ones([ len(y_pred_real_curr), ]), np.zeros([ len(y_pred_real_curr), ])), axis=0) # Compute the accuracy acc = accuracy_score(y_label_final, (y_pred_final > 0.5)) discriminative_score = np.abs(0.5 - acc) return discriminative_score
def timegan(ori_data, parameters): # Basic Parameters no, seq_len, dim = np.asarray(ori_data).shape # Maximum sequence length and each sequence length ori_time, max_seq_len = extract_time(ori_data) def MinMaxScaler(data): """Min-Max Normalizer. Args: - data: raw data Returns: - norm_data: normalized data - min_val: minimum values (for renormalization) - max_val: maximum values (for renormalization) """ min_val = np.nanmin(np.nanmin(data, axis = 0), axis = 0) data = data - min_val max_val = np.nanmax(np.nanmax(data, axis = 0), axis = 0) norm_data = data / (max_val + 1e-7) return norm_data, min_val, max_val # Normalization ori_data, min_val, max_val = MinMaxScaler(ori_data) ## Build a RNN networks # Network Parameters hidden_dim = parameters['hidden_dim'] num_layers = parameters['num_layer'] iterations = parameters['iterations'] batch_size = parameters['batch_size'] module_name = parameters['module'] loss_mode = parameters['loss'] z_dim = dim gamma = 1 gp_weight = 10 def make_embedder (): """Embedding network between original feature space to latent space. Args for model: - X: input time-series features Model returns: - H: embeddings """ embedder_model = tf.keras.Sequential(name='embedder') #embedder_model.add(tf.keras.layers.Masking(mask_value=-1, input_shape=(seq_len,dim))) embedder_model.add(rnn_cell(module_name, hidden_dim, return_sequences=True, input_shape=(seq_len,dim))) for i in range(num_layers-1): embedder_model.add(rnn_cell(module_name, hidden_dim, return_sequences=True, input_shape=(seq_len, hidden_dim))) embedder_model.add(tf.keras.layers.Dense(hidden_dim, activation='sigmoid')) return embedder_model def make_recovery (): """Recovery network from latent space to original space. Args for model: - H: latent representation Model returns: - X_tilde: recovered data """ recovery_model = tf.keras.Sequential(name='recovery') for i in range(num_layers): recovery_model.add(rnn_cell(module_name, hidden_dim, return_sequences=True, input_shape=(seq_len, hidden_dim))) recovery_model.add(tf.keras.layers.Dense(dim, activation='sigmoid')) return recovery_model def make_generator (): """Generator function: Generate time-series data in latent space. Args for model: - Z: random variables MOdel returns: - E: generated embedding """ generator_model = tf.keras.Sequential(name='generator') generator_model.add(rnn_cell(module_name, hidden_dim, return_sequences=True, input_shape=(seq_len, dim))) for i in range(num_layers-1): generator_model.add(rnn_cell(module_name, hidden_dim, return_sequences=True, input_shape=(seq_len, hidden_dim))) generator_model.add(tf.keras.layers.Dense(hidden_dim, activation='sigmoid')) return generator_model def make_supervisor (): """Generate next sequence using the previous sequence. Args for model: - H: latent representation Model returns: - S: generated sequence based on the latent representations generated by the generator """ supervisor_model = tf.keras.Sequential(name='supervisor') for i in range(num_layers-1): supervisor_model.add(rnn_cell(module_name, hidden_dim, return_sequences=True, input_shape=(seq_len, hidden_dim))) supervisor_model.add(tf.keras.layers.Dense(hidden_dim, activation='sigmoid')) return supervisor_model def make_discriminator (): """Recovery network from latent space to original space. Args for model: - H: latent representation MOdel returns: - Y_hat: classification results between original and synthetic time-series """ discriminator_model = tf.keras.Sequential(name='discriminator') for i in range(num_layers): discriminator_model.add(rnn_cell(module_name, hidden_dim, return_sequences=True, input_shape=(seq_len, hidden_dim))) discriminator_model.add(tf.keras.layers.Dense(1, activation=None)) return discriminator_model # make the models embedder_model = make_embedder() recovery_model = make_recovery() generator_model = make_generator() supervisor_model = make_supervisor() discriminator_model = make_discriminator() def get_embedder_T0_loss(X, X_tilde): mse = tf.keras.losses.MeanSquaredError() E_loss_T0 = mse(X, X_tilde) return E_loss_T0 def get_embedder_0_loss(X, X_tilde): E_loss_T0 = get_embedder_T0_loss(X, X_tilde) E_loss0 = 10*tf.sqrt(E_loss_T0) return E_loss0 def get_embedder_loss(X, X_tilde, H, H_hat_supervise): """ computes embedder network loss Args: - X: input time-series features - X_tilde: recovered data - H: latent representation - H_hat_supervise: generated sequence based on the latent representations generated by the generator Returns: - E_loss: embedder loss """ E_loss_T0 = get_embedder_T0_loss(X, X_tilde) E_loss0 = 10*tf.sqrt(E_loss_T0) #could use function above G_loss_S = get_generator_s_loss(H, H_hat_supervise) E_loss = E_loss0 + 0.1*G_loss_S return E_loss def get_generator_s_loss(H, H_hat_supervise): """ computes supervised loss Args: - H: latent representation - H_hat_supervise: generated sequence based on the latent representations generated by the generator Returns: - G_loss_s: supervised loss for generator """ mse = tf.keras.losses.MeanSquaredError() G_loss_S = mse(H[:,1:,:], H_hat_supervise[:,:-1,:]) return G_loss_S def get_generator_loss(Y_fake, Y_fake_e, X_hat, X, H, H_hat_supervise): """ computes generator loss Args: - Y_fake: classification results of latent synthetic time-series - Y_fake_e: classification results of generated sequence for latent synthetic time-series - X_hat: recovered data - X: input time-series data - H: latent representation - H_hat_supervise: generated sequence for latent representation Returns: - G_loss: generator loss - G_loss_U: unsupervised generator loss - G_loss_S: supervised generator loss - G_loss_V: moments loss for generator """ #1. Adversarial loss if loss_mode == "bce": bce = tf.keras.losses.BinaryCrossentropy(from_logits=True) G_loss_U = bce(tf.ones_like(Y_fake), Y_fake) G_loss_U_e = bce(tf.ones_like(Y_fake_e), Y_fake_e) elif loss_mode == "wgan_gp": G_loss_U = -tf.reduce_mean(Y_fake) G_loss_U_e = -tf.reduce_mean(Y_fake_e) else: raise Exception("Loss method should be specified") #2. Two Moments X = tf.convert_to_tensor(X) G_loss_V1 = tf.reduce_mean(tf.abs(tf.sqrt(tf.nn.moments(X_hat,[0])[1] + 1e-6) - tf.sqrt(tf.nn.moments(X,[0])[1] + 1e-6))) G_loss_V2 = tf.reduce_mean(tf.abs((tf.nn.moments(X_hat,[0])[0]) - (tf.nn.moments(X,[0])[0]))) G_loss_V = G_loss_V1 + G_loss_V2 #3. Supervised loss G_loss_S = get_generator_s_loss(H, H_hat_supervise) #4. Summation G_loss = G_loss_U + gamma * G_loss_U_e + 100 * tf.sqrt(G_loss_S) + 100*G_loss_V return G_loss, G_loss_U, G_loss_S, G_loss_V def get_discriminator_loss(Y_real, Y_fake, Y_fake_e): """ computes discrminator loss Args: - Y_real: classification results of latent real time-series - Y_fake: classification results of latent synthetic time-series - Y_fake_e: classification results of generated sequence for latent synthetic time-series Returns: - d_loss: discriminator loss """ if loss_mode == "bce": bce = tf.keras.losses.BinaryCrossentropy(from_logits=True) #loss for cls of latent real data seq #default arg for tf.keras.losses.BinaryCrossentropy reduction=losses_utils.ReductionV2.AUTO D_loss_real = bce(tf.ones_like(Y_real), Y_real) D_loss_fake = bce(tf.zeros_like(Y_fake), Y_fake) D_loss_fake_e = bce(tf.zeros_like(Y_fake_e), Y_fake_e) #Wasserstein loss elif loss_mode == "wgan_gp": D_loss_real = -tf.reduce_mean(Y_real) D_loss_fake = tf.reduce_mean(Y_fake) D_loss_fake_e = tf.reduce_mean(Y_fake_e) else: raise Exception("Loss method should be specified") D_loss = D_loss_real + D_loss_fake + gamma * D_loss_fake_e return D_loss def gradient_penalty(f, H_mb, H_hat_mb): """ Calculates the gradient penalty. This loss is calculated on an interpolated "patient" and added to the discriminator loss. """ # get the interplated patient alpha = tf.random.uniform([H_mb.shape[0], 1, 1], minval=0.0, maxval=1.0, dtype=tf.float64) diff = H_hat_mb - H_mb interpolated = H_mb + alpha * diff with tf.GradientTape() as tape: tape.watch(interpolated) predict = f(interpolated) # Calculate the gradients w.r.t to this interpolated patient grad = tape.gradient(predict, interpolated) norm = tf.norm(tf.reshape(grad, [tf.shape(grad)[0], -1]), axis=1) gp = tf.reduce_mean((norm - 1.)**2) return gp # optimizer embedder0_optimizer = tf.keras.optimizers.Adam() embedder_optimizer = tf.keras.optimizers.Adam() gen_s_optimizer = tf.keras.optimizers.Adam() generator_optimizer = tf.keras.optimizers.Adam() discriminator_optimizer = tf.keras.optimizers.Adam() @tf.function def train_step_embedder(X_mb): """ trains embedder model """ with tf.GradientTape() as embedder_tape: # Embedder & Recovery H_mb = embedder_model(X_mb) X_tilde_mb = recovery_model(H_mb) embedder_0_loss = get_embedder_0_loss(X_mb, X_tilde_mb) emb_vars = embedder_model.trainable_variables + recovery_model.trainable_variables gradients_of_embedder = embedder_tape.gradient(embedder_0_loss, emb_vars) embedder0_optimizer.apply_gradients(zip(gradients_of_embedder, emb_vars)) return embedder_0_loss @tf.function def train_step_generator_s(X_mb): """ supervised training for generator model """ with tf.GradientTape() as gen_s_tape: H_mb = embedder_model(X_mb) # Generator H_hat_supervise_mb = supervisor_model(H_mb) gen_s_loss = get_generator_s_loss(H_mb, H_hat_supervise_mb) gen_s_vars = supervisor_model.trainable_variables gradients_of_gen_s = gen_s_tape.gradient(gen_s_loss, gen_s_vars) gen_s_optimizer.apply_gradients(zip(gradients_of_gen_s, gen_s_vars)) return gen_s_loss @tf.function def train_step_joint(X_mb, Z_mb): """ join training for generator and supervisor model, embedder model """ #train generator with tf.GradientTape() as gen_tape: # Generator H_mb = embedder_model(X_mb) E_hat_mb = generator_model(Z_mb) H_hat_mb = supervisor_model(E_hat_mb) H_hat_supervise_mb = supervisor_model(H_mb) # Synthetic data X_hat_mb = recovery_model(H_hat_mb) # Discriminator Y_fake_mb = discriminator_model(H_hat_mb) Y_real_mb = discriminator_model(H_mb) Y_fake_e_mb = discriminator_model(E_hat_mb) gen_loss, g_loss_u, gen_s_loss, g_loss_v = get_generator_loss(Y_fake_mb, Y_fake_e_mb, X_hat_mb, X_mb, H_mb, H_hat_supervise_mb) gen_vars = generator_model.trainable_variables + supervisor_model.trainable_variables gradients_of_gen = gen_tape.gradient(gen_loss, gen_vars) generator_optimizer.apply_gradients(zip(gradients_of_gen, gen_vars)) #train embedder with tf.GradientTape() as embedder_tape: H_mb = embedder_model(X_mb) X_tilde_mb = recovery_model(H_mb) H_hat_supervise = supervisor_model(H_mb) #i think we are minimizing E_loss but printing out E_loss_T0?? emb_T0_loss = get_embedder_T0_loss(X_mb, X_tilde_mb) emb_loss = get_embedder_loss(X_mb, X_tilde_mb, H_mb, H_hat_supervise) emb_vars = embedder_model.trainable_variables + recovery_model.trainable_variables gradients_of_emb = embedder_tape.gradient(emb_loss, emb_vars) embedder_optimizer.apply_gradients(zip(gradients_of_emb, emb_vars)) return emb_T0_loss, emb_loss, g_loss_u, gen_s_loss, g_loss_v #H_hat_mb, E_hat_mb, @tf.function def train_step_discriminator(X_mb, Z_mb): """ trains discriminator model """ with tf.GradientTape() as disc_tape: H_mb = embedder_model(X_mb) E_hat_mb = generator_model(Z_mb) H_hat_mb = supervisor_model(E_hat_mb) # Synthetic data X_hat_mb = recovery_model(H_hat_mb) # Discriminator Y_fake_mb = discriminator_model(H_hat_mb) Y_real_mb = discriminator_model(H_mb) Y_fake_e_mb = discriminator_model(E_hat_mb) # Check discriminator loss before updating disc_loss = get_discriminator_loss(Y_real_mb, Y_fake_mb, Y_fake_e_mb) if loss_mode == "wgan_gp": gp = gradient_penalty(discriminator_model, H_mb, H_hat_mb) disc_loss = gp * gp_weight + disc_loss # Train discriminator (only when the discriminator does not work well) if (disc_loss > 0.15): disc_vars = discriminator_model.trainable_variables gradients_of_disc = disc_tape.gradient(disc_loss, disc_vars) discriminator_optimizer.apply_gradients(zip(gradients_of_disc, disc_vars)) return disc_loss #timeGAN training def train(): #1. Embedding network training print('Start Embedding Network Training') for itt in range(iterations): # Set mini-batch X_mb, T_mb = batch_generator(ori_data, ori_time, batch_size) # Train embedder step_e_loss = train_step_embedder(X_mb) # Checkpoint if itt % 100 == 0: print('step: '+ str(itt) + '/' + str(iterations) + ', e_loss: ' + str(np.round(np.sqrt(step_e_loss),4)) ) print('Finish Embedding Network Training') #2. Training only with supervised loss print('Start Training with Supervised Loss Only') for itt in range(iterations): # Set mini-batch X_mb, T_mb = batch_generator(ori_data, ori_time, batch_size) # Random vector generation # Train generator step_gen_s_loss = train_step_generator_s(X_mb) # Checkpoint if itt % 100 == 0: print('step: '+ str(itt) + '/' + str(iterations) +', s_loss: ' + str(np.round(np.sqrt(step_gen_s_loss),4)) ) print('Finish Training with Supervised Loss Only') # 3. Joint Training print('Start Joint Training') for itt in range(iterations): # Generator training (twice more than discriminator training) for kk in range(2): # Set mini-batch X_mb, T_mb = batch_generator(ori_data, ori_time, batch_size) # Random vector generation Z_mb = random_generator(batch_size, z_dim, T_mb, max_seq_len) # Train generator and embedder emb_T0_loss, emb_loss, g_loss_u, gen_s_loss, g_loss_v = train_step_joint(X_mb, Z_mb) # Discriminator training # Set mini-batch X_mb, T_mb = batch_generator(ori_data, ori_time, batch_size) # Random vector generation Z_mb = random_generator(batch_size, z_dim, T_mb, max_seq_len) #train discriminator d_loss = train_step_discriminator(X_mb, Z_mb) # Print multiple checkpoints if itt % 100 == 0: print('step: '+ str(itt) + '/' + str(iterations) + ', d_loss: ' + str(np.round(d_loss,4)) + ', g_loss_u: ' + str(np.round(g_loss_u,4)) + ', g_loss_s: ' + str(np.round(np.sqrt(gen_s_loss),4)) + ', g_loss_v: ' + str(np.round(g_loss_v,4)) + ', e_loss_t0: ' + str(np.round(np.sqrt(emb_T0_loss),4)) ) print('Finish Joint Training') ## Synthetic data generation Z_mb = random_generator(no, z_dim, ori_time, max_seq_len) E_hat_generated = generator_model(Z_mb) H_hat_generated = supervisor_model(E_hat_generated) generated_data_curr = recovery_model(H_hat_generated) generated_data = list() for i in range(no): temp = generated_data_curr[i,:ori_time[i],:] generated_data.append(temp) # Renormalization generated_data = generated_data * max_val generated_data = generated_data + min_val return generated_data return train() ####TESTING#### # from data_loading import real_data_loading, sine_data_generation # data_name = 'prism' # seq_len = 10 # if data_name in ['stock', 'energy', 'sine_sampling', 'prism']: # ori_data = real_data_loading(data_name, seq_len) # elif data_name == 'sine': # # Set number of samples and its dimensions # no, dim = 15, 5 # ori_data = sine_data_generation(no, seq_len, dim) # print(data_name + ' dataset is ready.') # ## Newtork parameters # parameters = dict() # parameters['module'] = 'lstm' # parameters['hidden_dim'] = 3 # parameters['num_layer'] = 3 # parameters['iterations'] = 2 # parameters['batch_size'] = 2 # parameters['loss'] = "bce" # generated_data = timegan(ori_data, parameters) # print('Finish Synthetic Data Generation') # print(generated_data)
def timegan(ori_data, parameters): """TimeGAN function. Use original data as training set to generater synthetic data (time-series) Args: - ori_data: original time-series data - parameters: TimeGAN network parameters Returns: - generated_data: generated time-series data """ # Initialization on the Graph tf.reset_default_graph() # Basic Parameters no, seq_len, dim = np.asarray(ori_data).shape # Maximum sequence length and each sequence length ori_time, max_seq_len = extract_time(ori_data) def MinMaxScaler(data): """Min-Max Normalizer. Args: - data: raw data Returns: - norm_data: normalized data - min_val: minimum values (for renormalization) - max_val: maximum values (for renormalization) """ min_val = np.min(np.min(data, axis=0), axis=0) data = data - min_val max_val = np.max(np.max(data, axis=0), axis=0) norm_data = data / (max_val + 1e-7) return norm_data, min_val, max_val # Normalization ori_data, min_val, max_val = MinMaxScaler(ori_data) # Build a RNN networks # Network Parameters hidden_dim = parameters["hidden_dim"] num_layers = parameters["num_layer"] iterations = parameters["iterations"] batch_size = parameters["batch_size"] module_name = parameters["module"] z_dim = dim gamma = 1 # Input place holders X = tf.placeholder(tf.float32, [None, max_seq_len, dim], name="myinput_x") Z = tf.placeholder(tf.float32, [None, max_seq_len, z_dim], name="myinput_z") T = tf.placeholder(tf.int32, [None], name="myinput_t") def embedder(X, T): """Embedding network between original feature space to latent space. Args: - X: input time-series features - T: input time information Returns: - H: embeddings """ with tf.variable_scope("embedder", reuse=tf.AUTO_REUSE): e_cell = tf.nn.rnn_cell.MultiRNNCell( [rnn_cell(module_name, hidden_dim) for _ in range(num_layers)]) e_outputs, e_last_states = tf.nn.dynamic_rnn(e_cell, X, dtype=tf.float32, sequence_length=T) H = tf.contrib.layers.fully_connected(e_outputs, hidden_dim, activation_fn=tf.nn.sigmoid) return H def recovery(H, T): """Recovery network from latent space to original space. Args: - H: latent representation - T: input time information Returns: - X_tilde: recovered data """ with tf.variable_scope("recovery", reuse=tf.AUTO_REUSE): r_cell = tf.nn.rnn_cell.MultiRNNCell( [rnn_cell(module_name, hidden_dim) for _ in range(num_layers)]) r_outputs, r_last_states = tf.nn.dynamic_rnn(r_cell, H, dtype=tf.float32, sequence_length=T) X_tilde = tf.contrib.layers.fully_connected( r_outputs, dim, activation_fn=tf.nn.sigmoid) return X_tilde def generator(Z, T): """Generator function: Generate time-series data in latent space. Args: - Z: random variables - T: input time information Returns: - E: generated embedding """ with tf.variable_scope("generator", reuse=tf.AUTO_REUSE): e_cell = tf.nn.rnn_cell.MultiRNNCell( [rnn_cell(module_name, hidden_dim) for _ in range(num_layers)]) e_outputs, e_last_states = tf.nn.dynamic_rnn(e_cell, Z, dtype=tf.float32, sequence_length=T) E = tf.contrib.layers.fully_connected(e_outputs, hidden_dim, activation_fn=tf.nn.sigmoid) return E def supervisor(H, T): """Generate next sequence using the previous sequence. Args: - H: latent representation - T: input time information Returns: - S: generated sequence based on the latent representations generated by the generator """ with tf.variable_scope("supervisor", reuse=tf.AUTO_REUSE): e_cell = tf.nn.rnn_cell.MultiRNNCell([ rnn_cell(module_name, hidden_dim) for _ in range(num_layers - 1) ]) e_outputs, e_last_states = tf.nn.dynamic_rnn(e_cell, H, dtype=tf.float32, sequence_length=T) S = tf.contrib.layers.fully_connected(e_outputs, hidden_dim, activation_fn=tf.nn.sigmoid) return S def discriminator(H, T): """Discriminate the original and synthetic time-series data. Args: - H: latent representation - T: input time information Returns: - Y_hat: classification results between original and synthetic time-series """ with tf.variable_scope("discriminator", reuse=tf.AUTO_REUSE): d_cell = tf.nn.rnn_cell.MultiRNNCell( [rnn_cell(module_name, hidden_dim) for _ in range(num_layers)]) d_outputs, d_last_states = tf.nn.dynamic_rnn(d_cell, H, dtype=tf.float32, sequence_length=T) Y_hat = tf.contrib.layers.fully_connected(d_outputs, 1, activation_fn=None) return Y_hat # Embedder & Recovery H = embedder(X, T) X_tilde = recovery(H, T) # Generator E_hat = generator(Z, T) H_hat = supervisor(E_hat, T) H_hat_supervise = supervisor(H, T) # Synthetic data X_hat = recovery(H_hat, T) # Discriminator Y_fake = discriminator(H_hat, T) Y_real = discriminator(H, T) Y_fake_e = discriminator(E_hat, T) # Variables e_vars = [ v for v in tf.trainable_variables() if v.name.startswith("embedder") ] r_vars = [ v for v in tf.trainable_variables() if v.name.startswith("recovery") ] g_vars = [ v for v in tf.trainable_variables() if v.name.startswith("generator") ] s_vars = [ v for v in tf.trainable_variables() if v.name.startswith("supervisor") ] d_vars = [ v for v in tf.trainable_variables() if v.name.startswith("discriminator") ] # Discriminator loss D_loss_real = tf.losses.sigmoid_cross_entropy(tf.ones_like(Y_real), Y_real) D_loss_fake = tf.losses.sigmoid_cross_entropy(tf.zeros_like(Y_fake), Y_fake) D_loss_fake_e = tf.losses.sigmoid_cross_entropy(tf.zeros_like(Y_fake_e), Y_fake_e) D_loss = D_loss_real + D_loss_fake + gamma * D_loss_fake_e # Generator loss # 1. Adversarial loss G_loss_U = tf.losses.sigmoid_cross_entropy(tf.ones_like(Y_fake), Y_fake) G_loss_U_e = tf.losses.sigmoid_cross_entropy(tf.ones_like(Y_fake_e), Y_fake_e) # 2. Supervised loss G_loss_S = tf.losses.mean_squared_error(H[:, 1:, :], H_hat_supervise[:, :-1, :]) # 3. Two Momments G_loss_V1 = tf.reduce_mean( tf.abs( tf.sqrt(tf.nn.moments(X_hat, [0])[1] + 1e-6) - tf.sqrt(tf.nn.moments(X, [0])[1] + 1e-6))) G_loss_V2 = tf.reduce_mean( tf.abs((tf.nn.moments(X_hat, [0])[0]) - (tf.nn.moments(X, [0])[0]))) G_loss_V = G_loss_V1 + G_loss_V2 # 4. Summation G_loss = G_loss_U + gamma * G_loss_U_e + 100 * tf.sqrt( G_loss_S) + 100 * G_loss_V # Embedder network loss E_loss_T0 = tf.losses.mean_squared_error(X, X_tilde) E_loss0 = 10 * tf.sqrt(E_loss_T0) E_loss = E_loss0 + 0.1 * G_loss_S # optimizer E0_solver = tf.train.AdamOptimizer().minimize(E_loss0, var_list=e_vars + r_vars) E_solver = tf.train.AdamOptimizer().minimize(E_loss, var_list=e_vars + r_vars) D_solver = tf.train.AdamOptimizer().minimize(D_loss, var_list=d_vars) G_solver = tf.train.AdamOptimizer().minimize(G_loss, var_list=g_vars + s_vars) GS_solver = tf.train.AdamOptimizer().minimize(G_loss_S, var_list=g_vars + s_vars) # TimeGAN training sess = tf.Session() sess.run(tf.global_variables_initializer()) # 1. Embedding network training print("Start Embedding Network Training") for itt in range(iterations): # Set mini-batch X_mb, T_mb = batch_generator(ori_data, ori_time, batch_size) # Train embedder _, step_e_loss = sess.run([E0_solver, E_loss_T0], feed_dict={ X: X_mb, T: T_mb }) # Checkpoint if itt % 1000 == 0: print("step: " + str(itt) + "/" + str(iterations) + ", e_loss: " + str(np.round(np.sqrt(step_e_loss), 4))) print("Finish Embedding Network Training") # 2. Training only with supervised loss print("Start Training with Supervised Loss Only") for itt in range(iterations): # Set mini-batch X_mb, T_mb = batch_generator(ori_data, ori_time, batch_size) # Random vector generation Z_mb = random_generator(batch_size, z_dim, T_mb, max_seq_len) # Train generator _, step_g_loss_s = sess.run([GS_solver, G_loss_S], feed_dict={ Z: Z_mb, X: X_mb, T: T_mb }) # Checkpoint if itt % 1000 == 0: print("step: " + str(itt) + "/" + str(iterations) + ", s_loss: " + str(np.round(np.sqrt(step_g_loss_s), 4))) print("Finish Training with Supervised Loss Only") # 3. Joint Training print("Start Joint Training") for itt in range(iterations): # Generator training (twice more than discriminator training) for kk in range(2): # Set mini-batch X_mb, T_mb = batch_generator(ori_data, ori_time, batch_size) # Random vector generation Z_mb = random_generator(batch_size, z_dim, T_mb, max_seq_len) # Train generator _, step_g_loss_u, step_g_loss_s, step_g_loss_v = sess.run( [G_solver, G_loss_U, G_loss_S, G_loss_V], feed_dict={ Z: Z_mb, X: X_mb, T: T_mb }, ) # Train embedder _, step_e_loss_t0 = sess.run([E_solver, E_loss_T0], feed_dict={ Z: Z_mb, X: X_mb, T: T_mb }) # Discriminator training # Set mini-batch X_mb, T_mb = batch_generator(ori_data, ori_time, batch_size) # Random vector generation Z_mb = random_generator(batch_size, z_dim, T_mb, max_seq_len) # Check discriminator loss before updating check_d_loss = sess.run(D_loss, feed_dict={X: X_mb, T: T_mb, Z: Z_mb}) # Train discriminator (only when the discriminator does not work well) if check_d_loss > 0.15: _, step_d_loss = sess.run([D_solver, D_loss], feed_dict={ X: X_mb, T: T_mb, Z: Z_mb }) # Print multiple checkpoints if itt % 1000 == 0: print("step: " + str(itt) + "/" + str(iterations) + ", d_loss: " + str(np.round(step_d_loss, 4)) + ", g_loss_u: " + str(np.round(step_g_loss_u, 4)) + ", g_loss_s: " + str(np.round(np.sqrt(step_g_loss_s), 4)) + ", g_loss_v: " + str(np.round(step_g_loss_v, 4)) + ", e_loss_t0: " + str(np.round(np.sqrt(step_e_loss_t0), 4))) print("Finish Joint Training") # Synthetic data generation Z_mb = random_generator(no, z_dim, ori_time, max_seq_len) generated_data_curr = sess.run(X_hat, feed_dict={ Z: Z_mb, X: ori_data, T: ori_time }) generated_data = list() for _ in range(50): for i in range(no): temp = generated_data_curr[i, :ori_time[i], :] generated_data.append(temp) # Renormalization generated_data = generated_data * max_val generated_data = generated_data + min_val return generated_data
def timegan_static(ori_data, ori_data_static, ori_data_stack, parameters): # Basic Parameters no, seq_len, dim = np.asarray(ori_data).shape # Maximum sequence length and each sequence length ori_time, max_seq_len = extract_time(ori_data) ori_data = np.array(ori_data) ori_data_static = np.array(ori_data_static) no_static, dim_static = ori_data_static.shape dstack = np.dstack((ori_data, ori_data_stack)) no, seq_len, dim = np.asarray(ori_data).shape ori_time, max_seq_len = extract_time(ori_data) np.save('mix_data_no_seq_2k', dstack) def MinMaxScaler(data): """Min-Max Normalizer. Args: - data: raw data Returns: - norm_data: normalized data - min_val: minimum values (for renormalization) - max_val: maximum values (for renormalization) """ min_val = np.nanmin(np.nanmin(data, axis=0), axis=0) data = data - min_val max_val = np.nanmax(np.nanmax(data, axis=0), axis=0) norm_data = data / (max_val + 1e-7) return norm_data, min_val, max_val def MinMaxScaler_static(data): """Min-Max Normalizer. Args: - data: raw data Returns: - norm_data: normalized data - min_val: minimum values (for renormalization) - max_val: maximum values (for renormalization) """ min_val = np.nanmin(data, axis=0) data = data - min_val max_val = np.nanmax(data, axis=0) norm_data = data / (max_val + 1e-7) return norm_data, min_val, max_val # Normalization ori_data, min_val, max_val = MinMaxScaler(ori_data) ori_data_static, min_val_static, max_val_static = MinMaxScaler_static( ori_data_static) ## Build a RNN networks # Network Parameters hidden_dim = parameters['hidden_dim'] num_layers = parameters['num_layer'] iterations = parameters['iterations'] batch_size = parameters['batch_size'] module_name = parameters['module'] z_dim = dim z_dim_static = dim_static gamma = 1 def make_embedder(): """Embedding network between original feature space to latent space. Args for model: - X: input time-series features Model returns: - H: embeddings """ embedder_model = tf.keras.Sequential(name='embedder') embedder_model.add( rnn_cell(module_name, hidden_dim, return_sequences=True, input_shape=(seq_len, dim))) for i in range(num_layers - 1): embedder_model.add( rnn_cell(module_name, hidden_dim, return_sequences=True, input_shape=(seq_len, hidden_dim))) embedder_model.add( tf.keras.layers.Dense(hidden_dim, activation='sigmoid')) return embedder_model def make_embedder_static(): """ Embedder model for static values """ embedder_model_static = tf.keras.Sequential( name="embedder_static", layers=[ tf.keras.layers.Dense(hidden_dim, input_shape=(dim_static, )), tf.keras.layers.Dense(hidden_dim), tf.keras.layers.Dense(hidden_dim), tf.keras.layers.Dense(hidden_dim), tf.keras.layers.Dense(hidden_dim, activation=tf.nn.sigmoid) ]) return embedder_model_static def make_recovery(): """Recovery network from latent space to original space. Args for model: - H: latent representation Model returns: - X_tilde: recovered data """ recovery_model = tf.keras.Sequential(name='recovery') for i in range(num_layers): recovery_model.add( rnn_cell(module_name, hidden_dim, return_sequences=True, input_shape=(seq_len, hidden_dim))) recovery_model.add(tf.keras.layers.Dense(dim, activation='sigmoid')) return recovery_model def make_recovery_static(): """ recovery model for static values """ recovery_model_static = tf.keras.Sequential( name="recovery_static", layers=[ tf.keras.layers.Dense(dim_static, input_shape=(hidden_dim, )), tf.keras.layers.Dense(dim_static), tf.keras.layers.Dense(dim_static), tf.keras.layers.Dense(dim_static), tf.keras.layers.Dense(dim_static, activation=tf.nn.sigmoid) ]) return recovery_model_static def make_generator(): """Generator function: Generate time-series data in latent space. Args for model: - Z: random variables MOdel returns: - E: generated embedding """ generator_model = tf.keras.Sequential(name='generator') generator_model.add( rnn_cell(module_name, hidden_dim, return_sequences=True, input_shape=(seq_len, dim))) for i in range(num_layers - 1): generator_model.add( rnn_cell(module_name, hidden_dim, return_sequences=True, input_shape=(seq_len, hidden_dim))) generator_model.add( tf.keras.layers.Dense(hidden_dim, activation='sigmoid')) return generator_model def make_generator_static(): """ generator model for static values """ generator_model_static = tf.keras.Sequential( name="generator_static", layers=[ tf.keras.layers.Dense(hidden_dim, input_shape=(dim_static, )), tf.keras.layers.LeakyReLU(), tf.keras.layers.Dense(hidden_dim), tf.keras.layers.LeakyReLU(), tf.keras.layers.Dense(hidden_dim), tf.keras.layers.LeakyReLU(), tf.keras.layers.Dense(hidden_dim, activation='tanh'), ]) return generator_model_static def make_supervisor(): """Generate next sequence using the previous sequence. Args for model: - H: latent representation Model returns: - S: generated sequence based on the latent representations generated by the generator """ supervisor_model = tf.keras.Sequential(name='supervisor') for i in range(num_layers - 1): supervisor_model.add( rnn_cell(module_name, hidden_dim, return_sequences=True, input_shape=(seq_len, hidden_dim * 2))) supervisor_model.add( tf.keras.layers.Dense(hidden_dim, activation='sigmoid')) return supervisor_model def make_discriminator(): """Recovery network from latent space to original space. Args for model: - H: latent representation Model returns: - Y_hat: classification results between original and synthetic time-series """ discriminator_model = tf.keras.Sequential(name='discriminator') for i in range(num_layers): discriminator_model.add( rnn_cell(module_name, hidden_dim, return_sequences=True, input_shape=(seq_len, hidden_dim))) discriminator_model.add(tf.keras.layers.Dense(1, activation=None)) return discriminator_model def make_discriminator_static(): """ discirminator model for static values """ discriminator_model_static = tf.keras.Sequential( name="discriminator_static", layers=[ tf.keras.layers.Dense(hidden_dim, input_shape=(hidden_dim, )), tf.keras.layers.LeakyReLU(), tf.keras.layers.Dense(hidden_dim), tf.keras.layers.LeakyReLU(), tf.keras.layers.Dense(hidden_dim), tf.keras.layers.LeakyReLU(), tf.keras.layers.Dense(hidden_dim), tf.keras.layers.Dense(1, activation=None), ]) return discriminator_model_static # make the models embedder_model = make_embedder() recovery_model = make_recovery() generator_model = make_generator() supervisor_model = make_supervisor() discriminator_model = make_discriminator() embedder_model_static = make_embedder_static() recovery_model_static = make_recovery_static() generator_model_static = make_generator_static() discriminator_model_static = make_discriminator_static() def get_embedder_T0_loss(X, X_tilde): mse = tf.keras.losses.MeanSquaredError() E_loss_T0 = mse(X, X_tilde) return E_loss_T0 def get_embedder_0_loss(X, X_tilde): E_loss_T0 = get_embedder_T0_loss(X, X_tilde) E_loss0 = 10 * tf.sqrt(E_loss_T0) return E_loss0 def get_embedder_loss(X, X_tilde, H, H_hat_supervise): """ computes embedder network loss Args: - X: input time-series features - X_tilde: recovered data - H: latent representation - H_hat_supervise: generated sequence based on the latent representations generated by the generator Returns: - E_loss: embedder loss """ E_loss_T0 = get_embedder_T0_loss(X, X_tilde) E_loss0 = 10 * tf.sqrt(E_loss_T0) #could use function above G_loss_S = get_generator_s_loss(H, H_hat_supervise) E_loss = E_loss0 + 0.1 * G_loss_S return E_loss def get_generator_s_loss(H, H_hat_supervise): """ computes supervised loss Args: - H: latent representation - H_hat_supervise: generated sequence based on the latent representations generated by the generator Returns: - G_loss_s: supervised loss for generator """ mse = tf.keras.losses.MeanSquaredError() G_loss_S = mse(H[:, 1:, :], H_hat_supervise[:, :-1, :]) return G_loss_S def get_generator_loss(Y_fake, Y_fake_e, X_hat, X, H, H_hat_supervise): """ computes generator loss for time series variables Args: - Y_fake: classification results of latent synthetic time-series - Y_fake_e: classification results of generated sequence for latent synthetic time-series - X_hat: recovered data - X: input time-series data - H: latent representation - H_hat_supervise: generated sequence for latent representation Returns: - G_loss: generator loss - G_loss_U: unsupervised generator loss - G_loss_S: supervised generator loss - G_loss_V: moments loss for generator """ #1. Adversarial loss bce = tf.keras.losses.BinaryCrossentropy(from_logits=True) G_loss_U = bce(tf.ones_like(Y_fake), Y_fake) G_loss_U_e = bce(tf.ones_like(Y_fake_e), Y_fake_e) #2. Two Moments X = tf.convert_to_tensor(X) G_loss_V1 = tf.reduce_mean( tf.abs( tf.sqrt(tf.nn.moments(X_hat, [0])[1] + 1e-6) - tf.sqrt(tf.nn.moments(X, [0])[1] + 1e-6))) G_loss_V2 = tf.reduce_mean( tf.abs((tf.nn.moments(X_hat, [0])[0]) - (tf.nn.moments(X, [0])[0]))) G_loss_V = G_loss_V1 + G_loss_V2 #3. Supervised loss G_loss_S = get_generator_s_loss(H, H_hat_supervise) #4. Summation G_loss = G_loss_U + gamma * G_loss_U_e + 100 * tf.sqrt( G_loss_S) + 100 * G_loss_V return G_loss, G_loss_U, G_loss_S, G_loss_V def get_generator_loss_static(Y_fake_e, X_hat, X): """ returns generator loss for static values """ #1. Adversarial loss bce = tf.keras.losses.BinaryCrossentropy(from_logits=True) G_loss_U_e = bce(tf.ones_like(Y_fake_e), Y_fake_e) #2. Two Moments X = tf.convert_to_tensor(X) G_loss_V1 = tf.reduce_mean( tf.abs( tf.sqrt(tf.nn.moments(X_hat, [0])[1] + 1e-6) - tf.sqrt(tf.nn.moments(X, [0])[1] + 1e-6))) G_loss_V2 = tf.reduce_mean( tf.abs((tf.nn.moments(X_hat, [0])[0]) - (tf.nn.moments(X, [0])[0]))) G_loss_V = G_loss_V1 + G_loss_V2 #4. Summation G_loss = gamma * G_loss_U_e + 100 * G_loss_V return G_loss, G_loss_V def get_generator_loss_both(Y_fake, Y_fake_e, X_hat, X, H, H_hat_supervise, Y_fake_e_static, X_hat_static, X_static): """ returns generator loss for both static and time series variables """ #1. Adversarial loss bce = tf.keras.losses.BinaryCrossentropy(from_logits=True) G_loss_U = bce(tf.ones_like(Y_fake), Y_fake) G_loss_U_e = bce(tf.ones_like(Y_fake_e), Y_fake_e) G_loss_U_e_static = bce(tf.ones_like(Y_fake_e_static), Y_fake_e_static) #2. Two Moments X = tf.convert_to_tensor(X) G_loss_V1 = tf.reduce_mean( tf.abs( tf.sqrt(tf.nn.moments(X_hat, [0])[1] + 1e-6) - tf.sqrt(tf.nn.moments(X, [0])[1] + 1e-6))) G_loss_V2 = tf.reduce_mean( tf.abs((tf.nn.moments(X_hat, [0])[0]) - (tf.nn.moments(X, [0])[0]))) G_loss_V = G_loss_V1 + G_loss_V2 X_static = tf.convert_to_tensor(X_static) G_loss_V1_static = tf.reduce_mean( tf.abs( tf.sqrt(tf.nn.moments(X_hat_static, [0])[1] + 1e-6) - tf.sqrt(tf.nn.moments(X_static, [0])[1] + 1e-6))) G_loss_V2_static = tf.reduce_mean( tf.abs((tf.nn.moments(X_hat_static, [0])[0]) - (tf.nn.moments(X_static, [0])[0]))) G_loss_V_static = G_loss_V1_static + G_loss_V2_static #3. Supervised loss G_loss_S = get_generator_s_loss(H, H_hat_supervise) #4. Summation G_loss = G_loss_U + gamma * G_loss_U_e + 100 * tf.sqrt( G_loss_S ) + 100 * G_loss_V + gamma * G_loss_U_e_static + 100 * G_loss_V_static return G_loss, G_loss_U, G_loss_S, G_loss_V, G_loss_V_static def get_discriminator_loss(Y_real, Y_fake, Y_fake_e): """ computes discrminator loss for time series variables Args: - Y_real: classification results of latent real time-series - Y_fake: classification results of latent synthetic time-series - Y_fake_e: classification results of generated sequence for latent synthetic time-series Returns: - d_loss: discriminator loss """ bce = tf.keras.losses.BinaryCrossentropy( from_logits=True) #loss for cls of latent real data seq #default arg for tf.keras.losses.BinaryCrossentropy reduction=losses_utils.ReductionV2.AUTO D_loss_real = bce(tf.ones_like(Y_real), Y_real) D_loss_fake = bce(tf.zeros_like(Y_fake), Y_fake) #loss for cls of latent synthethic data seq D_loss_fake_e = bce(tf.zeros_like(Y_fake_e), Y_fake_e) #loss for cls of latent synthetic data D_loss = D_loss_real + D_loss_fake + gamma * D_loss_fake_e return D_loss def get_discriminator_loss_static(Y_real, Y_fake_e): """ returns discrminator loss for static values """ bce = tf.keras.losses.BinaryCrossentropy( from_logits=True) #loss for cls of latent real data seq #default arg for tf.keras.losses.BinaryCrossentropy reduction=losses_utils.ReductionV2.AUTO D_loss_real = bce(tf.ones_like(Y_real), Y_real) D_loss_fake_e = bce(tf.zeros_like(Y_fake_e), Y_fake_e) #loss for cls of latent synthetic data D_loss = D_loss_real + gamma * D_loss_fake_e return D_loss def get_discriminator_loss_both(Y_real, Y_fake, Y_fake_e, Y_real_static, Y_fake_e_static): """ returns discrminator loss for both static and temporal """ bce = tf.keras.losses.BinaryCrossentropy( from_logits=True) #loss for cls of latent real data seq #default arg for tf.keras.losses.BinaryCrossentropy reduction=losses_utils.ReductionV2.AUTO D_loss_real = bce(tf.ones_like(Y_real), Y_real) D_loss_fake = bce(tf.zeros_like(Y_fake), Y_fake) #loss for cls of latent synthethic data seq D_loss_fake_e = bce(tf.zeros_like(Y_fake_e), Y_fake_e) #loss for cls of latent synthetic data D_loss_real_static = bce(tf.ones_like(Y_real_static), Y_real_static) D_loss_fake_e_static = bce(tf.zeros_like(Y_fake_e_static), Y_fake_e_static) D_loss_temporal = D_loss_real + D_loss_fake + gamma * D_loss_fake_e D_loss_static = D_loss_real_static + D_loss_fake_e_static * gamma D_loss = D_loss_temporal + D_loss_static return D_loss, D_loss_static # optimizer embedder0_optimizer = tf.keras.optimizers.Adam() embedder_optimizer = tf.keras.optimizers.Adam() gen_s_optimizer = tf.keras.optimizers.Adam() generator_optimizer = tf.keras.optimizers.Adam() discriminator_optimizer = tf.keras.optimizers.Adam() embedder0_static_optimizer = tf.keras.optimizers.Adam() embedder_static_optimizer = tf.keras.optimizers.Adam() generator_static_optimizer = tf.keras.optimizers.Adam() discriminator_static_optimizer = tf.keras.optimizers.Adam() @tf.function def train_step_embedder_static(X_mb): """ trains static embedder model """ with tf.GradientTape() as embedder_static_tape: # Embedder & Recovery H_mb = embedder_model_static(X_mb) X_tilde_mb = recovery_model_static(H_mb) embedder_0_loss = get_embedder_0_loss(X_mb, X_tilde_mb) emb_vars = embedder_model_static.trainable_variables + recovery_model_static.trainable_variables gradients_of_embedder = embedder_static_tape.gradient( embedder_0_loss, emb_vars) embedder0_static_optimizer.apply_gradients( zip(gradients_of_embedder, emb_vars)) return embedder_0_loss @tf.function def train_step_embedder(X_mb): """ trains static embedder model """ with tf.GradientTape() as embedder_tape: # Embedder & Recovery H_mb = embedder_model(X_mb) X_tilde_mb = recovery_model(H_mb) embedder_0_loss = get_embedder_0_loss(X_mb, X_tilde_mb) emb_vars = embedder_model.trainable_variables + recovery_model.trainable_variables gradients_of_embedder = embedder_tape.gradient( embedder_0_loss, emb_vars) embedder0_optimizer.apply_gradients( zip(gradients_of_embedder, emb_vars)) return embedder_0_loss @tf.function def train_step_generator_s(X_mb, X_mb_static): """ supervised training for generator model """ with tf.GradientTape() as gen_s_tape: H_mb = embedder_model(X_mb) H_mb_static = embedder_model_static(X_mb_static) H_mb_static = tf.expand_dims(H_mb_static, axis=1) H_mb_static = tf.repeat(H_mb_static, seq_len, axis=1) #Embeddings of both static and temporal features H_mb_mix = tf.concat([H_mb, H_mb_static], axis=2) # Generator H_hat_supervise_mb = supervisor_model(H_mb_mix) gen_s_loss = get_generator_s_loss(H_mb, H_hat_supervise_mb) gen_s_vars = supervisor_model.trainable_variables gradients_of_gen_s = gen_s_tape.gradient(gen_s_loss, gen_s_vars) gen_s_optimizer.apply_gradients(zip(gradients_of_gen_s, gen_s_vars)) return gen_s_loss @tf.function def train_step_joint_static(X_mb, X_mb_static, Z_mb): """ joint training for static generator and supervisor model, embedder model """ #train generator ## STATIC Z with tf.GradientTape() as gen_tape: #Embedding H_mb_static = embedder_model_static(X_mb_static) #synthetic embedding E_hat_mb = generator_model_static(Z_mb) # Synthetic data X_hat_mb = recovery_model_static(E_hat_mb) # Discriminator Y_fake_e_mb = discriminator_model_static(E_hat_mb) gen_loss, g_loss_v = get_generator_loss_static( Y_fake_e_mb, X_hat_mb, X_mb_static) gen_vars = generator_model_static.trainable_variables gradients_of_gen = gen_tape.gradient(gen_loss, gen_vars) generator_static_optimizer.apply_gradients( zip(gradients_of_gen, gen_vars)) #train embedder with tf.GradientTape() as embedder_tape: H_mb_static = embedder_model_static(X_mb_static) X_tilde_mb = recovery_model_static(H_mb_static) emb_T0_loss = get_embedder_T0_loss(X_mb_static, X_tilde_mb) emb_loss = get_embedder_0_loss(X_mb_static, X_tilde_mb) emb_vars = embedder_model_static.trainable_variables + recovery_model_static.trainable_variables gradients_of_emb = embedder_tape.gradient(emb_loss, emb_vars) embedder_static_optimizer.apply_gradients( zip(gradients_of_emb, emb_vars)) return emb_T0_loss, emb_loss, g_loss_v @tf.function def train_step_joint_both(X_mb, X_mb_static, Z_mb, Z_mb_static): """ joint training for both static and temporal generator and supervisor model, embedder model """ #train generator - temporal + static with tf.GradientTape() as gen_tape: #Embeddings H_mb = embedder_model(X_mb) H_mb_static_1 = embedder_model_static(X_mb_static) H_mb_static = tf.expand_dims(H_mb_static_1, axis=1) H_mb_static = tf.repeat(H_mb_static, seq_len, axis=1) #Combine static and temporal features H_mb_mix = tf.concat([H_mb, H_mb_static], axis=2) E_hat_mb = generator_model(Z_mb) E_hat_mb_static_1 = generator_model_static(Z_mb_static) E_hat_mb_static = tf.expand_dims(E_hat_mb_static_1, axis=1) E_hat_mb_static = tf.repeat(E_hat_mb_static, seq_len, axis=1) #Combine static generator with temporal generator E_hat_mb_mix = tf.concat([E_hat_mb, E_hat_mb_static], axis=2) H_hat_mb = supervisor_model(E_hat_mb_mix) H_hat_supervise_mb = supervisor_model(H_mb_mix) # Synthetic data X_hat_mb = recovery_model(H_hat_mb) # Discriminator Y_fake_mb = discriminator_model(H_hat_mb) Y_fake_e_mb = discriminator_model(E_hat_mb) ###### STATIC # Synthetic data X_hat_mb_static = recovery_model_static(E_hat_mb_static_1) # Discriminator Y_fake_e_mb_static = discriminator_model_static(E_hat_mb_static_1) gen_loss, g_loss_u, gen_s_loss, g_loss_v, g_loss_v_static = get_generator_loss_both( Y_fake_mb, Y_fake_e_mb, X_hat_mb, X_mb, H_mb, H_hat_supervise_mb, Y_fake_e_mb_static, X_hat_mb_static, X_mb_static) gen_vars = generator_model.trainable_variables + supervisor_model.trainable_variables + generator_model_static.trainable_variables gradients_of_gen = gen_tape.gradient(gen_loss, gen_vars) generator_optimizer.apply_gradients(zip(gradients_of_gen, gen_vars)) #train embedder - temporal with tf.GradientTape() as embedder_tape: H_mb = embedder_model(X_mb) #recall H_mb_static = embedder_model_static(X_mb_static) X_tilde_mb = recovery_model(H_mb) H_mb_static = tf.expand_dims(H_mb_static, axis=1) H_mb_static = tf.repeat(H_mb_static, seq_len, axis=1) H_mb_mix = tf.concat([H_mb, H_mb_static], axis=2) H_hat_supervise = supervisor_model(H_mb_mix) emb_T0_loss = get_embedder_T0_loss(X_mb, X_tilde_mb) emb_loss = get_embedder_loss(X_mb, X_tilde_mb, H_mb, H_hat_supervise) emb_vars = embedder_model.trainable_variables + recovery_model.trainable_variables gradients_of_emb = embedder_tape.gradient(emb_loss, emb_vars) embedder_optimizer.apply_gradients(zip(gradients_of_emb, emb_vars)) #train embedder - static with tf.GradientTape() as embedder_tape: H_mb_static = embedder_model_static(X_mb_static) X_tilde_mb_static = recovery_model_static(H_mb_static) emb_T0_loss_static = get_embedder_T0_loss(X_mb_static, X_tilde_mb_static) emb_loss_static = get_embedder_0_loss( X_mb_static, X_tilde_mb_static) #Not sure which embedder loss to use emb_vars_static = embedder_model_static.trainable_variables + recovery_model_static.trainable_variables gradients_of_emb_static = embedder_tape.gradient( emb_loss_static, emb_vars_static) embedder_static_optimizer.apply_gradients( zip(gradients_of_emb_static, emb_vars_static)) return emb_T0_loss, emb_loss, g_loss_u, gen_s_loss, g_loss_v, emb_T0_loss_static, g_loss_v_static @tf.function def train_step_discriminator_static(X_mb, X_mb_static, Z_mb): """ trains static discriminator model """ with tf.GradientTape() as disc_tape: H_mb_static = embedder_model_static(X_mb_static) E_hat_mb = generator_model_static(Z_mb) # Synthetic data X_hat_mb = recovery_model_static(E_hat_mb) # Discriminator Y_real_mb = discriminator_model_static(H_mb_static) Y_fake_e_mb = discriminator_model_static(E_hat_mb) # Check discriminator loss before updating disc_loss = get_discriminator_loss_static(Y_real_mb, Y_fake_e_mb) # Train discriminator (only when the discriminator does not work well) if (disc_loss > 0.15): #disc_loss = get_discriminator_loss(Y_real_mb, Y_fake_mb, Y_fake_e_mb) disc_vars = discriminator_model_static.trainable_variables gradients_of_disc = disc_tape.gradient(disc_loss, disc_vars) discriminator_static_optimizer.apply_gradients( zip(gradients_of_disc, disc_vars)) return disc_loss @tf.function def train_step_discriminator_both(X_mb, X_mb_static, Z_mb, Z_mb_static): """ trains both static and temporal discriminator model """ #training discriminator - temporal + static with tf.GradientTape() as disc_tape: H_mb = embedder_model(X_mb) H_mb_static_1 = embedder_model_static(X_mb_static) H_mb_static = tf.expand_dims(H_mb_static_1, axis=1) H_mb_static = tf.repeat(H_mb_static, seq_len, axis=1) H_mb_mix = tf.concat([H_mb, H_mb_static], axis=2) E_hat_mb = generator_model(Z_mb) E_hat_mb_static_1 = generator_model_static(Z_mb_static) E_hat_mb_static = tf.expand_dims(E_hat_mb_static_1, axis=1) E_hat_mb_static = tf.repeat(E_hat_mb_static, seq_len, axis=1) E_hat_mb_mix = tf.concat([E_hat_mb, E_hat_mb_static], axis=2) H_hat_mb = supervisor_model(E_hat_mb_mix) # Synthetic data X_hat_mb = recovery_model(H_hat_mb) # Discriminator Y_fake_mb = discriminator_model(H_hat_mb) Y_real_mb = discriminator_model(H_mb) Y_fake_e_mb = discriminator_model(E_hat_mb) ### Discriminator static Y_real_mb_static = discriminator_model_static(H_mb_static_1) Y_fake_e_mb_static = discriminator_model_static(E_hat_mb_static_1) # Check discriminator loss before updating disc_loss, disc_loss_static = get_discriminator_loss_both( Y_real_mb, Y_fake_mb, Y_fake_e_mb, Y_real_mb_static, Y_fake_e_mb_static) # Train discriminator (only when the discriminator does not work well) if (disc_loss > 0.15): disc_vars = discriminator_model.trainable_variables + discriminator_model_static.trainable_variables gradients_of_disc = disc_tape.gradient(disc_loss, disc_vars) discriminator_optimizer.apply_gradients( zip(gradients_of_disc, disc_vars)) return disc_loss, disc_loss_static #timeGAN training def train(): #1. Embedding static network training print('Start Static Embedding Network Training') for itt in range(iterations): # Set mini-batch _, X_mb_static, _ = batch_generator_with_static( ori_data, ori_data_static, ori_time, batch_size) # Train embedder step_e_loss = train_step_embedder_static(X_mb_static) # Checkpoint if itt % 1000 == 0: print('step: ' + str(itt) + '/' + str(iterations) + ', e_loss: ' + str(np.round(np.sqrt(step_e_loss), 4))) print('Finish static Embedding Network Training') #1. Embedding network training print('Start Embedding Network Training') for itt in range(iterations): # Set mini-batch X_mb, _, T_mb = batch_generator_with_static( ori_data, ori_data_static, ori_time, batch_size) # Train embedder step_e_loss = train_step_embedder(X_mb) # Checkpoint if itt % 1000 == 0: print('step: ' + str(itt) + '/' + str(iterations) + ', e_loss: ' + str(np.round(np.sqrt(step_e_loss), 4))) #2. Training only with supervised loss print('Start Training with Supervised Loss Only') for itt in range(iterations): # Set mini-batch X_mb, X_mb_static, T_mb = batch_generator_with_static( ori_data, ori_data_static, ori_time, batch_size) # Train generator step_gen_s_loss = train_step_generator_s(X_mb, X_mb_static) # Checkpoint if itt % 1000 == 0: print('step: ' + str(itt) + '/' + str(iterations) + ', s_loss: ' + str(np.round(np.sqrt(step_gen_s_loss), 4))) print('Finish Training with Supervised Loss Only') # 3. Joint Training print('Start Joint Training') for itt in range(iterations): # Generator training (twice more than discriminator training) for kk in range(2): # Set mini-batch X_mb, X_mb_static, T_mb = batch_generator_with_static( ori_data, ori_data_static, ori_time, batch_size) # Random vector generation Z_mb = random_generator(batch_size, z_dim, T_mb, max_seq_len) Z_mb_static = random_generator_static(batch_size, z_dim_static, T_mb, max_seq_len) # Train generator and embedder emb_T0_loss, emb_loss, g_loss_u, gen_s_loss, g_loss_v, emb_T0_loss_static, g_loss_v_static = train_step_joint_both( X_mb, X_mb_static, Z_mb, Z_mb_static) # Discriminator training # Set mini-batch X_mb, X_mb_static, T_mb = batch_generator_with_static( ori_data, ori_data_static, ori_time, batch_size) # Random vector generation Z_mb = random_generator(batch_size, z_dim, T_mb, max_seq_len) Z_mb_static = random_generator_static(batch_size, z_dim_static, T_mb, max_seq_len) #train discriminator d_loss, d_loss_static = train_step_discriminator_both( X_mb, X_mb_static, Z_mb, Z_mb_static) # Print multiple checkpoints if itt % 200 == 0: print('step: ' + str(itt) + '/' + str(iterations) + ', d_loss: ' + str(np.round(d_loss, 4)) + ', g_loss_u: ' + str(np.round(g_loss_u, 4)) + ', g_loss_s: ' + str(np.round(np.sqrt(gen_s_loss), 4)) + ', g_loss_v: ' + str(np.round(g_loss_v, 4)) + ', e_loss_t0: ' + str(np.round(np.sqrt(emb_T0_loss), 4)) + ', d_loss_static: ' + str(np.round(d_loss_static, 4)) + ', g_loss_v_static: ' + str(np.round(g_loss_v_static, 4))) print('Finish Joint Training') ## Synthetic data generation Z_mb = random_generator(no, z_dim, ori_time, max_seq_len) Z_mb_static = random_generator_static(no, z_dim_static, ori_time, max_seq_len) # generate in latent dim E_hat_generated = generator_model(Z_mb) E_hat_generated_static = generator_model_static(Z_mb_static) # repeat for seq_len for static values E_hat_generated_static_ = tf.expand_dims(E_hat_generated_static, axis=1) E_hat_generated_static_ = tf.repeat(E_hat_generated_static_, seq_len, axis=1) # join static and temporal together E_hat_generated_mix = tf.concat( [E_hat_generated, E_hat_generated_static_], axis=2) H_hat_generated = supervisor_model(E_hat_generated_mix) # map up to original dimension generated_data_curr = recovery_model(H_hat_generated) generated_data_curr_static = recovery_model_static( E_hat_generated_static) generated_data_static = list() for i in range(no): temp = generated_data_curr_static[i, :] generated_data_static.append(temp) # Renormalization generated_data_static = generated_data_static * max_val_static generated_data_static = generated_data_static + min_val_static generated_data_seq = np.array( [[generated_data_static[i] for _ in range(seq_len)] for i in range(no)]) generated_data = list() for i in range(no): temp = generated_data_curr[i, :ori_time[i], :] generated_data.append(temp) # Renormalization generated_data = generated_data * max_val generated_data = generated_data + min_val generated_data = np.dstack((generated_data, generated_data_seq)) return generated_data return train()
def assert_extract_time(self, message, ftime): self.assertEqual(extract_time(message), ftime)
def timegan(ori_data, parameters): # Basic Parameters no, seq_len, dim = np.asarray(ori_data).shape # Maximum sequence length and each sequence length ori_time, max_seq_len = extract_time(ori_data) def MinMaxScaler(data): """Min-Max Normalizer. Args: - data: raw data Returns: - norm_data: normalized data - min_val: minimum values (for renormalization) - max_val: maximum values (for renormalization) """ min_val = np.nanmin(np.nanmin(data, axis=0), axis=0) data = data - min_val max_val = np.nanmax(np.nanmax(data, axis=0), axis=0) norm_data = data / (max_val + 1e-7) return norm_data, min_val, max_val # Normalization ori_data, min_val, max_val = MinMaxScaler(ori_data) ## Build a RNN networks # Network Parameters hidden_dim = parameters['hidden_dim'] num_layers = parameters['num_layer'] iterations = parameters['iterations'] batch_size = parameters['batch_size'] module_name = parameters['module'] z_dim = dim gamma = 1 mask_value = 0 def make_embedder(): """Embedding network between original feature space to latent space. Args: - X: input time-series features - T: input time information Returns: - H: embeddings """ embedder_model = tf.keras.Sequential(name='embedder') embedder_model.add( tf.keras.layers.Masking(mask_value=0, input_shape=(seq_len, dim))) embedder_model.add( rnn_cell(module_name, hidden_dim, return_sequences=True, input_shape=(seq_len, dim))) for i in range(num_layers - 1): embedder_model.add( rnn_cell(module_name, hidden_dim, return_sequences=True, input_shape=(seq_len, hidden_dim))) embedder_model.add( tf.keras.layers.Dense(hidden_dim, activation='sigmoid')) return embedder_model def make_recovery(): """Recovery network from latent space to original space. Args: - H: latent representation - T: input time information Returns: - X_tilde: recovered data """ recovery_model = tf.keras.Sequential(name='recovery') recovery_model.add( tf.keras.layers.Masking(mask_value=0, input_shape=(seq_len, hidden_dim))) for i in range(num_layers): recovery_model.add( rnn_cell(module_name, hidden_dim, return_sequences=True, input_shape=(seq_len, hidden_dim))) recovery_model.add(tf.keras.layers.Dense(dim, activation='sigmoid')) return recovery_model def make_generator(): """Generator function: Generate time-series data in latent space. Args: - Z: random variables - T: input time information Returns: - E: generated embedding """ generator_model = tf.keras.Sequential(name='generator') generator_model.add( rnn_cell(module_name, hidden_dim, return_sequences=True, input_shape=(seq_len, dim))) for i in range(num_layers - 1): generator_model.add( rnn_cell(module_name, hidden_dim, return_sequences=True, input_shape=(seq_len, hidden_dim))) generator_model.add( tf.keras.layers.Dense(hidden_dim, activation='sigmoid')) return generator_model def make_supervisor(): """Generate next sequence using the previous sequence. Args: - H: latent representation - T: input time information Returns: - S: generated sequence based on the latent representations generated by the generator """ supervisor_model = tf.keras.Sequential(name='supervisor') for i in range(num_layers - 1): supervisor_model.add( rnn_cell(module_name, hidden_dim, return_sequences=True, input_shape=(seq_len, hidden_dim))) supervisor_model.add( tf.keras.layers.Dense(hidden_dim, activation='sigmoid')) return supervisor_model def make_discriminator(): """Recovery network from latent space to original space. Args: - H: latent representation - T: input time information Returns: - X_tilde: recovered data """ discriminator_model = tf.keras.Sequential(name='discriminator') for i in range(num_layers): discriminator_model.add( rnn_cell(module_name, hidden_dim, return_sequences=True, input_shape=(seq_len, hidden_dim))) discriminator_model.add(tf.keras.layers.Dense(1, activation=None)) return discriminator_model embedder_model = make_embedder() recovery_model = make_recovery() generator_model = make_generator() supervisor_model = make_supervisor() discriminator_model = make_discriminator() def get_embedder_T0_loss(X, X_tilde, mask_slice): """ returns embedder_T0 loss Args: - X: input time-series information that is masked - X_tilde: X that has undergone embedding + recovery and is masked - mask_slice: (batch_size * seq_len * 1) tensor which contains information about masked rows returns: - E_loss_T0 : Scalar embedder_T0 loss """ #mse = tf.keras.losses.MeanSquaredError() #this automatically does reduction from array to scalar mse_loss = tf.keras.losses.mean_squared_error( X, X_tilde) #this is still an array and not reduced #reduce array to scalar #take mean over number of non-masked rows (not seq_length) E_loss_T0 = tf.reduce_sum(mse_loss) / tf.reduce_sum(mask_slice) return E_loss_T0 def get_embedder_0_loss(X, X_tilde, mask_slice): E_loss_T0 = get_embedder_T0_loss(X, X_tilde, mask_slice) E_loss0 = 10 * tf.sqrt(E_loss_T0) return E_loss0 def get_embedder_loss(X, X_tilde, H, H_hat_supervise, mask_slice): """ returns embedder network loss """ E_loss_T0 = get_embedder_T0_loss(X, X_tilde, mask_slice) E_loss0 = 10 * tf.sqrt(E_loss_T0) #could use function above G_loss_S = get_generator_s_loss(H, H_hat_supervise, mask_slice) E_loss = E_loss0 + 0.1 * G_loss_S return E_loss def get_generator_s_loss(H, H_hat_supervise, mask_slice): """ returns supervised loss """ #mse = tf.keras.losses.MeanSquaredError() #G_loss_S = mse(H[:,1:,:], H_hat_supervise[:,:-1,:]) mse_loss = tf.keras.losses.mean_squared_error( H[:, 1:, :], H_hat_supervise[:, :-1, :]) G_loss_S = tf.reduce_sum(mse_loss) / tf.reduce_sum(mask_slice) return G_loss_S def get_generator_loss(Y_fake, Y_fake_e, X_hat, X, H, H_hat_supervise, mask_slice): """ returns generator loss """ #1. Adversarial loss # bce = tf.keras.losses.BinaryCrossentropy(from_logits=True) # G_loss_U = bce(tf.ones_like(Y_fake), Y_fake) # G_loss_U_e = bce(tf.ones_like(Y_fake_e), Y_fake_e) # i think this does what we want? bce_loss_y_fake = tf.keras.losses.binary_crossentropy( tf.ones_like(Y_fake), Y_fake, from_logits=True) * tf.squeeze(mask_slice) G_loss_U = tf.reduce_sum(bce_loss_y_fake) / tf.reduce_sum(mask_slice) bce_loss_y_fake_e = tf.keras.losses.binary_crossentropy( tf.ones_like(Y_fake_e), Y_fake_e, from_logits=True) * tf.squeeze(mask_slice) G_loss_U_e = tf.reduce_sum(bce_loss_y_fake_e) / tf.reduce_sum( mask_slice) #2. Two Moments X = tf.convert_to_tensor(X) #G_loss_V1 = tf.reduce_mean(tf.abs(tf.sqrt(tf.nn.moments(X_hat,[0])[1] + 1e-6) - tf.sqrt(tf.nn.moments(X,[0])[1] + 1e-6))) #G_loss_V2 = tf.reduce_mean(tf.abs((tf.nn.moments(X_hat,[0])[0]) - (tf.nn.moments(X,[0])[0]))) #calculate mean- equivalent to masked version of tf.nn.moments(X_hat,[0])[0] #calculate variance- equivalent to masked version of tf.nn.moments(X_hat,[0])[1] mean_X_hat = tf.reduce_sum( X_hat, [0]) / (tf.reduce_sum(mask_slice, [0]) + 1e-6) squared_X_hat = tf.square(X_hat - mean_X_hat) * mask_slice variance_X_hat = (tf.reduce_sum(squared_X_hat, [0]) / (tf.reduce_sum(mask_slice, [0]) + 1e-6) ) #sample variance (biased) mean_X = tf.reduce_sum(X, [0]) / (tf.reduce_sum(mask_slice, [0]) + 1e-6) squared_X = tf.square(X - mean_X) * mask_slice variance_X = (tf.reduce_sum(squared_X, [0]) / (tf.reduce_sum(mask_slice, [0]) + 1e-6) ) #sample variance (biased) #get num unmasked value for reduced mean = total_num_of_values (seq_len*dim) - num_values_in_completely_masked_rows (rows that are masked in all patients in the batch * dim) num_unmasked_values = tf.reduce_sum( tf.clip_by_value(tf.reduce_sum(mask_slice, [2, 0]), clip_value_min=0, clip_value_max=1)) * dim G_loss_V1 = tf.reduce_sum( tf.abs(tf.sqrt(variance_X_hat) - tf.sqrt(variance_X))) / (num_unmasked_values + 1e-6) G_loss_V2 = tf.reduce_sum( tf.abs(mean_X_hat - mean_X)) / (num_unmasked_values + 1e-6) G_loss_V = G_loss_V1 + G_loss_V2 #3. Supervised loss G_loss_S = get_generator_s_loss(H, H_hat_supervise, mask_slice) #4. Summation G_loss = G_loss_U + gamma * G_loss_U_e + 100 * tf.sqrt( G_loss_S) + 100 * G_loss_V return G_loss, G_loss_U, G_loss_S, G_loss_V def get_discriminator_loss(Y_real, Y_fake, Y_fake_e, mask_slice): """ returns discrminator loss """ # bce = tf.keras.losses.BinaryCrossentropy(from_logits=True) #loss for cls of latent real data seq #default arg for tf.keras.losses.BinaryCrossentropy reduction=losses_utils.ReductionV2.AUTO # D_loss_real = bce(tf.ones_like(Y_real), Y_real) # D_loss_fake = bce(tf.zeros_like(Y_fake), Y_fake) #loss for cls of latent synthethic data seq # D_loss_fake_e = bce(tf.zeros_like(Y_fake_e), Y_fake_e) #loss for cls of latent synthetic data #following method in adversarial loss, i think this is what we want? bce_loss_y_real = tf.keras.losses.binary_crossentropy( tf.ones_like(Y_real), Y_real, from_logits=True) * tf.squeeze(mask_slice) D_loss_real = tf.reduce_sum(bce_loss_y_real) / tf.reduce_sum( mask_slice) bce_loss_y_fake = tf.keras.losses.binary_crossentropy( tf.ones_like(Y_fake), Y_fake, from_logits=True) * tf.squeeze(mask_slice) D_loss_fake = tf.reduce_sum(bce_loss_y_fake) / tf.reduce_sum( mask_slice) bce_loss_y_fake_e = tf.keras.losses.binary_crossentropy( tf.ones_like(Y_fake_e), Y_fake_e, from_logits=True) * tf.squeeze(mask_slice) D_loss_fake_e = tf.reduce_sum(bce_loss_y_fake_e) / tf.reduce_sum( mask_slice) D_loss = D_loss_real + D_loss_fake + gamma * D_loss_fake_e return D_loss # optimizer embedder0_optimizer = tf.keras.optimizers.Adam() embedder_optimizer = tf.keras.optimizers.Adam() gen_s_optimizer = tf.keras.optimizers.Adam() generator_optimizer = tf.keras.optimizers.Adam() discriminator_optimizer = tf.keras.optimizers.Adam() @tf.function def train_step_embedder(X_mb): with tf.GradientTape() as embedder_tape: #get a mask by looking at the first column of X_mb mask_slice = tf.slice( X_mb, [0, 0, 0], [batch_size, seq_len, 1]) #first column as mask slice mask_val = tf.ones([batch_size, seq_len, 1], dtype=tf.float64) * mask_value mask_slice = (mask_slice != mask_val) #False means masked mask_slice = tf.cast(mask_slice, tf.float64) X_mb = tf.multiply(X_mb, mask_slice) #masking # Embedder & Recovery H_mb = embedder_model(X_mb) H_mb = tf.multiply(H_mb, mask_slice) #masking X_tilde_mb = recovery_model(H_mb) X_tilde_mb = tf.multiply(X_tilde_mb, mask_slice) #masking #should i minimize embedder_0_loss but print out e_loss_T0? embedder_0_loss = get_embedder_0_loss(X_mb, X_tilde_mb, mask_slice) #minimize embedder_T0_loss = get_embedder_T0_loss(X_mb, X_tilde_mb, mask_slice) #print emb_vars = embedder_model.trainable_variables + recovery_model.trainable_variables gradients_of_embedder = embedder_tape.gradient(embedder_0_loss, emb_vars) embedder0_optimizer.apply_gradients( zip(gradients_of_embedder, emb_vars)) return embedder_T0_loss @tf.function def train_step_generator_s(X_mb): with tf.GradientTape() as gen_s_tape: #, tf.GradientTape() as s_tape: #get a mask slice for masked value in X_mb mask_slice = tf.slice( X_mb, [0, 0, 0], [batch_size, seq_len, 1]) #first column as mask slice mask_val = tf.ones([batch_size, seq_len, 1], dtype=tf.float64) * mask_value mask_slice = (mask_slice != mask_val) #False means masked mask_slice = tf.cast(mask_slice, tf.float64) X_mb = tf.multiply(X_mb, mask_slice) #masking H_mb = embedder_model(X_mb) #recall H_mb = tf.multiply(H_mb, mask_slice) #masking H_hat_supervise_mb = supervisor_model(H_mb) H_hat_supervise_mb = tf.multiply(H_hat_supervise_mb, mask_slice) gen_s_loss = get_generator_s_loss( H_mb, H_hat_supervise_mb, mask_slice ) #hot sure if i should do whole gen loss or only gen_s loss gen_s_vars = supervisor_model.trainable_variables #generator_model.trainable_variables + #vars = [generator_model.trainable_variables, supervisor_model.trainable_variables] gradients_of_gen_s = gen_s_tape.gradient(gen_s_loss, gen_s_vars) gen_s_optimizer.apply_gradients(zip(gradients_of_gen_s, gen_s_vars)) return gen_s_loss # E_hat_mb, H_hat_mb, H_hat_supervise_mb, #,generator_model, supervisor_model @tf.function def train_step_joint(X_mb, Z_mb): #train generator with tf.GradientTape() as gen_tape: #get a mask slice for masked value in X_mb mask_slice = tf.slice( X_mb, [0, 0, 0], [batch_size, seq_len, 1]) #first column as mask slice mask_val = tf.ones([batch_size, seq_len, 1], dtype=tf.float64) * mask_value mask_slice = (mask_slice != mask_val) #False means masked mask_slice = tf.cast(mask_slice, tf.float64) X_mb = tf.multiply(X_mb, mask_slice) #masking Z_mb = tf.multiply(Z_mb, mask_slice) #masking # Generator #not sure if i should call these generators and supervisors again #because returning models from train_step_generator_s and getting trainable variables does not work? #so called it again here H_mb = embedder_model(X_mb) #recall H_mb = tf.multiply(H_mb, mask_slice) #masking E_hat_mb = generator_model(Z_mb) #is this a recall? E_hat_mb = tf.multiply(E_hat_mb, mask_slice) #masking H_hat_mb = supervisor_model(E_hat_mb) #recall H_hat_mb = tf.multiply(H_hat_mb, mask_slice) #masking H_hat_supervise_mb = supervisor_model(H_mb) #recall H_hat_supervise_mb = tf.multiply(H_hat_supervise_mb, mask_slice) # Synthetic data X_hat_mb = recovery_model(H_hat_mb) X_hat_mb = tf.multiply(X_hat_mb, mask_slice) # Discriminator Y_fake_mb = discriminator_model(H_hat_mb) Y_fake_mb = tf.multiply(Y_fake_mb, mask_slice) Y_real_mb = discriminator_model(H_mb) Y_real_mb = tf.multiply(Y_real_mb, mask_slice) Y_fake_e_mb = discriminator_model(E_hat_mb) Y_fake_e_mb = tf.multiply(Y_fake_e_mb, mask_slice) gen_loss, g_loss_u, gen_s_loss, g_loss_v = get_generator_loss( Y_fake_mb, Y_fake_e_mb, X_hat_mb, X_mb, H_mb, H_hat_supervise_mb, mask_slice) gen_vars = generator_model.trainable_variables + supervisor_model.trainable_variables gradients_of_gen = gen_tape.gradient(gen_loss, gen_vars) generator_optimizer.apply_gradients(zip(gradients_of_gen, gen_vars)) #train embedder with tf.GradientTape() as embedder_tape: #get a mask slice for masked value in X_mb mask_slice = tf.slice( X_mb, [0, 0, 0], [batch_size, seq_len, 1]) #first column as mask slice mask_val = tf.ones([batch_size, seq_len, 1], dtype=tf.float64) * mask_value mask_slice = (mask_slice != mask_val) #False means masked mask_slice = tf.cast(mask_slice, tf.float64) X_mb = tf.multiply(X_mb, mask_slice) #masking H_mb = embedder_model(X_mb) #recall H_mb = tf.multiply(H_mb, mask_slice) #masking X_tilde_mb = recovery_model(H_mb) X_tilde_mb = tf.multiply(X_tilde_mb, mask_slice) #masking H_hat_supervise = supervisor_model( H_mb) #called in order to get emb_loss H_hat_supervise = tf.multiply(H_hat_supervise, mask_slice) #not sure if this should be E_loss or E_loss_T0 #i think we are minimizing E_loss but printing out E_loss_T0?? emb_T0_loss = get_embedder_T0_loss(X_mb, X_tilde_mb, mask_slice) emb_loss = get_embedder_loss(X_mb, X_tilde_mb, H_mb, H_hat_supervise, mask_slice) emb_vars = embedder_model.trainable_variables + recovery_model.trainable_variables gradients_of_emb = embedder_tape.gradient(emb_loss, emb_vars) embedder_optimizer.apply_gradients(zip(gradients_of_emb, emb_vars)) return emb_T0_loss, emb_loss, g_loss_u, gen_s_loss, g_loss_v #H_hat_mb, E_hat_mb, @tf.function def train_step_discriminator(X_mb, Z_mb): with tf.GradientTape() as disc_tape: #get a mask slice for masked value in X_mb mask_slice = tf.slice( X_mb, [0, 0, 0], [batch_size, seq_len, 1]) #first column as mask slice mask_val = tf.ones([batch_size, seq_len, 1], dtype=tf.float64) * mask_value mask_slice = (mask_slice != mask_val) #False means masked mask_slice = tf.cast(mask_slice, tf.float64) X_mb = tf.multiply(X_mb, mask_slice) #masking H_mb = embedder_model(X_mb) #recall H_mb = tf.multiply(H_mb, mask_slice) #masking E_hat_mb = generator_model(Z_mb) #recall E_hat_mb = tf.multiply(E_hat_mb, mask_slice) #masking H_hat_mb = supervisor_model(E_hat_mb) #recall H_hat_mb = tf.multiply(H_hat_mb, mask_slice) #masking # Synthetic data X_hat_mb = recovery_model(H_hat_mb) X_hat_mb = tf.multiply(X_hat_mb, mask_slice) #masking # Discriminator Y_fake_mb = discriminator_model(H_hat_mb) Y_fake_mb = tf.multiply(Y_fake_mb, mask_slice) #masking Y_real_mb = discriminator_model(H_mb) Y_real_mb = tf.multiply(Y_real_mb, mask_slice) #masking Y_fake_e_mb = discriminator_model(E_hat_mb) Y_fake_e_mb = tf.multiply(Y_fake_e_mb, mask_slice) #masking # Check discriminator loss before updating disc_loss = get_discriminator_loss(Y_real_mb, Y_fake_mb, Y_fake_e_mb, mask_slice) # Train discriminator (only when the discriminator does not work well) if (disc_loss > 0.15): #disc_loss = get_discriminator_loss(Y_real_mb, Y_fake_mb, Y_fake_e_mb) disc_vars = discriminator_model.trainable_variables gradients_of_disc = disc_tape.gradient(disc_loss, disc_vars) discriminator_optimizer.apply_gradients( zip(gradients_of_disc, disc_vars)) return disc_loss #timeGAN training def train(): #1. Embedding network training print('Start Embedding Network Training') for itt in range(iterations): # Set mini-batch X_mb, T_mb = batch_generator(ori_data, ori_time, batch_size) X_mb = np.nan_to_num(X_mb, nan=0) #model can't take in nans # Train embedder step_e_loss = train_step_embedder(X_mb) # Checkpoint if itt % 100 == 0: print('step: ' + str(itt) + '/' + str(iterations) + ', e_loss: ' + str(np.round(np.sqrt(step_e_loss), 4))) print('Finish Embedding Network Training') #2. Training only with supervised loss print('Start Training with Supervised Loss Only') for itt in range(iterations): # Set mini-batch X_mb, T_mb = batch_generator(ori_data, ori_time, batch_size) X_mb = np.nan_to_num(X_mb, nan=0) # Random vector generation Z_mb = random_generator(batch_size, z_dim, T_mb, max_seq_len) # Train generator step_gen_s_loss = train_step_generator_s(X_mb) # Checkpoint if itt % 100 == 0: print('step: ' + str(itt) + '/' + str(iterations) + ', s_loss: ' + str(np.round(np.sqrt(step_gen_s_loss), 4))) print('Finish Training with Supervised Loss Only') # 3. Joint Training print('Start Joint Training') for itt in range(iterations): # Generator training (twice more than discriminator training) for kk in range(2): # Set mini-batch X_mb, T_mb = batch_generator(ori_data, ori_time, batch_size) X_mb = np.nan_to_num(X_mb, nan=0) # Random vector generation Z_mb = random_generator(batch_size, z_dim, T_mb, max_seq_len) # Train generator and embedder emb_T0_loss, emb_loss, g_loss_u, gen_s_loss, g_loss_v = train_step_joint( X_mb, Z_mb) # Discriminator training # Set mini-batch X_mb, T_mb = batch_generator(ori_data, ori_time, batch_size) X_mb = np.nan_to_num(X_mb, nan=0) # Random vector generation Z_mb = random_generator(batch_size, z_dim, T_mb, max_seq_len) #train discriminator d_loss = train_step_discriminator(X_mb, Z_mb) # Print multiple checkpoints if itt % 100 == 0: print('step: ' + str(itt) + '/' + str(iterations) + ', d_loss: ' + str(np.round(d_loss, 4)) + ', g_loss_u: ' + str(np.round(g_loss_u, 4)) + ', g_loss_s: ' + str(np.round(np.sqrt(gen_s_loss), 4)) + ', g_loss_v: ' + str(np.round(g_loss_v, 4)) + ', e_loss_t0: ' + str(np.round(np.sqrt(emb_T0_loss), 4))) print('Finish Joint Training') ## Synthetic data generation Z_mb = random_generator(no, z_dim, ori_time, max_seq_len) E_hat_generated = generator_model(Z_mb) H_hat_generated = supervisor_model(E_hat_generated) generated_data_curr = recovery_model(H_hat_generated) generated_data = list() for i in range(no): temp = generated_data_curr[i, :ori_time[i], :] generated_data.append(temp) # Renormalization generated_data = generated_data * max_val generated_data = generated_data + min_val return generated_data return train()
def timegan(ori_data, parameters): """TimeGAN function. Use original data as training set to generater synthetic data (time-series) Args: - ori_data: original time-series data - parameters: TimeGAN network parameters Returns: - generated_data: generated time-series data """ # Basic Parameters no, seq_len, dim = np.asarray(ori_data).shape # Maximum sequence length and each sequence length ori_time, max_seq_len = extract_time(ori_data) def MinMaxScaler(data): """Min-Max Normalizer. Args: - data: raw data Returns: - norm_data: normalized data - min_val: minimum values (for renormalization) - max_val: maximum values (for renormalization) """ min_val = np.min(np.min(data, axis=0), axis=0) data = data - min_val max_val = np.max(np.max(data, axis=0), axis=0) norm_data = data / (max_val + 1e-7) return norm_data, min_val, max_val # Normalization ori_data, min_val, max_val = MinMaxScaler(ori_data) ## Build a RNN networks # Network Parameters hidden_dim = parameters['hidden_dim'] num_layers = parameters['num_layer'] iterations = parameters['iterations'] batch_size = parameters['batch_size'] module_name = parameters['module'] z_dim = dim gamma = 1 def embedder(): """Embedding network between original feature space to latent space. Args: - X: input time-series features - T: input time information Returns: - H: embeddings """ e_cell = tf.keras.layers.StackedRNNCells([ tf.keras.layers.GRUCell(hidden_dim, activation=tf.nn.tanh, input_shape=(seq_len, hidden_dim)) for _ in range(num_layers - 1) ]) model = tf.keras.Sequential([ rnn_cell(module_name, hidden_dim, return_sequences=True, input_shape=(seq_len, dim)), tf.keras.layers.RNN(e_cell, return_sequences=True), tf.keras.layers.Dense(hidden_dim, activation=tf.nn.sigmoid) ]) return model def recovery(): """Recovery network from latent space to original space. Args: - H: latent representation - T: input time information Returns: - X_tilde: recovered data """ r_cell = tf.keras.layers.StackedRNNCells([ tf.keras.layers.GRUCell(hidden_dim, activation=tf.nn.tanh, input_shape=(seq_len, hidden_dim)) for _ in range(num_layers) ]) model = tf.keras.Sequential([ tf.keras.layers.RNN(r_cell, return_sequences=True), tf.keras.layers.Dense(dim, activation=tf.nn.sigmoid) ]) return model def generator(): """Generator function: Generate time-series data in latent space. Args: - Z: random variables - T: input time information Returns: - E: generated embedding """ e_cell = tf.keras.layers.StackedRNNCells([ tf.keras.layers.GRUCell(hidden_dim, activation=tf.nn.tanh) for _ in range(num_layers) ]) model = tf.keras.Sequential([ tf.keras.layers.RNN(e_cell, return_sequences=True), tf.keras.layers.Dense(hidden_dim, activation=tf.nn.sigmoid) ]) return model def supervisor(): """Generate next sequence using the previous sequence. Args: - H: latent representation - T: input time information Returns: - S: generated sequence based on the latent representations generated by the generator """ e_cell = tf.keras.layers.StackedRNNCells([ tf.keras.layers.GRUCell(hidden_dim, activation=tf.nn.tanh, input_shape=(seq_len, hidden_dim)) for _ in range(num_layers - 1) ]) model = tf.keras.Sequential([ tf.keras.layers.RNN(e_cell, return_sequences=True), tf.keras.layers.Dense(hidden_dim, activation=tf.nn.sigmoid) ]) return model #sequential may not work def discriminator(): """Discriminate the original and synthetic time-series data. Args: - H: latent representation - T: input time information Returns: - Y_hat: classification results between original and synthetic time-series """ d_cell = tf.keras.layers.StackedRNNCells([ tf.keras.layers.GRUCell(hidden_dim, activation=tf.nn.tanh) for _ in range(num_layers) ]) model = tf.keras.Sequential([ tf.keras.layers.RNN(d_cell), tf.keras.layers.Dense(1, activation=None) ]) return model # Embedder & Recovery embedder = embedder() recovery = recovery() # Generator generator = generator() supervisor = supervisor() # Discriminator discriminator = discriminator() # Embedder network loss def embed_obj(X, X_tilde): return 10 * tf.sqrt(tf.compat.v1.losses.mean_squared_error(X, X_tilde)) def E_loss_T0(X, X_tilde): return tf.compat.v1.losses.mean_squared_error(X, X_tilde) def E_loss(E_loss0, G_loss_S): return E_loss0 + 0.1 * G_loss_S #Supervised loss def supervised_obj(H, H_hat_supervise): return tf.compat.v1.losses.mean_squared_error( H[:, 1:, :], H_hat_supervise[:, :-1, :]) #Generator loss def generator_obj(G_loss_U, G_loss_U_e, G_loss_S, G_loss_V): return G_loss_U + gamma * G_loss_U_e + 100 * tf.sqrt( G_loss_S) + 100 * G_loss_V #Discriminator loss def discriminator_obj(Y_real, Y_fake, Y_fake_e): D_loss_real = tf.compat.v1.losses.sigmoid_cross_entropy( tf.ones_like(Y_real), Y_real) D_loss_fake = tf.compat.v1.losses.sigmoid_cross_entropy( tf.zeros_like(Y_fake), Y_fake) D_loss_fake_e = tf.compat.v1.losses.sigmoid_cross_entropy( tf.zeros_like(Y_fake_e), Y_fake_e) return D_loss_real + D_loss_fake + gamma * D_loss_fake_e # optimizer EO_solver = tf.keras.optimizers.Adam() GS_solver = tf.keras.optimizers.Adam() G_solver = tf.keras.optimizers.Adam() E_solver = tf.keras.optimizers.Adam() D_solver = tf.keras.optimizers.Adam() # Train embedder @tf.function def train_step_embedder(X_mb): with tf.GradientTape() as embed_tape: embed = embedder(X_mb) recover = recovery(embed) loss = embed_obj(X_mb, recover) gradients_embed = embed_tape.gradient( loss, embedder.trainable_variables + recovery.trainable_variables) EO_solver.apply_gradients( zip( gradients_embed, embedder.trainable_variables + recovery.trainable_variables)) step_e_loss = E_loss_T0(X_mb, recover) return step_e_loss #Tensorflow 1 code was confusing so this is likely to be wrong. @tf.function def train_step_supervised(X_mb, Z_mb): with tf.GradientTape() as supervised_tape: embed = embedder(X_mb) generated = generator(Z_mb) supervised_g = supervisor(generated) supervised_e = supervisor(embed) loss = supervised_obj(embed, supervised_e) gradients_supervised = supervised_tape.gradient( loss, supervisor.trainable_variables + generator.trainable_variables) GS_solver.apply_gradients( zip( gradients_supervised, supervisor.trainable_variables + generator.trainable_variables)) return loss #Generator is supposed to take it's own previous generated data as input. @tf.function def train_step_generator(X_mb, Z_mb): with tf.GradientTape() as gen_tape: ##Not sure about feeding the generator embedded embed = embedder(X_mb) generated = generator(Z_mb) supervised_g = supervisor(generated) supervised_e = supervisor(embed) synth_data = recovery(supervised_g) Y_fake_mb = discriminator(supervised_g) Y_real_mb = discriminator(embed) Y_fake_e_mb = discriminator(generated) G_loss_U = tf.compat.v1.losses.sigmoid_cross_entropy( tf.ones_like(Y_fake_mb), Y_fake_mb) G_loss_U_e = tf.compat.v1.losses.sigmoid_cross_entropy( tf.ones_like(Y_fake_e_mb), Y_fake_e_mb) G_loss_S = supervised_obj(embed, supervised_e) G_loss_V1 = G_loss_V1 = tf.reduce_mean( tf.abs( tf.sqrt(tf.nn.moments(synth_data, [0])[1] + 1e-6) - tf.sqrt(tf.nn.moments(X_mb, [0])[1] + 1e-6))) G_loss_V2 = G_loss_V2 = tf.reduce_mean( tf.abs((tf.nn.moments(synth_data, [0])[0]) - (tf.nn.moments(X_mb, [0])[0]))) G_loss_V = G_loss_V1 + G_loss_V2 loss = generator_obj(G_loss_U, G_loss_U_e, G_loss_S, G_loss_V) gradients_generator = gen_tape.gradient( loss, generator.trainable_variables + supervisor.trainable_variables) GS_solver.apply_gradients( zip( gradients_generator, generator.trainable_variables + supervisor.trainable_variables)) return G_loss_U, G_loss_S, G_loss_V @tf.function def train_step_joint_embed(X_mb, Z_mb): with tf.GradientTape() as embed_tape: embed = embedder(X_mb) supervised = supervisor(embed) G_loss_S = supervised_obj(embed, supervised) recover = recovery(embed) E_loss0 = embed_obj(X_mb, recover) loss = E_loss0 + 0.1 * G_loss_S gradients_embed = embed_tape.gradient( loss, embedder.trainable_variables + recovery.trainable_variables) GS_solver.apply_gradients( zip( gradients_embed, embedder.trainable_variables + recovery.trainable_variables)) return tf.losses.mean_squared_error(X_mb, recover) @tf.function def train_step_discriminator(X_mb, Z_mb): with tf.GradientTape() as discrim_tape: # Check discriminator loss before updating embed = embedder(X_mb) generated = generator(Z_mb) supervised_g = supervisor(generated) Y_fake = discriminator(supervised_g) Y_real = discriminator(embed) Y_fake_e = discriminator(generated) loss = discriminator_obj(Y_real, Y_fake, Y_fake_e) if (loss > 0.15): gradients_discrim = discrim_tape.gradient( loss, discriminator.trainable_variables) GS_solver.apply_gradients( zip(gradients_discrim, discriminator.trainable_variables)) return loss ## TimeGAN training # 1. Embedding network training print('Start Embedding Network Training') def train(): for itt in range(iterations): pass # Set mini-batch #X_mb = ori_data X_mb, T_mb = batch_generator(ori_data, ori_time, batch_size) X_mb = X_mb.reshape(batch_size, seq_len, dim) step_e_loss = train_step_embedder(X_mb) # Checkpoint if itt % 1000 == 0: print('step: ' + str(itt) + '/' + str(iterations) + ', e_loss: ' + str(np.round(np.sqrt(step_e_loss), 4))) print('Finish Embedding Network Training') # 2. Training only with supervised loss print('Start Training with Supervised Loss Only') for itt in range(iterations): # Set mini-batch X_mb, T_mb = batch_generator(ori_data, ori_time, batch_size) # Random vector generation Z_mb = random_generator(batch_size, z_dim, T_mb, max_seq_len) # Train generator step_g_loss_s = train_step_supervised(X_mb, Z_mb) # Checkpoint if itt % 1000 == 0: print('step: ' + str(itt) + '/' + str(iterations) + ', s_loss: ' + str(np.round(np.sqrt(step_g_loss_s), 4))) print('Finish Training with Supervised Loss Only') # 3. Joint Training print('Start Joint Training') for itt in range(iterations): # Generator training (twice more than discriminator training) for kk in range(2): # Set mini-batch X_mb, T_mb = batch_generator(ori_data, ori_time, batch_size) # Random vector generation Z_mb = random_generator(batch_size, z_dim, T_mb, max_seq_len) # Train generator step_g_loss_u, step_g_loss_s, step_g_loss_v = train_step_generator( X_mb, Z_mb) #_, step_g_loss_u, step_g_loss_s, step_g_loss_v = sess.run([G_solver, G_loss_U, G_loss_S, G_loss_V], feed_dict={Z: Z_mb, X: X_mb, T: T_mb}) # Train embedder step_e_loss_t0 = train_step_joint_embed(X_mb, Z_mb) #_, step_e_loss_t0 = sess.run([E_solver, E_loss_T0], feed_dict={Z: Z_mb, X: X_mb, T: T_mb}) # Discriminator training # Set mini-batch X_mb, T_mb = batch_generator(ori_data, ori_time, batch_size) # Random vector generation Z_mb = random_generator(batch_size, z_dim, T_mb, max_seq_len) step_d_loss = train_step_discriminator(X_mb, Z_mb) # Print multiple checkpoints if itt % 1000 == 0: print('step: ' + str(itt) + '/' + str(iterations) + ', d_loss: ' + str(np.round(step_d_loss, 4)) + ', g_loss_u: ' + str(np.round(step_g_loss_u, 4)) + ', g_loss_s: ' + str(np.round(np.sqrt(step_g_loss_s), 4)) + ', g_loss_v: ' + str(np.round(step_g_loss_v, 4)) + ', e_loss_t0: ' + str(np.round(np.sqrt(step_e_loss_t0), 4))) print('Finish Joint Training') ## Synthetic data generation Z_mb = random_generator(no, z_dim, ori_time, max_seq_len) generated = generator(Z_mb) supervised = supervisor(generated) generated_data_curr = recovery(supervised) generated_data = list() for i in range(no): temp = generated_data_curr[i, :ori_time[i], :] generated_data.append(temp) # Renormalization generated_data = generated_data * max_val generated_data = generated_data + min_val return generated_data return train()