def __init__(self, pac_num, data_dim): self.D_W1 = tf.Variable(xavier_init([data_dim * pac_num, 128])) self.D_b1 = tf.Variable(tf.zeros(shape=[128])) self.D_W2 = tf.Variable(xavier_init([128, 1])) self.D_b2 = tf.Variable(tf.zeros(shape=[1])) self.theta_D = [self.D_W1, self.D_W2, self.D_b1, self.D_b2]
def __init__(self, Z_dim, data_dim, pac_num, mode): self.pac_num = pac_num self.mode = mode self.G_W1 = tf.Variable(xavier_init([Z_dim, 128])) self.G_b1 = tf.Variable(tf.zeros(shape=[128])) self.G_W2 = tf.Variable(xavier_init([128, data_dim])) self.G_b2 = tf.Variable(tf.zeros(shape=[data_dim])) self.Z = [] for i in range(self.pac_num): self.Z.append(tf.placeholder(tf.float32, shape=[None, Z_dim])) self.theta_G = [self.G_W1, self.G_W2, self.G_b1, self.G_b2]
def __init__(self, num_hidden, cat_index, num_index, all_levels, batch_size, hint_rate, alpha, iterations, num_imputations): self.cat_index = cat_index self.num_index = num_index self.all_levels = all_levels self.num_imputations = num_imputations self.num_hidden = num_hidden # System parameters self.batch_size = batch_size self.hint_rate = hint_rate self.alpha = alpha self.iterations = iterations input_dim = self.num_hidden # Hidden state dimensions h_Gdim = int(input_dim) h_Ddim = int(input_dim) ## GAIN architecture # Discriminator variables self.D_W1 = tf.Variable(xavier_init([input_dim * 2, h_Ddim])) # Data + Hint as inputs self.D_b1 = tf.Variable(tf.zeros(shape=[h_Ddim])) self.D_W2 = tf.Variable(xavier_init([h_Ddim, h_Ddim])) self.D_b2 = tf.Variable(tf.zeros(shape=[h_Ddim])) self.D_W3 = tf.Variable(xavier_init([h_Ddim, input_dim])) self.D_b3 = tf.Variable(tf.zeros(shape=[input_dim])) # Multi-variate outputs self.theta_D = [self.D_W1, self.D_W2, self.D_W3, self.D_b1, self.D_b2, self.D_b3] # Generator variables # Data + Mask as inputs (Random noise is in missing components) self.G_W1 = tf.Variable(xavier_init([input_dim * 2, h_Gdim])) self.G_b1 = tf.Variable(tf.zeros(shape=[h_Gdim])) self.G_W2 = tf.Variable(xavier_init([h_Gdim, h_Gdim])) self.G_b2 = tf.Variable(tf.zeros(shape=[h_Gdim])) self.G_W3 = tf.Variable(xavier_init([h_Gdim, input_dim])) self.G_b3 = tf.Variable(tf.zeros(shape=[input_dim])) self.theta_G = [self.G_W1, self.G_W3, self.G_b1, self.G_b3] ## GAIN solver self.D_solver = tf.optimizers.Adam() self.G_solver = tf.optimizers.Adam() pass
def __init__(self, size_input, size_output, use_bias=True, activ='ReLu'): super().__init__() self.size_input = size_input self.size_output = size_output self.activ = activ self.use_bias = use_bias self.batch_size = 0 self.weights = Parameters(size_input, size_output) if activ.lower() == 'relu': self.weights.data = kaiming_init(size_input, size_output) elif activ.lower() == 'tanh': self.weights.data = xavier_init(size_input, size_output) else: raise NotImplementedError if use_bias: self.bias = Parameters(1, size_output) self.bias.data.normal_(mean=0, std=0.1) else: self.bias = None
def gain(data_x, data_m, cat_index, num_index, all_levels, gain_parameters, num_imputations=10): # System parameters batch_size = gain_parameters['batch_size'] hint_rate = gain_parameters['hint_rate'] alpha = gain_parameters['alpha'] iterations = gain_parameters['iterations'] data_train = np.array([]) data_train_m = np.array([]) # preprocess categorical variables if cat_index: data_cat = data_x[:, cat_index] data_cat_m = data_m[:, cat_index] data_cat_enc, data_cat_enc_miss = onehot_encoding(data_cat, data_cat_m, all_levels, has_miss=True) data_cat_enc = np.nan_to_num(data_cat_enc, 0) data_train = data_cat_enc data_train_m = data_cat_enc_miss n_classes = list(map(lambda x: len(x), all_levels)) # preprocess numerical variables if num_index: data_num = data_x[:, num_index] data_num_m = data_m[:, num_index] data_num_norm, norm_parameters = normalization(data_num) data_num_norm = np.nan_to_num(data_num_norm, 0) data_train = np.concatenate( [data_train, data_num_norm], axis=1) if data_train.size else data_num_norm data_train_m = np.concatenate( [data_train_m, data_num_m], axis=1) if data_train_m.size else data_num_m # Other parameters no, dim = data_x.shape input_dim = data_train.shape[1] # Hidden state dimensions h_Gdim = int(input_dim) h_Ddim = int(input_dim) ## GAIN architecture # Discriminator variables D_W1 = tf.Variable(xavier_init([input_dim * 2, h_Ddim])) # Data + Hint as inputs D_b1 = tf.Variable(tf.zeros(shape=[h_Ddim])) D_W2 = tf.Variable(xavier_init([h_Ddim, h_Ddim])) D_b2 = tf.Variable(tf.zeros(shape=[h_Ddim])) D_W3 = tf.Variable(xavier_init([h_Ddim, input_dim])) D_b3 = tf.Variable(tf.zeros(shape=[input_dim])) # Multi-variate outputs theta_D = [D_W1, D_W2, D_W3, D_b1, D_b2, D_b3] #Generator variables # Data + Mask as inputs (Random noise is in missing components) G_W1 = tf.Variable(xavier_init([input_dim * 2, h_Gdim])) G_b1 = tf.Variable(tf.zeros(shape=[h_Gdim])) G_W2 = tf.Variable(xavier_init([h_Gdim, h_Gdim])) G_b2 = tf.Variable(tf.zeros(shape=[h_Gdim])) G_W3 = tf.Variable(xavier_init([h_Gdim, input_dim])) G_b3 = tf.Variable(tf.zeros(shape=[input_dim])) theta_G = [G_W1, G_W3, G_b1, G_b3] ## GAIN functions # Generator @tf.function def generator(x, m): # Concatenate Mask and Data inputs = tf.concat(values=[x, m], axis=1) G_h1 = tf.nn.leaky_relu(tf.matmul(inputs, G_W1) + G_b1) G_h2 = tf.nn.leaky_relu(tf.matmul(G_h1, G_W2) + G_b2) G_logit = tf.matmul(G_h2, G_W3) + G_b3 col_index = 0 empty_G_out = True # apply softmax to each categorical variable if cat_index: empty_G_out = False G_out = tf.nn.softmax(G_logit[:, :n_classes[0]]) col_index = n_classes[0] for j in range(1, len(n_classes)): G_out = tf.concat(values=[ G_out, tf.nn.softmax(G_logit[:, col_index:col_index + n_classes[j]]) ], axis=1) col_index += n_classes[j] # apply sigmoid to all numerical variables if num_index: G_out_num = tf.nn.sigmoid(G_logit[:, col_index:]) G_out = tf.concat(values=[G_out, G_out_num], axis=1) if not empty_G_out else G_out_num return G_out # Discriminator @tf.function def discriminator(x, h): # Concatenate Data and Hint inputs = tf.concat(values=[x, h], axis=1) D_h1 = tf.nn.leaky_relu(tf.matmul(inputs, D_W1) + D_b1) D_h2 = tf.nn.leaky_relu(tf.matmul(D_h1, D_W2) + D_b2) D_logit = tf.matmul(D_h2, D_W3) + D_b3 D_prob = tf.nn.sigmoid(D_logit) return D_prob # loss function @tf.function def gain_Dloss(D_prob, mask): D_loss_temp = -tf.reduce_mean(mask * tf.math.log(D_prob + 1e-7) + (1 - mask) * tf.math.log(1. - D_prob + 1e-7)) D_loss = D_loss_temp return D_loss @tf.function def gain_Gloss(sample, G_sample, D_prob, mask, n_classes): G_loss_temp = -tf.reduce_mean((1 - mask) * tf.math.log(D_prob + 1e-7)) reconstruct_loss = 0 # categorical loss current_ind = 0 if cat_index: for j in range(len(n_classes)): M_current = mask[:, current_ind:current_ind + n_classes[j]] G_sample_temp = G_sample[:, current_ind:current_ind + n_classes[j]] X_temp = sample[:, current_ind:current_ind + n_classes[j]] reconstruct_loss += -tf.reduce_mean( M_current * X_temp * tf.math.log(M_current * G_sample_temp + 1e-7)) / tf.reduce_mean(M_current) current_ind += n_classes[j] # numerical loss if num_index: M_current = mask[:, current_ind:] G_sample_temp = G_sample[:, current_ind:] X_temp = sample[:, current_ind:] reconstruct_loss += tf.reduce_mean( (M_current * X_temp - M_current * G_sample_temp)** 2) / tf.reduce_mean(M_current) return G_loss_temp, reconstruct_loss # optimizer @tf.function def optimize_step(X_mb, M_mb, H_mb, n_classes): with tf.GradientTape() as g: # Generator G_sample = generator(X_mb, M_mb) # Combine with observed data Hat_X = X_mb * M_mb + G_sample * (1 - M_mb) # Discriminator D_prob = discriminator(Hat_X, H_mb) D_loss = gain_Dloss(D_prob, M_mb) Dgradients = g.gradient(D_loss, theta_D) D_solver.apply_gradients(zip(Dgradients, theta_D)) for i in range(3): with tf.GradientTape() as g: # Generator G_sample = generator(X_mb, M_mb) # Combine with observed data Hat_X = X_mb * M_mb + G_sample * (1 - M_mb) # Discriminator D_prob = discriminator(Hat_X, H_mb) G_loss_temp, reconstructloss = gain_Gloss( X_mb, G_sample, D_prob, M_mb, n_classes) G_loss = G_loss_temp + alpha * reconstructloss Ggradients = g.gradient(G_loss, theta_G) G_solver.apply_gradients(zip(Ggradients, theta_G)) return D_loss, G_loss_temp, reconstructloss ## GAIN solver D_solver = tf.optimizers.Adam() G_solver = tf.optimizers.Adam() # Start Iterations Gloss_list = [] Dloss_list = [] pbar = tqdm(range(iterations)) for i in pbar: # create mini batch indices = np.arange(no) np.random.shuffle(indices) for start_idx in range(0, no - batch_size + 1, batch_size): batch_idx = indices[start_idx:start_idx + batch_size] X_mb = data_train[batch_idx, :] M_mb = data_train_m[batch_idx, :] # Sample random vectors Z_mb = uniform_sampler(0, 0.01, batch_size, input_dim) # Sample hint vectors H_mb_temp = binary_sampler(hint_rate, batch_size, input_dim) H_mb = M_mb * H_mb_temp # Combine random vectors with observed vectors X_mb = M_mb * X_mb + (1 - M_mb) * Z_mb D_loss_curr, G_loss_curr, reconstructloss = optimize_step( X_mb, M_mb, H_mb, n_classes) Gloss_list.append(G_loss_curr) Dloss_list.append(D_loss_curr) pbar.set_description( "D_loss: {:.3f}, G_loss: {:.3f}, Reconstruction loss: {:.3f}". format(D_loss_curr.numpy(), G_loss_curr.numpy(), reconstructloss.numpy())) ## Return imputed data imputed_list = [] for l in range(num_imputations): Z_mb = uniform_sampler(0, 0.01, no, input_dim) M_mb = data_train_m X_mb = data_train X_mb = M_mb * X_mb + (1 - M_mb) * Z_mb imputed_data = generator(X_mb, M_mb) imputed_data = data_train_m * data_train + ( 1 - data_train_m) * imputed_data # revert onehot and renormalize imputed = np.empty(shape=(no, dim)) if cat_index: imputed_cat = imputed_data[:, :data_cat_enc.shape[1]] imputed_cat = onehot_decoding(imputed_cat, data_cat_enc_miss, all_levels, has_miss=False) imputed[:, cat_index] = imputed_cat if num_index: imputed_num = imputed_data[:, -data_num.shape[1]:] imputed_num = renormalization(imputed_num.numpy(), norm_parameters) imputed[:, num_index] = imputed_num imputed_list.append(imputed) return imputed_list, Gloss_list, Dloss_list