def main():
''' Wrapper of Training Pipeline. '''
import os
import yaml, pickle
import numpy as np
import tensorflow as tf
# local modules
import model, train, holdout
print('Loading configuration parameters.')
params = yaml.load( open(os.getcwd() + '/config.yaml'), yaml.Loader )
print('Setting GPU configurations.')
### Sets GPU configurations
if params['use_gpu']:
# This prevents CuDNN 'Failed to get convolution algorithm' error
gpus = tf.config.experimental.list_physical_devices('GPU')
if gpus:
try:
# Currently, memory growth needs to be the same across GPUs
for gpu in gpus:
tf.config.experimental.set_memory_growth(gpu, True)
logical_gpus = tf.config.experimental.list_logical_devices('GPU')
print(len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPUs")
except RuntimeError as e:
# Memory growth must be set before GPUs have been initialized
print(e)
# To see list of allocated tensors in case of OOM
tf.compat.v1.RunOptions(report_tensor_allocations_upon_oom = True)
else:
try:
# Disable all GPUs
tf.config.set_visible_devices([], 'GPU')
visible_devices = tf.config.get_visible_devices()
for device in visible_devices:
assert device.device_type != 'GPU'
except:
print('Invalid device or cannot modify virtual devices once initialized.')
pass
print('\nStart Training Pipeline.\n')
if params['model_type'] == 1:
# If the model already exists load it, otherwise make a new one
if params['model_name']+'.h5' in os.listdir( os.getcwd() + '/saved_models/' ):
print('Loading existing model: {}.'.format(params['model_name']))
Imputer = tf.keras.models.load_model( os.getcwd() + '/saved_models/' + params['model_name'] + '.h5' )
else:
print('\nVanilla Seq2seq model instantiated as:\n')
Imputer = model.build_vanilla_seq2seq(params)
Imputer.summary()
print('\nStart training.\n')
train.train_vanilla_seq2seq(Imputer, params)
elif params['model_type'] == 2:
if params['model_name']+'.h5' in os.listdir( os.getcwd() + '/saved_models/' ):
print('Loading existing model: {}.'.format(params['model_name']))
Imputer = tf.keras.models.load_model( os.getcwd() + '/saved_models/' + params['model_name'] + '.h5' )
Discriminator = tf.keras.models.load_model( os.getcwd() + '/saved_models/' + params['model_name'] + '_discriminator.h5' )
else:
print('\nGAN Seq2seq model instantiated as:\n')
Imputer, Discriminator = model.build_GAN(params)
print('\nGenerator:\n')
Imputer.summary()
print('\n\nDiscriminator:\n')
Discriminator.summary()
print('\nStart GAN training.\n')
train.train_GAN(Imputer, Discriminator, params)
elif params['model_type'] == 3:
if params['model_name']+'.h5' in os.listdir( os.getcwd() + '/saved_models/' ):
print('Loading existing model: {}.'.format(params['model_name']))
Imputer = tf.keras.models.load_model( os.getcwd() + '/saved_models/' + params['model_name'] + '.h5' )
Discriminator = tf.keras.models.load_model( os.getcwd() + '/saved_models/' + params['model_name'] + '_discriminator.h5' )
else:
print('\nPartially adversarial Seq2seq model instantiated as:\n')
Imputer, Discriminator = model.build_GAN(params)
print('\nGenerator:\n')
Imputer.summary()
print('\nDiscriminator:\n')
Discriminator.summary()
print('\nStart partially adversarial training.\n')
train.train_partial_GAN(Imputer, Discriminator, params)
else:
print("ERROR:\nIn config.yaml, from 'model_type' parameter, specify one of the following model architectures:")
print("'model_type': 1\tVanilla Seq2seq")
print("'model_type': 2\tGAN Seq2seq")
print("'model_type': 3\tSeq2seq with partially adversarial training")
quit()
# Check performance on Validation data and Test, optionally
holdout.run_test(Imputer,
params,
check_test_performance = params['check_test_performance'])
return None
def main():
nrows = 30_000_000
train, val, ENDPOINT, AGE, SEX, vocab_size, sequence_length, n_individuals = get_dataset(nrows = nrows)
print('Data loaded, number of individuals:', n_individuals)
print('GAN type:', GAN_type)
print('Relativistic average:', relativistic_average)
# Train the GAN
G = RelationalMemoryGenerator(mem_slots, head_size, embed_size, vocab_size, temperature, num_heads, num_blocks)
D = RelGANDiscriminator(n_embeddings, vocab_size, embed_size, sequence_length, out_channels, filter_sizes, use_aux_info, use_mbd, mbd_out_features, mbd_kernel_dims)
'''
if torch.cuda.device_count() > 1:
print("Using", torch.cuda.device_count(), "GPUs")
G = nn.DataParallel(G)
D = nn.DataParallel(D)
elif cuda:
print("Using 1 GPU")
'''
N_max = 11
prefix = 'Before:'
G.eval()
data1, ages1, sexes1, data_fake1, ages_fake1, sexes_fake1 = get_real_and_fake_data(G, train, ignore_similar, dummy_batch_size, sequence_length, True)
data2, ages2, sexes2, data_fake2, ages_fake2, sexes_fake2 = get_real_and_fake_data(G, val, ignore_similar, dummy_batch_size, sequence_length, True)
'''
predictor_name = 'I9_STR_EXH'
event_name = 'I9_HEARTFAIL_NS'
analyse(data1, data_fake1, True, True, ENDPOINT, event_name, predictor_name)
analyse(data2, data_fake2, True, False, ENDPOINT, event_name, predictor_name)
predictor_name = 'I9_ANGINA'
event_name = 'I9_HYPTENS'
analyse(data1, data_fake1, True, True, ENDPOINT, event_name, predictor_name)
analyse(data2, data_fake2, True, False, ENDPOINT, event_name, predictor_name)
'''
save_frequency_comparisons(data_fake1, data_fake2, vocab_size, ENDPOINT, prefix, N_max)
test_generator(data1, ages1, sexes1, data_fake1, ages_fake1, sexes_fake1, True, True, ENDPOINT, SEX, vocab_size, sequence_length)
test_generator(data2, ages2, sexes2, data_fake2, ages_fake2, sexes_fake2, True, False, ENDPOINT, SEX, vocab_size, sequence_length)
G.train()
start_time = time.time()
# Call train function
scores1_train, transition_scores_mean_train, similarity_score1_train, similarity_score2_train, mode_collapse_score_train, \
scores1_val, transition_scores_mean_val, similarity_score1_val, similarity_score2_val, mode_collapse_score_val, \
accuracies_real, accuracies_fake = train_GAN(
G, D, train, val, ENDPOINT, batch_size, vocab_size, sequence_length, n_epochs, lr, temperature, GAN_type, n_critic, print_step, get_scores, ignore_time, dummy_batch_size, ignore_similar, one_sided_label_smoothing, relativistic_average, False, use_gp, lambda_gp
)
print('Time taken:', round_to_n(time.time() - start_time, n = 3), 'seconds')
torch.save(G.state_dict(), G_filename)
G.eval()
prefix = 'After:'
data1, ages1, sexes1, data_fake1, ages_fake1, sexes_fake1 = get_real_and_fake_data(G, train, ignore_similar, dummy_batch_size, sequence_length, True)
data2, ages2, sexes2, data_fake2, ages_fake2, sexes_fake2 = get_real_and_fake_data(G, val, ignore_similar, dummy_batch_size, sequence_length, True)
save_frequency_comparisons(data_fake1, data_fake2, vocab_size, ENDPOINT, prefix, N_max)
save_plots_of_train_scores(scores1_train, transition_scores_mean_train, similarity_score1_train, similarity_score2_train, mode_collapse_score_train, \
scores1_val, transition_scores_mean_val, similarity_score1_val, similarity_score2_val, mode_collapse_score_val, \
accuracies_real, accuracies_fake, ignore_time, sequence_length, vocab_size, ENDPOINT)
test_generator(data1, ages1, sexes1, data_fake1, ages_fake1, sexes_fake1, False, True, ENDPOINT, SEX, vocab_size, sequence_length)
test_generator(data2, ages2, sexes2, data_fake2, ages_fake2, sexes_fake2, False, False, ENDPOINT, SEX, vocab_size, sequence_length)
predictor_name = 'I9_STR_EXH'
event_name = 'I9_HEARTFAIL_NS'
analyse(data1, data_fake1, False, True, ENDPOINT, event_name, predictor_name)
analyse(data2, data_fake2, False, False, ENDPOINT, event_name, predictor_name)
predictor_name = 'I9_ANGINA'
event_name = 'I9_HYPTENS'
analyse(data1, data_fake1, False, True, ENDPOINT, event_name, predictor_name)
analyse(data2, data_fake2, False, False, ENDPOINT, event_name, predictor_name)
save(data1, data_fake1, train = True)
save(data2, data_fake2, train = False)
test_size = 10
visualize_output(G, test_size, val, sequence_length, ENDPOINT, SEX)
def main():
''' Wrapper of Training Pipeline. '''
import os
import yaml, pickle
import numpy as np
import tensorflow as tf
# local modules
import model, train, tools, holdout
print('Loading configuration parameters.')
params = yaml.load(open(os.getcwd() + '/config.yaml'), yaml.Loader)
# Fix number of NaN's to speed up training
params['size_nan'] = int(params['len_input'] * params['total_nan_share'])
tools.set_gpu_configurations(params)
print('\nStart Training Pipeline.\n')
if params['model_type'] == 1:
# If the model already exists load it, otherwise make a new one
if params['model_name'] + '.h5' in os.listdir(os.getcwd() +
'/saved_models/'):
print('Loading existing model: {}.'.format(params['model_name']))
Imputer = tf.keras.models.load_model(os.getcwd() +
'/saved_models/' +
params['model_name'] + '.h5')
else:
print('\nVanilla Seq2seq model instantiated as:\n')
Imputer = model.build_vanilla_seq2seq(params)
Imputer.summary()
print('\nStart training.\n')
train.train_vanilla_seq2seq(Imputer, params)
elif params['model_type'] == 2:
if params['model_name'] + '.h5' in os.listdir(os.getcwd() +
'/saved_models/'):
print('Loading existing model: {}.'.format(params['model_name']))
Imputer = tf.keras.models.load_model(os.getcwd() +
'/saved_models/' +
params['model_name'] + '.h5')
Discriminator = tf.keras.models.load_model(os.getcwd() +
'/saved_models/' +
params['model_name'] +
'_discriminator.h5')
else:
print('\nGAN Seq2seq model instantiated as:\n')
Imputer, Discriminator = model.build_GAN(params)
print('\nGenerator:\n')
Imputer.summary()
print('\n\nDiscriminator:\n')
Discriminator.summary()
print('\nStart GAN training.\n')
train.train_GAN(Imputer, Discriminator, params)
elif params['model_type'] == 3:
if params['model_name'] + '.h5' in os.listdir(os.getcwd() +
'/saved_models/'):
print('Loading existing model: {}.'.format(params['model_name']))
Imputer = tf.keras.models.load_model(os.getcwd() +
'/saved_models/' +
params['model_name'] + '.h5')
Discriminator = tf.keras.models.load_model(os.getcwd() +
'/saved_models/' +
params['model_name'] +
'_discriminator.h5')
else:
print('\nPartially adversarial Seq2seq model instantiated as:\n')
Imputer, Discriminator = model.build_GAN(params)
print('\nGenerator:\n')
Imputer.summary()
print('\nDiscriminator:\n')
Discriminator.summary()
print('\nStart partially adversarial training.\n')
train.train_partial_GAN(Imputer, Discriminator, params)
else:
print(
"ERROR:\nIn config.yaml, from 'model_type' parameter, specify one of the following model architectures:"
)
print("'model_type': 1\tVanilla Seq2seq")
print("'model_type': 2\tGAN Seq2seq")
print("'model_type': 3\tSeq2seq with partially adversarial training")
quit()
# Check performance on Validation data and Test, optionally
holdout.run_test(Imputer,
params,
check_test_performance=params['check_test_performance'])
return None
def random_search(n_runs):
train, val, ENDPOINT, AGE, SEX, vocab_size, sequence_length, n_individuals = get_dataset(nrows = 30_000_000)
print('Data loaded, number of individuals:', n_individuals)
# Generator params
mem_slots = np.arange(1, 21)
head_size = np.arange(1, 21)
embed_size = np.arange(2, vocab_size + 1) # Same for the discriminator
temperature = np.arange(1, 50)
num_heads = np.arange(1, 21)
num_blocks = np.arange(1, 21)
# Discriminator params
n_embeddings = np.arange(1, vocab_size + 1)
out_channels = np.arange(1, 21)
num_filters = np.arange(1, sequence_length - 1)
# Training params
batch_size = np.arange(32, 200)
n_epochs = 8
print_step = max(n_epochs // 2, 1)
lr = np.arange(4, 8)
params = dict()
params['mem_slots'] = mem_slots
params['head_size'] = head_size
params['embed_size'] = embed_size
params['temperature'] = temperature
params['num_heads'] = num_heads
params['num_blocks'] = num_blocks
params['n_embeddings'] = n_embeddings
params['out_channels'] = out_channels
params['num_filters'] = num_filters
params['batch_size'] = batch_size
params['lr'] = lr
filename = 'search_results/random_search_{}.csv'.format(n_individuals)
print(filename)
try:
resulting_df = pd.read_csv(filename, index_col = 0)
except FileNotFoundError as e:
print(e)
resulting_df = pd.DataFrame()
print(resulting_df)
for run in range(n_runs):
try:
chosen_params = dict()
for k, v in params.items():
chosen_params[k] = int(np.random.choice(v))
print('Params chosen:', chosen_params)
mem_slots, head_size, embed_size, temperature, num_heads, num_blocks, n_embeddings, \
out_channels, num_filters, batch_size, lr = tuple(chosen_params.values())
filter_sizes = list(range(2, 2 + num_filters)) # values can be at most the sequence_length
lr = 10 ** (-lr)
print('lr:', lr)
dummy_batch_size = 128
ignore_time = True
# Train the GAN
start_time = time.time()
G = RelationalMemoryGenerator(mem_slots, head_size, embed_size, vocab_size, temperature, num_heads, num_blocks)
D = RelGANDiscriminator(n_embeddings, vocab_size, embed_size, sequence_length, out_channels, filter_sizes, mbd_out_features, mbd_kernel_dims)
'''
if torch.cuda.device_count() > 1:
print("Using", torch.cuda.device_count(), "GPUs")
G = nn.DataParallel(G)
D = nn.DataParallel(D)
elif cuda:
print("Using 1 GPU")
'''
# Call train function
scores1, scores2_mean, similarity_score, mode_collapse_score, indv_score_mean, scores2, indv_score, accuracies_real, accuracies_fake = train_GAN(
G, D, train, val, ENDPOINT, batch_size, vocab_size, sequence_length, n_epochs, lr, temperature, GAN_type, print_step, get_scores, ignore_time, dummy_batch_size
)
chosen_params['chi-squared_score'] = float(scores1[-1])
chosen_params['transition_score'] = float(scores2_mean[-1])
chosen_params['similarity_score'] = float(similarity_score[-1])
chosen_params['indv_score'] = float(indv_score_mean[-1])
ser = pd.DataFrame({len(resulting_df): chosen_params}).T
resulting_df = pd.concat([resulting_df, ser], ignore_index=True)
print('Time taken:', round_to_n(time.time() - start_time, n = 3), 'seconds')
print(resulting_df)
resulting_df.to_csv(filename)
except RuntimeError as e:
print(e)
def objective_function(batch_size, lr, temperature):
try:
batch_size = int(batch_size)
lr = 10 ** (-lr)
scores = []
for i in range(n_sub_runs):
print('sub run {}'.format(i))
# Train the GAN
G = RelationalMemoryGenerator(mem_slots, head_size, embed_size, vocab_size, temperature, num_heads, num_blocks)
D = RelGANDiscriminator(n_embeddings, vocab_size, embed_size, sequence_length, out_channels, filter_sizes, use_aux_info, use_mbd, mbd_out_features, mbd_kernel_dims)
# Call train function
dist_score, transition_score, similarity_score, mode_collapse_score, indv_score, transition_score_full, _, _, _ = train_GAN(
G, D, train, val, ENDPOINT, batch_size, vocab_size, sequence_length, n_epochs, lr, temperature, GAN_type, n_critic, print_step, get_scores, ignore_time, dummy_batch_size, ignore_similar, one_sided_label_smoothing, relativistic_average, True, use_gp, lambda_gp
)
if score_type == 'general':
score = -(2 * dist_score[-1] + \
1 * transition_score[-1] + \
#1 * similarity_score[-1] + \
4 * mode_collapse_score[-1])
elif score_type == 'chd_and_br_cancer':
# minimize the transition score from chd to breast cancer
score = -transition_score_full[ \
-1, ENDPOINT.vocab.stoi['C3_BREAST'] - 3, ENDPOINT.vocab.stoi['I9_CHD'] - 3 \
]
elif score_type == 'transition':
score = -transition_score[-1]
if isnan(score):
score = min_score
score = max(min_score, score)
print('Score:', score)
scores.append(score)
score = np.mean(scores)
return score
except RuntimeError as e:
print(e)
return min_score
epochs = args.epochs_first
else:
epochs = args.epochs_active
# Randomly sample 10000 unlabeled data points
random.shuffle(unlabeled_set)#-----!!!!!
subset = unlabeled_set[:SUBSET]#-----!!!!!
# Create unlabeled dataloader for the unlabeled subset
#pdb.set_trace()
unlabeled_loader = DataLoader(train_dset, batch_size, num_workers=1,
sampler=SubsetRandomSampler(random.sample(unlabeled_set, min(len(labeled_set),len(unlabeled_set)) )))
model.train()
score = train_GAN(model,DD, train_loader, unlabeled_loader, eval_loader, epochs, args.output,cycle)#----!!!!
logger.write('eval score [CYCLE %d] : %.2f' % (cycle, 100 * score))
##
# Update the labeled dataset via loss prediction-based uncertainty measurement
if cycle<CYCLE-1:
# Create unlabeled dataloader for the unlabeled subset
unlabeled_loader = DataLoader(train_dset, batch_size*4, num_workers=1,sampler=SubsetSequentialSampler(subset))# non shuffle version
# Measure uncertainty of each data points in the subset
uncertainty = get_uncertainty(model, unlabeled_loader,len(subset),args)
# Index in ascending order
arg = np.argsort(uncertainty)
# Update the labeled dataset and the unlabeled dataset, respectivelypy()] #list(torch.Tensor(subset)[arg][-ADDENDUM:].numpy())