def test_adversarial(): results = {} p = process_cli_params(get_cli_params()) os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" os.environ["CUDA_VISIBLE_DEVICES"] = str(p.which_gpu) p.epsilon = parse_argstring('1.0-0.1-0.001-0.001-0.001-0.001-0.001', float) p.rc_weights = parse_argstring('1000-10-0.1-0.1-0.1-0.1-0.1', float) # p.num_labeled = 50 p.batch_size = 50 if p.num_labeled is 50 else 100 p.input_size = 784 p.encoder_layers = [p.input_size, 1000, 500, 250, 250, 250, 10] p.lrelu_a = 0.1 seeds = [8340, 8794, 2773, 967, 2368] models = ['n', 'nlw', 'ladder', 'c', 'clw', 'vat'] # models = ['vat'] for model in models: p.model = model results[model] = {} for seed in seeds: p.seed = seed p.id = 'full_{}_labeled-{}'.format(model, p.num_labeled) results[model][seed] = test_aer_on_normal_and_adv(p) save_obj(results, 'labeled-{}'.format(p.num_labeled)) return results
def test_hessian_ops(): p = process_cli_params(get_cli_params()) mnist = input_data.read_data_sets("MNIST_data", n_labeled=p.num_labeled, validation_size=p.validation, one_hot=True, disjoint=False) num_examples = mnist.train.num_examples p.num_examples = num_examples if p.validation > 0: mnist.test = mnist.validation p.iter_per_epoch = (num_examples // p.batch_size) p.num_iter = p.iter_per_epoch * p.end_epoch p.model = 'ladder' # Build graph inputs = tf.placeholder(tf.float32, shape=(784)) outputs = tf.placeholder(tf.float32) logits = tf.matmul(tf.expand_dims(inputs, axis=0), tf.get_variable('w', shape=(784, 10))) loss = softmax_cross_entropy_with_logits(outputs, logits) loss_grad = tf.gradients(loss, inputs, name='help') # hess = tf.hessians(loss, inputs) IPython.embed()
def test_data_splitting(): p = process_cli_params(get_cli_params()) global VERBOSE VERBOSE = p.verbose # ----------------------------- # Set GPU device to use os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" os.environ["CUDA_VISIBLE_DEVICES"] = str(p.which_gpu) config = tf.ConfigProto() config.gpu_options.allow_growth = True # Set seeds np.random.seed(p.seed) tf.set_random_seed(p.seed) # Load data print("=== Loading Data ===") if p.dataset == 'svhn': from src.svhn import read_data_sets dataset = read_data_sets("../../data/svhn/", n_labeled=p.num_labeled, validation_size=p.validation, one_hot=True, disjoint=False, downsample=True, download_and_extract=True) else: from src.mnist import read_data_sets dataset = read_data_sets("MNIST_data", n_labeled=p.num_labeled, validation_size=p.validation, one_hot=True, disjoint=False) print(dataset.train.l_idx) print(np.sum(dataset.train.labeled_ds.labels, axis=0))
def __init__(self): # Parse command line and default parameters self.params = process_cli_params(get_cli_params())
log_i = [int(now), ep] + test_aer_and_costs + train_aer + \ train_costs with open(log_file, 'a') as train_log: print(*log_i, sep=',', flush=True, file=train_log) with open(desc_file, 'a') as f: final_aer = eval_metric(dataset.test, sess, aer) print("Final AER: ", final_aer, "%", file=f, flush=True) sess.close() if __name__ == '__main__': p = get_cli_params() main(p)
def main(): params = process_cli_params(get_cli_params()) # ----------------------------- # Set GPU device to use os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" os.environ["CUDA_VISIBLE_DEVICES"] = str(params.which_gpu) # Set seeds np.random.seed(params.seed) tf.set_random_seed(params.seed) print("=== Loading Data ===") mnist = mnist.read_data_sets("MNIST_data", n_labeled=params.num_labeled, one_hot=True, disjoint=False) num_examples = mnist.train.num_examples # ----------------------------- # Parameter setup params.iter_per_epoch = (num_examples // params.batch_size) params.num_iter = params.iter_per_epoch * params.end_epoch params.encoder_layers = params.cnn_fan if params.cnn else \ params.encoder_layers # ----------------------------- # Placeholder setup inputs_placeholder = tf.placeholder(tf.float32, shape=(None, params.encoder_layers[ 0])) inputs = preprocess(inputs_placeholder, params) outputs = tf.placeholder(tf.float32) train_flag = tf.placeholder(tf.bool) # ----------------------------- # Ladder ladder = Ladder(inputs, outputs, train_flag, params) # ----------------------------- # Loss, accuracy and training steps loss = ladder.cost + ladder.u_cost accuracy = tf.reduce_mean( tf.cast( tf.equal(ladder.predict, tf.argmax(outputs, 1)), "float")) * tf.constant(100.0) learning_rate = tf.Variable(params.initial_learning_rate, trainable=False) train_step = tf.train.AdamOptimizer(learning_rate).minimize(loss) # add the updates of batch normalization statistics to train_step bn_updates = tf.group(*ladder.bn.bn_assigns) with tf.control_dependencies([train_step]): train_step = tf.group(bn_updates) saver = tf.train.Saver(keep_checkpoint_every_n_hours=0.5, max_to_keep=5) # ----------------------------- # Create logs after full graph created to count trainable parameters # Write logs to appropriate directory log_dir = params.logdir + params.id if not os.path.exists(log_dir): os.makedirs(log_dir) desc_file = log_dir + "/" + "description" with open(desc_file, 'a') as f: print(*order_param_settings(params), sep='\n', file=f, flush=True) print("Trainable parameters:", count_trainable_params(), file=f, flush=True) log_file = log_dir + "/" + "train_log" # ----------------------------- print("=== Starting Session ===") sess = tf.Session() i_iter = 0 # ----------------------------- # Resume from checkpoint ckpt_dir = "checkpoints/" + params.id + "/" ckpt = tf.train.get_checkpoint_state( ckpt_dir) # get latest checkpoint (if any) if ckpt and ckpt.model_checkpoint_path: # if checkpoint exists, restore the parameters and set epoch_n and i_iter saver.restore(sess, ckpt.model_checkpoint_path) epoch_n = int(ckpt.model_checkpoint_path.split('/')[-1].split('-')[1]) i_iter = (epoch_n + 1) * (num_examples // params.batch_size) print("Restored Epoch ", epoch_n) else: # no checkpoint exists. create checkpoints directory if it does not exist. if not os.path.exists(ckpt_dir): os.makedirs(ckpt_dir) init = tf.global_variables_initializer() sess.run(init) # ----------------------------- print("=== Training ===") def evaluate_metric(dataset, sess, op): metric = 0 num_eval_iters = dataset.num_examples // params.batch_size for _ in range(num_eval_iters): images, labels = dataset.next_batch(params.batch_size) init_feed = {inputs_placeholder: images, outputs: labels, train_flag: False} metric += sess.run(op, init_feed) metric /= num_eval_iters return metric def evaluate_metric_list(dataset, sess, ops): metrics = [0.0 for _ in ops] num_eval_iters = dataset.num_examples // params.batch_size for _ in range(num_eval_iters): images, labels = dataset.next_batch(params.batch_size) init_feed = {inputs_placeholder: images, outputs: labels, train_flag: False} op_eval = sess.run(ops, init_feed) for i, op in enumerate(op_eval): metrics[i] += op metrics = [metric/num_eval_iters for metric in metrics] return metrics # ----------------------------- # Evaluate initial training accuracy and losses # init_loss = evaluate_metric( # mnist.train.labeled_ds, sess, cost) with open(desc_file, 'a') as f: print('================================', file=f, flush=True) print("Initial Train Accuracy: ", sess.run(accuracy, feed_dict={ inputs_placeholder: mnist.train.labeled_ds.images, outputs: mnist.train.labeled_ds.labels, train_flag: False}), "%", file=f, flush=True) print("Initial Train Losses: ", *evaluate_metric_list( mnist.train, sess, [loss, ladder.cost, ladder.u_cost]), file=f, flush=True) # ----------------------------- # Evaluate initial testing accuracy and cross-entropy loss print("Initial Test Accuracy: ", sess.run(accuracy, feed_dict={ inputs_placeholder: mnist.test.images, outputs: mnist.test.labels, train_flag: False}), "%", file=f, flush=True) print("Initial Test Cross Entropy: ", evaluate_metric(mnist.test, sess, ladder.cost), file=f, flush=True) start = time.time() for i in tqdm(range(i_iter, params.num_iter)): images, labels = mnist.train.next_batch(params.batch_size) _ = sess.run( [train_step], feed_dict={inputs_placeholder: images, outputs: labels, train_flag: True}) # --------------------------------------------- # Epoch completed? if (i > 1) and ((i+1) % params.iter_per_epoch == 0): epoch_n = i // (num_examples // params.batch_size) update_decays(sess, epoch_n, iter=i, graph=g, params=p) # --------------------------------------------- # Evaluate every test_frequency_in_epochs if ((i + 1) % (params.test_frequency_in_epochs * params.iter_per_epoch) == 0): now = time.time() - start if not params.do_not_save: saver.save(sess, ckpt_dir + 'model.ckpt', epoch_n) # --------------------------------------------- # Compute error on testing set (10k examples) test_cost = evaluate_metric(mnist.test, sess, ladder.cost) # Create log of: # time, epoch number, test accuracy, test cross entropy, # train accuracy, train loss, train cross entropy, # train reconstruction loss log_i = [now, epoch_n] + sess.run( [accuracy], feed_dict={inputs_placeholder: mnist.test.images, outputs: mnist.test.labels, train_flag: False} ) + [test_cost] + sess.run( [accuracy], feed_dict={inputs_placeholder: mnist.train.labeled_ds.images, outputs: mnist.train.labeled_ds.labels, train_flag: False} ) + sess.run( [loss, ladder.cost, ladder.u_cost], feed_dict={inputs_placeholder: images, outputs: labels, train_flag: False}) with open(log_file, 'a') as train_log: print(*log_i, sep=',', flush=True, file=train_log) with open(desc_file, 'a') as f: print("Final Accuracy: ", sess.run(accuracy, feed_dict={ inputs_placeholder: mnist.test.images, outputs: mnist.test.labels, train_flag: False}), "%", file=f, flush=True) sess.close()