def dataset_iterator(args): if args.dataset == 'mnist': train_gen, dev_gen, test_gen = mnist.load(args.batch_size, args.batch_size) if args.dataset == 'cifar10': data_dir = '../../../images/cifar-10-batches-py/' train_gen, dev_gen = cifar10.load(args.batch_size, data_dir) test_gen = None return (train_gen, dev_gen, test_gen)
def main(): """Create a Neural Network with one hidden layer.""" training_data, validation_data, test_data = mnist.load() model = nn.NeuralNetwork([784, 100, 10], learning_rate=0.01, batch_size=50) model_training = training.EarlyStoppingRegularization( model, training_data, validation_data, test_data, max_steps_without_progression=2) result = model_training.train() result.save('models/mnist')
def dataset_iterator(args): transform = get_transform(args) if args.dataset == 'mnist': train_gen, dev_gen, test_gen = mnist.load(args.batch_size, args.batch_size) if args.dataset == 'cifar10': data_dir = '/data0/images/cifar-10-batches-py/' train_gen, dev_gen = cifar10.load(args.batch_size, data_dir) test_gen = None if args.dataset == 'celeba': data_dir = '/data0/images/celeba' data = datasets.ImageFolder(data_dir, transform=transform) data_loader = torch.utils.data.DataLoader(data, batch_size=args.batch_size, shuffle=True, drop_last=True, num_workers=4) return data_loader #train_gen = celeba.load(args.batch_size, data_dir+'/train/') #dev_gen = celeba.load(args.batch_size, data_dir+'/test/') #test_gen = None return (train_gen, dev_gen, test_gen)
def run(global_rank, world_size, local_rank, max_epoch, batch_size, model, data, sgd, graph, verbosity, dist_option='fp32', spars=None): dev = device.create_cuda_gpu_on(local_rank) dev.SetRandSeed(0) np.random.seed(0) if data == 'cifar10': from data import cifar10 train_x, train_y, val_x, val_y = cifar10.load() elif data == 'cifar100': from data import cifar100 train_x, train_y, val_x, val_y = cifar100.load() elif data == 'mnist': from data import mnist train_x, train_y, val_x, val_y = mnist.load() num_channels = train_x.shape[1] image_size = train_x.shape[2] data_size = np.prod(train_x.shape[1:train_x.ndim]).item() num_classes = (np.max(train_y) + 1).item() #print(num_classes) if model == 'resnet': from model import resnet model = resnet.resnet50(num_channels=num_channels, num_classes=num_classes) elif model == 'xceptionnet': from model import xceptionnet model = xceptionnet.create_model(num_channels=num_channels, num_classes=num_classes) elif model == 'cnn': from model import cnn model = cnn.create_model(num_channels=num_channels, num_classes=num_classes) elif model == 'alexnet': from model import alexnet model = alexnet.create_model(num_channels=num_channels, num_classes=num_classes) elif model == 'mlp': import os, sys, inspect current = os.path.dirname( os.path.abspath(inspect.getfile(inspect.currentframe()))) parent = os.path.dirname(current) sys.path.insert(0, parent) from mlp import module model = module.create_model(data_size=data_size, num_classes=num_classes) # For distributed training, sequential gives better performance if hasattr(sgd, "communicator"): DIST = True sequential = True else: DIST = False sequential = False if DIST: train_x, train_y, val_x, val_y = partition(global_rank, world_size, train_x, train_y, val_x, val_y) ''' # check dataset shape correctness if global_rank == 0: print("Check the shape of dataset:") print(train_x.shape) print(train_y.shape) ''' if model.dimension == 4: tx = tensor.Tensor( (batch_size, num_channels, model.input_size, model.input_size), dev, tensor.float32) elif model.dimension == 2: tx = tensor.Tensor((batch_size, data_size), dev, tensor.float32) np.reshape(train_x, (train_x.shape[0], -1)) np.reshape(val_x, (val_x.shape[0], -1)) ty = tensor.Tensor((batch_size, ), dev, tensor.int32) num_train_batch = train_x.shape[0] // batch_size num_val_batch = val_x.shape[0] // batch_size idx = np.arange(train_x.shape[0], dtype=np.int32) # attached model to graph model.set_optimizer(sgd) model.compile([tx], is_train=True, use_graph=graph, sequential=sequential) dev.SetVerbosity(verbosity) # Training and Evaluation Loop for epoch in range(max_epoch): start_time = time.time() np.random.shuffle(idx) if global_rank == 0: print('Starting Epoch %d:' % (epoch)) # Training Phase train_correct = np.zeros(shape=[1], dtype=np.float32) test_correct = np.zeros(shape=[1], dtype=np.float32) train_loss = np.zeros(shape=[1], dtype=np.float32) model.train() for b in range(num_train_batch): # Generate the patch data in this iteration x = train_x[idx[b * batch_size:(b + 1) * batch_size]] if model.dimension == 4: x = augmentation(x, batch_size) if (image_size != model.input_size): x = resize_dataset(x, model.input_size) y = train_y[idx[b * batch_size:(b + 1) * batch_size]] # Copy the patch data into input tensors tx.copy_from_numpy(x) ty.copy_from_numpy(y) # Train the model out, loss = model(tx, ty, dist_option, spars) train_correct += accuracy(tensor.to_numpy(out), y) train_loss += tensor.to_numpy(loss)[0] if DIST: # Reduce the Evaluation Accuracy and Loss from Multiple Devices reducer = tensor.Tensor((1, ), dev, tensor.float32) train_correct = reduce_variable(train_correct, sgd, reducer) train_loss = reduce_variable(train_loss, sgd, reducer) if global_rank == 0: print('Training loss = %f, training accuracy = %f' % (train_loss, train_correct / (num_train_batch * batch_size * world_size)), flush=True) # Evaluation Phase model.eval() for b in range(num_val_batch): x = val_x[b * batch_size:(b + 1) * batch_size] if model.dimension == 4: if (image_size != model.input_size): x = resize_dataset(x, model.input_size) y = val_y[b * batch_size:(b + 1) * batch_size] tx.copy_from_numpy(x) ty.copy_from_numpy(y) out_test = model(tx) test_correct += accuracy(tensor.to_numpy(out_test), y) if DIST: # Reduce the Evaulation Accuracy from Multiple Devices test_correct = reduce_variable(test_correct, sgd, reducer) # Output the Evaluation Accuracy if global_rank == 0: print('Evaluation accuracy = %f, Elapsed Time = %fs' % (test_correct / (num_val_batch * batch_size * world_size), time.time() - start_time), flush=True) dev.PrintTimeProfiling()
def main(argv): # load datasets datasets = mnist.load(FLAGS.batch_size) train_iterator = datasets['train'] valid_iterator = datasets['valid'] test_iterator = datasets['test'] # dataset switch (allows us to switch between train and valid set as will) dataset_switch = tf.placeholder(tf.string, shape=[]) data_iterator = tf.data.Iterator.from_string_handle( string_handle=dataset_switch, output_types=datasets['output_types'], output_shapes=datasets['output_shapes']) features = data_iterator.get_next() images, labels = itemgetter('image', 'label')(features) # ========================== Create computation graph ========================== # create model model = models[FLAGS.model]() # call the constructor of the model # create train op training = tf.placeholder_with_default(False, []) logits = model(images, training=training) predictions = tf.argmax(logits, axis=-1, output_type=tf.int32) loss = tf.losses.softmax_cross_entropy(tf.one_hot(labels, 10), logits) train_op = tf.train.AdamOptimizer(FLAGS.learning_rate).minimize(loss) # Accuracy metric tensorflow 2.0 style def accuracy_tf_1(predictions, labels, name): result, update_state = tf.metrics.accuracy(labels=labels, predictions=predictions, name=name) running_vars = tf.get_collection(tf.GraphKeys.LOCAL_VARIABLES, scope=name) reset_state = tf.variables_initializer(var_list=running_vars) return result, update_state, reset_state # metric for training, validation and testing: train_acc, update_train_acc, reset_train_acc = accuracy_tf_1( predictions, labels, 'train_accuracy') valid_acc, update_valid_acc, reset_valid_acc = accuracy_tf_1( predictions, labels, 'valid_accuracy') test_acc, update_test_acc, reset_test_acc = accuracy_tf_1( predictions, labels, 'test_accuracy') # ============================== Do the training =============================== with tf.Session() as sess: # initialize variables sess.run(tf.global_variables_initializer()) sess.run(tf.local_variables_initializer()) # create handles to choose the dataset to use train_switch = sess.run(train_iterator.string_handle()) valid_switch = sess.run(valid_iterator.string_handle()) test_switch = sess.run(test_iterator.string_handle()) step = FLAGS.initial_step while step != FLAGS.final_step: step += 1 sess.run([train_op, update_train_acc], feed_dict={ dataset_switch: train_switch, training: True, }) if step % FLAGS.train_info_freq == 0: loss_val, accuracy_val = sess.run( [loss, train_acc], feed_dict={dataset_switch: train_switch}) template = '| step {:6d} | loss: {:7.5f} | accuracy: {:7.5f} | valid accuracy: --- |' logging.info(template.format(step, loss_val, accuracy_val)) sess.run(reset_train_acc) if step % FLAGS.valid_info_freq == 0: sess.run([valid_iterator.initializer, reset_valid_acc]) while True: try: sess.run(update_valid_acc, feed_dict={dataset_switch: valid_switch}) except tf.errors.OutOfRangeError: break accuracy_val = sess.run(valid_acc) template = '| step {:6d} | loss: --- | accuracy: --- | valid accuracy: {:7.5f} |' logging.info(template.format(step, accuracy_val)) if FLAGS.final_test: sess.run([test_iterator.initializer, reset_test_acc]) while True: try: sess.run(update_test_acc, feed_dict={dataset_switch: test_switch}) except tf.errors.OutOfRangeError: break accuracy_val = sess.run(test_acc) template = 'final accuracy: {:7.5f}' logging.info(template.format(accuracy_val))
help='plot the results every {} batches') parser.add_argument('--batch_size', type=int, default=4, metavar='N', help='batch size.') parser.add_argument('--gp_lambda', type=int, default=10, metavar='N', help='the channel of each image.') parser.add_argument('--noise_dim', type=int, default=10, metavar='N', help='dimension of gaussian noise.') parser.add_argument('--ncritic', type=int, default= 5, metavar='N', help='the channel of each image.') parser.add_argument('--save_folder', type=str, default= 'tmp_images', metavar='N', help='folder to save the temper images.') args = parser.parse_args() args.cuda = args.cuda and torch.cuda.is_available() netD = Disc(input_dim = 28, num_chan =1, hid_dim = 64, out_dim = 16 ) netG = Gen(input_dim = 28, noise_dim = args.noise_dim, num_chan=1, hid_dim= 64) if args.cuda: netD = netD.cuda() netG = netG.cuda() train_gen, dev_gen, test_gen = mnist.load(args.batch_size) data_sampler = DataIterator(train_gen).next model_root, model_name = 'model', 'mnist_mlp' train_gans(data_sampler, model_root, model_name, netG, netD,args)
flags.DEFINE_float('learning_rate', 0.001, '') flags.DEFINE_integer('initial_step', 0, '') flags.DEFINE_integer('final_step', -1, 'set to `-1` to train indefinitely') flags.DEFINE_integer('info_freq', 10, '') flags.DEFINE_bool('fine_tune', False, '') flags.DEFINE_string('model', 'mlp', '') flags = tf.app.flags FLAGS = flags.FLAGS # example: # >> python main.py --batch_size=200 --final_step=2000 # ================================ Read data =================================== # load datasets datasets = mnist.load(FLAGS.batch_size) train_iterator = datasets['train'] valid_iterator = datasets['valid'] # dataset switch (allows us to switch between train and valid set as will) dataset_switch = tf.placeholder(tf.string, shape=[]) data_iterator = tf.data.Iterator.from_string_handle( string_handle=dataset_switch, output_types=datasets['output_types'], output_shapes=datasets['output_shapes'] ) features = data_iterator.get_next() images, labels = itemgetter('image', 'label')(features) # ========================== Create computation graph ==========================
import numpy as np import tensorflow as tf from models.classifiers import MNISTClassifier from components.learners import Learner import data.mnist as mnist datasets = mnist.load(data_dir="~/scikit_learn_data", num_classes=5, batch_size=100, split=[5. / 7, 1. / 7, 1. / 7]) dataset = datasets[0] val_dataset = datasets[1] model = MNISTClassifier(num_classes=5, inputs=None, targets=None) update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): optimizer = tf.train.AdamOptimizer(1e-4).minimize(model.loss) global_init_op = tf.global_variables_initializer() saver = tf.train.Saver() save_dir = "/data/ziz/jxu/hmaml-saved-models" config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: sess.run(global_init_op) ckpt_file = save_dir + '/params_' + "mnist" + '.ckpt' print('restoring parameters from', ckpt_file) saver.restore(sess, ckpt_file)
from models.classifiers import MNISTClassifier from components.learners import Learner import data.mnist as mnist parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--load_params', help='', action='store_true', default=False) parser.add_argument('--num_inner_iters', help='', default=10, type=int) args = parser.parse_args() meta_train_set, meta_val_set, meta_test_set = mnist.load( data_dir="~/scikit_learn_data", num_classes=5, batch_size=5, split=[5. / 7, 1. / 7, 1. / 7], return_meta=True) model = MNISTClassifier(num_classes=5, inputs=None, targets=None) update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): optimizer = tf.train.AdamOptimizer(1e-4).minimize(model.loss) global_init_op = tf.global_variables_initializer() saver = tf.train.Saver() save_dir = "/data/ziz/jxu/hmaml-saved-models" config = tf.ConfigProto() config.gpu_options.allow_growth = True