def dataset_iterator(args):
if args.dataset == 'mnist':
train_gen, dev_gen, test_gen = mnist.load(args.batch_size,
args.batch_size)
if args.dataset == 'cifar10':
data_dir = '../../../images/cifar-10-batches-py/'
train_gen, dev_gen = cifar10.load(args.batch_size, data_dir)
test_gen = None
return (train_gen, dev_gen, test_gen)
def main():
"""Create a Neural Network with one hidden layer."""
training_data, validation_data, test_data = mnist.load()
model = nn.NeuralNetwork([784, 100, 10], learning_rate=0.01, batch_size=50)
model_training = training.EarlyStoppingRegularization(
model,
training_data,
validation_data,
test_data,
max_steps_without_progression=2)
result = model_training.train()
result.save('models/mnist')
def dataset_iterator(args):
transform = get_transform(args)
if args.dataset == 'mnist':
train_gen, dev_gen, test_gen = mnist.load(args.batch_size,
args.batch_size)
if args.dataset == 'cifar10':
data_dir = '/data0/images/cifar-10-batches-py/'
train_gen, dev_gen = cifar10.load(args.batch_size, data_dir)
test_gen = None
if args.dataset == 'celeba':
data_dir = '/data0/images/celeba'
data = datasets.ImageFolder(data_dir, transform=transform)
data_loader = torch.utils.data.DataLoader(data,
batch_size=args.batch_size,
shuffle=True,
drop_last=True,
num_workers=4)
return data_loader
#train_gen = celeba.load(args.batch_size, data_dir+'/train/')
#dev_gen = celeba.load(args.batch_size, data_dir+'/test/')
#test_gen = None
return (train_gen, dev_gen, test_gen)
def run(global_rank,
world_size,
local_rank,
max_epoch,
batch_size,
model,
data,
sgd,
graph,
verbosity,
dist_option='fp32',
spars=None):
dev = device.create_cuda_gpu_on(local_rank)
dev.SetRandSeed(0)
np.random.seed(0)
if data == 'cifar10':
from data import cifar10
train_x, train_y, val_x, val_y = cifar10.load()
elif data == 'cifar100':
from data import cifar100
train_x, train_y, val_x, val_y = cifar100.load()
elif data == 'mnist':
from data import mnist
train_x, train_y, val_x, val_y = mnist.load()
num_channels = train_x.shape[1]
image_size = train_x.shape[2]
data_size = np.prod(train_x.shape[1:train_x.ndim]).item()
num_classes = (np.max(train_y) + 1).item()
#print(num_classes)
if model == 'resnet':
from model import resnet
model = resnet.resnet50(num_channels=num_channels,
num_classes=num_classes)
elif model == 'xceptionnet':
from model import xceptionnet
model = xceptionnet.create_model(num_channels=num_channels,
num_classes=num_classes)
elif model == 'cnn':
from model import cnn
model = cnn.create_model(num_channels=num_channels,
num_classes=num_classes)
elif model == 'alexnet':
from model import alexnet
model = alexnet.create_model(num_channels=num_channels,
num_classes=num_classes)
elif model == 'mlp':
import os, sys, inspect
current = os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe())))
parent = os.path.dirname(current)
sys.path.insert(0, parent)
from mlp import module
model = module.create_model(data_size=data_size,
num_classes=num_classes)
# For distributed training, sequential gives better performance
if hasattr(sgd, "communicator"):
DIST = True
sequential = True
else:
DIST = False
sequential = False
if DIST:
train_x, train_y, val_x, val_y = partition(global_rank, world_size,
train_x, train_y, val_x,
val_y)
'''
# check dataset shape correctness
if global_rank == 0:
print("Check the shape of dataset:")
print(train_x.shape)
print(train_y.shape)
'''
if model.dimension == 4:
tx = tensor.Tensor(
(batch_size, num_channels, model.input_size, model.input_size),
dev, tensor.float32)
elif model.dimension == 2:
tx = tensor.Tensor((batch_size, data_size), dev, tensor.float32)
np.reshape(train_x, (train_x.shape[0], -1))
np.reshape(val_x, (val_x.shape[0], -1))
ty = tensor.Tensor((batch_size, ), dev, tensor.int32)
num_train_batch = train_x.shape[0] // batch_size
num_val_batch = val_x.shape[0] // batch_size
idx = np.arange(train_x.shape[0], dtype=np.int32)
# attached model to graph
model.set_optimizer(sgd)
model.compile([tx], is_train=True, use_graph=graph, sequential=sequential)
dev.SetVerbosity(verbosity)
# Training and Evaluation Loop
for epoch in range(max_epoch):
start_time = time.time()
np.random.shuffle(idx)
if global_rank == 0:
print('Starting Epoch %d:' % (epoch))
# Training Phase
train_correct = np.zeros(shape=[1], dtype=np.float32)
test_correct = np.zeros(shape=[1], dtype=np.float32)
train_loss = np.zeros(shape=[1], dtype=np.float32)
model.train()
for b in range(num_train_batch):
# Generate the patch data in this iteration
x = train_x[idx[b * batch_size:(b + 1) * batch_size]]
if model.dimension == 4:
x = augmentation(x, batch_size)
if (image_size != model.input_size):
x = resize_dataset(x, model.input_size)
y = train_y[idx[b * batch_size:(b + 1) * batch_size]]
# Copy the patch data into input tensors
tx.copy_from_numpy(x)
ty.copy_from_numpy(y)
# Train the model
out, loss = model(tx, ty, dist_option, spars)
train_correct += accuracy(tensor.to_numpy(out), y)
train_loss += tensor.to_numpy(loss)[0]
if DIST:
# Reduce the Evaluation Accuracy and Loss from Multiple Devices
reducer = tensor.Tensor((1, ), dev, tensor.float32)
train_correct = reduce_variable(train_correct, sgd, reducer)
train_loss = reduce_variable(train_loss, sgd, reducer)
if global_rank == 0:
print('Training loss = %f, training accuracy = %f' %
(train_loss, train_correct /
(num_train_batch * batch_size * world_size)),
flush=True)
# Evaluation Phase
model.eval()
for b in range(num_val_batch):
x = val_x[b * batch_size:(b + 1) * batch_size]
if model.dimension == 4:
if (image_size != model.input_size):
x = resize_dataset(x, model.input_size)
y = val_y[b * batch_size:(b + 1) * batch_size]
tx.copy_from_numpy(x)
ty.copy_from_numpy(y)
out_test = model(tx)
test_correct += accuracy(tensor.to_numpy(out_test), y)
if DIST:
# Reduce the Evaulation Accuracy from Multiple Devices
test_correct = reduce_variable(test_correct, sgd, reducer)
# Output the Evaluation Accuracy
if global_rank == 0:
print('Evaluation accuracy = %f, Elapsed Time = %fs' %
(test_correct / (num_val_batch * batch_size * world_size),
time.time() - start_time),
flush=True)
dev.PrintTimeProfiling()
def main(argv):
# load datasets
datasets = mnist.load(FLAGS.batch_size)
train_iterator = datasets['train']
valid_iterator = datasets['valid']
test_iterator = datasets['test']
# dataset switch (allows us to switch between train and valid set as will)
dataset_switch = tf.placeholder(tf.string, shape=[])
data_iterator = tf.data.Iterator.from_string_handle(
string_handle=dataset_switch,
output_types=datasets['output_types'],
output_shapes=datasets['output_shapes'])
features = data_iterator.get_next()
images, labels = itemgetter('image', 'label')(features)
# ========================== Create computation graph ==========================
# create model
model = models[FLAGS.model]() # call the constructor of the model
# create train op
training = tf.placeholder_with_default(False, [])
logits = model(images, training=training)
predictions = tf.argmax(logits, axis=-1, output_type=tf.int32)
loss = tf.losses.softmax_cross_entropy(tf.one_hot(labels, 10), logits)
train_op = tf.train.AdamOptimizer(FLAGS.learning_rate).minimize(loss)
# Accuracy metric tensorflow 2.0 style
def accuracy_tf_1(predictions, labels, name):
result, update_state = tf.metrics.accuracy(labels=labels,
predictions=predictions,
name=name)
running_vars = tf.get_collection(tf.GraphKeys.LOCAL_VARIABLES,
scope=name)
reset_state = tf.variables_initializer(var_list=running_vars)
return result, update_state, reset_state
# metric for training, validation and testing:
train_acc, update_train_acc, reset_train_acc = accuracy_tf_1(
predictions, labels, 'train_accuracy')
valid_acc, update_valid_acc, reset_valid_acc = accuracy_tf_1(
predictions, labels, 'valid_accuracy')
test_acc, update_test_acc, reset_test_acc = accuracy_tf_1(
predictions, labels, 'test_accuracy')
# ============================== Do the training ===============================
with tf.Session() as sess:
# initialize variables
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
# create handles to choose the dataset to use
train_switch = sess.run(train_iterator.string_handle())
valid_switch = sess.run(valid_iterator.string_handle())
test_switch = sess.run(test_iterator.string_handle())
step = FLAGS.initial_step
while step != FLAGS.final_step:
step += 1
sess.run([train_op, update_train_acc],
feed_dict={
dataset_switch: train_switch,
training: True,
})
if step % FLAGS.train_info_freq == 0:
loss_val, accuracy_val = sess.run(
[loss, train_acc],
feed_dict={dataset_switch: train_switch})
template = '| step {:6d} | loss: {:7.5f} | accuracy: {:7.5f} | valid accuracy: --- |'
logging.info(template.format(step, loss_val, accuracy_val))
sess.run(reset_train_acc)
if step % FLAGS.valid_info_freq == 0:
sess.run([valid_iterator.initializer, reset_valid_acc])
while True:
try:
sess.run(update_valid_acc,
feed_dict={dataset_switch: valid_switch})
except tf.errors.OutOfRangeError:
break
accuracy_val = sess.run(valid_acc)
template = '| step {:6d} | loss: --- | accuracy: --- | valid accuracy: {:7.5f} |'
logging.info(template.format(step, accuracy_val))
if FLAGS.final_test:
sess.run([test_iterator.initializer, reset_test_acc])
while True:
try:
sess.run(update_test_acc,
feed_dict={dataset_switch: test_switch})
except tf.errors.OutOfRangeError:
break
accuracy_val = sess.run(test_acc)
template = 'final accuracy: {:7.5f}'
logging.info(template.format(accuracy_val))
help='plot the results every {} batches')
parser.add_argument('--batch_size', type=int, default=4, metavar='N',
help='batch size.')
parser.add_argument('--gp_lambda', type=int, default=10, metavar='N',
help='the channel of each image.')
parser.add_argument('--noise_dim', type=int, default=10, metavar='N',
help='dimension of gaussian noise.')
parser.add_argument('--ncritic', type=int, default= 5, metavar='N',
help='the channel of each image.')
parser.add_argument('--save_folder', type=str, default= 'tmp_images', metavar='N',
help='folder to save the temper images.')
args = parser.parse_args()
args.cuda = args.cuda and torch.cuda.is_available()
netD = Disc(input_dim = 28, num_chan =1, hid_dim = 64, out_dim = 16 )
netG = Gen(input_dim = 28, noise_dim = args.noise_dim, num_chan=1, hid_dim= 64)
if args.cuda:
netD = netD.cuda()
netG = netG.cuda()
train_gen, dev_gen, test_gen = mnist.load(args.batch_size)
data_sampler = DataIterator(train_gen).next
model_root, model_name = 'model', 'mnist_mlp'
train_gans(data_sampler, model_root, model_name, netG, netD,args)
flags.DEFINE_float('learning_rate', 0.001, '')
flags.DEFINE_integer('initial_step', 0, '')
flags.DEFINE_integer('final_step', -1, 'set to `-1` to train indefinitely')
flags.DEFINE_integer('info_freq', 10, '')
flags.DEFINE_bool('fine_tune', False, '')
flags.DEFINE_string('model', 'mlp', '')
flags = tf.app.flags
FLAGS = flags.FLAGS
# example:
# >> python main.py --batch_size=200 --final_step=2000
# ================================ Read data ===================================
# load datasets
datasets = mnist.load(FLAGS.batch_size)
train_iterator = datasets['train']
valid_iterator = datasets['valid']
# dataset switch (allows us to switch between train and valid set as will)
dataset_switch = tf.placeholder(tf.string, shape=[])
data_iterator = tf.data.Iterator.from_string_handle(
string_handle=dataset_switch,
output_types=datasets['output_types'],
output_shapes=datasets['output_shapes']
)
features = data_iterator.get_next()
images, labels = itemgetter('image', 'label')(features)
# ========================== Create computation graph ==========================
import numpy as np
import tensorflow as tf
from models.classifiers import MNISTClassifier
from components.learners import Learner
import data.mnist as mnist
datasets = mnist.load(data_dir="~/scikit_learn_data",
num_classes=5,
batch_size=100,
split=[5. / 7, 1. / 7, 1. / 7])
dataset = datasets[0]
val_dataset = datasets[1]
model = MNISTClassifier(num_classes=5, inputs=None, targets=None)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
optimizer = tf.train.AdamOptimizer(1e-4).minimize(model.loss)
global_init_op = tf.global_variables_initializer()
saver = tf.train.Saver()
save_dir = "/data/ziz/jxu/hmaml-saved-models"
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
sess.run(global_init_op)
ckpt_file = save_dir + '/params_' + "mnist" + '.ckpt'
print('restoring parameters from', ckpt_file)
saver.restore(sess, ckpt_file)
from models.classifiers import MNISTClassifier
from components.learners import Learner
import data.mnist as mnist
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--load_params',
help='',
action='store_true',
default=False)
parser.add_argument('--num_inner_iters', help='', default=10, type=int)
args = parser.parse_args()
meta_train_set, meta_val_set, meta_test_set = mnist.load(
data_dir="~/scikit_learn_data",
num_classes=5,
batch_size=5,
split=[5. / 7, 1. / 7, 1. / 7],
return_meta=True)
model = MNISTClassifier(num_classes=5, inputs=None, targets=None)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
optimizer = tf.train.AdamOptimizer(1e-4).minimize(model.loss)
global_init_op = tf.global_variables_initializer()
saver = tf.train.Saver()
save_dir = "/data/ziz/jxu/hmaml-saved-models"
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
