def test_iterate_scheme():
from fuel.datasets import IndexableDataset
from fuel.schemes import (SequentialScheme, ShuffledScheme,SequentialExampleScheme, ShuffledExampleScheme)
seed = 1234
rng = numpy.random.RandomState(seed)
features = rng.randint(256, size=(8, 2, 2))
targets = rng.randint(4, size=(8, 1))
dataset = IndexableDataset(indexables=OrderedDict([('features', features),
('targets', targets)]),
axis_labels=OrderedDict([('features', ('batch', 'height', 'width')),
('targets', ('batch', 'index'))]))
schemes = [SequentialScheme(examples=8, batch_size=5),
ShuffledScheme(examples=8, batch_size=3),
SequentialExampleScheme(examples=8),
ShuffledExampleScheme(examples=8)]
# for scheme in schemes:
# print(list(scheme.get_request_iterator()))
state = dataset.open()
scheme = ShuffledScheme(examples=dataset.num_examples, batch_size=3)
for request in scheme.get_request_iterator():
data = dataset.get_data(state=state, request=request)
print(data[0].shape, data[1].shape)
dataset.close(state)
def get_results(C=None, pt=None):
dual_class = KLD(C)
# Direct Otimizer
# model = CCmodel()
# Spade Optimizer
model = DeeNemBis(dual_class)
train_state = train_dataset.open()
test_state = test_dataset.open()
scheme = ShuffledScheme(examples=train_dataset.num_examples,
batch_size=batch_size)
# test_scheme = ShuffledScheme(examples=test_dataset.num_examples,
# batch_size=test_dataset.num_examples)
print "Input dim is " + str(input_dim)
print "p is " + str(p)
# Get the theano functions
train_fn, test_fn = model.get_fns(input_dim=input_dim, p=p)
num_epochs = 100
cost = []
time_arr = []
passed_time = 0
curr_time = time.time()
idx = 0
meas1 = []
meas2 = []
for epochs in tqdm(xrange(num_epochs)):
for request in scheme.get_request_iterator():
data = train_dataset.get_data(state=train_state, request=request)
r0, r1 = train_fn(data[0], data[1].ravel())
# print r0, r1
if idx % 100 == 0:
# print a, b
time_arr.append(passed_time + time.time() - curr_time)
passed_time = time_arr[-1]
test_req = range(test_dataset.num_examples)
test_data = test_dataset.get_data(state=test_state,
request=test_req)
minC, out = test_fn(test_data[0], test_data[1])
meas1.append(minC)
curr_time = time.time()
idx += 1
minC, out = test_fn(test_data[0], test_data[1])
np.savetxt('gold_' + location + '.csv', test_data[1])
np.savetxt('test_' + location + '.csv', out)
train_dataset.close(train_state)
test_dataset.close(test_state)
# print meas1, meas2
return time_arr, meas1
def get_curr_out(self, model, example_set, batch_size=256):
scheme = ShuffledScheme(examples=example_set.num_examples,
batch_size=batch_size)
example_state = example_set.open()
x = T.matrix('features')
out = model.apply(x)
pred_fn = theano.function([x], out)
y = np.zeros((example_set.num_examples))
y_hat = np.zeros((example_set.num_examples))
for idx, request in enumerate(scheme.get_request_iterator()):
data = example_set.get_data(state=example_state, request=request)
out_val = pred_fn(data[0].astype(np.float32))
end_idx = (idx + 1) * batch_size
if end_idx < example_set.num_examples:
y[idx * batch_size:end_idx] = data[1].flatten()
y_hat[idx * batch_size:end_idx] = out_val.flatten()
P = np.sum(y)
N = np.sum(1 - y)
return (y, y_hat), P, N
def main():
args = utils.get_args()
# Settings
BATCH_SIZE = 128
NB_EPOCHS = args.epochs # default 25
LRN_RATE = 0.0001 / 28
if args.verbose:
PRINT_DELAY = 1
else:
PRINT_DELAY = 500
# if running on server (MILA), copy dataset locally
dataset_path = utils.init_dataset(args, 'mnist')
# load dataset
data = load_mnist_dataset(dataset_path, args)
train_feats, train_targets, valid_feats, valid_targets, test_feats, test_targets = data
# Get theano functions
predict, preds_grad, network = build_theano_fn()
print 'Starting training...'
hist_errors = []
running_err_avg = 0
running_last_err_avg = 0
# Create mask that will be used to create sequences
mask = np.tril(np.ones((784, 784), dtype=theano.config.floatX), 0)
for i in xrange(NB_EPOCHS):
print 'Epoch #%s of %s' % ((i + 1), NB_EPOCHS)
epoch_err = []
num_batch = 0
t_epoch = time.time()
# iterate over minibatches for training
schemes_train = ShuffledScheme(examples=train_feats.shape[0], batch_size=1)
# We deal with 1 example as if it was an episode
for batch_idx in schemes_train.get_request_iterator():
batch_delta_params = []
batch_error = []
batch_grads = []
num_batch += 1
t_batch = time.time()
train_x = train_feats[batch_idx]
true_y = train_targets[batch_idx]
nb_seq = 0
for t in xrange(784):
# apply mask at fixed interval, making sequences of pixels appear
if (t + 1) % 28 == 0:
nb_seq += 1
seq_x = train_x * mask[t]
pred = predict(seq_x)[0, 0]
grad = preds_grad(seq_x)
# if nb_seq == 1:
# sum_grad = np.copy(grad)
# else:
# sum_grad += np.copy(grad)
if t < 783:
seq_x_prime = train_x * mask[t + 28]
pred_y_prime = predict(seq_x_prime)[0, 0]
TD_error = (pred_y_prime - pred)
error = (true_y - pred)
else:
TD_error = (true_y - pred)
error = (true_y - pred)
param_values = lyr.get_all_param_values(network)
delta_params = LRN_RATE * TD_error * grad
param_values += delta_params
lyr.set_all_param_values(network, param_values)
batch_error.append(error)
# update params based on experience
last_error = np.abs(error)[0]
sqrd_error = np.linalg.norm(batch_error, 2)
epoch_err.append(sqrd_error)
running_err_avg = 0.05 * sqrd_error + 0.95 * running_err_avg
running_last_err_avg = 0.05 * last_error + 0.95 * running_last_err_avg
if num_batch % PRINT_DELAY == 0:
print '- batch %s, err %s (avg %s), last %s (avg %s), in %s sec' % (num_batch, np.round(sqrd_error, 4),
np.round(running_err_avg, 4),
np.round(last_error, 4),
np.round(running_last_err_avg, 4),
np.round(time.time() - t_batch, 2))
print '- Epoch train (err %s) in %s sec' % (epoch_err, round(time.time() - t_epoch))
# hist_errors.append(epoch_err)
utils.dump_objects_output(args, epoch_err, 'epoch_%s_error_sqrd_norm.pkl' % (i + 1))
test_scheme = ShuffledScheme(examples=test_dataset.num_examples,
batch_size=test_dataset.num_examples)
print "Input dim is " + str(input_dim)
print "p is " + str(p)
# Get the theano functions
train_fn, test_fn = model.get_fns(input_dim=input_dim, p=p)
num_epochs = 10
cost = []
time_arr = []
passed_time = 0
curr_time = time.time()
idx = 0
for epochs in tqdm(xrange(num_epochs)):
for request in scheme.get_request_iterator():
data = train_dataset.get_data(state=train_state, request=request)
c = train_fn(data[0], data[1].ravel())
#print c
if idx % 100 == 0:
time_arr.append(passed_time + time.time() - curr_time)
passed_time = time_arr[-1]
test_req = range(test_dataset.num_examples)
test_data = test_dataset.get_data(state=test_state,
request=test_req)
minC = test_fn(test_data[0], test_data[1])
cost.append(minC)
curr_time = time.time()
idx += 1
train_dataset.close(train_state)
test_dataset.close(test_state)
# valid_acc = []
valid_loss = []
# train_acc = []
train_loss = []
best_valid_loss = float('inf')
for i in xrange(nb_epochs):
# iterate over minibatches for training
schemes = ShuffledScheme(examples=train_x.shape[0],
batch_size=batch_size)
epoch_acc = 0
epoch_loss = 0
num_batch = 0
for batch_idx in schemes.get_request_iterator():
batch_loss = train(batch_idx)
# epoch_acc += batch_acc
epoch_loss += batch_loss
num_batch += 1
# train_acc.append(np.round(epoch_acc / num_batch, 4))
train_loss.append(np.round(epoch_loss / num_batch, 4))
epoch_loss, preds = valid(valid_idx)
# valid_acc.append(np.round(batch_acc, 4))
valid_loss.append(np.round(epoch_loss, 4))
print 'Epoch #%s of %s' % ((i + 1), nb_epochs)
print '- Train (loss %s)' % (train_loss[i])
def main():
args = utils.get_args()
# Settings for training
BATCH_SIZE = 128
NB_EPOCHS = args.epochs # default 25
NB_GEN = args.gen # default 5
TRAIN_STEPS_DISCR = 15
TRAIN_STEPS_GEN = 10
if args.reload is not None:
RELOAD_SRC = args.reload[0]
RELOAD_ID = args.reload[1]
if args.verbose:
BATCH_PRINT_DELAY = 1
else:
BATCH_PRINT_DELAY = 100
# if running on server (MILA), copy dataset locally
dataset_path = utils.init_dataset(args, 'mscoco_inpainting/preprocessed')
train_path = os.path.join(dataset_path, 'train2014')
valid_path = os.path.join(dataset_path, 'val2014')
if args.captions:
t = time.time()
embedding_model = utils.init_google_word2vec_model(args)
print 'Embedding model was loaded in %s secs' % np.round(
time.time() - t, 0)
# build network and get theano functions for training
theano_fn = gen_theano_fn(args)
train_discr, train_gen, predict, reconstr_fn, reconstr_noise_shrd, model = theano_fn
# get different file names for the split data set
train_files = utils.get_preprocessed_files(train_path)
train_full_files, train_cter_files, train_capt_files = train_files
valid_files = utils.get_preprocessed_files(valid_path)
valid_full_files, valid_cter_files, valid_capt_files = valid_files
NB_TRAIN_FILES = len(train_full_files)
NB_VALID_FILES = len(valid_full_files)
print 'Starting training...'
train_loss = []
if args.reload is not None:
discriminator, generator = model
file_discr = 'discrminator_epoch_%s.pkl' % RELOAD_ID
file_gen = 'generator_epoch_%s.pkl' % RELOAD_ID
loaded_discr = utils.reload_model(args, discriminator, file_discr,
RELOAD_SRC)
loaded_gen = utils.reload_model(args, generator, file_gen, RELOAD_SRC)
for i in xrange(NB_EPOCHS):
print 'Epoch #%s of %s' % ((i + 1), NB_EPOCHS)
epoch_acc = 0
epoch_loss = 0
num_batch = 0
t_epoch = time.time()
d_batch_loss = 0
g_batch_loss = 0
steps_loss_g = [] # will store every loss of generator
steps_loss_d = [] # will store every loss of discriminator
d_train_step = 0
# iterate of split datasets
for file_id in np.random.choice(NB_TRAIN_FILES,
NB_TRAIN_FILES,
replace=False):
t_load = time.time()
# load file with full image
with open(train_full_files[file_id], 'r') as f:
train_full = np.load(f).astype(theano.config.floatX)
if args.captions:
# load file with the captions
with open(train_capt_files[file_id], 'rb') as f:
train_capt = pkl.load(f)
if args.verbose:
print 'file %s loaded in %s sec' % (
train_full_files[file_id], round(time.time() - t_load, 0))
# iterate over minibatches for training
schemes_train = ShuffledScheme(examples=len(train_full),
batch_size=BATCH_SIZE)
for batch_idx in schemes_train.get_request_iterator():
d_train_step += 1
t_batch = time.time()
# generate batch of uniform samples
rdm_d = np.random.uniform(-1., 1., size=(len(batch_idx), 100))
rdm_d = rdm_d.astype(theano.config.floatX)
# train with a minibatch on discriminator
if args.captions:
# generate embeddings for the batch
d_capts_batch = utils.captions_to_embedded_matrix(
embedding_model, batch_idx, train_capt)
d_batch_loss = train_discr(train_full[batch_idx], rdm_d,
d_capts_batch)
else:
d_batch_loss = train_discr(train_full[batch_idx], rdm_d)
steps_loss_d.append(d_batch_loss)
steps_loss_g.append(g_batch_loss)
if num_batch % BATCH_PRINT_DELAY == 0:
print '- train discr batch %s, loss %s in %s sec' % (
num_batch, np.round(d_batch_loss, 4),
np.round(time.time() - t_batch, 2))
# check if it is time to train the generator
if d_train_step >= TRAIN_STEPS_DISCR:
# reset discriminator step counter
d_train_step = 0
# train the generator for given number of steps
for _ in xrange(TRAIN_STEPS_GEN):
# generate batch of uniform samples
rdm_g = np.random.uniform(-1.,
1.,
size=(BATCH_SIZE, 100))
rdm_g = rdm_g.astype(theano.config.floatX)
# train with a minibatch on generator
if args.captions:
# sample a random set of captions from current training file
g_batch_idx = np.random.choice(len(train_full),
BATCH_SIZE,
replace=False)
g_capts_batch = utils.captions_to_embedded_matrix(
embedding_model, g_batch_idx, train_capt)
g_batch_loss = train_gen(rdm_g, g_capts_batch)
else:
g_batch_loss = train_gen(rdm_g)
steps_loss_d.append(d_batch_loss)
steps_loss_g.append(g_batch_loss)
if num_batch % BATCH_PRINT_DELAY == 0:
print '- train gen step %s, loss %s' % (
_ + 1, np.round(g_batch_loss, 4))
epoch_loss += d_batch_loss + g_batch_loss
num_batch += 1
train_loss.append(np.round(epoch_loss, 4))
if args.save > 0 and i % args.save == 0:
discriminator, generator = model
utils.save_model(args, discriminator,
'discrminator_epoch_%s.pkl' % i)
utils.save_model(args, generator, 'generator_epoch_%s.pkl' % i)
print '- Epoch train (loss %s) in %s sec' % (
train_loss[i], round(time.time() - t_epoch))
# save losses at each step
utils.dump_objects_output(args, (steps_loss_d, steps_loss_g),
'steps_loss_epoch_%s.pkl' % i)
print 'Training completed.'
# Generate images out of pure noise with random captions (if applicable from valid)
if NB_GEN > 0:
if args.reload is not None:
assert loaded_gen and loaded_discr, 'An error occured during loading, cannot generate.'
save_code = 'rload_%s_%s' % (RELOAD_SRC, RELOAD_ID)
else:
save_code = 'no_reload'
rdm_noise = np.random.uniform(-1., 1., size=(NB_GEN, 100))
rdm_noise = rdm_noise.astype(theano.config.floatX)
# choose random valid file
file_id = np.random.choice(NB_VALID_FILES, 1)
# load file
with open(valid_full_files[file_id], 'r') as f:
valid_full = np.load(f).astype(theano.config.floatX)
if args.captions:
# load file with the captions
with open(valid_capt_files[file_id], 'rb') as f:
valid_capt = pkl.load(f)
# pick a given number of images from that file
batch_valid = np.random.choice(len(valid_capt),
NB_GEN,
replace=False)
captions = utils.captions_to_embedded_matrix(
embedding_model, batch_valid, valid_capt)
# captions = np.empty((NB_GEN, 300), dtype=theano.config.floatX) # used for debugging
# make predictions
imgs_noise, probs_noise = predict(rdm_noise, captions)
else:
# make predictions
imgs_noise, probs_noise = predict(rdm_noise)
if args.verbose:
print probs_noise
# save images
true_imgs = valid_full[batch_valid]
utils.save_pics_gan(args,
imgs_noise,
'noise_caption_%s' % args.captions + save_code,
show=False,
save=True,
tanh=False)
utils.save_pics_gan(args,
true_imgs,
'true_caption_%s' % args.captions + save_code,
show=False,
save=True,
tanh=False)
if args.captions:
utils.save_captions(args, save_code, valid_capt, batch_valid)
if args.mila:
utils.move_results_from_local()
# if running on server (MILA), copy dataset locally
dataset_path = utils.init_dataset(args, 'mscoco_inpainting')
train_path = os.path.join(dataset_path, 'preprocessed/train2014')
valid_path = os.path.join(dataset_path, 'preprocessed/val2014')
NB_TRAIN = 82782
NB_VALID = 40504
MAX_IMGS = 15000
train_count = 0
valid_count = 0
# process for training dataset
schemes_train = ShuffledScheme(examples=NB_TRAIN, batch_size=MAX_IMGS)
for i, batch_idx in enumerate(schemes_train.get_request_iterator()):
print 'train file %s in progress...' % i
file_name_full_img = os.path.join(train_path, 'train_%s_full.npy' % i)
file_name_cter_img = os.path.join(train_path, 'train_%s_cter.npy' % i)
file_name_capt_dic = os.path.join(train_path, 'train_%s_capt.pkl' % i)
t = time.time()
# get large 'batch' of data for saving in file
full_img, targts, capts, color_count = utils.get_batch_data(
batch_idx, mscoco=dataset_path, split="train2014", crop=False)
train_count += color_count
print 'train file %s in completed.' % i
print '%s imgs saved, training total is now %s' % (color_count,