def test_process_parallel_data_source_with_index_mapping(process_pool):
from batchup import data_source
X = np.zeros((100,), dtype=int)
indices = np.random.RandomState(12345).permutation(100)
indices0 = indices[:50]
indices1 = indices[50:]
X[indices1] = 1
ds0 = data_source.ArrayDataSource([X], indices=indices0)
ds1 = data_source.ArrayDataSource([X], indices=indices1)
pds0 = process_pool.parallel_data_source(ds0)
pds1 = process_pool.parallel_data_source(ds1)
# Check number of samples
assert ds0.num_samples() == 50
assert ds1.num_samples() == 50
assert pds0.num_samples() == 50
assert pds1.num_samples() == 50
BATCHSIZE = 10
for batch_X, in pds0.batch_iterator(
batch_size=BATCHSIZE, shuffle=np.random.RandomState(12345)):
# Ensure that all the return values are 0
assert (batch_X == 0).all()
for batch_X, in pds1.batch_iterator(
batch_size=BATCHSIZE, shuffle=np.random.RandomState(12345)):
# Ensure that all the return values are 0
assert (batch_X == 1).all()
def predict(self, input_imgs, batch_size):
predict_ds = data_source.ArrayDataSource([input_imgs])
predict_batch_iter = predict_ds.batch_iterator(batch_size=batch_size)
predict_batch_total = len(input_imgs) // batch_size if len(
input_imgs
) % batch_size == 0 else len(input_imgs) // batch_size + 1
epoch_logits = []
predict_batch_tqdm = tqdm(predict_batch_iter,
total=predict_batch_total)
for predict_step, [minibatch_valid_imgs
] in enumerate(predict_batch_tqdm):
predict_batch_tqdm.set_description('predicting...')
train_feed = {
self._input: minibatch_valid_imgs,
self._dropout_rate: 1.0,
}
minibatch_logits = self._sess.run(self._y_logits_op,
feed_dict=train_feed)
epoch_logits.append(minibatch_logits)
return np.concatenate(epoch_logits, axis=0)
def predict(self, input_filenames, batch_size):
input_filename_idxs = np.array([i for i in range(0, len(input_filenames))])
predict_ds = data_source.ArrayDataSource([input_filename_idxs])
predict_batch_iter = predict_ds.batch_iterator(batch_size=batch_size)
predict_batch_total = len(input_filenames) // batch_size if len(input_filenames) % batch_size == 0 else len(input_filenames) // batch_size + 1
epoch_logits = []
predict_batch_tqdm = tqdm(predict_batch_iter, total=predict_batch_total)
for predict_step , [minibatch_predict_filename_idxs] in enumerate(predict_batch_tqdm):
predict_batch_tqdm.set_description('predicting...')
minibatch_predict_filenames = itemgetter(*minibatch_predict_filename_idxs)(input_filenames)
minibatch_predict_files = self.minibatch_preprocessing(minibatch_predict_filenames)
logit_feed = {
self._input: minibatch_predict_files,
}
minibatch_logits = self._sess.run(self._logits , feed_dict=logit_feed)
epoch_logits.append(minibatch_logits)
return np.concatenate(epoch_logits, axis=0)
def train(self, input_imgs, label_imgs, batch_size, dropout_rate,
learning_rate):
train_ds = data_source.ArrayDataSource([input_imgs, label_imgs])
train_batch_iter = train_ds.batch_iterator(batch_size=batch_size)
train_batch_total = len(input_imgs) // batch_size if len(
input_imgs
) % batch_size == 0 else len(input_imgs) // batch_size + 1
epoch_train_loss = []
epoch_train_iou = []
train_batch_tqdm = tqdm(train_batch_iter, total=train_batch_total)
for train_step, [minibatch_train_imgs, minibatch_train_labels
] in enumerate(train_batch_tqdm):
train_batch_tqdm.set_description('training...')
train_feed = {
self._input: minibatch_train_imgs,
self._labels: minibatch_train_labels,
self._dropout_rate: dropout_rate,
self._learning_rate: learning_rate
}
minibatch_loss, loss_summary, _ = \
self._sess.run([self._loss_op, self._loss_summary , self._train_op], feed_dict=train_feed)
eval_feed = {
self._input: minibatch_train_imgs,
self._labels: minibatch_train_labels,
self._dropout_rate: 1.0,
}
minibatch_eval, eval_summary = self._sess.run(
[self._eval_op, self._eval_summary], feed_dict=train_feed)
epoch_train_loss.append(minibatch_loss)
epoch_train_iou.append(minibatch_eval)
global_step = self._sess.run(self._global_step)
self._train_summary_writer.add_summary(loss_summary,
global_step=global_step)
self._train_summary_writer.flush()
self._train_summary_writer.add_summary(eval_summary,
global_step=global_step)
self._train_summary_writer.flush()
train_batch_tqdm.set_postfix(minibatch_loss=minibatch_loss,
minibatch_iou=minibatch_eval)
if self._hook and global_step % self._hook_every_n == 0:
self._hook()
epoch_train_loss = np.mean(epoch_train_loss)
epoch_train_iou = np.mean(epoch_train_iou)
return epoch_train_loss, epoch_train_iou
def train(self, input_filenames, labels, batch_size, learning_rate):
input_filename_idxs = np.array([i for i in range(0, len(input_filenames))])
train_ds = data_source.ArrayDataSource([input_filename_idxs, labels])
train_batch_iter = train_ds.batch_iterator(batch_size=batch_size)
train_batch_total = len(input_filenames) // batch_size if len(input_filenames) % batch_size == 0 else len(input_filenames) // batch_size + 1
epoch_train_loss = []
epoch_train_accuracy = []
train_batch_tqdm = tqdm(train_batch_iter, total=train_batch_total)
for train_step, [minibatch_filename_idxs, minibatch_train_labels] in enumerate(train_batch_tqdm):
train_batch_tqdm.set_description('training...')
minibatch_filenames = itemgetter(*minibatch_filename_idxs)(input_filenames)
minibatch_files = self.minibatch_preprocessing(minibatch_filenames)
train_feed = {
self._input: minibatch_files,
self._labels: minibatch_train_labels,
self._learning_rate: learning_rate,
self._is_train: True,
}
minibatch_loss, loss_summary, _ = \
self._sess.run([self._loss_op, self._loss_summary , self._train_op], feed_dict=train_feed)
train_feed = {
self._input: minibatch_files,
self._labels: minibatch_train_labels,
}
minibatch_accuracy, accuracy_summary = self._sess.run([self._accuracy_op, self._accuracy_summary], feed_dict=train_feed)
epoch_train_loss.append(minibatch_loss)
epoch_train_accuracy.append(minibatch_accuracy)
global_step = self._sess.run(self._global_step)
self._train_summary_writer.add_summary(loss_summary, global_step=global_step)
self._train_summary_writer.flush()
self._train_summary_writer.add_summary(accuracy_summary, global_step=global_step)
self._train_summary_writer.flush()
train_batch_tqdm.set_postfix(minibatch_loss=minibatch_loss, minibatch_accuracy=minibatch_accuracy)
if self._hook and global_step % self._hook_every_n == 0:
self._hook()
epoch_train_loss = np.mean(epoch_train_loss)
epoch_train_accuracy = np.mean(epoch_train_accuracy)
return epoch_train_loss, epoch_train_accuracy
def validate(self, input_imgs, label_imgs, batch_size):
valid_ds = data_source.ArrayDataSource([input_imgs, label_imgs])
valid_batch_iter = valid_ds.batch_iterator(batch_size=batch_size)
valid_batch_total = len(input_imgs) // batch_size if len(
input_imgs
) % batch_size == 0 else len(input_imgs) // batch_size + 1
epoch_valid_loss = []
epoch_valid_iou = []
valid_batch_tqdm = tqdm(valid_batch_iter, total=valid_batch_total)
for valid_step, [minibatch_valid_imgs, minibatch_valid_labels
] in enumerate(valid_batch_tqdm):
valid_batch_tqdm.set_description('validating...')
train_feed = {
self._input: minibatch_valid_imgs,
self._labels: minibatch_valid_labels,
self._dropout_rate: 1.0,
}
minibatch_loss, minibatch_eval = \
self._sess.run([self._loss_op, self._eval_op], feed_dict=train_feed)
epoch_valid_loss.append(minibatch_loss)
epoch_valid_iou.append(minibatch_eval)
valid_batch_tqdm.set_postfix(minibatch_loss=minibatch_loss,
minibatch_iou=minibatch_eval)
epoch_valid_loss = np.mean(epoch_valid_loss)
epoch_valid_iou = np.mean(epoch_valid_iou)
global_step = self._sess.run(self._global_step)
valid_loss_summary = tf.Summary()
valid_loss_summary.value.add(tag="loss", simple_value=epoch_valid_loss)
valid_loss_summary.value.add(tag="iou_metric",
simple_value=epoch_valid_iou)
self._validation_summary_writer.add_summary(valid_loss_summary,
global_step=global_step)
self._validation_summary_writer.flush()
return epoch_valid_loss, epoch_valid_iou
def validate(self, input_filenames, labels, batch_size):
input_filename_idxs = np.array([i for i in range(0, len(input_filenames))])
valid_ds = data_source.ArrayDataSource([input_filename_idxs, labels])
valid_batch_iter = valid_ds.batch_iterator(batch_size=batch_size)
valid_batch_total = len(input_filenames) // batch_size if len(input_filenames) % batch_size == 0 else len(input_filenames) // batch_size + 1
epoch_valid_loss = []
epoch_valid_accuracy = []
valid_batch_tqdm = tqdm(valid_batch_iter, total=valid_batch_total)
for valid_step, [minibatch_valid_filename_idxs, minibatch_valid_labels] in enumerate(valid_batch_tqdm):
valid_batch_tqdm.set_description('validating...')
minibatch_valid_filenames = itemgetter(*minibatch_valid_filename_idxs)(input_filenames)
minibatch_valid_files = self.minibatch_preprocessing(minibatch_valid_filenames)
valid_feed = {
self._input: minibatch_valid_files,
self._labels: minibatch_valid_labels,
}
minibatch_loss, minibatch_accuracy = \
self._sess.run([self._loss_op, self._accuracy_op], feed_dict=valid_feed)
epoch_valid_loss.append(minibatch_loss)
epoch_valid_accuracy.append(minibatch_accuracy)
valid_batch_tqdm.set_postfix(minibatch_loss=minibatch_loss, minibatch_accuracy=minibatch_accuracy)
epoch_valid_loss = np.mean(epoch_valid_loss)
epoch_valid_accuracy = np.mean(epoch_valid_accuracy)
global_step = self._sess.run(self._global_step)
valid_loss_summary = tf.Summary()
valid_loss_summary.value.add(tag="loss", simple_value=epoch_valid_loss)
valid_loss_summary.value.add(tag="accuracy", simple_value=epoch_valid_accuracy)
self._validation_summary_writer.add_summary(valid_loss_summary, global_step=global_step)
self._validation_summary_writer.flush()
return epoch_valid_loss, epoch_valid_accuracy
def test():
ds = data_source.ArrayDataSource([scaled_test, label_test])
# Iterate over samples, drawing batches of 64 elements in random order
for (test_data, test_target) in ds.batch_iterator(
batch_size=1000,
shuffle=True): # shuffle true will randomise every batch
test_data = test_data.reshape(-1, 1, 28, 28)
# cv2.imshow("image",np.reshape(test_data[5],newshape=(28,28,1)))
# cv2.waitKey(500)
# cv2.destroyAllWindows()
test_data1, target = torch.tensor(test_data), torch.tensor(
test_target).long()
optimizer.zero_grad()
output = model(test_data1)
out = torch.max(output, 1)[1]
match = torch.eq(out, target.squeeze())
all1 = torch.sum(match)
acc = all1.item() / len(match)
print("test accuracy is ", acc)
break
def train(loop):
run = 0
ds = data_source.ArrayDataSource([scaled_input, y])
# Iterate over samples, drawing batches of 64 elements in random order
for (data, target) in ds.batch_iterator(
batch_size=1000,
shuffle=True): # shuffle true will randomise every batch
new_data = data.reshape(-1, 1, 28, 28)
cool_data = new_data[0]
# cv2.imshow("image",np.reshape(cool_data,newshape=(28,28,1)))
# cv2.waitKey(500)
# cv2.destroyAllWindows()
# data, target=torch.from_numpy(new_data).double(),torch.from_numpy(target).double()
# inputs, labels = Variable(inputs.cuda()), Variable(labels.cuda())#this is for cuda gpu
data, target = torch.tensor(new_data), torch.tensor(target).long()
optimizer.zero_grad()
output = model(data)
# this is to find accuracy
out = torch.max(output, 1)[1]
match = torch.eq(out, target.squeeze())
all1 = torch.sum(match)
acc = all1.item() / len(match)
# _, predicted = torch.max(target.data, 1)
# total += labels.size(0)
# correct += (predicted == target).sum().item()
# print('Accuracy of the network on the 10000 test images: %d %%' % (100 * correct / total))
#
loss = F.nll_loss(output, target.squeeze())
# loss = criterea(output,torch.max(target, 1)[0])#max returns two values it means its return max value in each row and[0 ] means its take first return value
# loss = criterea(output, target.squeeze())
loss.backward()
optimizer.step()
run += 1
print(
"Epochs : {ep} - Run Cycle : {rc} - Training Loss : {tl} - Training accuracy: {ta} "
.format(ep=loop, rc=run, tl=loss.item(), ta=acc))
def experiment(exp, arch, learning_rate, standardise_samples, affine_std,
xlat_range, hflip, intens_flip, intens_scale_range,
intens_offset_range, gaussian_noise_std, num_epochs, batch_size,
seed, log_file, device):
import os
import sys
import cmdline_helpers
if log_file == '':
log_file = 'output_aug_log_{}.txt'.format(exp)
elif log_file == 'none':
log_file = None
if log_file is not None:
if os.path.exists(log_file):
print('Output log file {} already exists'.format(log_file))
return
intens_scale_range_lower, intens_scale_range_upper, intens_offset_range_lower, intens_offset_range_upper = \
cmdline_helpers.intens_aug_options(intens_scale_range, intens_offset_range)
import time
import math
import numpy as np
from batchup import data_source, work_pool
import data_loaders
import standardisation
import network_architectures
import augmentation
import torch, torch.cuda
from torch import nn
from torch.nn import functional as F
with torch.cuda.device(device):
pool = work_pool.WorkerThreadPool(2)
n_chn = 0
if exp == 'svhn_mnist':
d_source = data_loaders.load_svhn(zero_centre=False,
greyscale=True)
d_target = data_loaders.load_mnist(invert=False,
zero_centre=False,
pad32=True,
val=False)
elif exp == 'mnist_svhn':
d_source = data_loaders.load_mnist(invert=False,
zero_centre=False,
pad32=True)
d_target = data_loaders.load_svhn(zero_centre=False,
greyscale=True,
val=False)
elif exp == 'svhn_mnist_rgb':
d_source = data_loaders.load_svhn(zero_centre=False,
greyscale=False)
d_target = data_loaders.load_mnist(invert=False,
zero_centre=False,
pad32=True,
val=False,
rgb=True)
elif exp == 'mnist_svhn_rgb':
d_source = data_loaders.load_mnist(invert=False,
zero_centre=False,
pad32=True,
rgb=True)
d_target = data_loaders.load_svhn(zero_centre=False,
greyscale=False,
val=False)
elif exp == 'cifar_stl':
d_source = data_loaders.load_cifar10(range_01=False)
d_target = data_loaders.load_stl(zero_centre=False, val=False)
elif exp == 'stl_cifar':
d_source = data_loaders.load_stl(zero_centre=False)
d_target = data_loaders.load_cifar10(range_01=False, val=False)
elif exp == 'mnist_usps':
d_source = data_loaders.load_mnist(zero_centre=False)
d_target = data_loaders.load_usps(zero_centre=False,
scale28=True,
val=False)
elif exp == 'usps_mnist':
d_source = data_loaders.load_usps(zero_centre=False, scale28=True)
d_target = data_loaders.load_mnist(zero_centre=False, val=False)
elif exp == 'syndigits_svhn':
d_source = data_loaders.load_syn_digits(zero_centre=False)
d_target = data_loaders.load_svhn(zero_centre=False, val=False)
elif exp == 'svhn_syndigits':
d_source = data_loaders.load_svhn(zero_centre=False, val=False)
d_target = data_loaders.load_syn_digits(zero_centre=False)
elif exp == 'synsigns_gtsrb':
d_source = data_loaders.load_syn_signs(zero_centre=False)
d_target = data_loaders.load_gtsrb(zero_centre=False, val=False)
elif exp == 'gtsrb_synsigns':
d_source = data_loaders.load_gtsrb(zero_centre=False, val=False)
d_target = data_loaders.load_syn_signs(zero_centre=False)
else:
print('Unknown experiment type \'{}\''.format(exp))
return
# Delete the training ground truths as we should not be using them
del d_target.train_y
if standardise_samples:
standardisation.standardise_dataset(d_source)
standardisation.standardise_dataset(d_target)
n_classes = d_source.n_classes
print('Loaded data')
if arch == '':
if exp in {'mnist_usps', 'usps_mnist'}:
arch = 'mnist-bn-32-64-256'
if exp in {'svhn_mnist', 'mnist_svhn'}:
arch = 'grey-32-64-128-gp'
if exp in {
'cifar_stl', 'stl_cifar', 'syndigits_svhn',
'svhn_syndigits', 'svhn_mnist_rgb', 'mnist_svhn_rgb'
}:
arch = 'rgb-48-96-192-gp'
if exp in {'synsigns_gtsrb', 'gtsrb_synsigns'}:
arch = 'rgb40-48-96-192-384-gp'
net_class, expected_shape = network_architectures.get_net_and_shape_for_architecture(
arch)
if expected_shape != d_source.train_X.shape[1:]:
print(
'Architecture {} not compatible with experiment {}; it needs samples of shape {}, '
'data has samples of shape {}'.format(
arch, exp, expected_shape, d_source.train_X.shape[1:]))
return
net = net_class(n_classes).cuda()
params = list(net.parameters())
optimizer = torch.optim.Adam(params, lr=learning_rate)
classification_criterion = nn.CrossEntropyLoss()
print('Built network')
aug = augmentation.ImageAugmentation(
hflip,
xlat_range,
affine_std,
intens_scale_range_lower=intens_scale_range_lower,
intens_scale_range_upper=intens_scale_range_upper,
intens_offset_range_lower=intens_offset_range_lower,
intens_offset_range_upper=intens_offset_range_upper,
intens_flip=intens_flip,
gaussian_noise_std=gaussian_noise_std)
def augment(X_sup, y_sup):
X_sup = aug.augment(X_sup)
return [X_sup, y_sup]
def f_train(X_sup, y_sup):
X_sup = torch.autograd.Variable(torch.from_numpy(X_sup).cuda())
y_sup = torch.autograd.Variable(
torch.from_numpy(y_sup).long().cuda())
optimizer.zero_grad()
net.train(mode=True)
sup_logits_out = net(X_sup)
# Supervised classification loss
clf_loss = classification_criterion(sup_logits_out, y_sup)
loss_expr = clf_loss
loss_expr.backward()
optimizer.step()
n_samples = X_sup.size()[0]
return float(clf_loss.data.cpu().numpy()) * n_samples
print('Compiled training function')
def f_pred_src(X_sup):
X_var = torch.autograd.Variable(torch.from_numpy(X_sup).cuda())
net.train(mode=False)
return F.softmax(net(X_var)).data.cpu().numpy()
def f_pred_tgt(X_sup):
X_var = torch.autograd.Variable(torch.from_numpy(X_sup).cuda())
net.train(mode=False)
return F.softmax(net(X_var)).data.cpu().numpy()
def f_eval_src(X_sup, y_sup):
y_pred_prob = f_pred_src(X_sup)
y_pred = np.argmax(y_pred_prob, axis=1)
return float((y_pred != y_sup).sum())
def f_eval_tgt(X_sup, y_sup):
y_pred_prob = f_pred_tgt(X_sup)
y_pred = np.argmax(y_pred_prob, axis=1)
return float((y_pred != y_sup).sum())
print('Compiled evaluation function')
# Setup output
def log(text):
print(text)
if log_file is not None:
with open(log_file, 'a') as f:
f.write(text + '\n')
f.flush()
f.close()
cmdline_helpers.ensure_containing_dir_exists(log_file)
# Report setttings
log('sys.argv={}'.format(sys.argv))
# Report dataset size
log('Dataset:')
log('SOURCE Train: X.shape={}, y.shape={}'.format(
d_source.train_X.shape, d_source.train_y.shape))
log('SOURCE Test: X.shape={}, y.shape={}'.format(
d_source.test_X.shape, d_source.test_y.shape))
log('TARGET Train: X.shape={}'.format(d_target.train_X.shape))
log('TARGET Test: X.shape={}, y.shape={}'.format(
d_target.test_X.shape, d_target.test_y.shape))
print('Training...')
train_ds = data_source.ArrayDataSource(
[d_source.train_X, d_source.train_y]).map(augment)
source_test_ds = data_source.ArrayDataSource(
[d_source.test_X, d_source.test_y])
target_test_ds = data_source.ArrayDataSource(
[d_target.test_X, d_target.test_y])
if seed != 0:
shuffle_rng = np.random.RandomState(seed)
else:
shuffle_rng = np.random
best_src_test_err = 1.0
for epoch in range(num_epochs):
t1 = time.time()
train_res = train_ds.batch_map_mean(f_train,
batch_size=batch_size,
shuffle=shuffle_rng)
train_clf_loss = train_res[0]
src_test_err, = source_test_ds.batch_map_mean(
f_eval_src, batch_size=batch_size * 4)
tgt_test_err, = target_test_ds.batch_map_mean(
f_eval_tgt, batch_size=batch_size * 4)
t2 = time.time()
if src_test_err < best_src_test_err:
log('*** Epoch {} took {:.2f}s: TRAIN clf loss={:.6f}; '
'SRC TEST ERR={:.3%}, TGT TEST err={:.3%}'.format(
epoch, t2 - t1, train_clf_loss, src_test_err,
tgt_test_err))
best_src_test_err = src_test_err
else:
log('Epoch {} took {:.2f}s: TRAIN clf loss={:.6f}; '
'SRC TEST ERR={:.3%}, TGT TEST err={:.3%}'.format(
epoch, t2 - t1, train_clf_loss, src_test_err,
tgt_test_err))
def experiment(exp, arch, rnd_init, img_size, standardise_samples,
learning_rate, pretrained_lr_factor, fix_layers, double_softmax,
use_dropout, scale_u_range, scale_x_range, scale_y_range,
affine_std, xlat_range, rot_std, hflip, intens_scale_range,
colour_rot_std, colour_off_std, greyscale, img_pad_width,
num_epochs, batch_size, seed, log_file, result_file,
hide_progress_bar, subsetsize, subsetseed, device):
settings = locals().copy()
if rnd_init:
if fix_layers != '':
print('`rnd_init` and `fix_layers` are mutually exclusive')
return
import os
import sys
import cmdline_helpers
fix_layers = [lyr.strip() for lyr in fix_layers.split(',')]
if log_file == '':
log_file = 'output_aug_log_{}.txt'.format(exp)
elif log_file == 'none':
log_file = None
if log_file is not None:
if os.path.exists(log_file):
print('Output log file {} already exists'.format(log_file))
return
intens_scale_range_lower, intens_scale_range_upper = cmdline_helpers.colon_separated_range(
intens_scale_range)
scale_u_range = cmdline_helpers.colon_separated_range(scale_u_range)
scale_x_range = cmdline_helpers.colon_separated_range(scale_x_range)
scale_y_range = cmdline_helpers.colon_separated_range(scale_y_range)
import time
import tqdm
import math
import tables
import numpy as np
from batchup import data_source, work_pool
import image_dataset, visda17_dataset, office_dataset
import network_architectures
import augmentation
import image_transforms
from sklearn.model_selection import StratifiedShuffleSplit
import torch, torch.cuda
from torch import nn
from torch.nn import functional as F
if hide_progress_bar:
progress_bar = None
else:
progress_bar = tqdm.tqdm
with torch.cuda.device(device):
pool = work_pool.WorkerThreadPool(4)
n_chn = 0
half_batch_size = batch_size // 2
RESNET_ARCHS = {'resnet50', 'resnet101', 'resnet152'}
RNDINIT_ARCHS = {'vgg13_48_gp'}
if arch == '':
if exp in {'train_val', 'train_test'}:
arch = 'resnet50'
if arch in RESNET_ARCHS and not rnd_init:
mean_value = np.array([0.485, 0.456, 0.406])
std_value = np.array([0.229, 0.224, 0.225])
elif arch in RNDINIT_ARCHS:
mean_value = np.array([0.5, 0.5, 0.5])
std_value = np.array([0.5, 0.5, 0.5])
rnd_init = True
else:
mean_value = std_value = None
img_shape = (img_size, img_size)
img_padding = (img_pad_width, img_pad_width)
if exp == 'visda_train_val':
d_source = visda17_dataset.TrainDataset(img_size=img_shape,
range01=True,
rgb_order=True)
d_target = visda17_dataset.ValidationDataset(img_size=img_shape,
range01=True,
rgb_order=True)
elif exp == 'visda_train_test':
d_source = visda17_dataset.TrainDataset(img_size=img_shape,
range01=True,
rgb_order=True)
d_target = visda17_dataset.TestDataset(img_size=img_shape,
range01=True,
rgb_order=True)
elif exp == 'office_amazon_dslr':
d_source = office_dataset.OfficeAmazonDataset(img_size=img_shape,
range01=True,
rgb_order=True)
d_target = office_dataset.OfficeDSLRDataset(img_size=img_shape,
range01=True,
rgb_order=True)
elif exp == 'office_amazon_webcam':
d_source = office_dataset.OfficeAmazonDataset(img_size=img_shape,
range01=True,
rgb_order=True)
d_target = office_dataset.OfficeWebcamDataset(img_size=img_shape,
range01=True,
rgb_order=True)
elif exp == 'office_dslr_amazon':
d_source = office_dataset.OfficeDSLRDataset(img_size=img_shape,
range01=True,
rgb_order=True)
d_target = office_dataset.OfficeAmazonDataset(img_size=img_shape,
range01=True,
rgb_order=True)
elif exp == 'office_dslr_webcam':
d_source = office_dataset.OfficeDSLRDataset(img_size=img_shape,
range01=True,
rgb_order=True)
d_target = office_dataset.OfficeWebcamDataset(img_size=img_shape,
range01=True,
rgb_order=True)
elif exp == 'office_webcam_amazon':
d_source = office_dataset.OfficeWebcamDataset(img_size=img_shape,
range01=True,
rgb_order=True)
d_target = office_dataset.OfficeAmazonDataset(img_size=img_shape,
range01=True,
rgb_order=True)
elif exp == 'office_webcam_dslr':
d_source = office_dataset.OfficeWebcamDataset(img_size=img_shape,
range01=True,
rgb_order=True)
d_target = office_dataset.OfficeDSLRDataset(img_size=img_shape,
range01=True,
rgb_order=True)
else:
print('Unknown experiment type \'{}\''.format(exp))
return
#
# Result file
#
if result_file != '':
cmdline_helpers.ensure_containing_dir_exists(result_file)
h5_filters = tables.Filters(complevel=9, complib='blosc')
f_target_pred = tables.open_file(result_file, mode='w')
g_tgt_pred = f_target_pred.create_group(f_target_pred.root,
'target_pred_y',
'Target prediction')
arr_tgt_pred = f_target_pred.create_earray(
g_tgt_pred,
'y',
tables.Float32Atom(),
(0, len(d_target.images), d_target.n_classes),
filters=h5_filters)
else:
f_target_pred = None
g_tgt_pred = None
arr_tgt_pred = None
# Delete the training ground truths as we should not be using them
# del d_target.y
n_classes = d_source.n_classes
print('Loaded data')
net_class = network_architectures.get_build_fn_for_architecture(arch)
net = net_class(n_classes, img_size, use_dropout, not rnd_init).cuda()
if arch in RESNET_ARCHS and not rnd_init:
named_params = list(net.named_parameters())
new_params = []
pretrained_params = []
for name, param in named_params:
if name.startswith('new_'):
new_params.append(param)
else:
fix = False
for lyr in fix_layers:
if name.startswith(lyr + '.'):
fix = True
break
if not fix:
pretrained_params.append(param)
else:
print('Fixing param {}'.format(name))
param.requires_grad = False
new_optimizer = torch.optim.Adam(new_params, lr=learning_rate)
if len(pretrained_params) > 0:
pretrained_optimizer = torch.optim.Adam(pretrained_params,
lr=learning_rate *
pretrained_lr_factor)
else:
pretrained_optimizer = None
else:
new_optimizer = torch.optim.Adam(net.parameters(),
lr=learning_rate)
pretrained_optimizer = None
classification_criterion = nn.CrossEntropyLoss()
print('Built network')
# Image augmentation
aug = augmentation.ImageAugmentation(
hflip,
xlat_range,
affine_std,
rot_std=rot_std,
intens_scale_range_lower=intens_scale_range_lower,
intens_scale_range_upper=intens_scale_range_upper,
colour_rot_std=colour_rot_std,
colour_off_std=colour_off_std,
greyscale=greyscale,
scale_u_range=scale_u_range,
scale_x_range=scale_x_range,
scale_y_range=scale_y_range)
test_aug = augmentation.ImageAugmentation(hflip,
xlat_range,
0.0,
rot_std=0.0,
scale_u_range=scale_u_range,
scale_x_range=scale_x_range,
scale_y_range=scale_y_range)
border_value = int(np.mean(mean_value) * 255 + 0.5)
sup_xf = image_transforms.Compose(
image_transforms.ScaleCropAndAugmentAffine(img_shape, img_padding,
True, aug, border_value,
mean_value, std_value),
image_transforms.ToTensor(),
)
test_xf = image_transforms.Compose(
image_transforms.ScaleAndCrop(img_shape, img_padding, False),
image_transforms.ToTensor(),
image_transforms.Standardise(mean_value, std_value),
)
test_xf_aug_mult = image_transforms.Compose(
image_transforms.ScaleCropAndAugmentAffineMultiple(
16, img_shape, img_padding, True, test_aug, border_value,
mean_value, std_value),
image_transforms.ToTensorMultiple(),
)
def augment(X_sup, y_sup):
X_sup = sup_xf(X_sup)[0]
return X_sup, y_sup
_one = torch.autograd.Variable(
torch.from_numpy(np.array([1.0]).astype(np.float32)).cuda())
def f_train(X_sup, y_sup):
X_sup = torch.autograd.Variable(torch.from_numpy(X_sup).cuda())
y_sup = torch.autograd.Variable(
torch.from_numpy(y_sup).long().cuda())
if pretrained_optimizer is not None:
pretrained_optimizer.zero_grad()
new_optimizer.zero_grad()
net.train(mode=True)
sup_logits_out = net(X_sup)
# Supervised classification loss
if double_softmax:
clf_loss = classification_criterion(F.softmax(sup_logits_out),
y_sup)
else:
clf_loss = classification_criterion(sup_logits_out, y_sup)
loss_expr = clf_loss
loss_expr.backward()
if pretrained_optimizer is not None:
pretrained_optimizer.step()
new_optimizer.step()
n_samples = X_sup.size()[0]
return (float(clf_loss.data.cpu()[0]) * n_samples, )
print('Compiled training function')
def f_pred(X_sup):
X_var = torch.autograd.Variable(torch.from_numpy(X_sup).cuda())
net.train(mode=False)
return F.softmax(net(X_var)).data.cpu().numpy()
def f_pred_tgt_mult(X_sup):
net.train(mode=False)
y_pred_aug = []
for aug_i in range(len(X_sup)):
X_var = torch.autograd.Variable(
torch.from_numpy(X_sup[aug_i, ...]).cuda())
y_pred = F.softmax(net(X_var)).data.cpu().numpy()
y_pred_aug.append(y_pred[None, ...])
y_pred_aug = np.concatenate(y_pred_aug, axis=0)
return (y_pred_aug.mean(axis=0), )
print('Compiled evaluation function')
# Setup output
def log(text):
print(text)
if log_file is not None:
with open(log_file, 'a') as f:
f.write(text + '\n')
f.flush()
f.close()
cmdline_helpers.ensure_containing_dir_exists(log_file)
# Report setttings
log('Program = {}'.format(sys.argv[0]))
log('Settings: {}'.format(', '.join([
'{}={}'.format(key, settings[key])
for key in sorted(list(settings.keys()))
])))
# Report dataset size
log('Dataset:')
print('SOURCE len(X)={}, y.shape={}'.format(len(d_source.images),
d_source.y.shape))
print('TARGET len(X)={}'.format(len(d_target.images)))
# Subset
source_indices, target_indices, n_src, n_tgt = image_dataset.subset_indices(
d_source, d_target, subsetsize, subsetseed)
n_train_batches = n_src // batch_size + 1
n_test_batches = n_tgt // (batch_size * 2) + 1
print('Training...')
train_ds = data_source.ArrayDataSource([d_source.images, d_source.y],
indices=source_indices)
train_ds = train_ds.map(augment)
train_ds = pool.parallel_data_source(train_ds,
batch_buffer_size=min(
20, n_train_batches))
# source_test_ds = data_source.ArrayDataSource([d_source.images])
# source_test_ds = pool.parallel_data_source(source_test_ds)
target_ds_for_test = data_source.ArrayDataSource(
[d_target.images], indices=target_indices)
target_test_ds = target_ds_for_test.map(test_xf)
target_test_ds = pool.parallel_data_source(target_test_ds,
batch_buffer_size=min(
20, n_test_batches))
target_mult_test_ds = target_ds_for_test.map(test_xf_aug_mult)
target_mult_test_ds = pool.parallel_data_source(target_mult_test_ds,
batch_buffer_size=min(
20,
n_test_batches))
if seed != 0:
shuffle_rng = np.random.RandomState(seed)
else:
shuffle_rng = np.random
if d_target.has_ground_truth:
evaluator = d_target.prediction_evaluator(target_indices)
else:
evaluator = None
train_batch_iter = train_ds.batch_iterator(batch_size=batch_size,
shuffle=shuffle_rng)
for epoch in range(num_epochs):
t1 = time.time()
test_batch_iter = target_test_ds.batch_iterator(
batch_size=batch_size)
train_clf_loss, = data_source.batch_map_mean(
f_train,
train_batch_iter,
n_batches=n_train_batches,
progress_iter_func=progress_bar)
# train_clf_loss, train_unsup_loss, mask_rate, train_align_loss = train_ds.batch_map_mean(
# lambda *x: 1.0, batch_size=batch_size, shuffle=shuffle_rng, n_batches=n_train_batches,
# progress_iter_func=progress_bar)
if d_target.has_ground_truth or arr_tgt_pred is not None:
tgt_pred_prob_y, = data_source.batch_map_concat(
f_pred, test_batch_iter, progress_iter_func=progress_bar)
else:
tgt_pred_prob_y = None
train_batch_iter = train_ds.batch_iterator(batch_size=batch_size,
shuffle=shuffle_rng)
if d_target.has_ground_truth:
mean_class_acc, cls_acc_str = evaluator.evaluate(
tgt_pred_prob_y)
t2 = time.time()
log('Epoch {} took {:.2f}s: TRAIN clf loss={:.6f}; '
'TGT mean class acc={:.3%}'.format(epoch, t2 - t1,
train_clf_loss,
mean_class_acc))
log(' per class: {}'.format(cls_acc_str))
else:
t2 = time.time()
log('Epoch {} took {:.2f}s: TRAIN clf loss={:.6f}'.format(
epoch, t2 - t1, train_clf_loss))
# Save results
if arr_tgt_pred is not None:
arr_tgt_pred.append(tgt_pred_prob_y[None,
...].astype(np.float32))
# Predict on test set, using augmentation
tgt_aug_pred_prob_y, = target_mult_test_ds.batch_map_concat(
f_pred_tgt_mult,
batch_size=batch_size,
progress_iter_func=progress_bar)
if d_target.has_ground_truth:
aug_mean_class_acc, aug_cls_acc_str = evaluator.evaluate(
tgt_aug_pred_prob_y)
log('FINAL: TGT AUG mean class acc={:.3%}'.format(
aug_mean_class_acc))
log(' per class: {}'.format(aug_cls_acc_str))
if f_target_pred is not None:
f_target_pred.create_array(g_tgt_pred, 'y_prob', tgt_pred_prob_y)
f_target_pred.create_array(g_tgt_pred, 'y_prob_aug',
tgt_aug_pred_prob_y)
f_target_pred.close()
rpn = []
feat_stride = 16
anchor_scale = [8, 16, 32]
imageNameFile = "../../../Datasets/VOCdevkit/VOC2012/ImageSets/Main/aeroplane_train.txt"
vocPath = "../../../Datasets/VOCdevkit/VOC2012"
Image_data, boundingBX_labels, im_dims = read_pascal_voc.prepareBatch(
0, 5, imageNameFile, vocPath)
print(Image_data, boundingBX_labels, im_dims)
epochs = 1
for loop in range(epochs):
# ______________________________________________________________________
# my batch creater
# Construct an array data source
ds = data_source.ArrayDataSource([Image_data, boundingBX_labels, im_dims])
# Iterate over samples, drawing batches of 64 elements in random order
for (image_input, gt_box, image_dim) in ds.batch_iterator(
batch_size=1,
shuffle=True): # shuffle true will randomise every batch
# accuoutput=sess.run([rpn], feed_dict={input_x: image_input, gt_bbox: gt_box, im_dimsal: image_dim})
_label, rpn_bbox_targets, rpn_bbox_inside_weights, rpn_bbox_outside_weights = anchor_target_layer.anchor_target_layer_python(
rpn_cls_score=image_input,
gt_boxes=gt_box,
im_dims=image_dim,
feat_strides=feat_stride,
anchor_scales=anchor_scale)
# (rpn_label)
def test_thread_parallel_data_source(thread_pool):
from batchup import data_source
class _AbstractExampleBatchIterator (object):
def __init__(self, N):
self.N = N
# Define __len__ here to save us some work in the base classes that we
# will actually use
def __len__(self):
return self.N
class _ExampleBatchIteratorIdentity (_AbstractExampleBatchIterator):
def __getitem__(self, indices):
return indices
class _ExampleBatchIteratorSquare (_AbstractExampleBatchIterator):
def __getitem__(self, indices):
return indices**2
class _ExampleBatchIteratorError (_AbstractExampleBatchIterator):
def __getitem__(self, indices):
raise ValueError
ds = data_source.ArrayDataSource(
[_ExampleBatchIteratorIdentity(100),
_ExampleBatchIteratorSquare(100)]
)
pds = thread_pool.parallel_data_source(ds)
# Check number of samples
assert ds.num_samples() == 100
assert pds.num_samples() == 100
# Arrays of flags indicating which numbers we got back
n_flags = np.zeros((100,), dtype=bool)
n_sqr_flags = np.zeros((10000,), dtype=bool)
BATCHSIZE = 10
for batch in pds.batch_iterator(
batch_size=BATCHSIZE, shuffle=np.random.RandomState(12345)):
# batch should be a list
assert isinstance(batch, tuple)
# batch should contain two arrays
assert len(batch) == 2
# each array should be of length BATCHSIZE
assert batch[0].shape[0] == BATCHSIZE
assert batch[1].shape[0] == BATCHSIZE
# Check off the numbers we got back
n_flags[batch[0]] = True
n_sqr_flags[batch[1]] = True
# Check the flags arrays
assert n_flags.sum() == 100
assert n_sqr_flags.sum() == 100
expected_n_flags = np.ones((100,), dtype=bool)
expected_n_sqr_flags = np.zeros((10000,), dtype=bool)
expected_n_sqr_flags[np.arange(100)**2] = True
assert (n_flags == expected_n_flags).all()
assert (n_sqr_flags == expected_n_sqr_flags).all()
# Check that passing something that isn't a data source fails
with pytest.raises(TypeError):
thread_pool.parallel_data_source(_ExampleBatchIteratorIdentity(100))
# Check that passing a non random access data source fails
with pytest.raises(TypeError):
def make_batch_iter(**kwargs):
def batch_iterator(batch_size):
for i in range(0, 100, batch_size):
yield np.random.normal(size=(batch_size, 2))
cds = data_source.CallableDataSource(make_batch_iter)
thread_pool.parallel_data_source(cds)
# Check that errors raised by data sources are passed back
eds = data_source.ArrayDataSource([_ExampleBatchIteratorError(100)])
peds = thread_pool.parallel_data_source(eds)
peds_iter = peds.batch_iterator(batch_size=BATCHSIZE,
shuffle=np.random.RandomState(12345))
with pytest.raises(ValueError):
next(peds_iter)
prior_train_op = tf.train.GradientDescentOptimizer(
learning_rate=0.01).minimize(gen_loss, var_list=[prior_logit0])
with tf.control_dependencies([gen_train_op, inf_train_op, prior_train_op]):
#with tf.control_dependencies([gen_train_op, inf_train_op]):
train_op = tf.no_op()
init_op = tf.global_variables_initializer()
#%% TRAIN
# get data
train_data, valid_data, test_data = load_omniglot()
from batchup import data_source
train_ds = data_source.ArrayDataSource([train_data])
test_ds = data_source.ArrayDataSource([test_data[:8000]])
valid_ds = data_source.ArrayDataSource([valid_data[:1300]])
directory = os.getcwd() + '/discrete_out/'
if not os.path.exists(directory):
os.makedirs(directory)
batch_size = 25
total_points = train_data.shape[0]
total_batch = int(total_points / batch_size)
total_test_batch = int(test_data.shape[0] / batch_size)
training_epochs = 1500 #600epoch 42min
display_step = total_batch
def test_process_parallel_data_source(process_pool):
from batchup import data_source
ds = data_source.ArrayDataSource(
[_ExampleBatchIteratorIdentity(100),
_ExampleBatchIteratorSquare(100)]
)
pds = process_pool.parallel_data_source(ds)
# Check number of samples
assert ds.num_samples() == 100
assert pds.num_samples() == 100
# Arrays of flags indicating which numbers we got back
n_flags = np.zeros((100,), dtype=bool)
n_sqr_flags = np.zeros((10000,), dtype=bool)
BATCHSIZE = 10
for batch in pds.batch_iterator(
batch_size=BATCHSIZE, shuffle=np.random.RandomState(12345)):
# batch should be a list
assert isinstance(batch, tuple)
# batch should contain two arrays
assert len(batch) == 2
# each array should be of length BATCHSIZE
assert batch[0].shape[0] == BATCHSIZE
assert batch[1].shape[0] == BATCHSIZE
# Check off the numbers we got back
n_flags[batch[0]] = True
n_sqr_flags[batch[1]] = True
# Check the flags arrays
assert n_flags.sum() == 100
assert n_sqr_flags.sum() == 100
expected_n_flags = np.ones((100,), dtype=bool)
expected_n_sqr_flags = np.zeros((10000,), dtype=bool)
expected_n_sqr_flags[np.arange(100)**2] = True
assert (n_flags == expected_n_flags).all()
assert (n_sqr_flags == expected_n_sqr_flags).all()
# Check that passing something that isn't a data source fails
with pytest.raises(TypeError):
process_pool.parallel_data_source(_ExampleBatchIteratorIdentity(100))
# Check that passing a non random access data source fails
with pytest.raises(TypeError):
def make_batch_iter(**kwargs):
def batch_iterator(batch_size):
for i in range(0, 100, batch_size):
yield np.random.normal(size=(batch_size, 2))
cds = data_source.CallableDataSource(make_batch_iter)
process_pool.parallel_data_source(cds)
# Check that errors raised by data sources are passed back
eds = data_source.ArrayDataSource([_ExampleBatchIteratorError(100)])
peds = process_pool.parallel_data_source(eds)
peds_iter = peds.batch_iterator(batch_size=BATCHSIZE,
shuffle=np.random.RandomState(12345))
with pytest.raises(ValueError):
next(peds_iter)
def experiment(exp, arch, loss, double_softmax, confidence_thresh, rampup,
teacher_alpha, fix_ema, unsup_weight, cls_bal_scale,
cls_bal_scale_range, cls_balance, cls_balance_loss,
combine_batches, learning_rate, standardise_samples,
src_affine_std, src_xlat_range, src_hflip, src_intens_flip,
src_intens_scale_range, src_intens_offset_range,
src_gaussian_noise_std, tgt_affine_std, tgt_xlat_range,
tgt_hflip, tgt_intens_flip, tgt_intens_scale_range,
tgt_intens_offset_range, tgt_gaussian_noise_std, num_epochs,
batch_size, epoch_size, seed, log_file, model_file, device):
settings = locals().copy()
import os
import sys
import pickle
import cmdline_helpers
if log_file == '':
log_file = 'output_aug_log_{}.txt'.format(exp)
elif log_file == 'none':
log_file = None
if log_file is not None:
if os.path.exists(log_file):
print('Output log file {} already exists'.format(log_file))
return
use_rampup = rampup > 0
src_intens_scale_range_lower, src_intens_scale_range_upper, src_intens_offset_range_lower, src_intens_offset_range_upper = \
cmdline_helpers.intens_aug_options(src_intens_scale_range, src_intens_offset_range)
tgt_intens_scale_range_lower, tgt_intens_scale_range_upper, tgt_intens_offset_range_lower, tgt_intens_offset_range_upper = \
cmdline_helpers.intens_aug_options(tgt_intens_scale_range, tgt_intens_offset_range)
import time
import math
import numpy as np
from batchup import data_source, work_pool
import data_loaders
import standardisation
import network_architectures
import augmentation
import torch, torch.cuda
from torch import nn
from torch.nn import functional as F
import optim_weight_ema
torch_device = torch.device(device)
pool = work_pool.WorkerThreadPool(2)
n_chn = 0
if exp == 'svhn_mnist':
d_source = data_loaders.load_svhn(zero_centre=False, greyscale=True)
d_target = data_loaders.load_mnist(invert=False,
zero_centre=False,
pad32=True,
val=False)
elif exp == 'mnist_svhn':
d_source = data_loaders.load_mnist(invert=False,
zero_centre=False,
pad32=True)
d_target = data_loaders.load_svhn(zero_centre=False,
greyscale=True,
val=False)
elif exp == 'svhn_mnist_rgb':
d_source = data_loaders.load_svhn(zero_centre=False, greyscale=False)
d_target = data_loaders.load_mnist(invert=False,
zero_centre=False,
pad32=True,
val=False,
rgb=True)
elif exp == 'mnist_svhn_rgb':
d_source = data_loaders.load_mnist(invert=False,
zero_centre=False,
pad32=True,
rgb=True)
d_target = data_loaders.load_svhn(zero_centre=False,
greyscale=False,
val=False)
elif exp == 'cifar_stl':
d_source = data_loaders.load_cifar10(range_01=False)
d_target = data_loaders.load_stl(zero_centre=False, val=False)
elif exp == 'stl_cifar':
d_source = data_loaders.load_stl(zero_centre=False)
d_target = data_loaders.load_cifar10(range_01=False, val=False)
elif exp == 'mnist_usps':
d_source = data_loaders.load_mnist(zero_centre=False)
d_target = data_loaders.load_usps(zero_centre=False,
scale28=True,
val=False)
elif exp == 'usps_mnist':
d_source = data_loaders.load_usps(zero_centre=False, scale28=True)
d_target = data_loaders.load_mnist(zero_centre=False, val=False)
elif exp == 'syndigits_svhn':
d_source = data_loaders.load_syn_digits(zero_centre=False)
d_target = data_loaders.load_svhn(zero_centre=False, val=False)
elif exp == 'synsigns_gtsrb':
d_source = data_loaders.load_syn_signs(zero_centre=False)
d_target = data_loaders.load_gtsrb(zero_centre=False, val=False)
else:
print('Unknown experiment type \'{}\''.format(exp))
return
# Delete the training ground truths as we should not be using them
del d_target.train_y
if standardise_samples:
standardisation.standardise_dataset(d_source)
standardisation.standardise_dataset(d_target)
n_classes = d_source.n_classes
print('Loaded data')
if arch == '':
if exp in {'mnist_usps', 'usps_mnist'}:
arch = 'mnist-bn-32-64-256'
if exp in {'svhn_mnist', 'mnist_svhn'}:
arch = 'grey-32-64-128-gp'
if exp in {
'cifar_stl', 'stl_cifar', 'syndigits_svhn', 'svhn_mnist_rgb',
'mnist_svhn_rgb'
}:
arch = 'rgb-128-256-down-gp'
if exp in {'synsigns_gtsrb'}:
arch = 'rgb40-96-192-384-gp'
net_class, expected_shape = network_architectures.get_net_and_shape_for_architecture(
arch)
if expected_shape != d_source.train_X.shape[1:]:
print(
'Architecture {} not compatible with experiment {}; it needs samples of shape {}, '
'data has samples of shape {}'.format(arch, exp, expected_shape,
d_source.train_X.shape[1:]))
return
student_net = net_class(n_classes).to(torch_device)
teacher_net = net_class(n_classes).to(torch_device)
student_params = list(student_net.parameters())
teacher_params = list(teacher_net.parameters())
for param in teacher_params:
param.requires_grad = False
student_optimizer = torch.optim.Adam(student_params, lr=learning_rate)
if fix_ema:
teacher_optimizer = optim_weight_ema.EMAWeightOptimizer(
teacher_net, student_net, alpha=teacher_alpha)
else:
teacher_optimizer = optim_weight_ema.OldWeightEMA(teacher_net,
student_net,
alpha=teacher_alpha)
classification_criterion = nn.CrossEntropyLoss()
print('Built network')
src_aug = augmentation.ImageAugmentation(
src_hflip,
src_xlat_range,
src_affine_std,
intens_flip=src_intens_flip,
intens_scale_range_lower=src_intens_scale_range_lower,
intens_scale_range_upper=src_intens_scale_range_upper,
intens_offset_range_lower=src_intens_offset_range_lower,
intens_offset_range_upper=src_intens_offset_range_upper,
gaussian_noise_std=src_gaussian_noise_std)
tgt_aug = augmentation.ImageAugmentation(
tgt_hflip,
tgt_xlat_range,
tgt_affine_std,
intens_flip=tgt_intens_flip,
intens_scale_range_lower=tgt_intens_scale_range_lower,
intens_scale_range_upper=tgt_intens_scale_range_upper,
intens_offset_range_lower=tgt_intens_offset_range_lower,
intens_offset_range_upper=tgt_intens_offset_range_upper,
gaussian_noise_std=tgt_gaussian_noise_std)
if combine_batches:
def augment(X_sup, y_src, X_tgt):
X_src_stu, X_src_tea = src_aug.augment_pair(X_sup)
X_tgt_stu, X_tgt_tea = tgt_aug.augment_pair(X_tgt)
return X_src_stu, X_src_tea, y_src, X_tgt_stu, X_tgt_tea
else:
def augment(X_src, y_src, X_tgt):
X_src = src_aug.augment(X_src)
X_tgt_stu, X_tgt_tea = tgt_aug.augment_pair(X_tgt)
return X_src, y_src, X_tgt_stu, X_tgt_tea
rampup_weight_in_list = [0]
cls_bal_fn = network_architectures.get_cls_bal_function(cls_balance_loss)
def compute_aug_loss(stu_out, tea_out):
# Augmentation loss
if use_rampup:
unsup_mask = None
conf_mask_count = None
unsup_mask_count = None
else:
conf_tea = torch.max(tea_out, 1)[0]
unsup_mask = conf_mask = (conf_tea > confidence_thresh).float()
unsup_mask_count = conf_mask_count = conf_mask.sum()
if loss == 'bce':
aug_loss = network_architectures.robust_binary_crossentropy(
stu_out, tea_out)
else:
d_aug_loss = stu_out - tea_out
aug_loss = d_aug_loss * d_aug_loss
# Class balance scaling
if cls_bal_scale:
if use_rampup:
n_samples = float(aug_loss.shape[0])
else:
n_samples = unsup_mask.sum()
avg_pred = n_samples / float(n_classes)
bal_scale = avg_pred / torch.clamp(tea_out.sum(dim=0), min=1.0)
if cls_bal_scale_range != 0.0:
bal_scale = torch.clamp(bal_scale,
min=1.0 / cls_bal_scale_range,
max=cls_bal_scale_range)
bal_scale = bal_scale.detach()
aug_loss = aug_loss * bal_scale[None, :]
aug_loss = aug_loss.mean(dim=1)
if use_rampup:
unsup_loss = aug_loss.mean() * rampup_weight_in_list[0]
else:
unsup_loss = (aug_loss * unsup_mask).mean()
# Class balance loss
if cls_balance > 0.0:
# Compute per-sample average predicated probability
# Average over samples to get average class prediction
avg_cls_prob = stu_out.mean(dim=0)
# Compute loss
equalise_cls_loss = cls_bal_fn(avg_cls_prob,
float(1.0 / n_classes))
equalise_cls_loss = equalise_cls_loss.mean() * n_classes
if use_rampup:
equalise_cls_loss = equalise_cls_loss * rampup_weight_in_list[0]
else:
if rampup == 0:
equalise_cls_loss = equalise_cls_loss * unsup_mask.mean(
dim=0)
unsup_loss += equalise_cls_loss * cls_balance
return unsup_loss, conf_mask_count, unsup_mask_count
if combine_batches:
def f_train(X_src0, X_src1, y_src, X_tgt0, X_tgt1):
X_src0 = torch.tensor(X_src0,
dtype=torch.float,
device=torch_device)
X_src1 = torch.tensor(X_src1,
dtype=torch.float,
device=torch_device)
y_src = torch.tensor(y_src, dtype=torch.long, device=torch_device)
X_tgt0 = torch.tensor(X_tgt0,
dtype=torch.float,
device=torch_device)
X_tgt1 = torch.tensor(X_tgt1,
dtype=torch.float,
device=torch_device)
n_samples = X_src0.size()[0]
n_total = n_samples + X_tgt0.size()[0]
student_optimizer.zero_grad()
student_net.train()
teacher_net.train()
# Concatenate source and target mini-batches
X0 = torch.cat([X_src0, X_tgt0], 0)
X1 = torch.cat([X_src1, X_tgt1], 0)
student_logits_out = student_net(X0)
student_prob_out = F.softmax(student_logits_out, dim=1)
src_logits_out = student_logits_out[:n_samples]
src_prob_out = student_prob_out[:n_samples]
teacher_logits_out = teacher_net(X1)
teacher_prob_out = F.softmax(teacher_logits_out, dim=1)
# Supervised classification loss
if double_softmax:
clf_loss = classification_criterion(src_prob_out, y_src)
else:
clf_loss = classification_criterion(src_logits_out, y_src)
unsup_loss, conf_mask_count, unsup_mask_count = compute_aug_loss(
student_prob_out, teacher_prob_out)
loss_expr = clf_loss + unsup_loss * unsup_weight
loss_expr.backward()
student_optimizer.step()
teacher_optimizer.step()
outputs = [
float(clf_loss) * n_samples,
float(unsup_loss) * n_total
]
if not use_rampup:
mask_count = float(conf_mask_count) * 0.5
unsup_count = float(unsup_mask_count) * 0.5
outputs.append(mask_count)
outputs.append(unsup_count)
return tuple(outputs)
else:
def f_train(X_src, y_src, X_tgt0, X_tgt1):
X_src = torch.tensor(X_src, dtype=torch.float, device=torch_device)
y_src = torch.tensor(y_src, dtype=torch.long, device=torch_device)
X_tgt0 = torch.tensor(X_tgt0,
dtype=torch.float,
device=torch_device)
X_tgt1 = torch.tensor(X_tgt1,
dtype=torch.float,
device=torch_device)
student_optimizer.zero_grad()
student_net.train()
teacher_net.train()
src_logits_out = student_net(X_src)
student_tgt_logits_out = student_net(X_tgt0)
student_tgt_prob_out = F.softmax(student_tgt_logits_out, dim=1)
teacher_tgt_logits_out = teacher_net(X_tgt1)
teacher_tgt_prob_out = F.softmax(teacher_tgt_logits_out, dim=1)
# Supervised classification loss
if double_softmax:
clf_loss = classification_criterion(
F.softmax(src_logits_out, dim=1), y_src)
else:
clf_loss = classification_criterion(src_logits_out, y_src)
unsup_loss, conf_mask_count, unsup_mask_count = compute_aug_loss(
student_tgt_prob_out, teacher_tgt_prob_out)
loss_expr = clf_loss + unsup_loss * unsup_weight
loss_expr.backward()
student_optimizer.step()
teacher_optimizer.step()
n_samples = X_src.size()[0]
outputs = [
float(clf_loss) * n_samples,
float(unsup_loss) * n_samples
]
if not use_rampup:
mask_count = float(conf_mask_count)
unsup_count = float(unsup_mask_count)
outputs.append(mask_count)
outputs.append(unsup_count)
return tuple(outputs)
print('Compiled training function')
def f_pred_src(X_sup):
X_var = torch.tensor(X_sup, dtype=torch.float, device=torch_device)
student_net.eval()
teacher_net.eval()
return (F.softmax(student_net(X_var), dim=1).detach().cpu().numpy(),
F.softmax(teacher_net(X_var), dim=1).detach().cpu().numpy())
def f_pred_tgt(X_sup):
X_var = torch.tensor(X_sup, dtype=torch.float, device=torch_device)
student_net.eval()
teacher_net.eval()
return (F.softmax(student_net(X_var), dim=1).detach().cpu().numpy(),
F.softmax(teacher_net(X_var), dim=1).detach().cpu().numpy())
def f_eval_src(X_sup, y_sup):
y_pred_prob_stu, y_pred_prob_tea = f_pred_src(X_sup)
y_pred_stu = np.argmax(y_pred_prob_stu, axis=1)
y_pred_tea = np.argmax(y_pred_prob_tea, axis=1)
return (float(
(y_pred_stu != y_sup).sum()), float((y_pred_tea != y_sup).sum()))
def f_eval_tgt(X_sup, y_sup):
y_pred_prob_stu, y_pred_prob_tea = f_pred_tgt(X_sup)
y_pred_stu = np.argmax(y_pred_prob_stu, axis=1)
y_pred_tea = np.argmax(y_pred_prob_tea, axis=1)
return (float(
(y_pred_stu != y_sup).sum()), float((y_pred_tea != y_sup).sum()))
print('Compiled evaluation function')
# Setup output
def log(text):
print(text)
if log_file is not None:
with open(log_file, 'a') as f:
f.write(text + '\n')
f.flush()
f.close()
cmdline_helpers.ensure_containing_dir_exists(log_file)
# Report setttings
log('Settings: {}'.format(', '.join([
'{}={}'.format(key, settings[key])
for key in sorted(list(settings.keys()))
])))
# Report dataset size
log('Dataset:')
log('SOURCE Train: X.shape={}, y.shape={}'.format(d_source.train_X.shape,
d_source.train_y.shape))
log('SOURCE Test: X.shape={}, y.shape={}'.format(d_source.test_X.shape,
d_source.test_y.shape))
log('TARGET Train: X.shape={}'.format(d_target.train_X.shape))
log('TARGET Test: X.shape={}, y.shape={}'.format(d_target.test_X.shape,
d_target.test_y.shape))
print('Training...')
sup_ds = data_source.ArrayDataSource([d_source.train_X, d_source.train_y],
repeats=-1)
tgt_train_ds = data_source.ArrayDataSource([d_target.train_X], repeats=-1)
train_ds = data_source.CompositeDataSource([sup_ds,
tgt_train_ds]).map(augment)
train_ds = pool.parallel_data_source(train_ds)
if epoch_size == 'large':
n_samples = max(d_source.train_X.shape[0], d_target.train_X.shape[0])
elif epoch_size == 'small':
n_samples = min(d_source.train_X.shape[0], d_target.train_X.shape[0])
elif epoch_size == 'target':
n_samples = d_target.train_X.shape[0]
n_train_batches = n_samples // batch_size
source_test_ds = data_source.ArrayDataSource(
[d_source.test_X, d_source.test_y])
target_test_ds = data_source.ArrayDataSource(
[d_target.test_X, d_target.test_y])
if seed != 0:
shuffle_rng = np.random.RandomState(seed)
else:
shuffle_rng = np.random
train_batch_iter = train_ds.batch_iterator(batch_size=batch_size,
shuffle=shuffle_rng)
best_teacher_model_state = {
k: v.cpu().numpy()
for k, v in teacher_net.state_dict().items()
}
best_conf_mask_rate = 0.0
best_src_test_err = 1.0
for epoch in range(num_epochs):
t1 = time.time()
if use_rampup:
if epoch < rampup:
p = max(0.0, float(epoch)) / float(rampup)
p = 1.0 - p
rampup_value = math.exp(-p * p * 5.0)
else:
rampup_value = 1.0
rampup_weight_in_list[0] = rampup_value
train_res = data_source.batch_map_mean(f_train,
train_batch_iter,
n_batches=n_train_batches)
train_clf_loss = train_res[0]
if combine_batches:
unsup_loss_string = 'unsup (both) loss={:.6f}'.format(train_res[1])
else:
unsup_loss_string = 'unsup (tgt) loss={:.6f}'.format(train_res[1])
src_test_err_stu, src_test_err_tea = source_test_ds.batch_map_mean(
f_eval_src, batch_size=batch_size * 2)
tgt_test_err_stu, tgt_test_err_tea = target_test_ds.batch_map_mean(
f_eval_tgt, batch_size=batch_size * 2)
if use_rampup:
unsup_loss_string = '{}, rampup={:.3%}'.format(
unsup_loss_string, rampup_value)
if src_test_err_stu < best_src_test_err:
best_src_test_err = src_test_err_stu
best_teacher_model_state = {
k: v.cpu().numpy()
for k, v in teacher_net.state_dict().items()
}
improve = '*** '
else:
improve = ''
else:
conf_mask_rate = train_res[-2]
unsup_mask_rate = train_res[-1]
if conf_mask_rate > best_conf_mask_rate:
best_conf_mask_rate = conf_mask_rate
improve = '*** '
best_teacher_model_state = {
k: v.cpu().numpy()
for k, v in teacher_net.state_dict().items()
}
else:
improve = ''
unsup_loss_string = '{}, conf mask={:.3%}, unsup mask={:.3%}'.format(
unsup_loss_string, conf_mask_rate, unsup_mask_rate)
t2 = time.time()
log('{}Epoch {} took {:.2f}s: TRAIN clf loss={:.6f}, {}; '
'SRC TEST ERR={:.3%}, TGT TEST student err={:.3%}, TGT TEST teacher err={:.3%}'
.format(improve, epoch, t2 - t1, train_clf_loss, unsup_loss_string,
src_test_err_stu, tgt_test_err_stu, tgt_test_err_tea))
# Save network
if model_file != '':
cmdline_helpers.ensure_containing_dir_exists(model_file)
with open(model_file, 'wb') as f:
torch.save(best_teacher_model_state, f)
size = 0
for row in csvreader:
if size >= 90000:
test_X.append(row[1:])
test_Y.append([float(row[0])])
else:
train_X.append(row[1:])
train_Y.append([float(row[0])])
size+=1
train_X = np.array(train_X,dtype="float64")
train_Y = np.array(train_Y,dtype="int32")
test_X = np.array(test_X, dtype="float64")
test_Y = np.array(test_Y, dtype="int32")
ds = data_source.ArrayDataSource([train_X, train_Y])
class MultilayerPerceptron(nn.Module):
def __init__(self, input_size, hidden_size, out_classes):
super(MultilayerPerceptron, self).__init__()
self.fc1 = nn.Linear(input_size, hidden_size)
self.fc2 = nn.Linear(hidden_size, num_classes)
self.tanh = nn.Tanh()
self.sigmoid = nn.Sigmoid()
def forward(self, x):
out = self.fc1(x)
out = self.tanh(out)
out = self.fc2(out)
out = self.sigmoid(out)
# intialiasing all variable
init = tf.global_variables_initializer()
epochs = 20
sess = tf.Session()
sess.run(init)
sess = tf_debug.TensorBoardDebugWrapperSession(sess, "localhost:7000")
# X_train, X_test, y_train, y_test = train_test_split(new_image_input, y, test_size = .10, random_state = 4)#splitting data (no need if test data is present
for loop in range(epochs):
# ______________________________________________________________________
# my batch creater
# Construct an array data source
ds = data_source.ArrayDataSource([scaled_input, y_train])
# Iterate over samples, drawing batches of 64 elements in random order
for (batch_X, batch_y) in ds.batch_iterator(
batch_size=1000,
shuffle=True): # shuffle true will randomise every batch
# for (batch_X, batch_y) in ds.batch_iterator(batch_size=1000, shuffle=np.random.RandomState(12345)):
_, accuracy = sess.run([train, acc],
feed_dict={
x: batch_X,
y_true: batch_y,
hold_prob: 0.5
})
print("the train accuracy is:", accuracy)
# ________________________________________________________________________
# batch_x1, batch_y1 = tf.train.batch([feature_input, y_train], batch_size=50)
def experiment(n_classes, bilinear, dropout, learning_rate, gamma, alpha,
data_path, val_percent, batch_size, rep_num, logfile, augment,
date):
settings = locals().copy()
# Make results folder for specific parameters
savepath = 'results/{}/g{}_alp{}_bs{}_lr{}_bilin{}_do{}_aug{}'.format(
date, gamma, alpha, batch_size, learning_rate, bilinear, dropout,
augment)
if not os.path.exists(savepath):
os.makedirs(savepath)
if logfile != 'none':
try:
logger = loggers.Logger('{}/{}'.format(savepath, logfile))
except loggers.LogAlreadyExistsError as e:
print(e.message)
return
logger.connect()
print('Program: {}'.format(sys.argv[0]))
print('Command line: {}'.format(' '.join(sys.argv)))
print('Settings:')
print(', '.join(
['{}={}'.format(k, settings[k]) for k in sorted(settings.keys())]))
best_accuracy = float('inf')
# LOAD MODEL
input_channels = 5
print('LOAD VGG16 UNET MODEL')
torch_device = torch.device(
'cuda:0' if torch.cuda.is_available() else 'cpu')
model = vgg16_UNet(models.vgg16(), input_channels, n_classes, bilinear,
dropout).to(torch_device)
model.encoder.expand_input(input_channels)
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
criterion = FocalLoss(gamma=gamma, alpha=alpha)
print('Selecting dataset from training islands...')
trainset_X, valset_X, trainset_y, valset_y = new_data_splitter(
data_path, test_island, island_refs, observer_list, val_percent)
train_X = ImageAccessor(trainset_X, load_image_vgg16)
val_X = ImageAccessor(valset_X, load_image_vgg16)
train_y = ImageAccessor(trainset_y, load_tgt_vgg16)
val_y = ImageAccessor(valset_y, load_tgt_vgg16)
trainset = data_source.ArrayDataSource([train_X, train_y])
valset = data_source.ArrayDataSource([val_X, val_y])
# Augment
trainset = trainset.map(augment_batch)
valset = valset.map(augment_batch)
pool = work_pool.WorkerThreadPool(4)
trainset = pool.parallel_data_source(trainset)
valset = pool.parallel_data_source(valset)
data_loaders = {'train': trainset, 'val': valset}
print('BEGIN TRAINING...')
total_train_loss = []
total_val_loss = []
no_improvement = 0
epoch = 0
while no_improvement < 10:
t1 = time.time()
print('-' * 10)
# Train and validate
train_loss = train(model, optimizer, data_loaders, criterion,
batch_size, torch_device, 'train')
val_loss = train(model, optimizer, data_loaders, criterion, batch_size,
torch_device, 'val')
t2 = time.time()
print(
'Epoch {} took {:.3f}s; training loss = {:.6f}; validation loss = {:.6f}'
.format(epoch, t2 - t1, train_loss, val_loss))
# Save losses
total_train_loss.append(train_loss)
total_val_loss.append(val_loss)
# Check loss and save checkpoint
is_best = bool(val_loss < best_accuracy)
if is_best:
no_improvement = 0 # reset the counter after new best found
else:
no_improvement += 1 # count how many non-improvements
print('Current best loss: {}'.format(best_accuracy))
best_accuracy = min(val_loss, best_accuracy)
print('{}/checkpoint.pth.tar'.format(savepath))
save_checkpoint(
{
'epoch': epoch,
'state_dict': model.state_dict(),
'best_accuracy': torch.FloatTensor([best_accuracy])
}, is_best, '{}/checkpoint.pth.tar'.format(savepath))
print('No improvement in loss for {} epochs'.format(no_improvement))
if epoch == 5:
print('Setting new learning rate...')
optimizer = torch.optim.Adam(model.parameters(),
lr=learning_rate / 10.)
epoch += 1
np.savez('{}/TrainingLoss.npz'.format(savepath),
train_loss=total_train_loss,
val_loss=total_val_loss)
return
def experiment(exp, scale_u_range, scale_x_range, scale_y_range, affine_std,
xlat_range, rot_std, hflip, intens_scale_range, colour_rot_std,
colour_off_std, greyscale, cutout_prob, cutout_size, batch_size,
n_batches, seed):
import os
import sys
import cmdline_helpers
intens_scale_range_lower, intens_scale_range_upper = cmdline_helpers.colon_separated_range(
intens_scale_range)
scale_u_range = cmdline_helpers.colon_separated_range(scale_u_range)
scale_x_range = cmdline_helpers.colon_separated_range(scale_x_range)
scale_y_range = cmdline_helpers.colon_separated_range(scale_y_range)
import time
import tqdm
import math
import numpy as np
from matplotlib import pyplot as plt
from batchup import data_source, work_pool
import visda17_dataset
import augmentation, image_transforms
import itertools
n_chn = 0
mean_value = np.array([0.485, 0.456, 0.406])
std_value = np.array([0.229, 0.224, 0.225])
if exp == 'train_val':
d_source = visda17_dataset.TrainDataset(img_size=(96, 96),
mean_value=mean_value,
std_value=std_value,
range01=True,
rgb_order=True,
random_crop=False)
d_target = visda17_dataset.ValidationDataset(img_size=(96, 96),
mean_value=mean_value,
std_value=std_value,
range01=True,
rgb_order=True,
random_crop=False)
d_target_test = visda17_dataset.ValidationDataset(
img_size=(96, 96),
mean_value=mean_value,
std_value=std_value,
range01=True,
rgb_order=True,
random_crop=False)
elif exp == 'train_test':
print('train_test experiment not supported yet')
return
else:
print('Unknown experiment type \'{}\''.format(exp))
return
n_classes = d_source.n_classes
n_domains = 2
print('Loaded data')
arch = 'show-images'
# Image augmentation
aug = augmentation.ImageAugmentation(
hflip,
xlat_range,
affine_std,
rot_std=rot_std,
intens_scale_range_lower=intens_scale_range_lower,
intens_scale_range_upper=intens_scale_range_upper,
colour_rot_std=colour_rot_std,
colour_off_std=colour_off_std,
greyscale=greyscale,
scale_u_range=scale_u_range,
scale_x_range=scale_x_range,
scale_y_range=scale_y_range,
cutout_size=cutout_size,
cutout_probability=cutout_prob)
# Report setttings
print('sys.argv={}'.format(sys.argv))
# Report dataset size
print('Dataset:')
print('SOURCE len(X)={}, y.shape={}'.format(len(d_source.images),
d_source.y.shape))
print('TARGET len(X)={}'.format(len(d_target.images)))
print('Building data sources...')
source_train_ds = data_source.ArrayDataSource(
[d_source.images, d_source.y], repeats=-1)
target_train_ds = data_source.ArrayDataSource([d_target.images],
repeats=-1)
train_ds = data_source.CompositeDataSource(
[source_train_ds, target_train_ds])
border_value = int(np.mean(mean_value) * 255 + 0.5)
train_xf = image_transforms.Compose(
image_transforms.ScaleCropAndAugmentAffine(
(96, 96), (16, 16), True, aug, border_value, mean_value,
std_value),
image_transforms.ToTensor(),
)
test_xf = image_transforms.Compose(
image_transforms.ScaleAndCrop((96, 96), (16, 16), False),
image_transforms.ToTensor(),
image_transforms.Standardise(mean_value, std_value),
)
def augment(X_sup, y_sup, X_tgt):
X_sup = train_xf(X_sup)[0]
X_tgt_0 = train_xf(X_tgt)[0]
X_tgt_1 = train_xf(X_tgt)[0]
return [X_sup, y_sup, X_tgt_0, X_tgt_1]
train_ds = train_ds.map(augment)
test_ds = data_source.ArrayDataSource([d_target_test.images]).map(test_xf)
if seed != 0:
shuffle_rng = np.random.RandomState(seed)
else:
shuffle_rng = np.random
print('Showing...')
n_shown = 0
for (src_X, src_y, tgt_X0, tgt_X1), (te_X, ) in zip(
train_ds.batch_iterator(batch_size=batch_size,
shuffle=shuffle_rng),
test_ds.batch_iterator(batch_size=batch_size)):
print('Batch')
tgt_X = np.zeros(
(tgt_X0.shape[0] + tgt_X1.shape[0], ) + tgt_X0.shape[1:],
dtype=np.float32)
tgt_X[0::2] = tgt_X0
tgt_X[1::2] = tgt_X1
x = np.concatenate([src_X, tgt_X, te_X], axis=0)
n = x.shape[0]
n_sup = src_X.shape[0] + tgt_X.shape[0]
across = int(math.ceil(math.sqrt(float(n))))
plt.figure(figsize=(16, 16))
for i in tqdm.tqdm(range(n)):
plt.subplot(across, across, i + 1)
im_x = x[i] * std_value[:, None, None] + mean_value[:, None, None]
im_x = np.clip(im_x, 0.0, 1.0)
plt.imshow(im_x.transpose(1, 2, 0))
if i < src_y.shape[0]:
plt.title(str(src_y[i]))
elif i < n_sup:
plt.title('target')
else:
plt.title('test')
plt.show()
n_shown += 1
if n_shown >= n_batches:
break
bbox_data = val.get('boxes')
box_return = np.array(bbox_data)
print(box_return[0])
my_data.close()
print("closing hdf5 file")
print(image_return[7])
'''for loop,bbox_val in enumerate (box_return):
# for loop, image_val in enumerate(image_return):
# print(image_return[0])
# image_val= image_return[len]
# img = Image.open(io.BytesIO(image_val))
# file_name="file_"+str(loop) +'.jpeg'
print(bbox_val)
# image_path=os.path.join(output_path, file_name)
# img.save(imag'''
ds = data_source.ArrayDataSource([image_return, box_return])
# Iterate over samples, drawing batches of 64 elements in random order
for (batch_X, batch_y) in ds.batch_iterator(
batch_size=1, shuffle=True): # shuffle true will randomise every batch
print(batch_X.shape)
print(batch_y)
data = batch_y.reshape((batch_y.shape[0], 1))
print(data)
# box=np.reshape(-1,5)
# break
# e_path, 'JPEG')
if __name__ == "__main__":
# load_data()
# Map to [-1,1]
(train_images,
train_labels), (test_images,
test_labels) = mnist.load_data(path="mnist.npz")
train_images = (2 * np.array(train_images) / 255) - 1
i = 0
generator = build_generator()
discriminator = build_discriminator()
gan_model = build_gan_model(generator, discriminator)
batches = data_source.ArrayDataSource([train_images], repeats=epochs)
for batch in batches.batch_iterator(batch_size, True):
if i % 100 == 0:
samples = generator.predict(random_sample(1))
print(np.shape(samples))
fig = plot_single(samples[0])
plt.savefig('out_classic_single/{}.png'.format(str(i).zfill(3)),
bbox_inches='tight',
pad_inches=0,
transparent=True)
plt.close(fig)
real_data_input, real_data_label = batch[0].reshape(
-1, 28, 28, 1), np.repeat(np.random.uniform(0.3, 0.7), batch_size)
fake_data_input, fake_data_label = generator.predict(
if __name__ == "__main__":
result_real = discriminator_network(real_data)
result_fake = discriminator_network(generator_network(fake_data))
loss_discriminator = -tf.reduce_mean(tf.log(result_real) + tf.log(1. - result_fake))
loss_generator = -tf.reduce_mean(tf.log(result_fake))
disc_vars = [var for var in tf.trainable_variables() if var.name.startswith("disc")]
gen_vars = [var for var in tf.trainable_variables() if var.name.startswith("gen")]
discriminator_solver = tf.train.AdamOptimizer(learning_rate=0.0002, beta1=0.5).minimize(loss_discriminator, var_list=disc_vars)
generator_solver = tf.train.AdamOptimizer(learning_rate=0.0002, beta1=0.5).minimize(loss_generator, var_list=gen_vars)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
train_images = train_images.astype(np.float32) / 255
train_images = [np.reshape(np.array(val), (1,784)).flatten() for val in train_images]
batches = data_source.ArrayDataSource([np.array(train_images)], repeats=iterations)
i = 0
for batch in batches.batch_iterator(batch_size, True):
if i % 1000 == 0:
samples = sess.run(generator_network(fake_data), feed_dict={fake_data : random_sample(16)})
fig = plot(samples)
plt.savefig('out/{}.png'.format(str(i).zfill(3)), bbox_inches='tight')
plt.close(fig)
_, discriminator_curr = sess.run([discriminator_solver, loss_discriminator], feed_dict={real_data: batch[0], fake_data: random_sample(batch_size)})
_, generator_curr = sess.run([generator_solver, loss_generator], feed_dict={fake_data : random_sample(batch_size)})
print('D loss: {:.4}'. format(discriminator_curr))
i=i+1
return decoded
if __name__ == "__main__":
(train_images,
train_labels), (test_images,
test_labels) = mnist.load_data(path="mnist.npz")
train_images = (2 * np.array(train_images) / 255) - 1
autoencoder = Autoencoder(latent_dim)
autoencoder.compile(loss='mse',
optimizer=keras.optimizers.Adam(lr=0.0002, beta_1=0.5))
i = 0
for iterations in range(epochs):
batches = data_source.ArrayDataSource([train_images])
for batch in batches.batch_iterator(batch_size, True):
if (i % 100 == 0):
encoded_image = autoencoder.encoder(batch[0][0].reshape(
-1, 28, 28, 1))
print(np.shape(encoded_image))
decoded_image = autoencoder.decoder(encoded_image)
plt.imshow(np.reshape(decoded_image, (28, 28, 1)),
cmap='Greys_r')
plt.savefig('autoencoder/{}.png'.format(str(i).zfill(3)),
bbox_inches='tight')
plt.close()
i = i + 1
loss = autoencoder.train_on_batch(batch[0].reshape(-1, 28, 28, 1),
batch[0].reshape(-1, 28, 28, 1))
print(loss)
def experiment(plot_path, ds_name, no_aug, affine_std, scale_u_range,
scale_x_range, scale_y_range, xlat_range, hflip, intens_flip,
intens_scale_range, intens_offset_range, grid_h, grid_w, seed):
settings = locals().copy()
import os
import sys
import cmdline_helpers
intens_scale_range_lower, intens_scale_range_upper = cmdline_helpers.colon_separated_range(
intens_scale_range)
intens_offset_range_lower, intens_offset_range_upper = cmdline_helpers.colon_separated_range(
intens_offset_range)
scale_u_range = cmdline_helpers.colon_separated_range(scale_u_range)
scale_x_range = cmdline_helpers.colon_separated_range(scale_x_range)
scale_y_range = cmdline_helpers.colon_separated_range(scale_y_range)
import numpy as np
# from skimage.util import montage2d
from skimage.util import montage as montage2d
from PIL import Image
from batchup import data_source
import data_loaders
import augmentation
n_chn = 0
if ds_name == 'mnist':
d_source = data_loaders.load_mnist(zero_centre=False)
elif ds_name == 'usps':
d_source = data_loaders.load_usps(zero_centre=False, scale28=True)
elif ds_name == 'svhn_grey':
d_source = data_loaders.load_svhn(zero_centre=False, greyscale=True)
elif ds_name == 'svhn':
d_source = data_loaders.load_svhn(zero_centre=False, greyscale=False)
elif ds_name == 'cifar':
d_source = data_loaders.load_cifar10()
elif ds_name == 'stl':
d_source = data_loaders.load_stl()
elif ds_name == 'syndigits':
d_source = data_loaders.load_syn_digits(zero_centre=False,
greyscale=False)
elif ds_name == 'synsigns':
d_source = data_loaders.load_syn_signs(zero_centre=False,
greyscale=False)
elif ds_name == 'gtsrb':
d_source = data_loaders.load_gtsrb(zero_centre=False, greyscale=False)
else:
print('Unknown dataset \'{}\''.format(ds_name))
return
# Delete the training ground truths as we should not be using them
del d_source.train_y
n_classes = d_source.n_classes
print('Loaded data')
src_aug = augmentation.ImageAugmentation(
hflip,
xlat_range,
affine_std,
intens_flip=intens_flip,
intens_scale_range_lower=intens_scale_range_lower,
intens_scale_range_upper=intens_scale_range_upper,
intens_offset_range_lower=intens_offset_range_lower,
intens_offset_range_upper=intens_offset_range_upper,
scale_u_range=scale_u_range,
scale_x_range=scale_x_range,
scale_y_range=scale_y_range)
def augment(X):
if not no_aug:
X = src_aug.augment(X)
return X,
rampup_weight_in_list = [0]
print('Rendering...')
train_ds = data_source.ArrayDataSource([d_source.train_X],
repeats=-1).map(augment)
n_samples = len(d_source.train_X)
if seed != 0:
shuffle_rng = np.random.RandomState(seed)
else:
shuffle_rng = np.random
batch_size = grid_h * grid_w
display_batch_iter = train_ds.batch_iterator(batch_size=batch_size,
shuffle=shuffle_rng)
best_src_test_err = 1.0
x_batch, = next(display_batch_iter)
montage = []
for chn_i in range(x_batch.shape[1]):
m = montage2d(x_batch[:, chn_i, :, :], grid_shape=(grid_h, grid_w))
montage.append(m[:, :, None])
montage = np.concatenate(montage, axis=2)
if montage.shape[2] == 1:
montage = montage[:, :, 0]
lower = min(0.0, montage.min())
upper = max(1.0, montage.max())
montage = (montage - lower) / (upper - lower)
montage = (np.clip(montage, 0.0, 1.0) * 255.0).astype(np.uint8)
Image.fromarray(montage).save(plot_path)
biases['disc_hidden3'], biases['disc_out']
]
# Create training operations
train_gen = optimizer_gen.minimize(gen_loss, var_list=gen_vars)
train_disc = optimizer_disc.minimize(disc_loss, var_list=disc_vars)
# Initialize the variables (i.e. assign their default value)
init = tf.global_variables_initializer()
""" Start training """
with tf.Session() as sess:
# Run the initializer
sess.run(init)
trainDataSets = data_source.ArrayDataSource([x_train_data, y_train_data],
repeats=1)
i = 0
gl_append = []
dl_append = []
# f, a = plt.subplots(4, 4, figsize=(4, 4))
for e in range(epoch):
for (batch_x,
batch_y) in trainDataSets.batch_iterator(batch_size=batch_size):
i += 1
# Generate noise to feed to the generator
# z = np.random.uniform(-1., 1., size=[batch_size, noise_dim])
# Train
feed_dict = {disc_input: batch_y, gen_input: batch_x, dropout: 0.4}
def experiment(exp, arch, rnd_init, img_size, confidence_thresh, teacher_alpha,
unsup_weight, cls_balance, cls_balance_loss, learning_rate,
pretrained_lr_factor, fix_layers, double_softmax, use_dropout,
src_scale_u_range, src_scale_x_range, src_scale_y_range,
src_affine_std, src_xlat_range, src_rot_std, src_hflip,
src_intens_scale_range, src_colour_rot_std, src_colour_off_std,
src_greyscale, src_cutout_prob, src_cutout_size,
tgt_scale_u_range, tgt_scale_x_range, tgt_scale_y_range,
tgt_affine_std, tgt_xlat_range, tgt_rot_std, tgt_hflip,
tgt_intens_scale_range, tgt_colour_rot_std, tgt_colour_off_std,
tgt_greyscale, tgt_cutout_prob, tgt_cutout_size, constrain_crop,
img_pad_width, num_epochs, batch_size, epoch_size, seed,
log_file, skip_epoch_eval, result_file, record_history,
model_file, hide_progress_bar, subsetsize, subsetseed, device,
num_threads):
settings = locals().copy()
if rnd_init:
if fix_layers != '':
print('`rnd_init` and `fix_layers` are mutually exclusive')
return
if epoch_size not in {'source', 'target'}:
try:
epoch_size = int(epoch_size)
except ValueError:
print(
'epoch_size should be an integer, \'source\', or \'target\', not {}'
.format(epoch_size))
return
import os
import sys
import pickle
import cmdline_helpers
fix_layers = [lyr.strip() for lyr in fix_layers.split(',')]
if log_file == '':
log_file = 'output_aug_log_{}.txt'.format(exp)
elif log_file == 'none':
log_file = None
if log_file is not None:
if os.path.exists(log_file):
print('Output log file {} already exists'.format(log_file))
return
src_intens_scale_range_lower, src_intens_scale_range_upper = cmdline_helpers.colon_separated_range(
src_intens_scale_range)
tgt_intens_scale_range_lower, tgt_intens_scale_range_upper = cmdline_helpers.colon_separated_range(
tgt_intens_scale_range)
src_scale_u_range = cmdline_helpers.colon_separated_range(
src_scale_u_range)
tgt_scale_u_range = cmdline_helpers.colon_separated_range(
tgt_scale_u_range)
src_scale_x_range = cmdline_helpers.colon_separated_range(
src_scale_x_range)
tgt_scale_x_range = cmdline_helpers.colon_separated_range(
tgt_scale_x_range)
src_scale_y_range = cmdline_helpers.colon_separated_range(
src_scale_y_range)
tgt_scale_y_range = cmdline_helpers.colon_separated_range(
tgt_scale_y_range)
import time
import tqdm
import math
import tables
import numpy as np
from batchup import data_source, work_pool
import image_dataset, visda17_dataset, office_dataset
import network_architectures
import augmentation
import image_transforms
from sklearn.model_selection import StratifiedShuffleSplit, ShuffleSplit
import torch, torch.cuda
from torch import nn
from torch.nn import functional as F
import optim_weight_ema
if hide_progress_bar:
progress_bar = None
else:
progress_bar = tqdm.tqdm
with torch.cuda.device(device):
pool = work_pool.WorkerThreadPool(num_threads)
n_chn = 0
half_batch_size = batch_size // 2
if arch == '':
if exp in {'train_val', 'train_test'}:
arch = 'resnet50'
if rnd_init:
mean_value = np.array([0.5, 0.5, 0.5])
std_value = np.array([0.5, 0.5, 0.5])
else:
mean_value = np.array([0.485, 0.456, 0.406])
std_value = np.array([0.229, 0.224, 0.225])
img_shape = (img_size, img_size)
img_padding = (img_pad_width, img_pad_width)
if exp == 'visda_train_val':
d_source = visda17_dataset.TrainDataset(img_size=img_shape,
range01=True,
rgb_order=True)
d_target = visda17_dataset.ValidationDataset(img_size=img_shape,
range01=True,
rgb_order=True)
elif exp == 'visda_train_test':
d_source = visda17_dataset.TrainDataset(img_size=img_shape,
range01=True,
rgb_order=True)
d_target = visda17_dataset.TestDataset(img_size=img_shape,
range01=True,
rgb_order=True)
if not skip_epoch_eval:
print('WARNING: setting skip_epoch_eval to True')
skip_epoch_eval = True
elif exp == 'office_amazon_dslr':
d_source = office_dataset.OfficeAmazonDataset(img_size=img_shape,
range01=True,
rgb_order=True)
d_target = office_dataset.OfficeDSLRDataset(img_size=img_shape,
range01=True,
rgb_order=True)
elif exp == 'office_amazon_webcam':
d_source = office_dataset.OfficeAmazonDataset(img_size=img_shape,
range01=True,
rgb_order=True)
d_target = office_dataset.OfficeWebcamDataset(img_size=img_shape,
range01=True,
rgb_order=True)
elif exp == 'office_dslr_amazon':
d_source = office_dataset.OfficeDSLRDataset(img_size=img_shape,
range01=True,
rgb_order=True)
d_target = office_dataset.OfficeAmazonDataset(img_size=img_shape,
range01=True,
rgb_order=True)
elif exp == 'office_dslr_webcam':
d_source = office_dataset.OfficeDSLRDataset(img_size=img_shape,
range01=True,
rgb_order=True)
d_target = office_dataset.OfficeWebcamDataset(img_size=img_shape,
range01=True,
rgb_order=True)
elif exp == 'office_webcam_amazon':
d_source = office_dataset.OfficeWebcamDataset(img_size=img_shape,
range01=True,
rgb_order=True)
d_target = office_dataset.OfficeAmazonDataset(img_size=img_shape,
range01=True,
rgb_order=True)
elif exp == 'office_webcam_dslr':
d_source = office_dataset.OfficeWebcamDataset(img_size=img_shape,
range01=True,
rgb_order=True)
d_target = office_dataset.OfficeDSLRDataset(img_size=img_shape,
range01=True,
rgb_order=True)
else:
print('Unknown experiment type \'{}\''.format(exp))
return
# Tensorboard log
# Subset
source_indices, target_indices, n_src, n_tgt = image_dataset.subset_indices(
d_source, d_target, subsetsize, subsetseed)
#
# Result file
#
if result_file != '':
cmdline_helpers.ensure_containing_dir_exists(result_file)
h5_filters = tables.Filters(complevel=9, complib='blosc')
f_target_pred = tables.open_file(result_file, mode='w')
g_tgt_pred = f_target_pred.create_group(f_target_pred.root,
'target_pred_y',
'Target prediction')
if record_history:
arr_tgt_pred_history = f_target_pred.create_earray(
g_tgt_pred,
'y_prob_history',
tables.Float32Atom(), (0, n_tgt, d_target.n_classes),
filters=h5_filters)
else:
arr_tgt_pred_history = None
else:
arr_tgt_pred_history = None
f_target_pred = None
g_tgt_pred = None
n_classes = d_source.n_classes
print('Loaded data')
net_class = network_architectures.get_build_fn_for_architecture(arch)
student_net = net_class(n_classes, img_size, use_dropout,
not rnd_init).cuda()
teacher_net = net_class(n_classes, img_size, use_dropout,
not rnd_init).cuda()
student_params = list(student_net.parameters())
teacher_params = list(teacher_net.parameters())
for param in teacher_params:
param.requires_grad = False
if rnd_init:
new_student_optimizer = torch.optim.Adam(student_params,
lr=learning_rate)
pretrained_student_optimizer = None
else:
named_params = list(student_net.named_parameters())
new_params = []
pretrained_params = []
for name, param in named_params:
if name.startswith('new_'):
new_params.append(param)
else:
fix = False
for lyr in fix_layers:
if name.startswith(lyr + '.'):
fix = True
break
if not fix:
pretrained_params.append(param)
else:
print('Fixing param {}'.format(name))
param.requires_grad = False
new_student_optimizer = torch.optim.Adam(new_params,
lr=learning_rate)
if len(pretrained_params) > 0:
pretrained_student_optimizer = torch.optim.Adam(
pretrained_params, lr=learning_rate * pretrained_lr_factor)
else:
pretrained_student_optimizer = None
teacher_optimizer = optim_weight_ema.WeightEMA(teacher_params,
student_params,
alpha=teacher_alpha)
classification_criterion = nn.CrossEntropyLoss()
print('Built network')
# Image augmentation
src_aug = augmentation.ImageAugmentation(
src_hflip,
src_xlat_range,
src_affine_std,
rot_std=src_rot_std,
intens_scale_range_lower=src_intens_scale_range_lower,
intens_scale_range_upper=src_intens_scale_range_upper,
colour_rot_std=src_colour_rot_std,
colour_off_std=src_colour_off_std,
greyscale=src_greyscale,
scale_u_range=src_scale_u_range,
scale_x_range=src_scale_x_range,
scale_y_range=src_scale_y_range,
cutout_probability=src_cutout_prob,
cutout_size=src_cutout_size)
tgt_aug = augmentation.ImageAugmentation(
tgt_hflip,
tgt_xlat_range,
tgt_affine_std,
rot_std=tgt_rot_std,
intens_scale_range_lower=tgt_intens_scale_range_lower,
intens_scale_range_upper=tgt_intens_scale_range_upper,
colour_rot_std=tgt_colour_rot_std,
colour_off_std=tgt_colour_off_std,
greyscale=tgt_greyscale,
scale_u_range=tgt_scale_u_range,
scale_x_range=tgt_scale_x_range,
scale_y_range=tgt_scale_y_range,
cutout_probability=tgt_cutout_prob,
cutout_size=tgt_cutout_size)
test_aug = augmentation.ImageAugmentation(
tgt_hflip,
tgt_xlat_range,
0.0,
rot_std=0.0,
scale_u_range=tgt_scale_u_range,
scale_x_range=tgt_scale_x_range,
scale_y_range=tgt_scale_y_range)
border_value = int(np.mean(mean_value) * 255 + 0.5)
sup_xf = image_transforms.Compose(
image_transforms.ScaleCropAndAugmentAffine(img_shape, img_padding,
True, src_aug,
border_value,
mean_value, std_value),
image_transforms.ToTensor(),
)
if constrain_crop >= 0:
unsup_xf = image_transforms.Compose(
image_transforms.ScaleCropAndAugmentAffinePair(
img_shape, img_padding, constrain_crop, True, tgt_aug,
border_value, mean_value, std_value),
image_transforms.ToTensor(),
)
else:
unsup_xf = image_transforms.Compose(
image_transforms.ScaleCropAndAugmentAffine(
img_shape, img_padding, True, tgt_aug, border_value,
mean_value, std_value),
image_transforms.ToTensor(),
)
test_xf = image_transforms.Compose(
image_transforms.ScaleAndCrop(img_shape, img_padding, False),
image_transforms.ToTensor(),
image_transforms.Standardise(mean_value, std_value),
)
test_xf_aug_mult = image_transforms.Compose(
image_transforms.ScaleCropAndAugmentAffineMultiple(
16, img_shape, img_padding, True, test_aug, border_value,
mean_value, std_value),
image_transforms.ToTensorMultiple(),
)
if constrain_crop >= 0:
def augment(X_sup, y_sup, X_tgt):
X_sup = sup_xf(X_sup)[0]
X_unsup_both = unsup_xf(X_tgt)[0]
X_unsup_stu = X_unsup_both[:len(X_tgt)]
X_unsup_tea = X_unsup_both[len(X_tgt):]
return X_sup, y_sup, X_unsup_stu, X_unsup_tea
else:
def augment(X_sup, y_sup, X_tgt):
X_sup = sup_xf(X_sup)[0]
X_unsup_stu = unsup_xf(X_tgt)[0]
X_unsup_tea = unsup_xf(X_tgt)[0]
return X_sup, y_sup, X_unsup_stu, X_unsup_tea
cls_bal_fn = network_architectures.get_cls_bal_function(
cls_balance_loss)
def compute_aug_loss(stu_out, tea_out):
# Augmentation loss
conf_tea = torch.max(tea_out, 1)[0]
conf_mask = torch.gt(conf_tea, confidence_thresh).float()
d_aug_loss = stu_out - tea_out
aug_loss = d_aug_loss * d_aug_loss
aug_loss = torch.mean(aug_loss, 1) * conf_mask
# Class balance loss
if cls_balance > 0.0:
# Average over samples to get average class prediction
avg_cls_prob = torch.mean(stu_out, 0)
# Compute loss
equalise_cls_loss = cls_bal_fn(avg_cls_prob,
float(1.0 / n_classes))
equalise_cls_loss = torch.mean(equalise_cls_loss) * n_classes
equalise_cls_loss = equalise_cls_loss * torch.mean(
conf_mask, 0)
else:
equalise_cls_loss = None
return aug_loss, conf_mask, equalise_cls_loss
_one = torch.autograd.Variable(
torch.from_numpy(np.array([1.0]).astype(np.float32)).cuda())
def f_train(X_sup, y_sup, X_unsup0, X_unsup1):
X_sup = torch.autograd.Variable(torch.from_numpy(X_sup).cuda())
y_sup = torch.autograd.Variable(
torch.from_numpy(y_sup).long().cuda())
X_unsup0 = torch.autograd.Variable(
torch.from_numpy(X_unsup0).cuda())
X_unsup1 = torch.autograd.Variable(
torch.from_numpy(X_unsup1).cuda())
if pretrained_student_optimizer is not None:
pretrained_student_optimizer.zero_grad()
new_student_optimizer.zero_grad()
student_net.train(mode=True)
teacher_net.train(mode=True)
sup_logits_out = student_net(X_sup)
student_unsup_logits_out = student_net(X_unsup0)
student_unsup_prob_out = F.softmax(student_unsup_logits_out)
teacher_unsup_logits_out = teacher_net(X_unsup1)
teacher_unsup_prob_out = F.softmax(teacher_unsup_logits_out)
# Supervised classification loss
if double_softmax:
clf_loss = classification_criterion(F.softmax(sup_logits_out),
y_sup)
else:
clf_loss = classification_criterion(sup_logits_out, y_sup)
aug_loss, conf_mask, cls_bal_loss = compute_aug_loss(
student_unsup_prob_out, teacher_unsup_prob_out)
conf_mask_count = torch.sum(conf_mask)
unsup_loss = torch.mean(aug_loss)
loss_expr = clf_loss + unsup_loss * unsup_weight
if cls_bal_loss is not None:
loss_expr = loss_expr + cls_bal_loss * cls_balance * unsup_weight
loss_expr.backward()
if pretrained_student_optimizer is not None:
pretrained_student_optimizer.step()
new_student_optimizer.step()
teacher_optimizer.step()
n_samples = X_sup.size()[0]
mask_count = conf_mask_count.data.cpu()[0]
outputs = [
float(clf_loss.data.cpu()[0]) * n_samples,
float(unsup_loss.data.cpu()[0]) * n_samples, mask_count
]
return tuple(outputs)
print('Compiled training function')
def f_pred_src(X_sup):
X_var = torch.autograd.Variable(torch.from_numpy(X_sup).cuda())
teacher_net.train(mode=False)
return (F.softmax(teacher_net(X_var)).data.cpu().numpy(), )
def f_pred_tgt(X_sup):
X_var = torch.autograd.Variable(torch.from_numpy(X_sup).cuda())
teacher_net.train(mode=False)
return (F.softmax(teacher_net(X_var)).data.cpu().numpy(), )
def f_pred_tgt_mult(X_sup):
teacher_net.train(mode=False)
y_pred_aug = []
for aug_i in range(len(X_sup)):
X_var = torch.autograd.Variable(
torch.from_numpy(X_sup[aug_i, ...]).cuda())
y_pred = F.softmax(teacher_net(X_var)).data.cpu().numpy()
y_pred_aug.append(y_pred[None, ...])
y_pred_aug = np.concatenate(y_pred_aug, axis=0)
return (y_pred_aug.mean(axis=0), )
print('Compiled evaluation function')
# Setup output
cmdline_helpers.ensure_containing_dir_exists(log_file)
def log(text):
print(text)
if log_file is not None:
with open(log_file, 'a') as f:
f.write(text + '\n')
f.flush()
f.close()
# Report setttings
log('Program = {}'.format(sys.argv[0]))
log('Settings: {}'.format(', '.join([
'{}={}'.format(key, settings[key])
for key in sorted(list(settings.keys()))
])))
# Report dataset size
log('Dataset:')
log('SOURCE len(X)={}, y.shape={}'.format(len(d_source.images),
d_source.y.shape))
log('TARGET len(X)={}'.format(len(d_target.images)))
if epoch_size == 'source':
n_samples = n_src
elif epoch_size == 'target':
n_samples = n_tgt
else:
n_samples = epoch_size
n_train_batches = n_samples // batch_size
n_test_batches = n_tgt // (batch_size * 2) + 1
print('Training...')
sup_ds = data_source.ArrayDataSource([d_source.images, d_source.y],
repeats=-1,
indices=source_indices)
tgt_train_ds = data_source.ArrayDataSource([d_target.images],
repeats=-1,
indices=target_indices)
train_ds = data_source.CompositeDataSource([sup_ds,
tgt_train_ds]).map(augment)
train_ds = pool.parallel_data_source(train_ds,
batch_buffer_size=min(
20, n_train_batches))
target_ds_for_test = data_source.ArrayDataSource(
[d_target.images], indices=target_indices)
target_test_ds = target_ds_for_test.map(test_xf)
target_test_ds = pool.parallel_data_source(target_test_ds,
batch_buffer_size=min(
20, n_test_batches))
target_mult_test_ds = target_ds_for_test.map(test_xf_aug_mult)
target_mult_test_ds = pool.parallel_data_source(target_mult_test_ds,
batch_buffer_size=min(
20,
n_test_batches))
if seed != 0:
shuffle_rng = np.random.RandomState(seed)
else:
shuffle_rng = np.random
if d_target.has_ground_truth:
evaluator = d_target.prediction_evaluator(target_indices)
else:
evaluator = None
best_mask_rate = 0.0
best_teacher_model_state = {
k: v.cpu().numpy()
for k, v in teacher_net.state_dict().items()
}
train_batch_iter = train_ds.batch_iterator(batch_size=batch_size,
shuffle=shuffle_rng)
for epoch in range(num_epochs):
t1 = time.time()
if not skip_epoch_eval:
test_batch_iter = target_test_ds.batch_iterator(
batch_size=batch_size * 2)
else:
test_batch_iter = None
train_clf_loss, train_unsup_loss, mask_rate = data_source.batch_map_mean(
f_train,
train_batch_iter,
progress_iter_func=progress_bar,
n_batches=n_train_batches)
# train_clf_loss, train_unsup_loss, mask_rate = train_ds.batch_map_mean(
# f_train, batch_size=batch_size, shuffle=shuffle_rng, n_batches=n_train_batches,
# progress_iter_func=progress_bar)
if mask_rate > best_mask_rate:
best_mask_rate = mask_rate
improve = True
improve_str = '*** '
best_teacher_model_state = {
k: v.cpu().numpy()
for k, v in teacher_net.state_dict().items()
}
else:
improve = False
improve_str = ''
if not skip_epoch_eval:
tgt_pred_prob_y, = data_source.batch_map_concat(
f_pred_tgt,
test_batch_iter,
progress_iter_func=progress_bar)
mean_class_acc, cls_acc_str = evaluator.evaluate(
tgt_pred_prob_y)
t2 = time.time()
log('{}Epoch {} took {:.2f}s: TRAIN clf loss={:.6f}, unsup loss={:.6f}, mask={:.3%}; '
'TGT mean class acc={:.3%}'.format(improve_str, epoch,
t2 - t1, train_clf_loss,
train_unsup_loss,
mask_rate,
mean_class_acc))
log(' per class: {}'.format(cls_acc_str))
# Save results
if arr_tgt_pred_history is not None:
arr_tgt_pred_history.append(
tgt_pred_prob_y[None, ...].astype(np.float32))
else:
t2 = time.time()
log('{}Epoch {} took {:.2f}s: TRAIN clf loss={:.6f}, unsup loss={:.6f}, mask={:.3%}'
.format(improve_str, epoch, t2 - t1, train_clf_loss,
train_unsup_loss, mask_rate))
# Save network
if model_file != '':
cmdline_helpers.ensure_containing_dir_exists(model_file)
with open(model_file, 'wb') as f:
pickle.dump(best_teacher_model_state, f)
# Restore network to best state
teacher_net.load_state_dict({
k: torch.from_numpy(v)
for k, v in best_teacher_model_state.items()
})
# Predict on test set, without augmentation
tgt_pred_prob_y, = target_test_ds.batch_map_concat(
f_pred_tgt, batch_size=batch_size, progress_iter_func=progress_bar)
if d_target.has_ground_truth:
mean_class_acc, cls_acc_str = evaluator.evaluate(tgt_pred_prob_y)
log('FINAL: TGT mean class acc={:.3%}'.format(mean_class_acc))
log(' per class: {}'.format(cls_acc_str))
# Predict on test set, using augmentation
tgt_aug_pred_prob_y, = target_mult_test_ds.batch_map_concat(
f_pred_tgt_mult,
batch_size=batch_size,
progress_iter_func=progress_bar)
if d_target.has_ground_truth:
aug_mean_class_acc, aug_cls_acc_str = evaluator.evaluate(
tgt_aug_pred_prob_y)
log('FINAL: TGT AUG mean class acc={:.3%}'.format(
aug_mean_class_acc))
log(' per class: {}'.format(aug_cls_acc_str))
if f_target_pred is not None:
f_target_pred.create_array(g_tgt_pred, 'y_prob', tgt_pred_prob_y)
f_target_pred.create_array(g_tgt_pred, 'y_prob_aug',
tgt_aug_pred_prob_y)
f_target_pred.close()
if __name__ == "__main__":
(train_images, train_labels), (test_images,
test_labels) = mnist.load_data()
train_images = (2 * train_images.astype(np.float32) / 255) - 1
train_images = [
np.reshape(np.array(val), (1, 784)).flatten() for val in train_images
]
i = 0
generator = build_generator()
discriminator = build_discriminator()
gan_model = build_gan_model(generator, discriminator)
batches = data_source.ArrayDataSource([np.array(train_images)],
repeats=epochs)
for batch in batches.batch_iterator(batch_size, True):
if i % 1000 == 0:
samples = generator.predict(random_sample(1))
fig = plot_single(samples[0])
plt.savefig('out_classic_single/{}.png'.format(str(i).zfill(3)),
bbox_inches='tight',
pad_inches=0,
transparent=True)
plt.close(fig)
real_data_input, real_data_label = batch[0], np.ones(batch_size)
fake_data_input, fake_data_label = generator.predict(
random_sample(batch_size)), np.zeros(batch_size)
def main(relaxation=None, learn_prior=True, max_iters=None,
batch_size=25, num_latents=200, model_type=None, lr=None,
test_bias=False, train_dir=None, iwae_samples=1, dataset="mnist",
logf=None, var_lr_scale=10., Q_wd=.0001, Q_depth=-1, checkpoint_path=None):
tf.reset_default_graph()
#valid_batch_size = 100
if model_type == "L1":
num_layers = 1
layer_type = linear_layer
elif model_type == "L2":
num_layers = 2
layer_type = linear_layer
elif model_type == "NL1":
num_layers = 1
layer_type = nonlinear_layer
else:
assert False, "bad model type {}".format(model_type)
sess = tf.Session()
if dataset == "mnist":
#X_tr, X_va, X_te = datasets.load_mnist()
mnist = input_data.read_data_sets(os.getcwd()+'/MNIST', one_hot=True)
X_tr = mnist.train.images
X_te = mnist.test.images
X_va = mnist.validation.images
train_data = mnist.train
test_data = mnist.test
valid_data = mnist.validation
elif dataset == "omni":
X_tr, X_va, X_te = load_omniglot()
X_te = X_te[:8000]
X_va = X_va[:1300]
else:
assert False
train_mean = np.mean(X_tr, axis=0, keepdims=True)
train_output_bias = -np.log(1. / np.clip(train_mean, 0.001, 0.999) - 1.).astype(np.float32)
x = tf.placeholder(tf.float32, [None, 784])
x = tf.to_float(x > .5)
#x_im = tf.reshape(x, [-1, 28, 28, 1])
#tf.summary.image("x_true", x_im)
# make prior for top b
p_prior = tf.Variable(
tf.zeros([num_latents],
dtype=tf.float32),
trainable=learn_prior,
name='p_prior',
)
# create rebar specific variables temperature and eta
log_temperatures = [create_log_temp(1) for l in range(num_layers)]
temperatures = [tf.exp(log_temp) for log_temp in log_temperatures]
batch_temperatures = [tf.reshape(temp, [1, -1]) for temp in temperatures]
etas = [create_eta(1) for l in range(num_layers)]
batch_etas = [tf.reshape(eta, [1, -1]) for eta in etas]
# random uniform samples
u = [
tf.random_uniform([tf.shape(x)[0], num_latents], dtype=tf.float32)
for l in range(num_layers)
]
# create binary sampler
b_sampler = BSampler(u, "b_sampler")
gen_b_sampler = BSampler(u, "gen_b_sampler")
# generate hard forward pass
encoder_name = "encoder"
decoder_name = "decoder"
inf_la_b, samples_b = inference_network(
x, train_mean,
layer_type, num_layers,
num_latents, encoder_name, False, b_sampler
)
gen_la_b = generator_network(
samples_b, train_output_bias,
layer_type, num_layers,
num_latents, decoder_name, False
)
log_image(gen_la_b[-1], "x_pred")
# produce samples
_samples_la_b = generator_network(
None, train_output_bias,
layer_type, num_layers,
num_latents, decoder_name, True, sampler=gen_b_sampler, prior=p_prior
)
log_image(_samples_la_b[-1], "x_sample")
# hard loss evaluation and log probs
f_b, log_q_bs = neg_elbo(x, samples_b, inf_la_b, gen_la_b, p_prior, log=True)
batch_f_b = tf.expand_dims(f_b, 1)
total_loss = tf.reduce_mean(f_b)
#tf.summary.scalar("fb", total_loss)
# optimizer for model parameters
model_opt = tf.train.AdamOptimizer(lr, beta2=.99999)
# optimizer for variance reducing parameters
variance_opt = tf.train.AdamOptimizer(var_lr_scale * lr, beta2=.99999)
# get encoder and decoder variables
encoder_params = get_variables(encoder_name)
decoder_params = get_variables(decoder_name)
if learn_prior:
decoder_params.append(p_prior)
# compute and store gradients of hard loss with respect to encoder_parameters
encoder_loss_grads = {}
for g, v in model_opt.compute_gradients(total_loss, var_list=encoder_params):
encoder_loss_grads[v.name] = g
# get gradients for decoder parameters
decoder_gradvars = model_opt.compute_gradients(total_loss, var_list=decoder_params)
# will hold all gradvars for the model (non-variance adjusting variables)
model_gradvars = [gv for gv in decoder_gradvars]
# conditional samples
v = [v_from_u(_u, log_alpha) for _u, log_alpha in zip(u, inf_la_b)]
# need to create soft samplers
sig_z_sampler = SIGZSampler(u, batch_temperatures, "sig_z_sampler")
sig_zt_sampler = SIGZSampler(v, batch_temperatures, "sig_zt_sampler")
z_sampler = ZSampler(u, "z_sampler")
zt_sampler = ZSampler(v, "zt_sampler")
rebars = []
reinforces = []
variance_objectives = []
# have to produce 2 forward passes for each layer for z and zt samples
for l in range(num_layers):
cur_la_b = inf_la_b[l]
# if standard rebar or additive relaxation
if relaxation == "rebar" or relaxation == "add":
# compute soft samples and soft passes through model and soft elbos
cur_z_sample = sig_z_sampler.sample(cur_la_b, l)
prev_samples_z = samples_b[:l] + [cur_z_sample]
cur_zt_sample = sig_zt_sampler.sample(cur_la_b, l)
prev_samples_zt = samples_b[:l] + [cur_zt_sample]
prev_log_alphas = inf_la_b[:l] + [cur_la_b]
# soft forward passes
inf_la_z, samples_z = inference_network(
x, train_mean,
layer_type, num_layers,
num_latents, encoder_name, True, sig_z_sampler,
samples=prev_samples_z, log_alphas=prev_log_alphas
)
gen_la_z = generator_network(
samples_z, train_output_bias,
layer_type, num_layers,
num_latents, decoder_name, True
)
inf_la_zt, samples_zt = inference_network(
x, train_mean,
layer_type, num_layers,
num_latents, encoder_name, True, sig_zt_sampler,
samples=prev_samples_zt, log_alphas=prev_log_alphas
)
gen_la_zt = generator_network(
samples_zt, train_output_bias,
layer_type, num_layers,
num_latents, decoder_name, True
)
# soft loss evaluataions
f_z, _ = neg_elbo(x, samples_z, inf_la_z, gen_la_z, p_prior)
f_zt, _ = neg_elbo(x, samples_zt, inf_la_zt, gen_la_zt, p_prior)
if relaxation == "add" or relaxation == "all":
# sample z and zt
prev_bs = samples_b[:l]
cur_z_sample = z_sampler.sample(cur_la_b, l)
cur_zt_sample = zt_sampler.sample(cur_la_b, l)
q_z = Q_func(x, train_mean, cur_z_sample, prev_bs, Q_name(l), False, depth=Q_depth)
q_zt = Q_func(x, train_mean, cur_zt_sample, prev_bs, Q_name(l), True, depth=Q_depth)
#tf.summary.scalar("q_z_{}".format(l), tf.reduce_mean(q_z))
#tf.summary.scalar("q_zt_{}".format(l), tf.reduce_mean(q_zt))
if relaxation == "add":
f_z = f_z + q_z
f_zt = f_zt + q_zt
elif relaxation == "all":
f_z = q_z
f_zt = q_zt
else:
assert False
#tf.summary.scalar("f_z_{}".format(l), tf.reduce_mean(f_z))
#tf.summary.scalar("f_zt_{}".format(l), tf.reduce_mean(f_zt))
cur_samples_b = samples_b[l]
# get gradient of sample log-likelihood wrt current parameter
d_log_q_d_la = bernoulli_loglikelihood_derivitive(cur_samples_b, cur_la_b)
# get gradient of soft-losses wrt current parameter
d_f_z_d_la = tf.gradients(f_z, cur_la_b)[0]
d_f_zt_d_la = tf.gradients(f_zt, cur_la_b)[0]
batch_f_zt = tf.expand_dims(f_zt, 1)
eta = batch_etas[l]
# compute rebar and reinforce
#tf.summary.histogram("der_diff_{}".format(l), d_f_z_d_la - d_f_zt_d_la)
#tf.summary.histogram("d_log_q_d_la_{}".format(l), d_log_q_d_la)
rebar = ((batch_f_b - eta * batch_f_zt) * d_log_q_d_la + eta * (d_f_z_d_la - d_f_zt_d_la)) / batch_size
reinforce = batch_f_b * d_log_q_d_la / batch_size
rebars.append(rebar)
reinforces.append(reinforce)
#tf.summary.histogram("rebar_{}".format(l), rebar)
#tf.summary.histogram("reinforce_{}".format(l), reinforce)
# backpropogate rebar to individual layer parameters
layer_params = get_variables(layer_name(l), arr=encoder_params)
layer_rebar_grads = tf.gradients(cur_la_b, layer_params, grad_ys=rebar)
# get direct loss grads for each parameter
layer_loss_grads = [encoder_loss_grads[v.name] for v in layer_params]
# each param's gradient should be rebar + the direct loss gradient
layer_grads = [rg + lg for rg, lg in zip(layer_rebar_grads, layer_loss_grads)]
# for rg, lg, v in zip(layer_rebar_grads, layer_loss_grads, layer_params):
# tf.summary.histogram(v.name+"_grad_rebar", rg)
# tf.summary.histogram(v.name+"_grad_loss", lg)
layer_gradvars = list(zip(layer_grads, layer_params))
model_gradvars.extend(layer_gradvars)
variance_objective = tf.reduce_mean(tf.square(rebar))
variance_objectives.append(variance_objective)
variance_objective = tf.add_n(variance_objectives)
variance_vars = log_temperatures + etas
if relaxation != "rebar":
q_vars = get_variables("Q_")
wd = tf.add_n([Q_wd * tf.nn.l2_loss(v) for v in q_vars])
#tf.summary.scalar("Q_weight_decay", wd)
variance_vars = variance_vars + q_vars
else:
wd = 0.0
variance_gradvars = variance_opt.compute_gradients(variance_objective+wd, var_list=variance_vars)
variance_train_op = variance_opt.apply_gradients(variance_gradvars)
model_train_op = model_opt.apply_gradients(model_gradvars)
with tf.control_dependencies([model_train_op, variance_train_op]):
train_op = tf.no_op()
# for g, v in model_gradvars + variance_gradvars:
# print(g, v.name)
# if g is not None:
# tf.summary.histogram(v.name, v)
# tf.summary.histogram(v.name+"_grad", g)
# val_loss = tf.Variable(1000, trainable=False, name="val_loss", dtype=tf.float32)
# train_loss = tf.Variable(1000, trainable=False, name="train_loss", dtype=tf.float32)
# tf.summary.scalar("val_loss", val_loss)
# tf.summary.scalar("train_loss", train_loss)
# create savers
# iwae_elbo = -(tf.reduce_logsumexp(-f_b) - np.log(valid_batch_size))
sess.run(tf.global_variables_initializer())
iters_per_epoch = X_tr.shape[0] // batch_size
print("Train set has {} examples".format(X_tr.shape[0]))
if relaxation != "rebar":
print("Pretraining Q network")
for i in range(1000):
if i % 100 == 0:
print(i)
idx = np.random.randint(0, iters_per_epoch-1)
batch_xs = X_tr[idx * batch_size: (idx + 1) * batch_size]
sess.run(variance_train_op, feed_dict={x: batch_xs})
def get_loss(sess,ds,batch_size):
cost_eval = []
for xs in ds.batch_iterator(batch_size=batch_size, shuffle=True):
xs = np.squeeze(np.array(xs))
cost_eval.append(sess.run(f_b,{x:xs}))
return np.mean(cost_eval)
start = time.time()
from batchup import data_source
train_ds = data_source.ArrayDataSource([X_tr])
test_ds = data_source.ArrayDataSource([X_te])
valid_ds = data_source.ArrayDataSource([X_va])
# total_points = X_tr.shape[0]
# total_batch = int(total_points / batch_size)
# total_test_batch = int(X_te.shape[0] / batch_size)
#best_val_loss = np.inf
COUNT=[]; COST=[]; TIME=[];COST_TEST=[];COST_VALID=[];epoch_list=[];time_list=[]
EXPERIMENT = 'layer2_OMNI_Bernoulli_' + 'REBAR' if relaxation == 'rebar' else 'layer2_OMNI_Bernoulli_' + 'RELAX'
print('Training stats....',EXPERIMENT)
record = []; step = 0
for epoch in range(5000):
for batch_xs in train_ds.batch_iterator(batch_size=batch_size, shuffle=True):
batch_xs = np.squeeze(np.array(batch_xs))
cur_iter = step
if cur_iter > max_iters:
print("Training Completed")
return
#batch_xs = X_tr[i*batch_size: (i+1) * batch_size]
#batch_xs,_ = train_data.next_batch(batch_size)
loss, _ = sess.run([total_loss, train_op], feed_dict={x: batch_xs})
record.append(loss)
step += 1
if epoch%1 == 0:
valid_loss = get_loss(sess,valid_ds,batch_size=batch_size)
COUNT.append(step); COST.append(np.mean(record)); TIME.append(time.time()-start)
COST_VALID.append(valid_loss)
print(epoch,'valid_cost=',valid_loss)
print(epoch,'train_cost=',np.mean(record),'with std=',np.std(record))
if epoch%5 == 0:
COST_TEST.append(get_loss(sess,test_ds,batch_size=batch_size))
epoch_list.append(epoch)
time_list.append(time.time()-start)
import cPickle
directory = os.getcwd()+'/discrete_out/'
if not os.path.exists(directory):
os.makedirs(directory)
all_ = [COUNT,COST,TIME,COST_TEST,COST_VALID,epoch_list,time_list]
cPickle.dump(all_, open(directory+EXPERIMENT, 'w'))
record=[]
# model.cuda()
model = model.double()
criterea = torch.nn.CrossEntropyLoss(
) # this include softmax and cross entropy
criterea = torch.nn.BCELoss(
) # this is for binary cross entropy when you have only binary output (0/1 or True/False)
# criterea=torch.nn.MSELoss(size_average=False)#cross entropy loss
optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.5)
run = 0
epochs = 10
for loop in range(epochs):
# ______________________________________________________________________
# my batch creater
# Construct an array data source
ds = data_source.ArrayDataSource([scaled_input, y])
# Iterate over samples, drawing batches of 64 elements in random order
for (data, target) in ds.batch_iterator(
batch_size=1,
shuffle=True): # shuffle true will randomise every batch
new_data = data.reshape(-1, 1, 28, 28)
cool_data = new_data[0]
# cv2.imshow("image",np.reshape(cool_data,newshape=(28,28,1)))
# cv2.waitKey(500)
# cv2.destroyAllWindows()
# data, target=torch.from_numpy(new_data).double(),torch.from_numpy(target).double()
# inputs, labels = Variable(inputs.cuda()), Variable(labels.cuda())#this is for cuda gpu
data, target = torch.tensor(new_data), torch.tensor(target).long()
optimizer.zero_grad()
output = model(data)
