def test_process_parallel_data_source_with_index_mapping(process_pool): from batchup import data_source X = np.zeros((100,), dtype=int) indices = np.random.RandomState(12345).permutation(100) indices0 = indices[:50] indices1 = indices[50:] X[indices1] = 1 ds0 = data_source.ArrayDataSource([X], indices=indices0) ds1 = data_source.ArrayDataSource([X], indices=indices1) pds0 = process_pool.parallel_data_source(ds0) pds1 = process_pool.parallel_data_source(ds1) # Check number of samples assert ds0.num_samples() == 50 assert ds1.num_samples() == 50 assert pds0.num_samples() == 50 assert pds1.num_samples() == 50 BATCHSIZE = 10 for batch_X, in pds0.batch_iterator( batch_size=BATCHSIZE, shuffle=np.random.RandomState(12345)): # Ensure that all the return values are 0 assert (batch_X == 0).all() for batch_X, in pds1.batch_iterator( batch_size=BATCHSIZE, shuffle=np.random.RandomState(12345)): # Ensure that all the return values are 0 assert (batch_X == 1).all()
def predict(self, input_imgs, batch_size): predict_ds = data_source.ArrayDataSource([input_imgs]) predict_batch_iter = predict_ds.batch_iterator(batch_size=batch_size) predict_batch_total = len(input_imgs) // batch_size if len( input_imgs ) % batch_size == 0 else len(input_imgs) // batch_size + 1 epoch_logits = [] predict_batch_tqdm = tqdm(predict_batch_iter, total=predict_batch_total) for predict_step, [minibatch_valid_imgs ] in enumerate(predict_batch_tqdm): predict_batch_tqdm.set_description('predicting...') train_feed = { self._input: minibatch_valid_imgs, self._dropout_rate: 1.0, } minibatch_logits = self._sess.run(self._y_logits_op, feed_dict=train_feed) epoch_logits.append(minibatch_logits) return np.concatenate(epoch_logits, axis=0)
def predict(self, input_filenames, batch_size): input_filename_idxs = np.array([i for i in range(0, len(input_filenames))]) predict_ds = data_source.ArrayDataSource([input_filename_idxs]) predict_batch_iter = predict_ds.batch_iterator(batch_size=batch_size) predict_batch_total = len(input_filenames) // batch_size if len(input_filenames) % batch_size == 0 else len(input_filenames) // batch_size + 1 epoch_logits = [] predict_batch_tqdm = tqdm(predict_batch_iter, total=predict_batch_total) for predict_step , [minibatch_predict_filename_idxs] in enumerate(predict_batch_tqdm): predict_batch_tqdm.set_description('predicting...') minibatch_predict_filenames = itemgetter(*minibatch_predict_filename_idxs)(input_filenames) minibatch_predict_files = self.minibatch_preprocessing(minibatch_predict_filenames) logit_feed = { self._input: minibatch_predict_files, } minibatch_logits = self._sess.run(self._logits , feed_dict=logit_feed) epoch_logits.append(minibatch_logits) return np.concatenate(epoch_logits, axis=0)
def train(self, input_imgs, label_imgs, batch_size, dropout_rate, learning_rate): train_ds = data_source.ArrayDataSource([input_imgs, label_imgs]) train_batch_iter = train_ds.batch_iterator(batch_size=batch_size) train_batch_total = len(input_imgs) // batch_size if len( input_imgs ) % batch_size == 0 else len(input_imgs) // batch_size + 1 epoch_train_loss = [] epoch_train_iou = [] train_batch_tqdm = tqdm(train_batch_iter, total=train_batch_total) for train_step, [minibatch_train_imgs, minibatch_train_labels ] in enumerate(train_batch_tqdm): train_batch_tqdm.set_description('training...') train_feed = { self._input: minibatch_train_imgs, self._labels: minibatch_train_labels, self._dropout_rate: dropout_rate, self._learning_rate: learning_rate } minibatch_loss, loss_summary, _ = \ self._sess.run([self._loss_op, self._loss_summary , self._train_op], feed_dict=train_feed) eval_feed = { self._input: minibatch_train_imgs, self._labels: minibatch_train_labels, self._dropout_rate: 1.0, } minibatch_eval, eval_summary = self._sess.run( [self._eval_op, self._eval_summary], feed_dict=train_feed) epoch_train_loss.append(minibatch_loss) epoch_train_iou.append(minibatch_eval) global_step = self._sess.run(self._global_step) self._train_summary_writer.add_summary(loss_summary, global_step=global_step) self._train_summary_writer.flush() self._train_summary_writer.add_summary(eval_summary, global_step=global_step) self._train_summary_writer.flush() train_batch_tqdm.set_postfix(minibatch_loss=minibatch_loss, minibatch_iou=minibatch_eval) if self._hook and global_step % self._hook_every_n == 0: self._hook() epoch_train_loss = np.mean(epoch_train_loss) epoch_train_iou = np.mean(epoch_train_iou) return epoch_train_loss, epoch_train_iou
def train(self, input_filenames, labels, batch_size, learning_rate): input_filename_idxs = np.array([i for i in range(0, len(input_filenames))]) train_ds = data_source.ArrayDataSource([input_filename_idxs, labels]) train_batch_iter = train_ds.batch_iterator(batch_size=batch_size) train_batch_total = len(input_filenames) // batch_size if len(input_filenames) % batch_size == 0 else len(input_filenames) // batch_size + 1 epoch_train_loss = [] epoch_train_accuracy = [] train_batch_tqdm = tqdm(train_batch_iter, total=train_batch_total) for train_step, [minibatch_filename_idxs, minibatch_train_labels] in enumerate(train_batch_tqdm): train_batch_tqdm.set_description('training...') minibatch_filenames = itemgetter(*minibatch_filename_idxs)(input_filenames) minibatch_files = self.minibatch_preprocessing(minibatch_filenames) train_feed = { self._input: minibatch_files, self._labels: minibatch_train_labels, self._learning_rate: learning_rate, self._is_train: True, } minibatch_loss, loss_summary, _ = \ self._sess.run([self._loss_op, self._loss_summary , self._train_op], feed_dict=train_feed) train_feed = { self._input: minibatch_files, self._labels: minibatch_train_labels, } minibatch_accuracy, accuracy_summary = self._sess.run([self._accuracy_op, self._accuracy_summary], feed_dict=train_feed) epoch_train_loss.append(minibatch_loss) epoch_train_accuracy.append(minibatch_accuracy) global_step = self._sess.run(self._global_step) self._train_summary_writer.add_summary(loss_summary, global_step=global_step) self._train_summary_writer.flush() self._train_summary_writer.add_summary(accuracy_summary, global_step=global_step) self._train_summary_writer.flush() train_batch_tqdm.set_postfix(minibatch_loss=minibatch_loss, minibatch_accuracy=minibatch_accuracy) if self._hook and global_step % self._hook_every_n == 0: self._hook() epoch_train_loss = np.mean(epoch_train_loss) epoch_train_accuracy = np.mean(epoch_train_accuracy) return epoch_train_loss, epoch_train_accuracy
def validate(self, input_imgs, label_imgs, batch_size): valid_ds = data_source.ArrayDataSource([input_imgs, label_imgs]) valid_batch_iter = valid_ds.batch_iterator(batch_size=batch_size) valid_batch_total = len(input_imgs) // batch_size if len( input_imgs ) % batch_size == 0 else len(input_imgs) // batch_size + 1 epoch_valid_loss = [] epoch_valid_iou = [] valid_batch_tqdm = tqdm(valid_batch_iter, total=valid_batch_total) for valid_step, [minibatch_valid_imgs, minibatch_valid_labels ] in enumerate(valid_batch_tqdm): valid_batch_tqdm.set_description('validating...') train_feed = { self._input: minibatch_valid_imgs, self._labels: minibatch_valid_labels, self._dropout_rate: 1.0, } minibatch_loss, minibatch_eval = \ self._sess.run([self._loss_op, self._eval_op], feed_dict=train_feed) epoch_valid_loss.append(minibatch_loss) epoch_valid_iou.append(minibatch_eval) valid_batch_tqdm.set_postfix(minibatch_loss=minibatch_loss, minibatch_iou=minibatch_eval) epoch_valid_loss = np.mean(epoch_valid_loss) epoch_valid_iou = np.mean(epoch_valid_iou) global_step = self._sess.run(self._global_step) valid_loss_summary = tf.Summary() valid_loss_summary.value.add(tag="loss", simple_value=epoch_valid_loss) valid_loss_summary.value.add(tag="iou_metric", simple_value=epoch_valid_iou) self._validation_summary_writer.add_summary(valid_loss_summary, global_step=global_step) self._validation_summary_writer.flush() return epoch_valid_loss, epoch_valid_iou
def validate(self, input_filenames, labels, batch_size): input_filename_idxs = np.array([i for i in range(0, len(input_filenames))]) valid_ds = data_source.ArrayDataSource([input_filename_idxs, labels]) valid_batch_iter = valid_ds.batch_iterator(batch_size=batch_size) valid_batch_total = len(input_filenames) // batch_size if len(input_filenames) % batch_size == 0 else len(input_filenames) // batch_size + 1 epoch_valid_loss = [] epoch_valid_accuracy = [] valid_batch_tqdm = tqdm(valid_batch_iter, total=valid_batch_total) for valid_step, [minibatch_valid_filename_idxs, minibatch_valid_labels] in enumerate(valid_batch_tqdm): valid_batch_tqdm.set_description('validating...') minibatch_valid_filenames = itemgetter(*minibatch_valid_filename_idxs)(input_filenames) minibatch_valid_files = self.minibatch_preprocessing(minibatch_valid_filenames) valid_feed = { self._input: minibatch_valid_files, self._labels: minibatch_valid_labels, } minibatch_loss, minibatch_accuracy = \ self._sess.run([self._loss_op, self._accuracy_op], feed_dict=valid_feed) epoch_valid_loss.append(minibatch_loss) epoch_valid_accuracy.append(minibatch_accuracy) valid_batch_tqdm.set_postfix(minibatch_loss=minibatch_loss, minibatch_accuracy=minibatch_accuracy) epoch_valid_loss = np.mean(epoch_valid_loss) epoch_valid_accuracy = np.mean(epoch_valid_accuracy) global_step = self._sess.run(self._global_step) valid_loss_summary = tf.Summary() valid_loss_summary.value.add(tag="loss", simple_value=epoch_valid_loss) valid_loss_summary.value.add(tag="accuracy", simple_value=epoch_valid_accuracy) self._validation_summary_writer.add_summary(valid_loss_summary, global_step=global_step) self._validation_summary_writer.flush() return epoch_valid_loss, epoch_valid_accuracy
def test(): ds = data_source.ArrayDataSource([scaled_test, label_test]) # Iterate over samples, drawing batches of 64 elements in random order for (test_data, test_target) in ds.batch_iterator( batch_size=1000, shuffle=True): # shuffle true will randomise every batch test_data = test_data.reshape(-1, 1, 28, 28) # cv2.imshow("image",np.reshape(test_data[5],newshape=(28,28,1))) # cv2.waitKey(500) # cv2.destroyAllWindows() test_data1, target = torch.tensor(test_data), torch.tensor( test_target).long() optimizer.zero_grad() output = model(test_data1) out = torch.max(output, 1)[1] match = torch.eq(out, target.squeeze()) all1 = torch.sum(match) acc = all1.item() / len(match) print("test accuracy is ", acc) break
def train(loop): run = 0 ds = data_source.ArrayDataSource([scaled_input, y]) # Iterate over samples, drawing batches of 64 elements in random order for (data, target) in ds.batch_iterator( batch_size=1000, shuffle=True): # shuffle true will randomise every batch new_data = data.reshape(-1, 1, 28, 28) cool_data = new_data[0] # cv2.imshow("image",np.reshape(cool_data,newshape=(28,28,1))) # cv2.waitKey(500) # cv2.destroyAllWindows() # data, target=torch.from_numpy(new_data).double(),torch.from_numpy(target).double() # inputs, labels = Variable(inputs.cuda()), Variable(labels.cuda())#this is for cuda gpu data, target = torch.tensor(new_data), torch.tensor(target).long() optimizer.zero_grad() output = model(data) # this is to find accuracy out = torch.max(output, 1)[1] match = torch.eq(out, target.squeeze()) all1 = torch.sum(match) acc = all1.item() / len(match) # _, predicted = torch.max(target.data, 1) # total += labels.size(0) # correct += (predicted == target).sum().item() # print('Accuracy of the network on the 10000 test images: %d %%' % (100 * correct / total)) # loss = F.nll_loss(output, target.squeeze()) # loss = criterea(output,torch.max(target, 1)[0])#max returns two values it means its return max value in each row and[0 ] means its take first return value # loss = criterea(output, target.squeeze()) loss.backward() optimizer.step() run += 1 print( "Epochs : {ep} - Run Cycle : {rc} - Training Loss : {tl} - Training accuracy: {ta} " .format(ep=loop, rc=run, tl=loss.item(), ta=acc))
def experiment(exp, arch, learning_rate, standardise_samples, affine_std, xlat_range, hflip, intens_flip, intens_scale_range, intens_offset_range, gaussian_noise_std, num_epochs, batch_size, seed, log_file, device): import os import sys import cmdline_helpers if log_file == '': log_file = 'output_aug_log_{}.txt'.format(exp) elif log_file == 'none': log_file = None if log_file is not None: if os.path.exists(log_file): print('Output log file {} already exists'.format(log_file)) return intens_scale_range_lower, intens_scale_range_upper, intens_offset_range_lower, intens_offset_range_upper = \ cmdline_helpers.intens_aug_options(intens_scale_range, intens_offset_range) import time import math import numpy as np from batchup import data_source, work_pool import data_loaders import standardisation import network_architectures import augmentation import torch, torch.cuda from torch import nn from torch.nn import functional as F with torch.cuda.device(device): pool = work_pool.WorkerThreadPool(2) n_chn = 0 if exp == 'svhn_mnist': d_source = data_loaders.load_svhn(zero_centre=False, greyscale=True) d_target = data_loaders.load_mnist(invert=False, zero_centre=False, pad32=True, val=False) elif exp == 'mnist_svhn': d_source = data_loaders.load_mnist(invert=False, zero_centre=False, pad32=True) d_target = data_loaders.load_svhn(zero_centre=False, greyscale=True, val=False) elif exp == 'svhn_mnist_rgb': d_source = data_loaders.load_svhn(zero_centre=False, greyscale=False) d_target = data_loaders.load_mnist(invert=False, zero_centre=False, pad32=True, val=False, rgb=True) elif exp == 'mnist_svhn_rgb': d_source = data_loaders.load_mnist(invert=False, zero_centre=False, pad32=True, rgb=True) d_target = data_loaders.load_svhn(zero_centre=False, greyscale=False, val=False) elif exp == 'cifar_stl': d_source = data_loaders.load_cifar10(range_01=False) d_target = data_loaders.load_stl(zero_centre=False, val=False) elif exp == 'stl_cifar': d_source = data_loaders.load_stl(zero_centre=False) d_target = data_loaders.load_cifar10(range_01=False, val=False) elif exp == 'mnist_usps': d_source = data_loaders.load_mnist(zero_centre=False) d_target = data_loaders.load_usps(zero_centre=False, scale28=True, val=False) elif exp == 'usps_mnist': d_source = data_loaders.load_usps(zero_centre=False, scale28=True) d_target = data_loaders.load_mnist(zero_centre=False, val=False) elif exp == 'syndigits_svhn': d_source = data_loaders.load_syn_digits(zero_centre=False) d_target = data_loaders.load_svhn(zero_centre=False, val=False) elif exp == 'svhn_syndigits': d_source = data_loaders.load_svhn(zero_centre=False, val=False) d_target = data_loaders.load_syn_digits(zero_centre=False) elif exp == 'synsigns_gtsrb': d_source = data_loaders.load_syn_signs(zero_centre=False) d_target = data_loaders.load_gtsrb(zero_centre=False, val=False) elif exp == 'gtsrb_synsigns': d_source = data_loaders.load_gtsrb(zero_centre=False, val=False) d_target = data_loaders.load_syn_signs(zero_centre=False) else: print('Unknown experiment type \'{}\''.format(exp)) return # Delete the training ground truths as we should not be using them del d_target.train_y if standardise_samples: standardisation.standardise_dataset(d_source) standardisation.standardise_dataset(d_target) n_classes = d_source.n_classes print('Loaded data') if arch == '': if exp in {'mnist_usps', 'usps_mnist'}: arch = 'mnist-bn-32-64-256' if exp in {'svhn_mnist', 'mnist_svhn'}: arch = 'grey-32-64-128-gp' if exp in { 'cifar_stl', 'stl_cifar', 'syndigits_svhn', 'svhn_syndigits', 'svhn_mnist_rgb', 'mnist_svhn_rgb' }: arch = 'rgb-48-96-192-gp' if exp in {'synsigns_gtsrb', 'gtsrb_synsigns'}: arch = 'rgb40-48-96-192-384-gp' net_class, expected_shape = network_architectures.get_net_and_shape_for_architecture( arch) if expected_shape != d_source.train_X.shape[1:]: print( 'Architecture {} not compatible with experiment {}; it needs samples of shape {}, ' 'data has samples of shape {}'.format( arch, exp, expected_shape, d_source.train_X.shape[1:])) return net = net_class(n_classes).cuda() params = list(net.parameters()) optimizer = torch.optim.Adam(params, lr=learning_rate) classification_criterion = nn.CrossEntropyLoss() print('Built network') aug = augmentation.ImageAugmentation( hflip, xlat_range, affine_std, intens_scale_range_lower=intens_scale_range_lower, intens_scale_range_upper=intens_scale_range_upper, intens_offset_range_lower=intens_offset_range_lower, intens_offset_range_upper=intens_offset_range_upper, intens_flip=intens_flip, gaussian_noise_std=gaussian_noise_std) def augment(X_sup, y_sup): X_sup = aug.augment(X_sup) return [X_sup, y_sup] def f_train(X_sup, y_sup): X_sup = torch.autograd.Variable(torch.from_numpy(X_sup).cuda()) y_sup = torch.autograd.Variable( torch.from_numpy(y_sup).long().cuda()) optimizer.zero_grad() net.train(mode=True) sup_logits_out = net(X_sup) # Supervised classification loss clf_loss = classification_criterion(sup_logits_out, y_sup) loss_expr = clf_loss loss_expr.backward() optimizer.step() n_samples = X_sup.size()[0] return float(clf_loss.data.cpu().numpy()) * n_samples print('Compiled training function') def f_pred_src(X_sup): X_var = torch.autograd.Variable(torch.from_numpy(X_sup).cuda()) net.train(mode=False) return F.softmax(net(X_var)).data.cpu().numpy() def f_pred_tgt(X_sup): X_var = torch.autograd.Variable(torch.from_numpy(X_sup).cuda()) net.train(mode=False) return F.softmax(net(X_var)).data.cpu().numpy() def f_eval_src(X_sup, y_sup): y_pred_prob = f_pred_src(X_sup) y_pred = np.argmax(y_pred_prob, axis=1) return float((y_pred != y_sup).sum()) def f_eval_tgt(X_sup, y_sup): y_pred_prob = f_pred_tgt(X_sup) y_pred = np.argmax(y_pred_prob, axis=1) return float((y_pred != y_sup).sum()) print('Compiled evaluation function') # Setup output def log(text): print(text) if log_file is not None: with open(log_file, 'a') as f: f.write(text + '\n') f.flush() f.close() cmdline_helpers.ensure_containing_dir_exists(log_file) # Report setttings log('sys.argv={}'.format(sys.argv)) # Report dataset size log('Dataset:') log('SOURCE Train: X.shape={}, y.shape={}'.format( d_source.train_X.shape, d_source.train_y.shape)) log('SOURCE Test: X.shape={}, y.shape={}'.format( d_source.test_X.shape, d_source.test_y.shape)) log('TARGET Train: X.shape={}'.format(d_target.train_X.shape)) log('TARGET Test: X.shape={}, y.shape={}'.format( d_target.test_X.shape, d_target.test_y.shape)) print('Training...') train_ds = data_source.ArrayDataSource( [d_source.train_X, d_source.train_y]).map(augment) source_test_ds = data_source.ArrayDataSource( [d_source.test_X, d_source.test_y]) target_test_ds = data_source.ArrayDataSource( [d_target.test_X, d_target.test_y]) if seed != 0: shuffle_rng = np.random.RandomState(seed) else: shuffle_rng = np.random best_src_test_err = 1.0 for epoch in range(num_epochs): t1 = time.time() train_res = train_ds.batch_map_mean(f_train, batch_size=batch_size, shuffle=shuffle_rng) train_clf_loss = train_res[0] src_test_err, = source_test_ds.batch_map_mean( f_eval_src, batch_size=batch_size * 4) tgt_test_err, = target_test_ds.batch_map_mean( f_eval_tgt, batch_size=batch_size * 4) t2 = time.time() if src_test_err < best_src_test_err: log('*** Epoch {} took {:.2f}s: TRAIN clf loss={:.6f}; ' 'SRC TEST ERR={:.3%}, TGT TEST err={:.3%}'.format( epoch, t2 - t1, train_clf_loss, src_test_err, tgt_test_err)) best_src_test_err = src_test_err else: log('Epoch {} took {:.2f}s: TRAIN clf loss={:.6f}; ' 'SRC TEST ERR={:.3%}, TGT TEST err={:.3%}'.format( epoch, t2 - t1, train_clf_loss, src_test_err, tgt_test_err))
def experiment(exp, arch, rnd_init, img_size, standardise_samples, learning_rate, pretrained_lr_factor, fix_layers, double_softmax, use_dropout, scale_u_range, scale_x_range, scale_y_range, affine_std, xlat_range, rot_std, hflip, intens_scale_range, colour_rot_std, colour_off_std, greyscale, img_pad_width, num_epochs, batch_size, seed, log_file, result_file, hide_progress_bar, subsetsize, subsetseed, device): settings = locals().copy() if rnd_init: if fix_layers != '': print('`rnd_init` and `fix_layers` are mutually exclusive') return import os import sys import cmdline_helpers fix_layers = [lyr.strip() for lyr in fix_layers.split(',')] if log_file == '': log_file = 'output_aug_log_{}.txt'.format(exp) elif log_file == 'none': log_file = None if log_file is not None: if os.path.exists(log_file): print('Output log file {} already exists'.format(log_file)) return intens_scale_range_lower, intens_scale_range_upper = cmdline_helpers.colon_separated_range( intens_scale_range) scale_u_range = cmdline_helpers.colon_separated_range(scale_u_range) scale_x_range = cmdline_helpers.colon_separated_range(scale_x_range) scale_y_range = cmdline_helpers.colon_separated_range(scale_y_range) import time import tqdm import math import tables import numpy as np from batchup import data_source, work_pool import image_dataset, visda17_dataset, office_dataset import network_architectures import augmentation import image_transforms from sklearn.model_selection import StratifiedShuffleSplit import torch, torch.cuda from torch import nn from torch.nn import functional as F if hide_progress_bar: progress_bar = None else: progress_bar = tqdm.tqdm with torch.cuda.device(device): pool = work_pool.WorkerThreadPool(4) n_chn = 0 half_batch_size = batch_size // 2 RESNET_ARCHS = {'resnet50', 'resnet101', 'resnet152'} RNDINIT_ARCHS = {'vgg13_48_gp'} if arch == '': if exp in {'train_val', 'train_test'}: arch = 'resnet50' if arch in RESNET_ARCHS and not rnd_init: mean_value = np.array([0.485, 0.456, 0.406]) std_value = np.array([0.229, 0.224, 0.225]) elif arch in RNDINIT_ARCHS: mean_value = np.array([0.5, 0.5, 0.5]) std_value = np.array([0.5, 0.5, 0.5]) rnd_init = True else: mean_value = std_value = None img_shape = (img_size, img_size) img_padding = (img_pad_width, img_pad_width) if exp == 'visda_train_val': d_source = visda17_dataset.TrainDataset(img_size=img_shape, range01=True, rgb_order=True) d_target = visda17_dataset.ValidationDataset(img_size=img_shape, range01=True, rgb_order=True) elif exp == 'visda_train_test': d_source = visda17_dataset.TrainDataset(img_size=img_shape, range01=True, rgb_order=True) d_target = visda17_dataset.TestDataset(img_size=img_shape, range01=True, rgb_order=True) elif exp == 'office_amazon_dslr': d_source = office_dataset.OfficeAmazonDataset(img_size=img_shape, range01=True, rgb_order=True) d_target = office_dataset.OfficeDSLRDataset(img_size=img_shape, range01=True, rgb_order=True) elif exp == 'office_amazon_webcam': d_source = office_dataset.OfficeAmazonDataset(img_size=img_shape, range01=True, rgb_order=True) d_target = office_dataset.OfficeWebcamDataset(img_size=img_shape, range01=True, rgb_order=True) elif exp == 'office_dslr_amazon': d_source = office_dataset.OfficeDSLRDataset(img_size=img_shape, range01=True, rgb_order=True) d_target = office_dataset.OfficeAmazonDataset(img_size=img_shape, range01=True, rgb_order=True) elif exp == 'office_dslr_webcam': d_source = office_dataset.OfficeDSLRDataset(img_size=img_shape, range01=True, rgb_order=True) d_target = office_dataset.OfficeWebcamDataset(img_size=img_shape, range01=True, rgb_order=True) elif exp == 'office_webcam_amazon': d_source = office_dataset.OfficeWebcamDataset(img_size=img_shape, range01=True, rgb_order=True) d_target = office_dataset.OfficeAmazonDataset(img_size=img_shape, range01=True, rgb_order=True) elif exp == 'office_webcam_dslr': d_source = office_dataset.OfficeWebcamDataset(img_size=img_shape, range01=True, rgb_order=True) d_target = office_dataset.OfficeDSLRDataset(img_size=img_shape, range01=True, rgb_order=True) else: print('Unknown experiment type \'{}\''.format(exp)) return # # Result file # if result_file != '': cmdline_helpers.ensure_containing_dir_exists(result_file) h5_filters = tables.Filters(complevel=9, complib='blosc') f_target_pred = tables.open_file(result_file, mode='w') g_tgt_pred = f_target_pred.create_group(f_target_pred.root, 'target_pred_y', 'Target prediction') arr_tgt_pred = f_target_pred.create_earray( g_tgt_pred, 'y', tables.Float32Atom(), (0, len(d_target.images), d_target.n_classes), filters=h5_filters) else: f_target_pred = None g_tgt_pred = None arr_tgt_pred = None # Delete the training ground truths as we should not be using them # del d_target.y n_classes = d_source.n_classes print('Loaded data') net_class = network_architectures.get_build_fn_for_architecture(arch) net = net_class(n_classes, img_size, use_dropout, not rnd_init).cuda() if arch in RESNET_ARCHS and not rnd_init: named_params = list(net.named_parameters()) new_params = [] pretrained_params = [] for name, param in named_params: if name.startswith('new_'): new_params.append(param) else: fix = False for lyr in fix_layers: if name.startswith(lyr + '.'): fix = True break if not fix: pretrained_params.append(param) else: print('Fixing param {}'.format(name)) param.requires_grad = False new_optimizer = torch.optim.Adam(new_params, lr=learning_rate) if len(pretrained_params) > 0: pretrained_optimizer = torch.optim.Adam(pretrained_params, lr=learning_rate * pretrained_lr_factor) else: pretrained_optimizer = None else: new_optimizer = torch.optim.Adam(net.parameters(), lr=learning_rate) pretrained_optimizer = None classification_criterion = nn.CrossEntropyLoss() print('Built network') # Image augmentation aug = augmentation.ImageAugmentation( hflip, xlat_range, affine_std, rot_std=rot_std, intens_scale_range_lower=intens_scale_range_lower, intens_scale_range_upper=intens_scale_range_upper, colour_rot_std=colour_rot_std, colour_off_std=colour_off_std, greyscale=greyscale, scale_u_range=scale_u_range, scale_x_range=scale_x_range, scale_y_range=scale_y_range) test_aug = augmentation.ImageAugmentation(hflip, xlat_range, 0.0, rot_std=0.0, scale_u_range=scale_u_range, scale_x_range=scale_x_range, scale_y_range=scale_y_range) border_value = int(np.mean(mean_value) * 255 + 0.5) sup_xf = image_transforms.Compose( image_transforms.ScaleCropAndAugmentAffine(img_shape, img_padding, True, aug, border_value, mean_value, std_value), image_transforms.ToTensor(), ) test_xf = image_transforms.Compose( image_transforms.ScaleAndCrop(img_shape, img_padding, False), image_transforms.ToTensor(), image_transforms.Standardise(mean_value, std_value), ) test_xf_aug_mult = image_transforms.Compose( image_transforms.ScaleCropAndAugmentAffineMultiple( 16, img_shape, img_padding, True, test_aug, border_value, mean_value, std_value), image_transforms.ToTensorMultiple(), ) def augment(X_sup, y_sup): X_sup = sup_xf(X_sup)[0] return X_sup, y_sup _one = torch.autograd.Variable( torch.from_numpy(np.array([1.0]).astype(np.float32)).cuda()) def f_train(X_sup, y_sup): X_sup = torch.autograd.Variable(torch.from_numpy(X_sup).cuda()) y_sup = torch.autograd.Variable( torch.from_numpy(y_sup).long().cuda()) if pretrained_optimizer is not None: pretrained_optimizer.zero_grad() new_optimizer.zero_grad() net.train(mode=True) sup_logits_out = net(X_sup) # Supervised classification loss if double_softmax: clf_loss = classification_criterion(F.softmax(sup_logits_out), y_sup) else: clf_loss = classification_criterion(sup_logits_out, y_sup) loss_expr = clf_loss loss_expr.backward() if pretrained_optimizer is not None: pretrained_optimizer.step() new_optimizer.step() n_samples = X_sup.size()[0] return (float(clf_loss.data.cpu()[0]) * n_samples, ) print('Compiled training function') def f_pred(X_sup): X_var = torch.autograd.Variable(torch.from_numpy(X_sup).cuda()) net.train(mode=False) return F.softmax(net(X_var)).data.cpu().numpy() def f_pred_tgt_mult(X_sup): net.train(mode=False) y_pred_aug = [] for aug_i in range(len(X_sup)): X_var = torch.autograd.Variable( torch.from_numpy(X_sup[aug_i, ...]).cuda()) y_pred = F.softmax(net(X_var)).data.cpu().numpy() y_pred_aug.append(y_pred[None, ...]) y_pred_aug = np.concatenate(y_pred_aug, axis=0) return (y_pred_aug.mean(axis=0), ) print('Compiled evaluation function') # Setup output def log(text): print(text) if log_file is not None: with open(log_file, 'a') as f: f.write(text + '\n') f.flush() f.close() cmdline_helpers.ensure_containing_dir_exists(log_file) # Report setttings log('Program = {}'.format(sys.argv[0])) log('Settings: {}'.format(', '.join([ '{}={}'.format(key, settings[key]) for key in sorted(list(settings.keys())) ]))) # Report dataset size log('Dataset:') print('SOURCE len(X)={}, y.shape={}'.format(len(d_source.images), d_source.y.shape)) print('TARGET len(X)={}'.format(len(d_target.images))) # Subset source_indices, target_indices, n_src, n_tgt = image_dataset.subset_indices( d_source, d_target, subsetsize, subsetseed) n_train_batches = n_src // batch_size + 1 n_test_batches = n_tgt // (batch_size * 2) + 1 print('Training...') train_ds = data_source.ArrayDataSource([d_source.images, d_source.y], indices=source_indices) train_ds = train_ds.map(augment) train_ds = pool.parallel_data_source(train_ds, batch_buffer_size=min( 20, n_train_batches)) # source_test_ds = data_source.ArrayDataSource([d_source.images]) # source_test_ds = pool.parallel_data_source(source_test_ds) target_ds_for_test = data_source.ArrayDataSource( [d_target.images], indices=target_indices) target_test_ds = target_ds_for_test.map(test_xf) target_test_ds = pool.parallel_data_source(target_test_ds, batch_buffer_size=min( 20, n_test_batches)) target_mult_test_ds = target_ds_for_test.map(test_xf_aug_mult) target_mult_test_ds = pool.parallel_data_source(target_mult_test_ds, batch_buffer_size=min( 20, n_test_batches)) if seed != 0: shuffle_rng = np.random.RandomState(seed) else: shuffle_rng = np.random if d_target.has_ground_truth: evaluator = d_target.prediction_evaluator(target_indices) else: evaluator = None train_batch_iter = train_ds.batch_iterator(batch_size=batch_size, shuffle=shuffle_rng) for epoch in range(num_epochs): t1 = time.time() test_batch_iter = target_test_ds.batch_iterator( batch_size=batch_size) train_clf_loss, = data_source.batch_map_mean( f_train, train_batch_iter, n_batches=n_train_batches, progress_iter_func=progress_bar) # train_clf_loss, train_unsup_loss, mask_rate, train_align_loss = train_ds.batch_map_mean( # lambda *x: 1.0, batch_size=batch_size, shuffle=shuffle_rng, n_batches=n_train_batches, # progress_iter_func=progress_bar) if d_target.has_ground_truth or arr_tgt_pred is not None: tgt_pred_prob_y, = data_source.batch_map_concat( f_pred, test_batch_iter, progress_iter_func=progress_bar) else: tgt_pred_prob_y = None train_batch_iter = train_ds.batch_iterator(batch_size=batch_size, shuffle=shuffle_rng) if d_target.has_ground_truth: mean_class_acc, cls_acc_str = evaluator.evaluate( tgt_pred_prob_y) t2 = time.time() log('Epoch {} took {:.2f}s: TRAIN clf loss={:.6f}; ' 'TGT mean class acc={:.3%}'.format(epoch, t2 - t1, train_clf_loss, mean_class_acc)) log(' per class: {}'.format(cls_acc_str)) else: t2 = time.time() log('Epoch {} took {:.2f}s: TRAIN clf loss={:.6f}'.format( epoch, t2 - t1, train_clf_loss)) # Save results if arr_tgt_pred is not None: arr_tgt_pred.append(tgt_pred_prob_y[None, ...].astype(np.float32)) # Predict on test set, using augmentation tgt_aug_pred_prob_y, = target_mult_test_ds.batch_map_concat( f_pred_tgt_mult, batch_size=batch_size, progress_iter_func=progress_bar) if d_target.has_ground_truth: aug_mean_class_acc, aug_cls_acc_str = evaluator.evaluate( tgt_aug_pred_prob_y) log('FINAL: TGT AUG mean class acc={:.3%}'.format( aug_mean_class_acc)) log(' per class: {}'.format(aug_cls_acc_str)) if f_target_pred is not None: f_target_pred.create_array(g_tgt_pred, 'y_prob', tgt_pred_prob_y) f_target_pred.create_array(g_tgt_pred, 'y_prob_aug', tgt_aug_pred_prob_y) f_target_pred.close()
rpn = [] feat_stride = 16 anchor_scale = [8, 16, 32] imageNameFile = "../../../Datasets/VOCdevkit/VOC2012/ImageSets/Main/aeroplane_train.txt" vocPath = "../../../Datasets/VOCdevkit/VOC2012" Image_data, boundingBX_labels, im_dims = read_pascal_voc.prepareBatch( 0, 5, imageNameFile, vocPath) print(Image_data, boundingBX_labels, im_dims) epochs = 1 for loop in range(epochs): # ______________________________________________________________________ # my batch creater # Construct an array data source ds = data_source.ArrayDataSource([Image_data, boundingBX_labels, im_dims]) # Iterate over samples, drawing batches of 64 elements in random order for (image_input, gt_box, image_dim) in ds.batch_iterator( batch_size=1, shuffle=True): # shuffle true will randomise every batch # accuoutput=sess.run([rpn], feed_dict={input_x: image_input, gt_bbox: gt_box, im_dimsal: image_dim}) _label, rpn_bbox_targets, rpn_bbox_inside_weights, rpn_bbox_outside_weights = anchor_target_layer.anchor_target_layer_python( rpn_cls_score=image_input, gt_boxes=gt_box, im_dims=image_dim, feat_strides=feat_stride, anchor_scales=anchor_scale) # (rpn_label)
def test_thread_parallel_data_source(thread_pool): from batchup import data_source class _AbstractExampleBatchIterator (object): def __init__(self, N): self.N = N # Define __len__ here to save us some work in the base classes that we # will actually use def __len__(self): return self.N class _ExampleBatchIteratorIdentity (_AbstractExampleBatchIterator): def __getitem__(self, indices): return indices class _ExampleBatchIteratorSquare (_AbstractExampleBatchIterator): def __getitem__(self, indices): return indices**2 class _ExampleBatchIteratorError (_AbstractExampleBatchIterator): def __getitem__(self, indices): raise ValueError ds = data_source.ArrayDataSource( [_ExampleBatchIteratorIdentity(100), _ExampleBatchIteratorSquare(100)] ) pds = thread_pool.parallel_data_source(ds) # Check number of samples assert ds.num_samples() == 100 assert pds.num_samples() == 100 # Arrays of flags indicating which numbers we got back n_flags = np.zeros((100,), dtype=bool) n_sqr_flags = np.zeros((10000,), dtype=bool) BATCHSIZE = 10 for batch in pds.batch_iterator( batch_size=BATCHSIZE, shuffle=np.random.RandomState(12345)): # batch should be a list assert isinstance(batch, tuple) # batch should contain two arrays assert len(batch) == 2 # each array should be of length BATCHSIZE assert batch[0].shape[0] == BATCHSIZE assert batch[1].shape[0] == BATCHSIZE # Check off the numbers we got back n_flags[batch[0]] = True n_sqr_flags[batch[1]] = True # Check the flags arrays assert n_flags.sum() == 100 assert n_sqr_flags.sum() == 100 expected_n_flags = np.ones((100,), dtype=bool) expected_n_sqr_flags = np.zeros((10000,), dtype=bool) expected_n_sqr_flags[np.arange(100)**2] = True assert (n_flags == expected_n_flags).all() assert (n_sqr_flags == expected_n_sqr_flags).all() # Check that passing something that isn't a data source fails with pytest.raises(TypeError): thread_pool.parallel_data_source(_ExampleBatchIteratorIdentity(100)) # Check that passing a non random access data source fails with pytest.raises(TypeError): def make_batch_iter(**kwargs): def batch_iterator(batch_size): for i in range(0, 100, batch_size): yield np.random.normal(size=(batch_size, 2)) cds = data_source.CallableDataSource(make_batch_iter) thread_pool.parallel_data_source(cds) # Check that errors raised by data sources are passed back eds = data_source.ArrayDataSource([_ExampleBatchIteratorError(100)]) peds = thread_pool.parallel_data_source(eds) peds_iter = peds.batch_iterator(batch_size=BATCHSIZE, shuffle=np.random.RandomState(12345)) with pytest.raises(ValueError): next(peds_iter)
prior_train_op = tf.train.GradientDescentOptimizer( learning_rate=0.01).minimize(gen_loss, var_list=[prior_logit0]) with tf.control_dependencies([gen_train_op, inf_train_op, prior_train_op]): #with tf.control_dependencies([gen_train_op, inf_train_op]): train_op = tf.no_op() init_op = tf.global_variables_initializer() #%% TRAIN # get data train_data, valid_data, test_data = load_omniglot() from batchup import data_source train_ds = data_source.ArrayDataSource([train_data]) test_ds = data_source.ArrayDataSource([test_data[:8000]]) valid_ds = data_source.ArrayDataSource([valid_data[:1300]]) directory = os.getcwd() + '/discrete_out/' if not os.path.exists(directory): os.makedirs(directory) batch_size = 25 total_points = train_data.shape[0] total_batch = int(total_points / batch_size) total_test_batch = int(test_data.shape[0] / batch_size) training_epochs = 1500 #600epoch 42min display_step = total_batch
def test_process_parallel_data_source(process_pool): from batchup import data_source ds = data_source.ArrayDataSource( [_ExampleBatchIteratorIdentity(100), _ExampleBatchIteratorSquare(100)] ) pds = process_pool.parallel_data_source(ds) # Check number of samples assert ds.num_samples() == 100 assert pds.num_samples() == 100 # Arrays of flags indicating which numbers we got back n_flags = np.zeros((100,), dtype=bool) n_sqr_flags = np.zeros((10000,), dtype=bool) BATCHSIZE = 10 for batch in pds.batch_iterator( batch_size=BATCHSIZE, shuffle=np.random.RandomState(12345)): # batch should be a list assert isinstance(batch, tuple) # batch should contain two arrays assert len(batch) == 2 # each array should be of length BATCHSIZE assert batch[0].shape[0] == BATCHSIZE assert batch[1].shape[0] == BATCHSIZE # Check off the numbers we got back n_flags[batch[0]] = True n_sqr_flags[batch[1]] = True # Check the flags arrays assert n_flags.sum() == 100 assert n_sqr_flags.sum() == 100 expected_n_flags = np.ones((100,), dtype=bool) expected_n_sqr_flags = np.zeros((10000,), dtype=bool) expected_n_sqr_flags[np.arange(100)**2] = True assert (n_flags == expected_n_flags).all() assert (n_sqr_flags == expected_n_sqr_flags).all() # Check that passing something that isn't a data source fails with pytest.raises(TypeError): process_pool.parallel_data_source(_ExampleBatchIteratorIdentity(100)) # Check that passing a non random access data source fails with pytest.raises(TypeError): def make_batch_iter(**kwargs): def batch_iterator(batch_size): for i in range(0, 100, batch_size): yield np.random.normal(size=(batch_size, 2)) cds = data_source.CallableDataSource(make_batch_iter) process_pool.parallel_data_source(cds) # Check that errors raised by data sources are passed back eds = data_source.ArrayDataSource([_ExampleBatchIteratorError(100)]) peds = process_pool.parallel_data_source(eds) peds_iter = peds.batch_iterator(batch_size=BATCHSIZE, shuffle=np.random.RandomState(12345)) with pytest.raises(ValueError): next(peds_iter)
def experiment(exp, arch, loss, double_softmax, confidence_thresh, rampup, teacher_alpha, fix_ema, unsup_weight, cls_bal_scale, cls_bal_scale_range, cls_balance, cls_balance_loss, combine_batches, learning_rate, standardise_samples, src_affine_std, src_xlat_range, src_hflip, src_intens_flip, src_intens_scale_range, src_intens_offset_range, src_gaussian_noise_std, tgt_affine_std, tgt_xlat_range, tgt_hflip, tgt_intens_flip, tgt_intens_scale_range, tgt_intens_offset_range, tgt_gaussian_noise_std, num_epochs, batch_size, epoch_size, seed, log_file, model_file, device): settings = locals().copy() import os import sys import pickle import cmdline_helpers if log_file == '': log_file = 'output_aug_log_{}.txt'.format(exp) elif log_file == 'none': log_file = None if log_file is not None: if os.path.exists(log_file): print('Output log file {} already exists'.format(log_file)) return use_rampup = rampup > 0 src_intens_scale_range_lower, src_intens_scale_range_upper, src_intens_offset_range_lower, src_intens_offset_range_upper = \ cmdline_helpers.intens_aug_options(src_intens_scale_range, src_intens_offset_range) tgt_intens_scale_range_lower, tgt_intens_scale_range_upper, tgt_intens_offset_range_lower, tgt_intens_offset_range_upper = \ cmdline_helpers.intens_aug_options(tgt_intens_scale_range, tgt_intens_offset_range) import time import math import numpy as np from batchup import data_source, work_pool import data_loaders import standardisation import network_architectures import augmentation import torch, torch.cuda from torch import nn from torch.nn import functional as F import optim_weight_ema torch_device = torch.device(device) pool = work_pool.WorkerThreadPool(2) n_chn = 0 if exp == 'svhn_mnist': d_source = data_loaders.load_svhn(zero_centre=False, greyscale=True) d_target = data_loaders.load_mnist(invert=False, zero_centre=False, pad32=True, val=False) elif exp == 'mnist_svhn': d_source = data_loaders.load_mnist(invert=False, zero_centre=False, pad32=True) d_target = data_loaders.load_svhn(zero_centre=False, greyscale=True, val=False) elif exp == 'svhn_mnist_rgb': d_source = data_loaders.load_svhn(zero_centre=False, greyscale=False) d_target = data_loaders.load_mnist(invert=False, zero_centre=False, pad32=True, val=False, rgb=True) elif exp == 'mnist_svhn_rgb': d_source = data_loaders.load_mnist(invert=False, zero_centre=False, pad32=True, rgb=True) d_target = data_loaders.load_svhn(zero_centre=False, greyscale=False, val=False) elif exp == 'cifar_stl': d_source = data_loaders.load_cifar10(range_01=False) d_target = data_loaders.load_stl(zero_centre=False, val=False) elif exp == 'stl_cifar': d_source = data_loaders.load_stl(zero_centre=False) d_target = data_loaders.load_cifar10(range_01=False, val=False) elif exp == 'mnist_usps': d_source = data_loaders.load_mnist(zero_centre=False) d_target = data_loaders.load_usps(zero_centre=False, scale28=True, val=False) elif exp == 'usps_mnist': d_source = data_loaders.load_usps(zero_centre=False, scale28=True) d_target = data_loaders.load_mnist(zero_centre=False, val=False) elif exp == 'syndigits_svhn': d_source = data_loaders.load_syn_digits(zero_centre=False) d_target = data_loaders.load_svhn(zero_centre=False, val=False) elif exp == 'synsigns_gtsrb': d_source = data_loaders.load_syn_signs(zero_centre=False) d_target = data_loaders.load_gtsrb(zero_centre=False, val=False) else: print('Unknown experiment type \'{}\''.format(exp)) return # Delete the training ground truths as we should not be using them del d_target.train_y if standardise_samples: standardisation.standardise_dataset(d_source) standardisation.standardise_dataset(d_target) n_classes = d_source.n_classes print('Loaded data') if arch == '': if exp in {'mnist_usps', 'usps_mnist'}: arch = 'mnist-bn-32-64-256' if exp in {'svhn_mnist', 'mnist_svhn'}: arch = 'grey-32-64-128-gp' if exp in { 'cifar_stl', 'stl_cifar', 'syndigits_svhn', 'svhn_mnist_rgb', 'mnist_svhn_rgb' }: arch = 'rgb-128-256-down-gp' if exp in {'synsigns_gtsrb'}: arch = 'rgb40-96-192-384-gp' net_class, expected_shape = network_architectures.get_net_and_shape_for_architecture( arch) if expected_shape != d_source.train_X.shape[1:]: print( 'Architecture {} not compatible with experiment {}; it needs samples of shape {}, ' 'data has samples of shape {}'.format(arch, exp, expected_shape, d_source.train_X.shape[1:])) return student_net = net_class(n_classes).to(torch_device) teacher_net = net_class(n_classes).to(torch_device) student_params = list(student_net.parameters()) teacher_params = list(teacher_net.parameters()) for param in teacher_params: param.requires_grad = False student_optimizer = torch.optim.Adam(student_params, lr=learning_rate) if fix_ema: teacher_optimizer = optim_weight_ema.EMAWeightOptimizer( teacher_net, student_net, alpha=teacher_alpha) else: teacher_optimizer = optim_weight_ema.OldWeightEMA(teacher_net, student_net, alpha=teacher_alpha) classification_criterion = nn.CrossEntropyLoss() print('Built network') src_aug = augmentation.ImageAugmentation( src_hflip, src_xlat_range, src_affine_std, intens_flip=src_intens_flip, intens_scale_range_lower=src_intens_scale_range_lower, intens_scale_range_upper=src_intens_scale_range_upper, intens_offset_range_lower=src_intens_offset_range_lower, intens_offset_range_upper=src_intens_offset_range_upper, gaussian_noise_std=src_gaussian_noise_std) tgt_aug = augmentation.ImageAugmentation( tgt_hflip, tgt_xlat_range, tgt_affine_std, intens_flip=tgt_intens_flip, intens_scale_range_lower=tgt_intens_scale_range_lower, intens_scale_range_upper=tgt_intens_scale_range_upper, intens_offset_range_lower=tgt_intens_offset_range_lower, intens_offset_range_upper=tgt_intens_offset_range_upper, gaussian_noise_std=tgt_gaussian_noise_std) if combine_batches: def augment(X_sup, y_src, X_tgt): X_src_stu, X_src_tea = src_aug.augment_pair(X_sup) X_tgt_stu, X_tgt_tea = tgt_aug.augment_pair(X_tgt) return X_src_stu, X_src_tea, y_src, X_tgt_stu, X_tgt_tea else: def augment(X_src, y_src, X_tgt): X_src = src_aug.augment(X_src) X_tgt_stu, X_tgt_tea = tgt_aug.augment_pair(X_tgt) return X_src, y_src, X_tgt_stu, X_tgt_tea rampup_weight_in_list = [0] cls_bal_fn = network_architectures.get_cls_bal_function(cls_balance_loss) def compute_aug_loss(stu_out, tea_out): # Augmentation loss if use_rampup: unsup_mask = None conf_mask_count = None unsup_mask_count = None else: conf_tea = torch.max(tea_out, 1)[0] unsup_mask = conf_mask = (conf_tea > confidence_thresh).float() unsup_mask_count = conf_mask_count = conf_mask.sum() if loss == 'bce': aug_loss = network_architectures.robust_binary_crossentropy( stu_out, tea_out) else: d_aug_loss = stu_out - tea_out aug_loss = d_aug_loss * d_aug_loss # Class balance scaling if cls_bal_scale: if use_rampup: n_samples = float(aug_loss.shape[0]) else: n_samples = unsup_mask.sum() avg_pred = n_samples / float(n_classes) bal_scale = avg_pred / torch.clamp(tea_out.sum(dim=0), min=1.0) if cls_bal_scale_range != 0.0: bal_scale = torch.clamp(bal_scale, min=1.0 / cls_bal_scale_range, max=cls_bal_scale_range) bal_scale = bal_scale.detach() aug_loss = aug_loss * bal_scale[None, :] aug_loss = aug_loss.mean(dim=1) if use_rampup: unsup_loss = aug_loss.mean() * rampup_weight_in_list[0] else: unsup_loss = (aug_loss * unsup_mask).mean() # Class balance loss if cls_balance > 0.0: # Compute per-sample average predicated probability # Average over samples to get average class prediction avg_cls_prob = stu_out.mean(dim=0) # Compute loss equalise_cls_loss = cls_bal_fn(avg_cls_prob, float(1.0 / n_classes)) equalise_cls_loss = equalise_cls_loss.mean() * n_classes if use_rampup: equalise_cls_loss = equalise_cls_loss * rampup_weight_in_list[0] else: if rampup == 0: equalise_cls_loss = equalise_cls_loss * unsup_mask.mean( dim=0) unsup_loss += equalise_cls_loss * cls_balance return unsup_loss, conf_mask_count, unsup_mask_count if combine_batches: def f_train(X_src0, X_src1, y_src, X_tgt0, X_tgt1): X_src0 = torch.tensor(X_src0, dtype=torch.float, device=torch_device) X_src1 = torch.tensor(X_src1, dtype=torch.float, device=torch_device) y_src = torch.tensor(y_src, dtype=torch.long, device=torch_device) X_tgt0 = torch.tensor(X_tgt0, dtype=torch.float, device=torch_device) X_tgt1 = torch.tensor(X_tgt1, dtype=torch.float, device=torch_device) n_samples = X_src0.size()[0] n_total = n_samples + X_tgt0.size()[0] student_optimizer.zero_grad() student_net.train() teacher_net.train() # Concatenate source and target mini-batches X0 = torch.cat([X_src0, X_tgt0], 0) X1 = torch.cat([X_src1, X_tgt1], 0) student_logits_out = student_net(X0) student_prob_out = F.softmax(student_logits_out, dim=1) src_logits_out = student_logits_out[:n_samples] src_prob_out = student_prob_out[:n_samples] teacher_logits_out = teacher_net(X1) teacher_prob_out = F.softmax(teacher_logits_out, dim=1) # Supervised classification loss if double_softmax: clf_loss = classification_criterion(src_prob_out, y_src) else: clf_loss = classification_criterion(src_logits_out, y_src) unsup_loss, conf_mask_count, unsup_mask_count = compute_aug_loss( student_prob_out, teacher_prob_out) loss_expr = clf_loss + unsup_loss * unsup_weight loss_expr.backward() student_optimizer.step() teacher_optimizer.step() outputs = [ float(clf_loss) * n_samples, float(unsup_loss) * n_total ] if not use_rampup: mask_count = float(conf_mask_count) * 0.5 unsup_count = float(unsup_mask_count) * 0.5 outputs.append(mask_count) outputs.append(unsup_count) return tuple(outputs) else: def f_train(X_src, y_src, X_tgt0, X_tgt1): X_src = torch.tensor(X_src, dtype=torch.float, device=torch_device) y_src = torch.tensor(y_src, dtype=torch.long, device=torch_device) X_tgt0 = torch.tensor(X_tgt0, dtype=torch.float, device=torch_device) X_tgt1 = torch.tensor(X_tgt1, dtype=torch.float, device=torch_device) student_optimizer.zero_grad() student_net.train() teacher_net.train() src_logits_out = student_net(X_src) student_tgt_logits_out = student_net(X_tgt0) student_tgt_prob_out = F.softmax(student_tgt_logits_out, dim=1) teacher_tgt_logits_out = teacher_net(X_tgt1) teacher_tgt_prob_out = F.softmax(teacher_tgt_logits_out, dim=1) # Supervised classification loss if double_softmax: clf_loss = classification_criterion( F.softmax(src_logits_out, dim=1), y_src) else: clf_loss = classification_criterion(src_logits_out, y_src) unsup_loss, conf_mask_count, unsup_mask_count = compute_aug_loss( student_tgt_prob_out, teacher_tgt_prob_out) loss_expr = clf_loss + unsup_loss * unsup_weight loss_expr.backward() student_optimizer.step() teacher_optimizer.step() n_samples = X_src.size()[0] outputs = [ float(clf_loss) * n_samples, float(unsup_loss) * n_samples ] if not use_rampup: mask_count = float(conf_mask_count) unsup_count = float(unsup_mask_count) outputs.append(mask_count) outputs.append(unsup_count) return tuple(outputs) print('Compiled training function') def f_pred_src(X_sup): X_var = torch.tensor(X_sup, dtype=torch.float, device=torch_device) student_net.eval() teacher_net.eval() return (F.softmax(student_net(X_var), dim=1).detach().cpu().numpy(), F.softmax(teacher_net(X_var), dim=1).detach().cpu().numpy()) def f_pred_tgt(X_sup): X_var = torch.tensor(X_sup, dtype=torch.float, device=torch_device) student_net.eval() teacher_net.eval() return (F.softmax(student_net(X_var), dim=1).detach().cpu().numpy(), F.softmax(teacher_net(X_var), dim=1).detach().cpu().numpy()) def f_eval_src(X_sup, y_sup): y_pred_prob_stu, y_pred_prob_tea = f_pred_src(X_sup) y_pred_stu = np.argmax(y_pred_prob_stu, axis=1) y_pred_tea = np.argmax(y_pred_prob_tea, axis=1) return (float( (y_pred_stu != y_sup).sum()), float((y_pred_tea != y_sup).sum())) def f_eval_tgt(X_sup, y_sup): y_pred_prob_stu, y_pred_prob_tea = f_pred_tgt(X_sup) y_pred_stu = np.argmax(y_pred_prob_stu, axis=1) y_pred_tea = np.argmax(y_pred_prob_tea, axis=1) return (float( (y_pred_stu != y_sup).sum()), float((y_pred_tea != y_sup).sum())) print('Compiled evaluation function') # Setup output def log(text): print(text) if log_file is not None: with open(log_file, 'a') as f: f.write(text + '\n') f.flush() f.close() cmdline_helpers.ensure_containing_dir_exists(log_file) # Report setttings log('Settings: {}'.format(', '.join([ '{}={}'.format(key, settings[key]) for key in sorted(list(settings.keys())) ]))) # Report dataset size log('Dataset:') log('SOURCE Train: X.shape={}, y.shape={}'.format(d_source.train_X.shape, d_source.train_y.shape)) log('SOURCE Test: X.shape={}, y.shape={}'.format(d_source.test_X.shape, d_source.test_y.shape)) log('TARGET Train: X.shape={}'.format(d_target.train_X.shape)) log('TARGET Test: X.shape={}, y.shape={}'.format(d_target.test_X.shape, d_target.test_y.shape)) print('Training...') sup_ds = data_source.ArrayDataSource([d_source.train_X, d_source.train_y], repeats=-1) tgt_train_ds = data_source.ArrayDataSource([d_target.train_X], repeats=-1) train_ds = data_source.CompositeDataSource([sup_ds, tgt_train_ds]).map(augment) train_ds = pool.parallel_data_source(train_ds) if epoch_size == 'large': n_samples = max(d_source.train_X.shape[0], d_target.train_X.shape[0]) elif epoch_size == 'small': n_samples = min(d_source.train_X.shape[0], d_target.train_X.shape[0]) elif epoch_size == 'target': n_samples = d_target.train_X.shape[0] n_train_batches = n_samples // batch_size source_test_ds = data_source.ArrayDataSource( [d_source.test_X, d_source.test_y]) target_test_ds = data_source.ArrayDataSource( [d_target.test_X, d_target.test_y]) if seed != 0: shuffle_rng = np.random.RandomState(seed) else: shuffle_rng = np.random train_batch_iter = train_ds.batch_iterator(batch_size=batch_size, shuffle=shuffle_rng) best_teacher_model_state = { k: v.cpu().numpy() for k, v in teacher_net.state_dict().items() } best_conf_mask_rate = 0.0 best_src_test_err = 1.0 for epoch in range(num_epochs): t1 = time.time() if use_rampup: if epoch < rampup: p = max(0.0, float(epoch)) / float(rampup) p = 1.0 - p rampup_value = math.exp(-p * p * 5.0) else: rampup_value = 1.0 rampup_weight_in_list[0] = rampup_value train_res = data_source.batch_map_mean(f_train, train_batch_iter, n_batches=n_train_batches) train_clf_loss = train_res[0] if combine_batches: unsup_loss_string = 'unsup (both) loss={:.6f}'.format(train_res[1]) else: unsup_loss_string = 'unsup (tgt) loss={:.6f}'.format(train_res[1]) src_test_err_stu, src_test_err_tea = source_test_ds.batch_map_mean( f_eval_src, batch_size=batch_size * 2) tgt_test_err_stu, tgt_test_err_tea = target_test_ds.batch_map_mean( f_eval_tgt, batch_size=batch_size * 2) if use_rampup: unsup_loss_string = '{}, rampup={:.3%}'.format( unsup_loss_string, rampup_value) if src_test_err_stu < best_src_test_err: best_src_test_err = src_test_err_stu best_teacher_model_state = { k: v.cpu().numpy() for k, v in teacher_net.state_dict().items() } improve = '*** ' else: improve = '' else: conf_mask_rate = train_res[-2] unsup_mask_rate = train_res[-1] if conf_mask_rate > best_conf_mask_rate: best_conf_mask_rate = conf_mask_rate improve = '*** ' best_teacher_model_state = { k: v.cpu().numpy() for k, v in teacher_net.state_dict().items() } else: improve = '' unsup_loss_string = '{}, conf mask={:.3%}, unsup mask={:.3%}'.format( unsup_loss_string, conf_mask_rate, unsup_mask_rate) t2 = time.time() log('{}Epoch {} took {:.2f}s: TRAIN clf loss={:.6f}, {}; ' 'SRC TEST ERR={:.3%}, TGT TEST student err={:.3%}, TGT TEST teacher err={:.3%}' .format(improve, epoch, t2 - t1, train_clf_loss, unsup_loss_string, src_test_err_stu, tgt_test_err_stu, tgt_test_err_tea)) # Save network if model_file != '': cmdline_helpers.ensure_containing_dir_exists(model_file) with open(model_file, 'wb') as f: torch.save(best_teacher_model_state, f)
size = 0 for row in csvreader: if size >= 90000: test_X.append(row[1:]) test_Y.append([float(row[0])]) else: train_X.append(row[1:]) train_Y.append([float(row[0])]) size+=1 train_X = np.array(train_X,dtype="float64") train_Y = np.array(train_Y,dtype="int32") test_X = np.array(test_X, dtype="float64") test_Y = np.array(test_Y, dtype="int32") ds = data_source.ArrayDataSource([train_X, train_Y]) class MultilayerPerceptron(nn.Module): def __init__(self, input_size, hidden_size, out_classes): super(MultilayerPerceptron, self).__init__() self.fc1 = nn.Linear(input_size, hidden_size) self.fc2 = nn.Linear(hidden_size, num_classes) self.tanh = nn.Tanh() self.sigmoid = nn.Sigmoid() def forward(self, x): out = self.fc1(x) out = self.tanh(out) out = self.fc2(out) out = self.sigmoid(out)
# intialiasing all variable init = tf.global_variables_initializer() epochs = 20 sess = tf.Session() sess.run(init) sess = tf_debug.TensorBoardDebugWrapperSession(sess, "localhost:7000") # X_train, X_test, y_train, y_test = train_test_split(new_image_input, y, test_size = .10, random_state = 4)#splitting data (no need if test data is present for loop in range(epochs): # ______________________________________________________________________ # my batch creater # Construct an array data source ds = data_source.ArrayDataSource([scaled_input, y_train]) # Iterate over samples, drawing batches of 64 elements in random order for (batch_X, batch_y) in ds.batch_iterator( batch_size=1000, shuffle=True): # shuffle true will randomise every batch # for (batch_X, batch_y) in ds.batch_iterator(batch_size=1000, shuffle=np.random.RandomState(12345)): _, accuracy = sess.run([train, acc], feed_dict={ x: batch_X, y_true: batch_y, hold_prob: 0.5 }) print("the train accuracy is:", accuracy) # ________________________________________________________________________ # batch_x1, batch_y1 = tf.train.batch([feature_input, y_train], batch_size=50)
def experiment(n_classes, bilinear, dropout, learning_rate, gamma, alpha, data_path, val_percent, batch_size, rep_num, logfile, augment, date): settings = locals().copy() # Make results folder for specific parameters savepath = 'results/{}/g{}_alp{}_bs{}_lr{}_bilin{}_do{}_aug{}'.format( date, gamma, alpha, batch_size, learning_rate, bilinear, dropout, augment) if not os.path.exists(savepath): os.makedirs(savepath) if logfile != 'none': try: logger = loggers.Logger('{}/{}'.format(savepath, logfile)) except loggers.LogAlreadyExistsError as e: print(e.message) return logger.connect() print('Program: {}'.format(sys.argv[0])) print('Command line: {}'.format(' '.join(sys.argv))) print('Settings:') print(', '.join( ['{}={}'.format(k, settings[k]) for k in sorted(settings.keys())])) best_accuracy = float('inf') # LOAD MODEL input_channels = 5 print('LOAD VGG16 UNET MODEL') torch_device = torch.device( 'cuda:0' if torch.cuda.is_available() else 'cpu') model = vgg16_UNet(models.vgg16(), input_channels, n_classes, bilinear, dropout).to(torch_device) model.encoder.expand_input(input_channels) optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) criterion = FocalLoss(gamma=gamma, alpha=alpha) print('Selecting dataset from training islands...') trainset_X, valset_X, trainset_y, valset_y = new_data_splitter( data_path, test_island, island_refs, observer_list, val_percent) train_X = ImageAccessor(trainset_X, load_image_vgg16) val_X = ImageAccessor(valset_X, load_image_vgg16) train_y = ImageAccessor(trainset_y, load_tgt_vgg16) val_y = ImageAccessor(valset_y, load_tgt_vgg16) trainset = data_source.ArrayDataSource([train_X, train_y]) valset = data_source.ArrayDataSource([val_X, val_y]) # Augment trainset = trainset.map(augment_batch) valset = valset.map(augment_batch) pool = work_pool.WorkerThreadPool(4) trainset = pool.parallel_data_source(trainset) valset = pool.parallel_data_source(valset) data_loaders = {'train': trainset, 'val': valset} print('BEGIN TRAINING...') total_train_loss = [] total_val_loss = [] no_improvement = 0 epoch = 0 while no_improvement < 10: t1 = time.time() print('-' * 10) # Train and validate train_loss = train(model, optimizer, data_loaders, criterion, batch_size, torch_device, 'train') val_loss = train(model, optimizer, data_loaders, criterion, batch_size, torch_device, 'val') t2 = time.time() print( 'Epoch {} took {:.3f}s; training loss = {:.6f}; validation loss = {:.6f}' .format(epoch, t2 - t1, train_loss, val_loss)) # Save losses total_train_loss.append(train_loss) total_val_loss.append(val_loss) # Check loss and save checkpoint is_best = bool(val_loss < best_accuracy) if is_best: no_improvement = 0 # reset the counter after new best found else: no_improvement += 1 # count how many non-improvements print('Current best loss: {}'.format(best_accuracy)) best_accuracy = min(val_loss, best_accuracy) print('{}/checkpoint.pth.tar'.format(savepath)) save_checkpoint( { 'epoch': epoch, 'state_dict': model.state_dict(), 'best_accuracy': torch.FloatTensor([best_accuracy]) }, is_best, '{}/checkpoint.pth.tar'.format(savepath)) print('No improvement in loss for {} epochs'.format(no_improvement)) if epoch == 5: print('Setting new learning rate...') optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate / 10.) epoch += 1 np.savez('{}/TrainingLoss.npz'.format(savepath), train_loss=total_train_loss, val_loss=total_val_loss) return
def experiment(exp, scale_u_range, scale_x_range, scale_y_range, affine_std, xlat_range, rot_std, hflip, intens_scale_range, colour_rot_std, colour_off_std, greyscale, cutout_prob, cutout_size, batch_size, n_batches, seed): import os import sys import cmdline_helpers intens_scale_range_lower, intens_scale_range_upper = cmdline_helpers.colon_separated_range( intens_scale_range) scale_u_range = cmdline_helpers.colon_separated_range(scale_u_range) scale_x_range = cmdline_helpers.colon_separated_range(scale_x_range) scale_y_range = cmdline_helpers.colon_separated_range(scale_y_range) import time import tqdm import math import numpy as np from matplotlib import pyplot as plt from batchup import data_source, work_pool import visda17_dataset import augmentation, image_transforms import itertools n_chn = 0 mean_value = np.array([0.485, 0.456, 0.406]) std_value = np.array([0.229, 0.224, 0.225]) if exp == 'train_val': d_source = visda17_dataset.TrainDataset(img_size=(96, 96), mean_value=mean_value, std_value=std_value, range01=True, rgb_order=True, random_crop=False) d_target = visda17_dataset.ValidationDataset(img_size=(96, 96), mean_value=mean_value, std_value=std_value, range01=True, rgb_order=True, random_crop=False) d_target_test = visda17_dataset.ValidationDataset( img_size=(96, 96), mean_value=mean_value, std_value=std_value, range01=True, rgb_order=True, random_crop=False) elif exp == 'train_test': print('train_test experiment not supported yet') return else: print('Unknown experiment type \'{}\''.format(exp)) return n_classes = d_source.n_classes n_domains = 2 print('Loaded data') arch = 'show-images' # Image augmentation aug = augmentation.ImageAugmentation( hflip, xlat_range, affine_std, rot_std=rot_std, intens_scale_range_lower=intens_scale_range_lower, intens_scale_range_upper=intens_scale_range_upper, colour_rot_std=colour_rot_std, colour_off_std=colour_off_std, greyscale=greyscale, scale_u_range=scale_u_range, scale_x_range=scale_x_range, scale_y_range=scale_y_range, cutout_size=cutout_size, cutout_probability=cutout_prob) # Report setttings print('sys.argv={}'.format(sys.argv)) # Report dataset size print('Dataset:') print('SOURCE len(X)={}, y.shape={}'.format(len(d_source.images), d_source.y.shape)) print('TARGET len(X)={}'.format(len(d_target.images))) print('Building data sources...') source_train_ds = data_source.ArrayDataSource( [d_source.images, d_source.y], repeats=-1) target_train_ds = data_source.ArrayDataSource([d_target.images], repeats=-1) train_ds = data_source.CompositeDataSource( [source_train_ds, target_train_ds]) border_value = int(np.mean(mean_value) * 255 + 0.5) train_xf = image_transforms.Compose( image_transforms.ScaleCropAndAugmentAffine( (96, 96), (16, 16), True, aug, border_value, mean_value, std_value), image_transforms.ToTensor(), ) test_xf = image_transforms.Compose( image_transforms.ScaleAndCrop((96, 96), (16, 16), False), image_transforms.ToTensor(), image_transforms.Standardise(mean_value, std_value), ) def augment(X_sup, y_sup, X_tgt): X_sup = train_xf(X_sup)[0] X_tgt_0 = train_xf(X_tgt)[0] X_tgt_1 = train_xf(X_tgt)[0] return [X_sup, y_sup, X_tgt_0, X_tgt_1] train_ds = train_ds.map(augment) test_ds = data_source.ArrayDataSource([d_target_test.images]).map(test_xf) if seed != 0: shuffle_rng = np.random.RandomState(seed) else: shuffle_rng = np.random print('Showing...') n_shown = 0 for (src_X, src_y, tgt_X0, tgt_X1), (te_X, ) in zip( train_ds.batch_iterator(batch_size=batch_size, shuffle=shuffle_rng), test_ds.batch_iterator(batch_size=batch_size)): print('Batch') tgt_X = np.zeros( (tgt_X0.shape[0] + tgt_X1.shape[0], ) + tgt_X0.shape[1:], dtype=np.float32) tgt_X[0::2] = tgt_X0 tgt_X[1::2] = tgt_X1 x = np.concatenate([src_X, tgt_X, te_X], axis=0) n = x.shape[0] n_sup = src_X.shape[0] + tgt_X.shape[0] across = int(math.ceil(math.sqrt(float(n)))) plt.figure(figsize=(16, 16)) for i in tqdm.tqdm(range(n)): plt.subplot(across, across, i + 1) im_x = x[i] * std_value[:, None, None] + mean_value[:, None, None] im_x = np.clip(im_x, 0.0, 1.0) plt.imshow(im_x.transpose(1, 2, 0)) if i < src_y.shape[0]: plt.title(str(src_y[i])) elif i < n_sup: plt.title('target') else: plt.title('test') plt.show() n_shown += 1 if n_shown >= n_batches: break
bbox_data = val.get('boxes') box_return = np.array(bbox_data) print(box_return[0]) my_data.close() print("closing hdf5 file") print(image_return[7]) '''for loop,bbox_val in enumerate (box_return): # for loop, image_val in enumerate(image_return): # print(image_return[0]) # image_val= image_return[len] # img = Image.open(io.BytesIO(image_val)) # file_name="file_"+str(loop) +'.jpeg' print(bbox_val) # image_path=os.path.join(output_path, file_name) # img.save(imag''' ds = data_source.ArrayDataSource([image_return, box_return]) # Iterate over samples, drawing batches of 64 elements in random order for (batch_X, batch_y) in ds.batch_iterator( batch_size=1, shuffle=True): # shuffle true will randomise every batch print(batch_X.shape) print(batch_y) data = batch_y.reshape((batch_y.shape[0], 1)) print(data) # box=np.reshape(-1,5) # break # e_path, 'JPEG')
if __name__ == "__main__": # load_data() # Map to [-1,1] (train_images, train_labels), (test_images, test_labels) = mnist.load_data(path="mnist.npz") train_images = (2 * np.array(train_images) / 255) - 1 i = 0 generator = build_generator() discriminator = build_discriminator() gan_model = build_gan_model(generator, discriminator) batches = data_source.ArrayDataSource([train_images], repeats=epochs) for batch in batches.batch_iterator(batch_size, True): if i % 100 == 0: samples = generator.predict(random_sample(1)) print(np.shape(samples)) fig = plot_single(samples[0]) plt.savefig('out_classic_single/{}.png'.format(str(i).zfill(3)), bbox_inches='tight', pad_inches=0, transparent=True) plt.close(fig) real_data_input, real_data_label = batch[0].reshape( -1, 28, 28, 1), np.repeat(np.random.uniform(0.3, 0.7), batch_size) fake_data_input, fake_data_label = generator.predict(
if __name__ == "__main__": result_real = discriminator_network(real_data) result_fake = discriminator_network(generator_network(fake_data)) loss_discriminator = -tf.reduce_mean(tf.log(result_real) + tf.log(1. - result_fake)) loss_generator = -tf.reduce_mean(tf.log(result_fake)) disc_vars = [var for var in tf.trainable_variables() if var.name.startswith("disc")] gen_vars = [var for var in tf.trainable_variables() if var.name.startswith("gen")] discriminator_solver = tf.train.AdamOptimizer(learning_rate=0.0002, beta1=0.5).minimize(loss_discriminator, var_list=disc_vars) generator_solver = tf.train.AdamOptimizer(learning_rate=0.0002, beta1=0.5).minimize(loss_generator, var_list=gen_vars) sess = tf.Session() sess.run(tf.global_variables_initializer()) train_images = train_images.astype(np.float32) / 255 train_images = [np.reshape(np.array(val), (1,784)).flatten() for val in train_images] batches = data_source.ArrayDataSource([np.array(train_images)], repeats=iterations) i = 0 for batch in batches.batch_iterator(batch_size, True): if i % 1000 == 0: samples = sess.run(generator_network(fake_data), feed_dict={fake_data : random_sample(16)}) fig = plot(samples) plt.savefig('out/{}.png'.format(str(i).zfill(3)), bbox_inches='tight') plt.close(fig) _, discriminator_curr = sess.run([discriminator_solver, loss_discriminator], feed_dict={real_data: batch[0], fake_data: random_sample(batch_size)}) _, generator_curr = sess.run([generator_solver, loss_generator], feed_dict={fake_data : random_sample(batch_size)}) print('D loss: {:.4}'. format(discriminator_curr)) i=i+1
return decoded if __name__ == "__main__": (train_images, train_labels), (test_images, test_labels) = mnist.load_data(path="mnist.npz") train_images = (2 * np.array(train_images) / 255) - 1 autoencoder = Autoencoder(latent_dim) autoencoder.compile(loss='mse', optimizer=keras.optimizers.Adam(lr=0.0002, beta_1=0.5)) i = 0 for iterations in range(epochs): batches = data_source.ArrayDataSource([train_images]) for batch in batches.batch_iterator(batch_size, True): if (i % 100 == 0): encoded_image = autoencoder.encoder(batch[0][0].reshape( -1, 28, 28, 1)) print(np.shape(encoded_image)) decoded_image = autoencoder.decoder(encoded_image) plt.imshow(np.reshape(decoded_image, (28, 28, 1)), cmap='Greys_r') plt.savefig('autoencoder/{}.png'.format(str(i).zfill(3)), bbox_inches='tight') plt.close() i = i + 1 loss = autoencoder.train_on_batch(batch[0].reshape(-1, 28, 28, 1), batch[0].reshape(-1, 28, 28, 1)) print(loss)
def experiment(plot_path, ds_name, no_aug, affine_std, scale_u_range, scale_x_range, scale_y_range, xlat_range, hflip, intens_flip, intens_scale_range, intens_offset_range, grid_h, grid_w, seed): settings = locals().copy() import os import sys import cmdline_helpers intens_scale_range_lower, intens_scale_range_upper = cmdline_helpers.colon_separated_range( intens_scale_range) intens_offset_range_lower, intens_offset_range_upper = cmdline_helpers.colon_separated_range( intens_offset_range) scale_u_range = cmdline_helpers.colon_separated_range(scale_u_range) scale_x_range = cmdline_helpers.colon_separated_range(scale_x_range) scale_y_range = cmdline_helpers.colon_separated_range(scale_y_range) import numpy as np # from skimage.util import montage2d from skimage.util import montage as montage2d from PIL import Image from batchup import data_source import data_loaders import augmentation n_chn = 0 if ds_name == 'mnist': d_source = data_loaders.load_mnist(zero_centre=False) elif ds_name == 'usps': d_source = data_loaders.load_usps(zero_centre=False, scale28=True) elif ds_name == 'svhn_grey': d_source = data_loaders.load_svhn(zero_centre=False, greyscale=True) elif ds_name == 'svhn': d_source = data_loaders.load_svhn(zero_centre=False, greyscale=False) elif ds_name == 'cifar': d_source = data_loaders.load_cifar10() elif ds_name == 'stl': d_source = data_loaders.load_stl() elif ds_name == 'syndigits': d_source = data_loaders.load_syn_digits(zero_centre=False, greyscale=False) elif ds_name == 'synsigns': d_source = data_loaders.load_syn_signs(zero_centre=False, greyscale=False) elif ds_name == 'gtsrb': d_source = data_loaders.load_gtsrb(zero_centre=False, greyscale=False) else: print('Unknown dataset \'{}\''.format(ds_name)) return # Delete the training ground truths as we should not be using them del d_source.train_y n_classes = d_source.n_classes print('Loaded data') src_aug = augmentation.ImageAugmentation( hflip, xlat_range, affine_std, intens_flip=intens_flip, intens_scale_range_lower=intens_scale_range_lower, intens_scale_range_upper=intens_scale_range_upper, intens_offset_range_lower=intens_offset_range_lower, intens_offset_range_upper=intens_offset_range_upper, scale_u_range=scale_u_range, scale_x_range=scale_x_range, scale_y_range=scale_y_range) def augment(X): if not no_aug: X = src_aug.augment(X) return X, rampup_weight_in_list = [0] print('Rendering...') train_ds = data_source.ArrayDataSource([d_source.train_X], repeats=-1).map(augment) n_samples = len(d_source.train_X) if seed != 0: shuffle_rng = np.random.RandomState(seed) else: shuffle_rng = np.random batch_size = grid_h * grid_w display_batch_iter = train_ds.batch_iterator(batch_size=batch_size, shuffle=shuffle_rng) best_src_test_err = 1.0 x_batch, = next(display_batch_iter) montage = [] for chn_i in range(x_batch.shape[1]): m = montage2d(x_batch[:, chn_i, :, :], grid_shape=(grid_h, grid_w)) montage.append(m[:, :, None]) montage = np.concatenate(montage, axis=2) if montage.shape[2] == 1: montage = montage[:, :, 0] lower = min(0.0, montage.min()) upper = max(1.0, montage.max()) montage = (montage - lower) / (upper - lower) montage = (np.clip(montage, 0.0, 1.0) * 255.0).astype(np.uint8) Image.fromarray(montage).save(plot_path)
biases['disc_hidden3'], biases['disc_out'] ] # Create training operations train_gen = optimizer_gen.minimize(gen_loss, var_list=gen_vars) train_disc = optimizer_disc.minimize(disc_loss, var_list=disc_vars) # Initialize the variables (i.e. assign their default value) init = tf.global_variables_initializer() """ Start training """ with tf.Session() as sess: # Run the initializer sess.run(init) trainDataSets = data_source.ArrayDataSource([x_train_data, y_train_data], repeats=1) i = 0 gl_append = [] dl_append = [] # f, a = plt.subplots(4, 4, figsize=(4, 4)) for e in range(epoch): for (batch_x, batch_y) in trainDataSets.batch_iterator(batch_size=batch_size): i += 1 # Generate noise to feed to the generator # z = np.random.uniform(-1., 1., size=[batch_size, noise_dim]) # Train feed_dict = {disc_input: batch_y, gen_input: batch_x, dropout: 0.4}
def experiment(exp, arch, rnd_init, img_size, confidence_thresh, teacher_alpha, unsup_weight, cls_balance, cls_balance_loss, learning_rate, pretrained_lr_factor, fix_layers, double_softmax, use_dropout, src_scale_u_range, src_scale_x_range, src_scale_y_range, src_affine_std, src_xlat_range, src_rot_std, src_hflip, src_intens_scale_range, src_colour_rot_std, src_colour_off_std, src_greyscale, src_cutout_prob, src_cutout_size, tgt_scale_u_range, tgt_scale_x_range, tgt_scale_y_range, tgt_affine_std, tgt_xlat_range, tgt_rot_std, tgt_hflip, tgt_intens_scale_range, tgt_colour_rot_std, tgt_colour_off_std, tgt_greyscale, tgt_cutout_prob, tgt_cutout_size, constrain_crop, img_pad_width, num_epochs, batch_size, epoch_size, seed, log_file, skip_epoch_eval, result_file, record_history, model_file, hide_progress_bar, subsetsize, subsetseed, device, num_threads): settings = locals().copy() if rnd_init: if fix_layers != '': print('`rnd_init` and `fix_layers` are mutually exclusive') return if epoch_size not in {'source', 'target'}: try: epoch_size = int(epoch_size) except ValueError: print( 'epoch_size should be an integer, \'source\', or \'target\', not {}' .format(epoch_size)) return import os import sys import pickle import cmdline_helpers fix_layers = [lyr.strip() for lyr in fix_layers.split(',')] if log_file == '': log_file = 'output_aug_log_{}.txt'.format(exp) elif log_file == 'none': log_file = None if log_file is not None: if os.path.exists(log_file): print('Output log file {} already exists'.format(log_file)) return src_intens_scale_range_lower, src_intens_scale_range_upper = cmdline_helpers.colon_separated_range( src_intens_scale_range) tgt_intens_scale_range_lower, tgt_intens_scale_range_upper = cmdline_helpers.colon_separated_range( tgt_intens_scale_range) src_scale_u_range = cmdline_helpers.colon_separated_range( src_scale_u_range) tgt_scale_u_range = cmdline_helpers.colon_separated_range( tgt_scale_u_range) src_scale_x_range = cmdline_helpers.colon_separated_range( src_scale_x_range) tgt_scale_x_range = cmdline_helpers.colon_separated_range( tgt_scale_x_range) src_scale_y_range = cmdline_helpers.colon_separated_range( src_scale_y_range) tgt_scale_y_range = cmdline_helpers.colon_separated_range( tgt_scale_y_range) import time import tqdm import math import tables import numpy as np from batchup import data_source, work_pool import image_dataset, visda17_dataset, office_dataset import network_architectures import augmentation import image_transforms from sklearn.model_selection import StratifiedShuffleSplit, ShuffleSplit import torch, torch.cuda from torch import nn from torch.nn import functional as F import optim_weight_ema if hide_progress_bar: progress_bar = None else: progress_bar = tqdm.tqdm with torch.cuda.device(device): pool = work_pool.WorkerThreadPool(num_threads) n_chn = 0 half_batch_size = batch_size // 2 if arch == '': if exp in {'train_val', 'train_test'}: arch = 'resnet50' if rnd_init: mean_value = np.array([0.5, 0.5, 0.5]) std_value = np.array([0.5, 0.5, 0.5]) else: mean_value = np.array([0.485, 0.456, 0.406]) std_value = np.array([0.229, 0.224, 0.225]) img_shape = (img_size, img_size) img_padding = (img_pad_width, img_pad_width) if exp == 'visda_train_val': d_source = visda17_dataset.TrainDataset(img_size=img_shape, range01=True, rgb_order=True) d_target = visda17_dataset.ValidationDataset(img_size=img_shape, range01=True, rgb_order=True) elif exp == 'visda_train_test': d_source = visda17_dataset.TrainDataset(img_size=img_shape, range01=True, rgb_order=True) d_target = visda17_dataset.TestDataset(img_size=img_shape, range01=True, rgb_order=True) if not skip_epoch_eval: print('WARNING: setting skip_epoch_eval to True') skip_epoch_eval = True elif exp == 'office_amazon_dslr': d_source = office_dataset.OfficeAmazonDataset(img_size=img_shape, range01=True, rgb_order=True) d_target = office_dataset.OfficeDSLRDataset(img_size=img_shape, range01=True, rgb_order=True) elif exp == 'office_amazon_webcam': d_source = office_dataset.OfficeAmazonDataset(img_size=img_shape, range01=True, rgb_order=True) d_target = office_dataset.OfficeWebcamDataset(img_size=img_shape, range01=True, rgb_order=True) elif exp == 'office_dslr_amazon': d_source = office_dataset.OfficeDSLRDataset(img_size=img_shape, range01=True, rgb_order=True) d_target = office_dataset.OfficeAmazonDataset(img_size=img_shape, range01=True, rgb_order=True) elif exp == 'office_dslr_webcam': d_source = office_dataset.OfficeDSLRDataset(img_size=img_shape, range01=True, rgb_order=True) d_target = office_dataset.OfficeWebcamDataset(img_size=img_shape, range01=True, rgb_order=True) elif exp == 'office_webcam_amazon': d_source = office_dataset.OfficeWebcamDataset(img_size=img_shape, range01=True, rgb_order=True) d_target = office_dataset.OfficeAmazonDataset(img_size=img_shape, range01=True, rgb_order=True) elif exp == 'office_webcam_dslr': d_source = office_dataset.OfficeWebcamDataset(img_size=img_shape, range01=True, rgb_order=True) d_target = office_dataset.OfficeDSLRDataset(img_size=img_shape, range01=True, rgb_order=True) else: print('Unknown experiment type \'{}\''.format(exp)) return # Tensorboard log # Subset source_indices, target_indices, n_src, n_tgt = image_dataset.subset_indices( d_source, d_target, subsetsize, subsetseed) # # Result file # if result_file != '': cmdline_helpers.ensure_containing_dir_exists(result_file) h5_filters = tables.Filters(complevel=9, complib='blosc') f_target_pred = tables.open_file(result_file, mode='w') g_tgt_pred = f_target_pred.create_group(f_target_pred.root, 'target_pred_y', 'Target prediction') if record_history: arr_tgt_pred_history = f_target_pred.create_earray( g_tgt_pred, 'y_prob_history', tables.Float32Atom(), (0, n_tgt, d_target.n_classes), filters=h5_filters) else: arr_tgt_pred_history = None else: arr_tgt_pred_history = None f_target_pred = None g_tgt_pred = None n_classes = d_source.n_classes print('Loaded data') net_class = network_architectures.get_build_fn_for_architecture(arch) student_net = net_class(n_classes, img_size, use_dropout, not rnd_init).cuda() teacher_net = net_class(n_classes, img_size, use_dropout, not rnd_init).cuda() student_params = list(student_net.parameters()) teacher_params = list(teacher_net.parameters()) for param in teacher_params: param.requires_grad = False if rnd_init: new_student_optimizer = torch.optim.Adam(student_params, lr=learning_rate) pretrained_student_optimizer = None else: named_params = list(student_net.named_parameters()) new_params = [] pretrained_params = [] for name, param in named_params: if name.startswith('new_'): new_params.append(param) else: fix = False for lyr in fix_layers: if name.startswith(lyr + '.'): fix = True break if not fix: pretrained_params.append(param) else: print('Fixing param {}'.format(name)) param.requires_grad = False new_student_optimizer = torch.optim.Adam(new_params, lr=learning_rate) if len(pretrained_params) > 0: pretrained_student_optimizer = torch.optim.Adam( pretrained_params, lr=learning_rate * pretrained_lr_factor) else: pretrained_student_optimizer = None teacher_optimizer = optim_weight_ema.WeightEMA(teacher_params, student_params, alpha=teacher_alpha) classification_criterion = nn.CrossEntropyLoss() print('Built network') # Image augmentation src_aug = augmentation.ImageAugmentation( src_hflip, src_xlat_range, src_affine_std, rot_std=src_rot_std, intens_scale_range_lower=src_intens_scale_range_lower, intens_scale_range_upper=src_intens_scale_range_upper, colour_rot_std=src_colour_rot_std, colour_off_std=src_colour_off_std, greyscale=src_greyscale, scale_u_range=src_scale_u_range, scale_x_range=src_scale_x_range, scale_y_range=src_scale_y_range, cutout_probability=src_cutout_prob, cutout_size=src_cutout_size) tgt_aug = augmentation.ImageAugmentation( tgt_hflip, tgt_xlat_range, tgt_affine_std, rot_std=tgt_rot_std, intens_scale_range_lower=tgt_intens_scale_range_lower, intens_scale_range_upper=tgt_intens_scale_range_upper, colour_rot_std=tgt_colour_rot_std, colour_off_std=tgt_colour_off_std, greyscale=tgt_greyscale, scale_u_range=tgt_scale_u_range, scale_x_range=tgt_scale_x_range, scale_y_range=tgt_scale_y_range, cutout_probability=tgt_cutout_prob, cutout_size=tgt_cutout_size) test_aug = augmentation.ImageAugmentation( tgt_hflip, tgt_xlat_range, 0.0, rot_std=0.0, scale_u_range=tgt_scale_u_range, scale_x_range=tgt_scale_x_range, scale_y_range=tgt_scale_y_range) border_value = int(np.mean(mean_value) * 255 + 0.5) sup_xf = image_transforms.Compose( image_transforms.ScaleCropAndAugmentAffine(img_shape, img_padding, True, src_aug, border_value, mean_value, std_value), image_transforms.ToTensor(), ) if constrain_crop >= 0: unsup_xf = image_transforms.Compose( image_transforms.ScaleCropAndAugmentAffinePair( img_shape, img_padding, constrain_crop, True, tgt_aug, border_value, mean_value, std_value), image_transforms.ToTensor(), ) else: unsup_xf = image_transforms.Compose( image_transforms.ScaleCropAndAugmentAffine( img_shape, img_padding, True, tgt_aug, border_value, mean_value, std_value), image_transforms.ToTensor(), ) test_xf = image_transforms.Compose( image_transforms.ScaleAndCrop(img_shape, img_padding, False), image_transforms.ToTensor(), image_transforms.Standardise(mean_value, std_value), ) test_xf_aug_mult = image_transforms.Compose( image_transforms.ScaleCropAndAugmentAffineMultiple( 16, img_shape, img_padding, True, test_aug, border_value, mean_value, std_value), image_transforms.ToTensorMultiple(), ) if constrain_crop >= 0: def augment(X_sup, y_sup, X_tgt): X_sup = sup_xf(X_sup)[0] X_unsup_both = unsup_xf(X_tgt)[0] X_unsup_stu = X_unsup_both[:len(X_tgt)] X_unsup_tea = X_unsup_both[len(X_tgt):] return X_sup, y_sup, X_unsup_stu, X_unsup_tea else: def augment(X_sup, y_sup, X_tgt): X_sup = sup_xf(X_sup)[0] X_unsup_stu = unsup_xf(X_tgt)[0] X_unsup_tea = unsup_xf(X_tgt)[0] return X_sup, y_sup, X_unsup_stu, X_unsup_tea cls_bal_fn = network_architectures.get_cls_bal_function( cls_balance_loss) def compute_aug_loss(stu_out, tea_out): # Augmentation loss conf_tea = torch.max(tea_out, 1)[0] conf_mask = torch.gt(conf_tea, confidence_thresh).float() d_aug_loss = stu_out - tea_out aug_loss = d_aug_loss * d_aug_loss aug_loss = torch.mean(aug_loss, 1) * conf_mask # Class balance loss if cls_balance > 0.0: # Average over samples to get average class prediction avg_cls_prob = torch.mean(stu_out, 0) # Compute loss equalise_cls_loss = cls_bal_fn(avg_cls_prob, float(1.0 / n_classes)) equalise_cls_loss = torch.mean(equalise_cls_loss) * n_classes equalise_cls_loss = equalise_cls_loss * torch.mean( conf_mask, 0) else: equalise_cls_loss = None return aug_loss, conf_mask, equalise_cls_loss _one = torch.autograd.Variable( torch.from_numpy(np.array([1.0]).astype(np.float32)).cuda()) def f_train(X_sup, y_sup, X_unsup0, X_unsup1): X_sup = torch.autograd.Variable(torch.from_numpy(X_sup).cuda()) y_sup = torch.autograd.Variable( torch.from_numpy(y_sup).long().cuda()) X_unsup0 = torch.autograd.Variable( torch.from_numpy(X_unsup0).cuda()) X_unsup1 = torch.autograd.Variable( torch.from_numpy(X_unsup1).cuda()) if pretrained_student_optimizer is not None: pretrained_student_optimizer.zero_grad() new_student_optimizer.zero_grad() student_net.train(mode=True) teacher_net.train(mode=True) sup_logits_out = student_net(X_sup) student_unsup_logits_out = student_net(X_unsup0) student_unsup_prob_out = F.softmax(student_unsup_logits_out) teacher_unsup_logits_out = teacher_net(X_unsup1) teacher_unsup_prob_out = F.softmax(teacher_unsup_logits_out) # Supervised classification loss if double_softmax: clf_loss = classification_criterion(F.softmax(sup_logits_out), y_sup) else: clf_loss = classification_criterion(sup_logits_out, y_sup) aug_loss, conf_mask, cls_bal_loss = compute_aug_loss( student_unsup_prob_out, teacher_unsup_prob_out) conf_mask_count = torch.sum(conf_mask) unsup_loss = torch.mean(aug_loss) loss_expr = clf_loss + unsup_loss * unsup_weight if cls_bal_loss is not None: loss_expr = loss_expr + cls_bal_loss * cls_balance * unsup_weight loss_expr.backward() if pretrained_student_optimizer is not None: pretrained_student_optimizer.step() new_student_optimizer.step() teacher_optimizer.step() n_samples = X_sup.size()[0] mask_count = conf_mask_count.data.cpu()[0] outputs = [ float(clf_loss.data.cpu()[0]) * n_samples, float(unsup_loss.data.cpu()[0]) * n_samples, mask_count ] return tuple(outputs) print('Compiled training function') def f_pred_src(X_sup): X_var = torch.autograd.Variable(torch.from_numpy(X_sup).cuda()) teacher_net.train(mode=False) return (F.softmax(teacher_net(X_var)).data.cpu().numpy(), ) def f_pred_tgt(X_sup): X_var = torch.autograd.Variable(torch.from_numpy(X_sup).cuda()) teacher_net.train(mode=False) return (F.softmax(teacher_net(X_var)).data.cpu().numpy(), ) def f_pred_tgt_mult(X_sup): teacher_net.train(mode=False) y_pred_aug = [] for aug_i in range(len(X_sup)): X_var = torch.autograd.Variable( torch.from_numpy(X_sup[aug_i, ...]).cuda()) y_pred = F.softmax(teacher_net(X_var)).data.cpu().numpy() y_pred_aug.append(y_pred[None, ...]) y_pred_aug = np.concatenate(y_pred_aug, axis=0) return (y_pred_aug.mean(axis=0), ) print('Compiled evaluation function') # Setup output cmdline_helpers.ensure_containing_dir_exists(log_file) def log(text): print(text) if log_file is not None: with open(log_file, 'a') as f: f.write(text + '\n') f.flush() f.close() # Report setttings log('Program = {}'.format(sys.argv[0])) log('Settings: {}'.format(', '.join([ '{}={}'.format(key, settings[key]) for key in sorted(list(settings.keys())) ]))) # Report dataset size log('Dataset:') log('SOURCE len(X)={}, y.shape={}'.format(len(d_source.images), d_source.y.shape)) log('TARGET len(X)={}'.format(len(d_target.images))) if epoch_size == 'source': n_samples = n_src elif epoch_size == 'target': n_samples = n_tgt else: n_samples = epoch_size n_train_batches = n_samples // batch_size n_test_batches = n_tgt // (batch_size * 2) + 1 print('Training...') sup_ds = data_source.ArrayDataSource([d_source.images, d_source.y], repeats=-1, indices=source_indices) tgt_train_ds = data_source.ArrayDataSource([d_target.images], repeats=-1, indices=target_indices) train_ds = data_source.CompositeDataSource([sup_ds, tgt_train_ds]).map(augment) train_ds = pool.parallel_data_source(train_ds, batch_buffer_size=min( 20, n_train_batches)) target_ds_for_test = data_source.ArrayDataSource( [d_target.images], indices=target_indices) target_test_ds = target_ds_for_test.map(test_xf) target_test_ds = pool.parallel_data_source(target_test_ds, batch_buffer_size=min( 20, n_test_batches)) target_mult_test_ds = target_ds_for_test.map(test_xf_aug_mult) target_mult_test_ds = pool.parallel_data_source(target_mult_test_ds, batch_buffer_size=min( 20, n_test_batches)) if seed != 0: shuffle_rng = np.random.RandomState(seed) else: shuffle_rng = np.random if d_target.has_ground_truth: evaluator = d_target.prediction_evaluator(target_indices) else: evaluator = None best_mask_rate = 0.0 best_teacher_model_state = { k: v.cpu().numpy() for k, v in teacher_net.state_dict().items() } train_batch_iter = train_ds.batch_iterator(batch_size=batch_size, shuffle=shuffle_rng) for epoch in range(num_epochs): t1 = time.time() if not skip_epoch_eval: test_batch_iter = target_test_ds.batch_iterator( batch_size=batch_size * 2) else: test_batch_iter = None train_clf_loss, train_unsup_loss, mask_rate = data_source.batch_map_mean( f_train, train_batch_iter, progress_iter_func=progress_bar, n_batches=n_train_batches) # train_clf_loss, train_unsup_loss, mask_rate = train_ds.batch_map_mean( # f_train, batch_size=batch_size, shuffle=shuffle_rng, n_batches=n_train_batches, # progress_iter_func=progress_bar) if mask_rate > best_mask_rate: best_mask_rate = mask_rate improve = True improve_str = '*** ' best_teacher_model_state = { k: v.cpu().numpy() for k, v in teacher_net.state_dict().items() } else: improve = False improve_str = '' if not skip_epoch_eval: tgt_pred_prob_y, = data_source.batch_map_concat( f_pred_tgt, test_batch_iter, progress_iter_func=progress_bar) mean_class_acc, cls_acc_str = evaluator.evaluate( tgt_pred_prob_y) t2 = time.time() log('{}Epoch {} took {:.2f}s: TRAIN clf loss={:.6f}, unsup loss={:.6f}, mask={:.3%}; ' 'TGT mean class acc={:.3%}'.format(improve_str, epoch, t2 - t1, train_clf_loss, train_unsup_loss, mask_rate, mean_class_acc)) log(' per class: {}'.format(cls_acc_str)) # Save results if arr_tgt_pred_history is not None: arr_tgt_pred_history.append( tgt_pred_prob_y[None, ...].astype(np.float32)) else: t2 = time.time() log('{}Epoch {} took {:.2f}s: TRAIN clf loss={:.6f}, unsup loss={:.6f}, mask={:.3%}' .format(improve_str, epoch, t2 - t1, train_clf_loss, train_unsup_loss, mask_rate)) # Save network if model_file != '': cmdline_helpers.ensure_containing_dir_exists(model_file) with open(model_file, 'wb') as f: pickle.dump(best_teacher_model_state, f) # Restore network to best state teacher_net.load_state_dict({ k: torch.from_numpy(v) for k, v in best_teacher_model_state.items() }) # Predict on test set, without augmentation tgt_pred_prob_y, = target_test_ds.batch_map_concat( f_pred_tgt, batch_size=batch_size, progress_iter_func=progress_bar) if d_target.has_ground_truth: mean_class_acc, cls_acc_str = evaluator.evaluate(tgt_pred_prob_y) log('FINAL: TGT mean class acc={:.3%}'.format(mean_class_acc)) log(' per class: {}'.format(cls_acc_str)) # Predict on test set, using augmentation tgt_aug_pred_prob_y, = target_mult_test_ds.batch_map_concat( f_pred_tgt_mult, batch_size=batch_size, progress_iter_func=progress_bar) if d_target.has_ground_truth: aug_mean_class_acc, aug_cls_acc_str = evaluator.evaluate( tgt_aug_pred_prob_y) log('FINAL: TGT AUG mean class acc={:.3%}'.format( aug_mean_class_acc)) log(' per class: {}'.format(aug_cls_acc_str)) if f_target_pred is not None: f_target_pred.create_array(g_tgt_pred, 'y_prob', tgt_pred_prob_y) f_target_pred.create_array(g_tgt_pred, 'y_prob_aug', tgt_aug_pred_prob_y) f_target_pred.close()
if __name__ == "__main__": (train_images, train_labels), (test_images, test_labels) = mnist.load_data() train_images = (2 * train_images.astype(np.float32) / 255) - 1 train_images = [ np.reshape(np.array(val), (1, 784)).flatten() for val in train_images ] i = 0 generator = build_generator() discriminator = build_discriminator() gan_model = build_gan_model(generator, discriminator) batches = data_source.ArrayDataSource([np.array(train_images)], repeats=epochs) for batch in batches.batch_iterator(batch_size, True): if i % 1000 == 0: samples = generator.predict(random_sample(1)) fig = plot_single(samples[0]) plt.savefig('out_classic_single/{}.png'.format(str(i).zfill(3)), bbox_inches='tight', pad_inches=0, transparent=True) plt.close(fig) real_data_input, real_data_label = batch[0], np.ones(batch_size) fake_data_input, fake_data_label = generator.predict( random_sample(batch_size)), np.zeros(batch_size)
def main(relaxation=None, learn_prior=True, max_iters=None, batch_size=25, num_latents=200, model_type=None, lr=None, test_bias=False, train_dir=None, iwae_samples=1, dataset="mnist", logf=None, var_lr_scale=10., Q_wd=.0001, Q_depth=-1, checkpoint_path=None): tf.reset_default_graph() #valid_batch_size = 100 if model_type == "L1": num_layers = 1 layer_type = linear_layer elif model_type == "L2": num_layers = 2 layer_type = linear_layer elif model_type == "NL1": num_layers = 1 layer_type = nonlinear_layer else: assert False, "bad model type {}".format(model_type) sess = tf.Session() if dataset == "mnist": #X_tr, X_va, X_te = datasets.load_mnist() mnist = input_data.read_data_sets(os.getcwd()+'/MNIST', one_hot=True) X_tr = mnist.train.images X_te = mnist.test.images X_va = mnist.validation.images train_data = mnist.train test_data = mnist.test valid_data = mnist.validation elif dataset == "omni": X_tr, X_va, X_te = load_omniglot() X_te = X_te[:8000] X_va = X_va[:1300] else: assert False train_mean = np.mean(X_tr, axis=0, keepdims=True) train_output_bias = -np.log(1. / np.clip(train_mean, 0.001, 0.999) - 1.).astype(np.float32) x = tf.placeholder(tf.float32, [None, 784]) x = tf.to_float(x > .5) #x_im = tf.reshape(x, [-1, 28, 28, 1]) #tf.summary.image("x_true", x_im) # make prior for top b p_prior = tf.Variable( tf.zeros([num_latents], dtype=tf.float32), trainable=learn_prior, name='p_prior', ) # create rebar specific variables temperature and eta log_temperatures = [create_log_temp(1) for l in range(num_layers)] temperatures = [tf.exp(log_temp) for log_temp in log_temperatures] batch_temperatures = [tf.reshape(temp, [1, -1]) for temp in temperatures] etas = [create_eta(1) for l in range(num_layers)] batch_etas = [tf.reshape(eta, [1, -1]) for eta in etas] # random uniform samples u = [ tf.random_uniform([tf.shape(x)[0], num_latents], dtype=tf.float32) for l in range(num_layers) ] # create binary sampler b_sampler = BSampler(u, "b_sampler") gen_b_sampler = BSampler(u, "gen_b_sampler") # generate hard forward pass encoder_name = "encoder" decoder_name = "decoder" inf_la_b, samples_b = inference_network( x, train_mean, layer_type, num_layers, num_latents, encoder_name, False, b_sampler ) gen_la_b = generator_network( samples_b, train_output_bias, layer_type, num_layers, num_latents, decoder_name, False ) log_image(gen_la_b[-1], "x_pred") # produce samples _samples_la_b = generator_network( None, train_output_bias, layer_type, num_layers, num_latents, decoder_name, True, sampler=gen_b_sampler, prior=p_prior ) log_image(_samples_la_b[-1], "x_sample") # hard loss evaluation and log probs f_b, log_q_bs = neg_elbo(x, samples_b, inf_la_b, gen_la_b, p_prior, log=True) batch_f_b = tf.expand_dims(f_b, 1) total_loss = tf.reduce_mean(f_b) #tf.summary.scalar("fb", total_loss) # optimizer for model parameters model_opt = tf.train.AdamOptimizer(lr, beta2=.99999) # optimizer for variance reducing parameters variance_opt = tf.train.AdamOptimizer(var_lr_scale * lr, beta2=.99999) # get encoder and decoder variables encoder_params = get_variables(encoder_name) decoder_params = get_variables(decoder_name) if learn_prior: decoder_params.append(p_prior) # compute and store gradients of hard loss with respect to encoder_parameters encoder_loss_grads = {} for g, v in model_opt.compute_gradients(total_loss, var_list=encoder_params): encoder_loss_grads[v.name] = g # get gradients for decoder parameters decoder_gradvars = model_opt.compute_gradients(total_loss, var_list=decoder_params) # will hold all gradvars for the model (non-variance adjusting variables) model_gradvars = [gv for gv in decoder_gradvars] # conditional samples v = [v_from_u(_u, log_alpha) for _u, log_alpha in zip(u, inf_la_b)] # need to create soft samplers sig_z_sampler = SIGZSampler(u, batch_temperatures, "sig_z_sampler") sig_zt_sampler = SIGZSampler(v, batch_temperatures, "sig_zt_sampler") z_sampler = ZSampler(u, "z_sampler") zt_sampler = ZSampler(v, "zt_sampler") rebars = [] reinforces = [] variance_objectives = [] # have to produce 2 forward passes for each layer for z and zt samples for l in range(num_layers): cur_la_b = inf_la_b[l] # if standard rebar or additive relaxation if relaxation == "rebar" or relaxation == "add": # compute soft samples and soft passes through model and soft elbos cur_z_sample = sig_z_sampler.sample(cur_la_b, l) prev_samples_z = samples_b[:l] + [cur_z_sample] cur_zt_sample = sig_zt_sampler.sample(cur_la_b, l) prev_samples_zt = samples_b[:l] + [cur_zt_sample] prev_log_alphas = inf_la_b[:l] + [cur_la_b] # soft forward passes inf_la_z, samples_z = inference_network( x, train_mean, layer_type, num_layers, num_latents, encoder_name, True, sig_z_sampler, samples=prev_samples_z, log_alphas=prev_log_alphas ) gen_la_z = generator_network( samples_z, train_output_bias, layer_type, num_layers, num_latents, decoder_name, True ) inf_la_zt, samples_zt = inference_network( x, train_mean, layer_type, num_layers, num_latents, encoder_name, True, sig_zt_sampler, samples=prev_samples_zt, log_alphas=prev_log_alphas ) gen_la_zt = generator_network( samples_zt, train_output_bias, layer_type, num_layers, num_latents, decoder_name, True ) # soft loss evaluataions f_z, _ = neg_elbo(x, samples_z, inf_la_z, gen_la_z, p_prior) f_zt, _ = neg_elbo(x, samples_zt, inf_la_zt, gen_la_zt, p_prior) if relaxation == "add" or relaxation == "all": # sample z and zt prev_bs = samples_b[:l] cur_z_sample = z_sampler.sample(cur_la_b, l) cur_zt_sample = zt_sampler.sample(cur_la_b, l) q_z = Q_func(x, train_mean, cur_z_sample, prev_bs, Q_name(l), False, depth=Q_depth) q_zt = Q_func(x, train_mean, cur_zt_sample, prev_bs, Q_name(l), True, depth=Q_depth) #tf.summary.scalar("q_z_{}".format(l), tf.reduce_mean(q_z)) #tf.summary.scalar("q_zt_{}".format(l), tf.reduce_mean(q_zt)) if relaxation == "add": f_z = f_z + q_z f_zt = f_zt + q_zt elif relaxation == "all": f_z = q_z f_zt = q_zt else: assert False #tf.summary.scalar("f_z_{}".format(l), tf.reduce_mean(f_z)) #tf.summary.scalar("f_zt_{}".format(l), tf.reduce_mean(f_zt)) cur_samples_b = samples_b[l] # get gradient of sample log-likelihood wrt current parameter d_log_q_d_la = bernoulli_loglikelihood_derivitive(cur_samples_b, cur_la_b) # get gradient of soft-losses wrt current parameter d_f_z_d_la = tf.gradients(f_z, cur_la_b)[0] d_f_zt_d_la = tf.gradients(f_zt, cur_la_b)[0] batch_f_zt = tf.expand_dims(f_zt, 1) eta = batch_etas[l] # compute rebar and reinforce #tf.summary.histogram("der_diff_{}".format(l), d_f_z_d_la - d_f_zt_d_la) #tf.summary.histogram("d_log_q_d_la_{}".format(l), d_log_q_d_la) rebar = ((batch_f_b - eta * batch_f_zt) * d_log_q_d_la + eta * (d_f_z_d_la - d_f_zt_d_la)) / batch_size reinforce = batch_f_b * d_log_q_d_la / batch_size rebars.append(rebar) reinforces.append(reinforce) #tf.summary.histogram("rebar_{}".format(l), rebar) #tf.summary.histogram("reinforce_{}".format(l), reinforce) # backpropogate rebar to individual layer parameters layer_params = get_variables(layer_name(l), arr=encoder_params) layer_rebar_grads = tf.gradients(cur_la_b, layer_params, grad_ys=rebar) # get direct loss grads for each parameter layer_loss_grads = [encoder_loss_grads[v.name] for v in layer_params] # each param's gradient should be rebar + the direct loss gradient layer_grads = [rg + lg for rg, lg in zip(layer_rebar_grads, layer_loss_grads)] # for rg, lg, v in zip(layer_rebar_grads, layer_loss_grads, layer_params): # tf.summary.histogram(v.name+"_grad_rebar", rg) # tf.summary.histogram(v.name+"_grad_loss", lg) layer_gradvars = list(zip(layer_grads, layer_params)) model_gradvars.extend(layer_gradvars) variance_objective = tf.reduce_mean(tf.square(rebar)) variance_objectives.append(variance_objective) variance_objective = tf.add_n(variance_objectives) variance_vars = log_temperatures + etas if relaxation != "rebar": q_vars = get_variables("Q_") wd = tf.add_n([Q_wd * tf.nn.l2_loss(v) for v in q_vars]) #tf.summary.scalar("Q_weight_decay", wd) variance_vars = variance_vars + q_vars else: wd = 0.0 variance_gradvars = variance_opt.compute_gradients(variance_objective+wd, var_list=variance_vars) variance_train_op = variance_opt.apply_gradients(variance_gradvars) model_train_op = model_opt.apply_gradients(model_gradvars) with tf.control_dependencies([model_train_op, variance_train_op]): train_op = tf.no_op() # for g, v in model_gradvars + variance_gradvars: # print(g, v.name) # if g is not None: # tf.summary.histogram(v.name, v) # tf.summary.histogram(v.name+"_grad", g) # val_loss = tf.Variable(1000, trainable=False, name="val_loss", dtype=tf.float32) # train_loss = tf.Variable(1000, trainable=False, name="train_loss", dtype=tf.float32) # tf.summary.scalar("val_loss", val_loss) # tf.summary.scalar("train_loss", train_loss) # create savers # iwae_elbo = -(tf.reduce_logsumexp(-f_b) - np.log(valid_batch_size)) sess.run(tf.global_variables_initializer()) iters_per_epoch = X_tr.shape[0] // batch_size print("Train set has {} examples".format(X_tr.shape[0])) if relaxation != "rebar": print("Pretraining Q network") for i in range(1000): if i % 100 == 0: print(i) idx = np.random.randint(0, iters_per_epoch-1) batch_xs = X_tr[idx * batch_size: (idx + 1) * batch_size] sess.run(variance_train_op, feed_dict={x: batch_xs}) def get_loss(sess,ds,batch_size): cost_eval = [] for xs in ds.batch_iterator(batch_size=batch_size, shuffle=True): xs = np.squeeze(np.array(xs)) cost_eval.append(sess.run(f_b,{x:xs})) return np.mean(cost_eval) start = time.time() from batchup import data_source train_ds = data_source.ArrayDataSource([X_tr]) test_ds = data_source.ArrayDataSource([X_te]) valid_ds = data_source.ArrayDataSource([X_va]) # total_points = X_tr.shape[0] # total_batch = int(total_points / batch_size) # total_test_batch = int(X_te.shape[0] / batch_size) #best_val_loss = np.inf COUNT=[]; COST=[]; TIME=[];COST_TEST=[];COST_VALID=[];epoch_list=[];time_list=[] EXPERIMENT = 'layer2_OMNI_Bernoulli_' + 'REBAR' if relaxation == 'rebar' else 'layer2_OMNI_Bernoulli_' + 'RELAX' print('Training stats....',EXPERIMENT) record = []; step = 0 for epoch in range(5000): for batch_xs in train_ds.batch_iterator(batch_size=batch_size, shuffle=True): batch_xs = np.squeeze(np.array(batch_xs)) cur_iter = step if cur_iter > max_iters: print("Training Completed") return #batch_xs = X_tr[i*batch_size: (i+1) * batch_size] #batch_xs,_ = train_data.next_batch(batch_size) loss, _ = sess.run([total_loss, train_op], feed_dict={x: batch_xs}) record.append(loss) step += 1 if epoch%1 == 0: valid_loss = get_loss(sess,valid_ds,batch_size=batch_size) COUNT.append(step); COST.append(np.mean(record)); TIME.append(time.time()-start) COST_VALID.append(valid_loss) print(epoch,'valid_cost=',valid_loss) print(epoch,'train_cost=',np.mean(record),'with std=',np.std(record)) if epoch%5 == 0: COST_TEST.append(get_loss(sess,test_ds,batch_size=batch_size)) epoch_list.append(epoch) time_list.append(time.time()-start) import cPickle directory = os.getcwd()+'/discrete_out/' if not os.path.exists(directory): os.makedirs(directory) all_ = [COUNT,COST,TIME,COST_TEST,COST_VALID,epoch_list,time_list] cPickle.dump(all_, open(directory+EXPERIMENT, 'w')) record=[]
# model.cuda() model = model.double() criterea = torch.nn.CrossEntropyLoss( ) # this include softmax and cross entropy criterea = torch.nn.BCELoss( ) # this is for binary cross entropy when you have only binary output (0/1 or True/False) # criterea=torch.nn.MSELoss(size_average=False)#cross entropy loss optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.5) run = 0 epochs = 10 for loop in range(epochs): # ______________________________________________________________________ # my batch creater # Construct an array data source ds = data_source.ArrayDataSource([scaled_input, y]) # Iterate over samples, drawing batches of 64 elements in random order for (data, target) in ds.batch_iterator( batch_size=1, shuffle=True): # shuffle true will randomise every batch new_data = data.reshape(-1, 1, 28, 28) cool_data = new_data[0] # cv2.imshow("image",np.reshape(cool_data,newshape=(28,28,1))) # cv2.waitKey(500) # cv2.destroyAllWindows() # data, target=torch.from_numpy(new_data).double(),torch.from_numpy(target).double() # inputs, labels = Variable(inputs.cuda()), Variable(labels.cuda())#this is for cuda gpu data, target = torch.tensor(new_data), torch.tensor(target).long() optimizer.zero_grad() output = model(data)