def random_classification_datasets(n_samples,
features=100,
classes=2,
informative=.1,
partition_proportions=(.5, .3),
rnd=None,
one_hot=True,
**mk_cls_kwargs):
rnd_state = em.get_rand_state(rnd)
X, Y = make_classification(n_samples,
features,
n_classes=classes,
random_state=rnd_state,
**mk_cls_kwargs)
if one_hot:
Y = utils.to_one_hot_enc(Y)
print('range of Y', np.min(Y), np.max(Y))
info = utils.merge_dicts({
'informative': informative,
'random_seed': rnd
}, mk_cls_kwargs)
name = em.utils.name_from_dict(info, 'w')
dt = em.Dataset(X, Y, name=name, info=info)
datasets = em.Datasets.from_list(redivide_data([dt],
partition_proportions))
print('conditioning of X^T X',
np.linalg.cond(datasets.train.data.T @ datasets.train.data))
return datasets
def random_regression_datasets(n_samples,
features=100,
outs=1,
informative=.1,
partition_proportions=(.5, .3),
rnd=None,
**mk_rgr_kwargs):
rnd_state = em.get_rand_state(rnd)
X, Y, w = make_regression(n_samples,
features,
int(features * informative),
outs,
random_state=rnd_state,
coef=True,
**mk_rgr_kwargs)
if outs == 1:
Y = np.reshape(Y, (n_samples, 1))
print('range of Y', np.min(Y), np.max(Y))
info = utils.merge_dicts(
{
'informative': informative,
'random_seed': rnd,
'w': w
}, mk_rgr_kwargs)
name = em.utils.name_from_dict(info, 'w')
dt = em.Dataset(X, Y, name=name, info=info)
datasets = em.Datasets.from_list(redivide_data([dt],
partition_proportions))
print('conditioning of X^T X',
np.linalg.cond(datasets.train.data.T @ datasets.train.data))
return datasets
def generate_datasets(self,
rand=None,
num_classes=None,
num_examples=None,
wait_for_n_min=None):
rand = em.get_rand_state(rand)
if wait_for_n_min:
import time
while not self.check_loaded_images(wait_for_n_min):
time.sleep(5)
if not num_examples: num_examples = self.kwargs['num_examples']
if not num_classes: num_classes = self.kwargs['num_classes']
clss = self._loaded_images if self._loaded_images else self.info[
'classes']
random_classes = rand.choice(list(clss.keys()),
size=(num_classes, ),
replace=False)
rand_class_dict = {rnd: k for k, rnd in enumerate(random_classes)}
_dts = []
for ns in em.as_tuple_or_list(num_examples):
classes = balanced_choice_wr(random_classes, ns, rand)
all_images = {cls: list(clss[cls]) for cls in classes}
data, targets, sample_info = [], [], []
for c in classes:
rand.shuffle(all_images[c])
img_name = all_images[c][0]
all_images[c].remove(img_name)
sample_info.append({'name': img_name, 'label': c})
if self._loaded_images:
data.append(clss[c][img_name])
else:
from scipy.misc import imread, imresize
data.append(
imresize(imread(join(self.info['base_folder'],
join(c, img_name)),
mode='RGB'),
size=(self.info['resize'],
self.info['resize'], 3)) / 255.)
targets.append(rand_class_dict[c])
if self.info['one_hot_enc']:
targets = em.to_one_hot_enc(targets, dimension=num_classes)
_dts.append(
em.Dataset(data=np.array(np.stack(data)),
target=targets,
sample_info=sample_info,
info={'all_classes': random_classes}))
return em.Datasets.from_list(_dts)
def generate_datasets(self,
rand=None,
num_classes=None,
num_examples=None):
rand = em.get_rand_state(rand)
if not num_examples: num_examples = self.kwargs['num_examples']
if not num_classes: num_classes = self.kwargs['num_classes']
clss = self._loaded_images if self._loaded_images else self.info[
'classes']
random_classes = rand.choice(list(clss.keys()),
size=(num_classes, ),
replace=False)
rand_class_dict = {rnd: k for k, rnd in enumerate(random_classes)}
_dts = []
for ns in em.as_tuple_or_list(num_examples):
classes = balanced_choice_wr(random_classes, ns, rand)
all_images = {cls: list(clss[cls]) for cls in classes}
data, targets, sample_info = [], [], []
for c in classes:
rand.shuffle(all_images[c])
img_name = all_images[c][0]
all_images[c].remove(img_name)
sample_info.append({'name': img_name, 'label': c})
data.append(clss[c][img_name])
targets.append(rand_class_dict[c])
if self.info['one_hot_enc']:
targets = em.to_one_hot_enc(targets, dimension=num_classes)
_dts.append(
em.Dataset(data=np.array(np.stack(data)),
target=targets,
sample_info=sample_info,
info={'all_classes': random_classes}))
return em.Datasets.from_list(_dts)
def train(metasets,
ex_name,
hyper_repr_model_builder,
classifier_builder=None,
saver=None,
seed=0,
MBS=4,
available_devices=('/gpu:0', '/gpu:1'),
mlr0=.001,
mlr_decay=1.e-5,
T=4,
n_episodes_testing=600,
print_every=1000,
patience=40,
restore_model=False,
lr=0.1,
learn_lr=True,
process_fn=None):
"""
Function for training an hyper-representation network.
:param metasets: Datasets of MetaDatasets
:param ex_name: name of the experiment
:param hyper_repr_model_builder: builder for the representation model
:param classifier_builder: optional builder for classifier model (if None then builds a linear model)
:param saver: experiment_manager.Saver object
:param seed:
:param MBS: meta-batch size
:param available_devices: distribute the computation among different GPUS!
:param mlr0: initial meta learning rate
:param mlr_decay:
:param T: number of gradient steps for training ground models
:param n_episodes_testing:
:param print_every:
:param patience:
:param restore_model:
:param lr: initial ground models learning rate
:param learn_lr: True for optimizing the ground models learning rate
:param process_fn: optinal hypergradient process function (like gradient clipping)
:return: tuple: the saver object, the hyper-representation model and the list of experiments objects
"""
if saver is None:
saver = SAVER_EXP(metasets)
T, ss, n_episodes_testing = setup(T, seed, n_episodes_testing, MBS)
exs = [em.SLExperiment(metasets) for _ in range(MBS)]
hyper_repr_model = hyper_repr_model_builder(exs[0].x, name=ex_name)
if classifier_builder is None:
classifier_builder = lambda inp, name: models.FeedForwardNet(
inp, metasets.train.dim_target, name=name)
io_optim, gs, meta_lr, oo_optim, farho = _optimizers(
lr, mlr0, mlr_decay, learn_lr)
for k, ex in enumerate(exs):
with tf.device(available_devices[k % len(available_devices)]):
ex.model = classifier_builder(
hyper_repr_model.for_input(ex.x).out, 'Classifier_%s' % k)
ex.errors['training'] = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=ex.y,
logits=ex.model.out))
ex.errors['validation'] = ex.errors['training']
ex.scores['accuracy'] = tf.reduce_mean(tf.cast(
tf.equal(tf.argmax(ex.y, 1), tf.argmax(ex.model.out, 1)),
tf.float32),
name='accuracy')
optim_dict = farho.inner_problem(ex.errors['training'],
io_optim,
var_list=ex.model.var_list)
farho.outer_problem(ex.errors['validation'],
optim_dict,
oo_optim,
global_step=gs)
farho.finalize(process_fn=process_fn)
feed_dicts, just_train_on_dataset, mean_acc_on, cond = _helper_function(
exs, n_episodes_testing, MBS, ss, farho, T)
rand = em.get_rand_state(0)
with saver.record(*_records(metasets, saver, hyper_repr_model, cond, ss,
mean_acc_on, ex_name, meta_lr),
where='far',
every=print_every,
append_string=ex_name):
tf.global_variables_initializer().run()
if restore_model:
saver.restore_model(hyper_repr_model)
# ADD ONLY TESTING
for _ in cond.early_stopping_sv(saver, patience):
trfd, vfd = feed_dicts(
metasets.train.generate_batch(MBS, rand=rand))
farho.run(
T[0], trfd, vfd
) # one iteration of optimization of representation variables (hyperparameters)
return saver, hyper_repr_model, exs
def meta_omniglot_v2(folder=OMNIGLOT_RESIZED,
std_num_classes=None,
std_num_examples=None,
one_hot_enc=True,
_rand=0,
n_splits=None):
"""
Loading function for Omniglot dataset in learning-to-learn version. Use image data as obtained from
https://github.com/cbfinn/maml/blob/master/data/omniglot_resized/resize_images.py
:param folder: root folder name.
:param std_num_classes: standard number of classes for N-way classification
:param std_num_examples: standard number of examples (e.g. for 1-shot 5-way should be 5)
:param one_hot_enc: one hot encoding
:param _rand: random seed or RandomState for generate training, validation, testing meta-datasets
split
:param n_splits: num classes per split
:return: a Datasets of MetaDataset s
"""
class OmniglotMetaDataset(em.MetaDataset):
def __init__(self,
info=None,
rotations=None,
name='Omniglot',
num_classes=None,
num_examples=None):
super().__init__(info,
name=name,
num_classes=num_classes,
num_examples=num_examples)
self._loaded_images = defaultdict(lambda: {})
self._rotations = rotations or [0, 90, 180, 270]
self._img_array = None
self.load_all()
def generate_datasets(self,
rand=None,
num_classes=None,
num_examples=None):
rand = em.get_rand_state(rand)
if not num_examples: num_examples = self.kwargs['num_examples']
if not num_classes: num_classes = self.kwargs['num_classes']
clss = self._loaded_images if self._loaded_images else self.info[
'classes']
random_classes = rand.choice(list(clss.keys()),
size=(num_classes, ),
replace=False)
rand_class_dict = {rnd: k for k, rnd in enumerate(random_classes)}
_dts = []
for ns in em.as_tuple_or_list(num_examples):
classes = balanced_choice_wr(random_classes, ns, rand)
all_images = {cls: list(clss[cls]) for cls in classes}
indices, targets = [], []
for c in classes:
rand.shuffle(all_images[c])
img_name = all_images[c][0]
all_images[c].remove(img_name)
# sample_info.append({'name': img_name, 'label': c})
indices.append(clss[c][img_name])
targets.append(rand_class_dict[c])
if self.info['one_hot_enc']:
targets = em.to_one_hot_enc(targets, dimension=num_classes)
data = self._img_array[indices]
_dts.append(em.Dataset(data=data, target=targets))
return em.Datasets.from_list(_dts)
def load_all(self):
from scipy.ndimage import imread
from scipy.ndimage.interpolation import rotate
_cls = self.info['classes']
_base_folder = self.info['base_folder']
_id = 0
flat_data = []
flat_targets = []
for c in _cls:
all_images = list(_cls[c])
for img_name in all_images:
img = imread(join(_base_folder, join(c, img_name)))
img = 1. - np.reshape(img, (28, 28, 1)) / 255.
for rot in self._rotations:
img = rotate(img, rot, reshape=False)
self._loaded_images[c + os.path.sep + 'rot_' +
str(rot)][img_name] = _id
_id += 1
flat_data.append(img)
# flat_targets maybe... no flat targets... they depend on the episode!!
self._img_array = np.stack(flat_data)
# end of class
alphabets = os.listdir(folder)
labels_and_images = OrderedDict()
for alphabet in alphabets:
base_folder = join(folder, alphabet)
label_names = os.listdir(base_folder) # all characters in one alphabet
labels_and_images.update({
alphabet + os.path.sep + ln: os.listdir(join(base_folder, ln))
# all examples of each character
for ln in label_names
})
# divide between training validation and test meta-datasets
_rand = em.get_rand_state(_rand)
all_clss = list(labels_and_images.keys())
_rand.shuffle(all_clss)
n_splits = n_splits or (0, 1200, 1300, len(all_clss))
meta_dts = []
for start, end in zip(n_splits, n_splits[1:]):
meta_dts.append(
OmniglotMetaDataset(info={
'base_folder': folder,
'classes':
{k: labels_and_images[k]
for k in all_clss[start:end]},
'one_hot_enc': one_hot_enc
},
num_classes=std_num_classes,
num_examples=std_num_examples))
return em.Datasets.from_list(meta_dts)
def redivide_data(datasets,
partition_proportions=None,
shuffle=False,
filters=None,
maps=None,
balance_classes=False,
rand=None):
"""
Function that redivides datasets. Can be use also to shuffle or filter or map examples.
:param rand:
:param balance_classes: # TODO RICCARDO
:param datasets: original datasets, instances of class Dataset (works with get_data and get_targets for
compatibility with mnist datasets
:param partition_proportions: (optional, default None) list of fractions that can either sum up to 1 or less
then one, in which case one additional partition is created with
proportion 1 - sum(partition proportions).
If None it will retain the same proportion of samples found in datasets
:param shuffle: (optional, default False) if True shuffles the examples
:param filters: (optional, default None) filter or list of filters: functions with signature
(data, target, index) -> boolean (accept or reject the sample)
:param maps: (optional, default None) map or list of maps: functions with signature
(data, target, index) -> (new_data, new_target) (maps the old sample to a new one,
possibly also to more
than one sample, for data augmentation)
:return: a list of datasets of length equal to the (possibly augmented) partition_proportion
"""
rnd = em.get_rand_state(rand)
all_data = vstack([get_data(d) for d in datasets])
all_labels = stack_or_concat([get_targets(d) for d in datasets])
all_infos = np.concatenate([d.sample_info for d in datasets])
N = all_data.shape[0]
if partition_proportions: # argument check
partition_proportions = list([partition_proportions] if isinstance(
partition_proportions, float) else partition_proportions)
sum_proportions = sum(partition_proportions)
assert sum_proportions <= 1, "partition proportions must sum up to at most one: %d" % sum_proportions
if sum_proportions < 1.:
partition_proportions += [1. - sum_proportions]
else:
partition_proportions = [
1. * get_data(d).shape[0] / N for d in datasets
]
if shuffle:
if sp and isinstance(all_data, sp.sparse.csr.csr_matrix):
raise NotImplementedError()
# if sk_shuffle: # TODO this does not work!!! find a way to shuffle these matrices while
# keeping compatibility with tensorflow!
# all_data, all_labels, all_infos = sk_shuffle(all_data, all_labels, all_infos)
# else:
permutation = np.arange(all_data.shape[0])
rnd.shuffle(permutation)
all_data = all_data[permutation]
all_labels = np.array(all_labels[permutation])
all_infos = np.array(all_infos[permutation])
if filters:
if sp and isinstance(all_data, sp.sparse.csr.csr_matrix):
raise NotImplementedError()
filters = as_list(filters)
data_triple = [(x, y, d)
for x, y, d in zip(all_data, all_labels, all_infos)]
for fiat in filters:
data_triple = [
xy for i, xy in enumerate(data_triple)
if fiat(xy[0], xy[1], xy[2], i)
]
all_data = np.vstack([e[0] for e in data_triple])
all_labels = np.vstack([e[1] for e in data_triple])
all_infos = np.vstack([e[2] for e in data_triple])
if maps:
if sp and isinstance(all_data, sp.sparse.csr.csr_matrix):
raise NotImplementedError()
maps = as_list(maps)
data_triple = [(x, y, d)
for x, y, d in zip(all_data, all_labels, all_infos)]
for _map in maps:
data_triple = [
_map(xy[0], xy[1], xy[2], i)
for i, xy in enumerate(data_triple)
]
all_data = np.vstack([e[0] for e in data_triple])
all_labels = np.vstack([e[1] for e in data_triple])
all_infos = np.vstack([e[2] for e in data_triple])
N = all_data.shape[0]
assert N == all_labels.shape[0]
calculated_partitions = reduce(
lambda v1, v2: v1 + [sum(v1) + v2],
[int(N * prp) for prp in partition_proportions], [0])
calculated_partitions[-1] = N
print('datasets.redivide_data:, computed partitions numbers -',
calculated_partitions,
'len all',
N,
end=' ')
new_general_info_dict = {}
for data in datasets:
new_general_info_dict = {**new_general_info_dict, **data.info}
if balance_classes:
new_datasets = []
forbidden_indices = np.empty(0, dtype=np.int64)
for d1, d2 in zip(calculated_partitions[:-1],
calculated_partitions[1:-1]):
indices = np.array(
get_indices_balanced_classes(d2 - d1, all_labels,
forbidden_indices))
dataset = em.Dataset(data=all_data[indices],
target=all_labels[indices],
sample_info=all_infos[indices],
info=new_general_info_dict)
new_datasets.append(dataset)
forbidden_indices = np.append(forbidden_indices, indices)
test_if_balanced(dataset)
remaining_indices = np.array(
list(set(list(range(N))) - set(forbidden_indices)))
new_datasets.append(
em.Dataset(data=all_data[remaining_indices],
target=all_labels[remaining_indices],
sample_info=all_infos[remaining_indices],
info=new_general_info_dict))
else:
new_datasets = [
em.Dataset(data=all_data[d1:d2],
target=all_labels[d1:d2],
sample_info=all_infos[d1:d2],
info=new_general_info_dict) for d1, d2 in
zip(calculated_partitions, calculated_partitions[1:])
]
print('DONE')
return new_datasets
def train(metasets, ex_name, hyper_repr_model_builder, classifier_builder=None, saver=None, seed=0, MBS=4,
available_devices=('/gpu:0', '/gpu:1'),
mlr0=.001, mlr_decay=1.e-5, T=4, n_episodes_testing=600,
print_every=1000, patience=40, restore_model=False,
lr=0.1, learn_lr=True, process_fn=None):
"""
Function for training an hyper-representation network.
:param metasets: Datasets of MetaDatasets
:param ex_name: name of the experiment
:param hyper_repr_model_builder: builder for the representation model,
function (input, name) -> `experiment_manager.Network`
:param classifier_builder: optional builder for classifier model (if None then builds a linear model)
:param saver: experiment_manager.Saver object
:param seed:
:param MBS: meta-batch size
:param available_devices: distribute the computation among different GPUS!
:param mlr0: initial meta learning rate
:param mlr_decay:
:param T: number of gradient steps for training ground models
:param n_episodes_testing:
:param print_every:
:param patience:
:param restore_model:
:param lr: initial ground models learning rate
:param learn_lr: True for optimizing the ground models learning rate
:param process_fn: optinal hypergradient process function (like gradient clipping)
:return: tuple: the saver object, the hyper-representation model and the list of experiments objects
"""
if saver is None:
saver = SAVER_EXP(metasets)
T, ss, n_episodes_testing = setup(T, seed, n_episodes_testing, MBS)
exs = [em.SLExperiment(metasets) for _ in range(MBS)]
hyper_repr_model = hyper_repr_model_builder(exs[0].x, name=ex_name)
if classifier_builder is None: classifier_builder = lambda inp, name: models.FeedForwardNet(
inp, metasets.train.dim_target, name=name)
io_optim, gs, meta_lr, oo_optim, farho = _optimizers(lr, mlr0, mlr_decay, learn_lr)
for k, ex in enumerate(exs):
with tf.device(available_devices[k % len(available_devices)]):
ex.model = classifier_builder(hyper_repr_model.for_input(ex.x).out, 'Classifier_%s' % k)
ex.errors['training'] = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=ex.y, logits=ex.model.out)
)
ex.errors['validation'] = ex.errors['training']
ex.scores['accuracy'] = tf.reduce_mean(tf.cast(
tf.equal(tf.argmax(ex.y, 1), tf.argmax(ex.model.out, 1)), tf.float32),
name='accuracy')
optim_dict = farho.inner_problem(ex.errors['training'], io_optim, var_list=ex.model.var_list)
farho.outer_problem(ex.errors['validation'], optim_dict, oo_optim, global_step=gs)
farho.finalize(process_fn=process_fn)
feed_dicts, just_train_on_dataset, mean_acc_on, cond = _helper_function(
exs, n_episodes_testing, MBS, ss, farho, T)
rand = em.get_rand_state(0)
with saver.record(*_records(metasets, saver, hyper_repr_model, cond, ss, mean_acc_on, ex_name, meta_lr),
where='far', every=print_every, append_string=ex_name):
tf.global_variables_initializer().run()
if restore_model:
saver.restore_model(hyper_repr_model)
# ADD ONLY TESTING
for _ in cond.early_stopping_sv(saver, patience):
trfd, vfd = feed_dicts(metasets.train.generate_batch(MBS, rand=rand))
farho.run(T[0], trfd, vfd) # one iteration of optimization of representation variables (hyperparameters)
return saver, hyper_repr_model, exs
def meta_test_up_to_T(exp_dir,
metasets,
exs,
far_ho,
saver,
sess,
c_way,
k_shot,
lr,
n_test_episodes,
MBS,
seed,
T,
iterations=list(range(10000))):
meta_test_str = str(c_way) + 'way_' + str(k_shot) + 'shot_' + str(
lr) + 'lr' + str(n_test_episodes) + 'ep'
n_test_batches = n_test_episodes // MBS
rand = em.get_rand_state(seed)
valid_batches = BatchQueueMock(metasets.validation, n_test_batches, MBS,
rand)
test_batches = BatchQueueMock(metasets.test, n_test_batches, MBS, rand)
train_batches = BatchQueueMock(metasets.train, n_test_batches, MBS, rand)
print('\nMeta-testing {} (over {} eps)...'.format(meta_test_str,
n_test_episodes))
test_results = {
'valid_test': [],
'test_test': [],
'train_test': [],
'time': [],
'n_test_episodes': n_test_episodes,
'episodes': [],
'iterations': []
}
test_result_path = os.path.join(exp_dir,
meta_test_str + 'noTrain_results.pickle')
start_time = time.time()
for i in iterations:
model_file = os.path.join(exp_dir, 'model' + str(i))
if tf.train.checkpoint_exists(model_file):
print("Restoring model weights from " + model_file)
saver.restore(sess, model_file)
test_results['iterations'].append(i)
test_results['episodes'].append(i * MBS)
valid_result = accuracy_on_up_to_T(valid_batches, exs, far_ho,
sess, T)
test_result = accuracy_on_up_to_T(test_batches, exs, far_ho, sess,
T)
train_result = accuracy_on_up_to_T(train_batches, exs, far_ho,
sess, T)
duration = time.time() - start_time
test_results['time'].append(duration)
for t in range(T):
valid_test = (np.mean(valid_result[1][t]),
np.std(valid_result[1][t]))
test_test = (np.mean(test_result[1][t]),
np.std(test_result[1][t]))
train_test = (np.mean(train_result[1][t]),
np.std(train_result[1][t]))
if t >= len(test_results['valid_test']):
test_results['valid_test'].append({'mean': [], 'std': []})
test_results['test_test'].append({'mean': [], 'std': []})
test_results['train_test'].append({'mean': [], 'std': []})
test_results['valid_test'][t]['mean'].append(valid_test[0])
test_results['test_test'][t]['mean'].append(test_test[0])
test_results['train_test'][t]['mean'].append(train_test[0])
test_results['valid_test'][t]['std'].append(valid_test[1])
test_results['test_test'][t]['std'].append(test_test[1])
test_results['train_test'][t]['std'].append(train_test[1])
print(
'valid-test_test acc T=%d (%d meta_it)(%.2fs): %.4f (%.4f), %.4f (%.4f),'
' %.4f (%.4f)' %
(t + 1, i, duration, train_test[0], train_test[1],
valid_test[0], valid_test[1], test_test[0], test_test[1]))
#print('valid-test_test acc T=%d (%d meta_it)(%.2fs): %.4f (%.4f),'
# ' %.4f (%.4f)' % (t+1, i, duration, valid_test[0], valid_test[1],
# test_test[0], test_test[1]))
save_obj(test_result_path, test_results)
return test_results
def meta_train(exp_dir, metasets, exs, far_ho, saver, sess, n_test_episodes,
MBS, seed, resume, T, n_meta_iterations, print_interval,
save_interval):
# use workers to fill the batches queues (is it worth it?)
result_path = os.path.join(exp_dir, 'results.pickle')
tf.global_variables_initializer().run(session=sess)
n_test_batches = n_test_episodes // MBS
rand = em.get_rand_state(seed)
results = {
'train_train': {
'mean': [],
'std': []
},
'train_test': {
'mean': [],
'std': []
},
'test_test': {
'mean': [],
'std': []
},
'valid_test': {
'mean': [],
'std': []
},
'outer_losses': {
'mean': [],
'std': []
},
'learning_rate': [],
'iterations': [],
'episodes': [],
'time': []
}
start_time = time.time()
resume_itr = 0
if resume:
model_file = tf.train.latest_checkpoint(exp_dir)
if model_file:
print("Restoring results from " + result_path)
results = load_obj(result_path)
start_time = results['time'][-1]
ind1 = model_file.index('model')
resume_itr = int(model_file[ind1 + 5:]) + 1
print("Restoring model weights from " + model_file)
saver.restore(sess, model_file)
''' Meta-Train '''
train_batches = BatchQueueMock(metasets.train, 1, MBS, rand)
valid_batches = BatchQueueMock(metasets.validation, n_test_batches, MBS,
rand)
test_batches = BatchQueueMock(metasets.test, n_test_batches, MBS, rand)
print(
'\nIteration quantities: train_train acc, train_test acc, valid_test, acc'
' test_test acc mean(std) over %d episodes' % n_test_episodes)
with sess.as_default():
inner_losses = []
for meta_it in range(resume_itr, n_meta_iterations):
tr_fd, v_fd = feed_dicts(train_batches.get()[0], exs)
far_ho.run(T, tr_fd, v_fd)
# inner_losses.append(far_ho.inner_losses)
outer_losses = [
sess.run(ex.errors['validation'], v_fd) for ex in exs
]
outer_losses_moments = (np.mean(outer_losses),
np.std(outer_losses))
results['outer_losses']['mean'].append(outer_losses_moments[0])
results['outer_losses']['std'].append(outer_losses_moments[1])
# print('inner_losses: ', inner_losses[-1])
if meta_it % print_interval == 0 or meta_it == n_meta_iterations - 1:
results['iterations'].append(meta_it)
results['episodes'].append(meta_it * MBS)
train_result = accuracy_on(train_batches, exs, far_ho, sess, T)
test_result = accuracy_on(test_batches, exs, far_ho, sess, T)
valid_result = accuracy_on(valid_batches, exs, far_ho, sess, T)
train_train = (np.mean(train_result[0]),
np.std(train_result[0]))
train_test = (np.mean(train_result[1]),
np.std(train_result[1]))
valid_test = (np.mean(valid_result[1]),
np.std(valid_result[1]))
test_test = (np.mean(test_result[1]), np.std(test_result[1]))
duration = time.time() - start_time
results['time'].append(duration)
results['train_train']['mean'].append(train_train[0])
results['train_test']['mean'].append(train_test[0])
results['valid_test']['mean'].append(valid_test[0])
results['test_test']['mean'].append(test_test[0])
results['train_train']['std'].append(train_train[1])
results['train_test']['std'].append(train_test[1])
results['valid_test']['std'].append(valid_test[1])
results['test_test']['std'].append(test_test[1])
results['inner_losses'] = inner_losses
print('mean outer losses: {}'.format(outer_losses_moments[1]))
print('it %d, ep %d (%.2fs): %.3f, %.3f, %.3f, %.3f' %
(meta_it, meta_it * MBS, duration, train_train[0],
train_test[0], valid_test[0], test_test[0]))
lr = sess.run(["lr:0"])[0]
print('lr: {}'.format(lr))
# do_plot(logdir, results)
if meta_it % save_interval == 0 or meta_it == n_meta_iterations - 1:
saver.save(sess, exp_dir + '/model' + str(meta_it))
save_obj(result_path, results)
return results
