def test_seed_class_splitter():
dataset_transform = ClassSplitter(shuffle=True,
num_train_per_class=5, num_test_per_class=5)
dataset = Sinusoid(10, num_tasks=1000, noise_std=0.1,
dataset_transform=dataset_transform)
dataset.seed(1)
expected_train_inputs = np.array([1.08565437,-1.56211897,4.62078213,-2.03870077,0.76977846])
expected_train_targets = np.array([-0.00309463,-1.37650356,-0.9346262,-0.1031986,-0.4698061])
expected_test_inputs = np.array([-2.48340416,3.75388738,-3.15504396,0.09898378,0.32922559])
expected_test_targets = np.array([0.73113509,0.91773121,1.86656819,-1.61885041,-1.52508997])
task = dataset[0]
train_dataset, test_dataset = task['train'], task['test']
assert len(train_dataset) == 5
assert len(test_dataset) == 5
for i, (train_input, train_target) in enumerate(train_dataset):
assert np.isclose(train_input, expected_train_inputs[i])
assert np.isclose(train_target, expected_train_targets[i])
for i, (test_input, test_target) in enumerate(test_dataset):
assert np.isclose(test_input, expected_test_inputs[i])
assert np.isclose(test_target, expected_test_targets[i])
def sinusoid(shots, shuffle=True, test_shots=None, seed=None, **kwargs):
"""Helper function to create a meta-dataset for the Sinusoid toy dataset.
Parameters
----------
shots : int
Number of (training) examples in each task. This corresponds to `k` in
`k-shot` classification.
shuffle : bool (default: `True`)
Shuffle the examples when creating the tasks.
test_shots : int, optional
Number of test examples in each task. If `None`, then the number of test
examples is equal to the number of training examples in each task.
seed : int, optional
Random seed to be used in the meta-dataset.
kwargs
Additional arguments passed to the `Sinusoid` class.
See also
--------
`torchmeta.toy.Sinusoid` : Meta-dataset for the Sinusoid toy dataset.
"""
if 'num_samples_per_task' in kwargs:
warnings.warn(
'Both arguments `shots` and `num_samples_per_task` were '
'set in the helper function for the number of samples in each task. '
'Ignoring the argument `shots`.',
stacklevel=2)
if test_shots is not None:
shots = kwargs['num_samples_per_task'] - test_shots
if shots <= 0:
raise ValueError(
'The argument `test_shots` ({0}) is greater '
'than the number of samples per task ({1}). Either use the '
'argument `shots` instead of `num_samples_per_task`, or '
'increase the value of `num_samples_per_task`.'.format(
test_shots, kwargs['num_samples_per_task']))
else:
shots = kwargs['num_samples_per_task'] // 2
if test_shots is None:
test_shots = shots
dataset = Sinusoid(num_samples_per_task=shots + test_shots, **kwargs)
dataset = ClassSplitter(dataset,
shuffle=shuffle,
num_train_per_class=shots,
num_test_per_class=test_shots)
dataset.seed(seed)
return dataset
def main(
shots=10,
tasks_per_batch=16,
num_tasks=160000,
adapt_lr=0.01,
meta_lr=0.001,
adapt_steps=5,
hidden_dim=32,
):
# load the dataset
tasksets = Sinusoid(num_samples_per_task=2 * shots, num_tasks=num_tasks)
dataloader = BatchMetaDataLoader(tasksets, batch_size=tasks_per_batch)
# create the model
model = SineModel(dim=hidden_dim)
maml = l2l.algorithms.MAML(model,
lr=adapt_lr,
first_order=False,
allow_unused=True)
opt = optim.Adam(maml.parameters(), meta_lr)
lossfn = nn.MSELoss(reduction='mean')
# for each iteration
for iter, batch in enumerate(dataloader): # num_tasks/batch_size
meta_train_loss = 0.0
# for each task in the batch
effective_batch_size = batch[0].shape[0]
for i in range(effective_batch_size):
learner = maml.clone()
# divide the data into support and query sets
train_inputs, train_targets = batch[0][i].float(
), batch[1][i].float()
x_support, y_support = train_inputs[::2], train_targets[::2]
x_query, y_query = train_inputs[1::2], train_targets[1::2]
for _ in range(adapt_steps): # adaptation_steps
support_preds = learner(x_support)
support_loss = lossfn(support_preds, y_support)
learner.adapt(support_loss)
query_preds = learner(x_query)
query_loss = lossfn(query_preds, y_query)
meta_train_loss += query_loss
meta_train_loss = meta_train_loss / effective_batch_size
if iter % 200 == 0:
print('Iteration:', iter, 'Meta Train Loss',
meta_train_loss.item())
opt.zero_grad()
meta_train_loss.backward()
opt.step()
def test_batch_meta_dataloader():
dataset = Sinusoid(10, num_tasks=1000, noise_std=None)
meta_dataloader = BatchMetaDataLoader(dataset, batch_size=4)
assert isinstance(meta_dataloader, DataLoader)
assert len(meta_dataloader) == 250 # 1000 / 4
inputs, targets = next(iter(meta_dataloader))
assert isinstance(inputs, torch.Tensor)
assert isinstance(targets, torch.Tensor)
assert inputs.shape == (4, 10, 1)
assert targets.shape == (4, 10, 1)
def test_meta_dataloader():
dataset = Sinusoid(10, num_tasks=1000, noise_std=None)
meta_dataloader = MetaDataLoader(dataset, batch_size=4)
assert isinstance(meta_dataloader, DataLoader)
assert len(meta_dataloader) == 250 # 1000 / 4
batch = next(iter(meta_dataloader))
assert isinstance(batch, list)
assert len(batch) == 4
task = batch[0]
assert isinstance(task, Task)
assert len(task) == 10
def test_meta_dataloader_task_loader():
dataset = Sinusoid(10, num_tasks=1000, noise_std=None)
meta_dataloader = MetaDataLoader(dataset, batch_size=4)
batch = next(iter(meta_dataloader))
dataloader = DataLoader(batch[0], batch_size=5)
inputs, targets = next(iter(dataloader))
assert len(dataloader) == 2 # 10 / 5
# PyTorch dataloaders convert numpy array to tensors
assert isinstance(inputs, torch.Tensor)
assert isinstance(targets, torch.Tensor)
assert inputs.shape == (5, 1)
assert targets.shape == (5, 1)
def get_sine_loader(batch_size, num_steps, shots=10, test_shots=15):
dataset_transform = ClassSplitter(
shuffle=True, num_train_per_class=shots, num_test_per_class=test_shots
)
transform = ToTensor1D()
dataset = Sinusoid(
shots + test_shots,
num_tasks=batch_size * num_steps,
transform=transform,
target_transform=transform,
dataset_transform=dataset_transform,
)
loader = BatchMetaDataLoader(
dataset, batch_size=batch_size, shuffle=True, num_workers=4, pin_memory=True,
)
return loader
def test_batch_meta_dataloader_splitter():
dataset = Sinusoid(20, num_tasks=1000, noise_std=None)
dataset = ClassSplitter(dataset, num_train_per_class=5,
num_test_per_class=15)
meta_dataloader = BatchMetaDataLoader(dataset, batch_size=4)
batch = next(iter(meta_dataloader))
assert isinstance(batch, dict)
assert 'train' in batch
assert 'test' in batch
train_inputs, train_targets = batch['train']
test_inputs, test_targets = batch['test']
assert isinstance(train_inputs, torch.Tensor)
assert isinstance(train_targets, torch.Tensor)
assert train_inputs.shape == (4, 5, 1)
assert train_targets.shape == (4, 5, 1)
assert isinstance(test_inputs, torch.Tensor)
assert isinstance(test_targets, torch.Tensor)
assert test_inputs.shape == (4, 15, 1)
assert test_targets.shape == (4, 15, 1)
def main(args):
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO)
device = torch.device(
'cuda' if args.use_cuda and torch.cuda.is_available() else 'cpu')
if (args.output_folder is not None):
if not os.path.exists(args.output_folder):
os.makedirs(args.output_folder)
logging.debug('Creating folder `{0}`'.format(args.output_folder))
folder = os.path.join(args.output_folder,
time.strftime('%Y-%m-%d_%H%M%S'))
os.makedirs(folder)
logging.debug('Creating folder `{0}`'.format(folder))
args.folder = os.path.abspath(args.folder)
args.model_path = os.path.abspath(os.path.join(folder, 'model.th'))
# Save the configuration in a config.json file
with open(os.path.join(folder, 'config.json'), 'w') as f:
json.dump(vars(args), f, indent=2)
logging.info('Saving configuration file in `{0}`'.format(
os.path.abspath(os.path.join(folder, 'config.json'))))
dataset_transform = ClassSplitter(shuffle=True,
num_train_per_class=args.num_shots,
num_test_per_class=args.num_shots_test)
class_augmentations = [Rotation([90, 180, 270])]
if args.dataset == 'sinusoid':
transform = ToTensor()
meta_train_dataset = Sinusoid(args.num_shots + args.num_shots_test,
num_tasks=1000000,
transform=transform,
target_transform=transform,
dataset_transform=dataset_transform)
meta_val_dataset = Sinusoid(args.num_shots + args.num_shots_test,
num_tasks=1000000,
transform=transform,
target_transform=transform,
dataset_transform=dataset_transform)
model = ModelMLPSinusoid(hidden_sizes=[40, 40])
loss_function = F.mse_loss
elif args.dataset == 'omniglot':
transform = Compose([Resize(28), ToTensor()])
meta_train_dataset = Omniglot(args.folder,
transform=transform,
target_transform=Categorical(
args.num_ways),
num_classes_per_task=args.num_ways,
meta_train=True,
class_augmentations=class_augmentations,
dataset_transform=dataset_transform,
download=True)
meta_val_dataset = Omniglot(args.folder,
transform=transform,
target_transform=Categorical(
args.num_ways),
num_classes_per_task=args.num_ways,
meta_val=True,
class_augmentations=class_augmentations,
dataset_transform=dataset_transform)
model = ModelConvOmniglot(args.num_ways, hidden_size=args.hidden_size)
loss_function = F.cross_entropy
elif args.dataset == 'miniimagenet':
transform = Compose([Resize(84), ToTensor()])
meta_train_dataset = MiniImagenet(
args.folder,
transform=transform,
target_transform=Categorical(args.num_ways),
num_classes_per_task=args.num_ways,
meta_train=True,
class_augmentations=class_augmentations,
dataset_transform=dataset_transform,
download=True)
meta_val_dataset = MiniImagenet(
args.folder,
transform=transform,
target_transform=Categorical(args.num_ways),
num_classes_per_task=args.num_ways,
meta_val=True,
class_augmentations=class_augmentations,
dataset_transform=dataset_transform)
model = ModelConvMiniImagenet(args.num_ways,
hidden_size=args.hidden_size)
loss_function = F.cross_entropy
else:
raise NotImplementedError('Unknown dataset `{0}`.'.format(
args.dataset))
meta_train_dataloader = BatchMetaDataLoader(meta_train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers,
pin_memory=True)
meta_val_dataloader = BatchMetaDataLoader(meta_val_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers,
pin_memory=True)
meta_optimizer = torch.optim.Adam(model.parameters(), lr=args.meta_lr)
metalearner = ModelAgnosticMetaLearning(
model,
meta_optimizer,
first_order=args.first_order,
num_adaptation_steps=args.num_steps,
step_size=args.step_size,
loss_function=loss_function,
device=device)
best_val_accuracy = None
# Training loop
epoch_desc = 'Epoch {{0: <{0}d}}'.format(1 +
int(math.log10(args.num_epochs)))
for epoch in range(args.num_epochs):
metalearner.train(meta_train_dataloader,
max_batches=args.num_batches,
verbose=args.verbose,
desc='Training',
leave=False)
results = metalearner.evaluate(meta_val_dataloader,
max_batches=args.num_batches,
verbose=args.verbose,
desc=epoch_desc.format(epoch + 1))
if (best_val_accuracy is None) \
or (best_val_accuracy < results['accuracies_after']):
best_val_accuracy = results['accuracies_after']
if args.output_folder is not None:
with open(args.model_path, 'wb') as f:
torch.save(model.state_dict(), f)
if hasattr(meta_train_dataset, 'close'):
meta_train_dataset.close()
meta_val_dataset.close()
def main(
shots=10,
tasks_per_batch=16,
num_tasks=16000,
num_test_tasks=32,
adapt_lr=0.01,
meta_lr=0.001,
adapt_steps=5,
hidden_dim=32,
):
exp_name = input(
"Enter Experiment NAME (should be unique, else overwrites): ")
EXPERIMENT_DIR = "./experiments/MAML_Sine_exps" + exp_name
if os.path.isdir(EXPERIMENT_DIR):
print("Experiment folder opened ...")
else:
os.mkdir(EXPERIMENT_DIR)
print("New Experiment folder started ...")
MODEL_CHECKPOINT_DIR = EXPERIMENT_DIR + "/model"
if os.path.isdir(MODEL_CHECKPOINT_DIR):
print("Model Checkpoint folder opened ...")
else:
os.mkdir(MODEL_CHECKPOINT_DIR)
print("New Model checkpoint folder made ...")
PLOT_RESULTS_DIR = EXPERIMENT_DIR + "/plot_results"
if os.path.isdir(PLOT_RESULTS_DIR):
print("Image results folder opened ...")
else:
os.mkdir(PLOT_RESULTS_DIR)
print("New Image results folder made ...")
# load the dataset
tasksets = Sinusoid(num_samples_per_task=2 * shots, num_tasks=num_tasks)
dataloader = BatchMetaDataLoader(tasksets, batch_size=tasks_per_batch)
# create the model
model = SineModel(dim=hidden_dim, experiment_dir=EXPERIMENT_DIR)
maml = l2l.algorithms.MAML(model,
lr=adapt_lr,
first_order=False,
allow_unused=True)
opt = optim.Adam(maml.parameters(), meta_lr)
lossfn = nn.MSELoss(reduction='mean')
# for each iteration
for iter, batch in enumerate(dataloader): # num_tasks/batch_size
meta_train_loss = 0.0
# for each task in the batch
effective_batch_size = batch[0].shape[0]
for i in range(effective_batch_size):
learner = maml.clone()
# divide the data into support and query sets
train_inputs, train_targets = batch[0][i].float(
), batch[1][i].float()
x_support, y_support = train_inputs[::2], train_targets[::2]
x_query, y_query = train_inputs[1::2], train_targets[1::2]
for _ in range(adapt_steps): # adaptation_steps
support_preds = learner(x_support)
support_loss = lossfn(support_preds, y_support)
learner.adapt(support_loss)
query_preds = learner(x_query)
query_loss = lossfn(query_preds, y_query)
meta_train_loss += query_loss
meta_train_loss = meta_train_loss / effective_batch_size
opt.zero_grad()
meta_train_loss.backward()
opt.step()
if iter % 100 == 0:
print('Iteration:', iter, 'Meta Train Loss',
meta_train_loss.item())
# print(x_query.requires_grad, y_query.requires_grad,query_preds.requires_grad,meta_train_loss.item())
plotter(x_query, y_query,
query_preds.detach().numpy(), iter, 'Train',
meta_train_loss.item(), model.plot_results)
#save current model
model.save_checkpoint()
#meta-testing
test_tasks = Sinusoid(num_samples_per_task=shots, num_tasks=num_test_tasks)
test_dataloader = BatchMetaDataLoader(test_tasks,
batch_size=tasks_per_batch)
#load learned model
test_model = SineModel(dim=hidden_dim, experiment_dir=EXPERIMENT_DIR)
test_model.load_checkpoint()
for iter, batch in enumerate(test_dataloader):
meta_test_loss = 0.0
# for each task in the batch
effective_batch_size = batch[0].shape[0]
for i in range(effective_batch_size):
learner = maml.clone()
# divide the data into support and query sets
test_inputs, test_targets = batch[0][i].float(), batch[1][i].float(
)
test_preds = test_model(test_inputs)
test_loss = lossfn(test_preds, test_targets)
meta_test_loss += test_loss
meta_test_loss = meta_test_loss / effective_batch_size
if iter % 20 == 0:
print('Iteration:', iter, 'Meta Test Loss', meta_test_loss.item())
plotter(test_inputs, test_targets,
test_preds.detach().numpy(), iter, 'Test',
meta_test_loss.item(), test_model.plot_results)
def get_benchmark_by_name(name,
folder,
num_ways,
num_shots,
num_shots_test,
hidden_size=None,
meta_batch_size=1,
ensemble_size=0
):
dataset_transform = ClassSplitter(shuffle=True,
num_train_per_class=num_shots,
num_test_per_class=num_shots_test)
if name == 'sinusoid':
transform = ToTensor1D()
meta_train_dataset = Sinusoid(num_shots + num_shots_test,
num_tasks=1000000,
transform=transform,
target_transform=transform,
dataset_transform=dataset_transform)
meta_val_dataset = Sinusoid(num_shots + num_shots_test,
num_tasks=1000000,
transform=transform,
target_transform=transform,
dataset_transform=dataset_transform)
meta_test_dataset = Sinusoid(num_shots + num_shots_test,
num_tasks=1000000,
transform=transform,
target_transform=transform,
dataset_transform=dataset_transform)
model = ModelMLPSinusoid(hidden_sizes=[40, 40], meta_batch_size=meta_batch_size, ensemble_size=ensemble_size)
loss_function = F.mse_loss
elif name == 'omniglot':
class_augmentations = [Rotation([90, 180, 270])]
transform = Compose([Resize(28), ToTensor()])
meta_train_dataset = Omniglot(folder,
transform=transform,
target_transform=Categorical(num_ways),
num_classes_per_task=num_ways,
meta_train=True,
class_augmentations=class_augmentations,
dataset_transform=dataset_transform,
download=True)
meta_val_dataset = Omniglot(folder,
transform=transform,
target_transform=Categorical(num_ways),
num_classes_per_task=num_ways,
meta_val=True,
class_augmentations=class_augmentations,
dataset_transform=dataset_transform)
meta_test_dataset = Omniglot(folder,
transform=transform,
target_transform=Categorical(num_ways),
num_classes_per_task=num_ways,
meta_test=True,
dataset_transform=dataset_transform)
model = ModelConvOmniglot(num_ways, hidden_size=hidden_size, meta_batch_size=meta_batch_size, ensemble_size=ensemble_size)
loss_function = batch_cross_entropy
elif name == 'miniimagenet':
transform = Compose([Resize(84), ToTensor()])
meta_train_dataset = MiniImagenet(folder,
transform=transform,
target_transform=Categorical(num_ways),
num_classes_per_task=num_ways,
meta_train=True,
dataset_transform=dataset_transform,
download=True)
meta_val_dataset = MiniImagenet(folder,
transform=transform,
target_transform=Categorical(num_ways),
num_classes_per_task=num_ways,
meta_val=True,
dataset_transform=dataset_transform)
meta_test_dataset = MiniImagenet(folder,
transform=transform,
target_transform=Categorical(num_ways),
num_classes_per_task=num_ways,
meta_test=True,
dataset_transform=dataset_transform)
model = ModelConvMiniImagenet(num_ways, hidden_size=hidden_size, meta_batch_size=meta_batch_size, ensemble_size=ensemble_size)
loss_function = batch_cross_entropy
else:
raise NotImplementedError('Unknown dataset `{0}`.'.format(name))
return Benchmark(meta_train_dataset=meta_train_dataset,
meta_val_dataset=meta_val_dataset,
meta_test_dataset=meta_test_dataset,
model=model,
loss_function=loss_function)
def get_benchmark_by_name(name,
folder,
num_ways,
num_shots,
num_shots_test,
hidden_size=None):
dataset_transform = ClassSplitter(shuffle=True,
num_train_per_class=num_shots,
num_test_per_class=num_shots_test)
if name == 'sinusoid':
transform = ToTensor1D()
meta_train_dataset = Sinusoid(num_shots + num_shots_test,
num_tasks=1000000,
transform=transform,
target_transform=transform,
dataset_transform=dataset_transform)
meta_val_dataset = Sinusoid(num_shots + num_shots_test,
num_tasks=1000000,
transform=transform,
target_transform=transform,
dataset_transform=dataset_transform)
meta_test_dataset = Sinusoid(num_shots + num_shots_test,
num_tasks=1000000,
transform=transform,
target_transform=transform,
dataset_transform=dataset_transform)
model = ModelMLPSinusoid(hidden_sizes=[40, 40])
loss_function = F.mse_loss
elif name == 'omniglot':
class_augmentations = [Rotation([90, 180, 270])]
transform = Compose([Resize(28), ToTensor()])
meta_train_dataset = Omniglot(folder,
transform=transform,
target_transform=Categorical(num_ways),
num_classes_per_task=num_ways,
meta_train=True,
class_augmentations=class_augmentations,
dataset_transform=dataset_transform,
download=True)
meta_val_dataset = Omniglot(folder,
transform=transform,
target_transform=Categorical(num_ways),
num_classes_per_task=num_ways,
meta_val=True,
class_augmentations=class_augmentations,
dataset_transform=dataset_transform)
meta_test_dataset = Omniglot(folder,
transform=transform,
target_transform=Categorical(num_ways),
num_classes_per_task=num_ways,
meta_test=True,
dataset_transform=dataset_transform)
model = ModelConvOmniglot(num_ways, hidden_size=hidden_size)
loss_function = F.cross_entropy
elif name == 'miniimagenet':
transform = Compose([Resize(84), ToTensor()])
meta_train_dataset = MiniImagenet(
folder,
transform=transform,
target_transform=Categorical(num_ways),
num_classes_per_task=num_ways,
meta_train=True,
dataset_transform=dataset_transform,
download=True)
meta_val_dataset = MiniImagenet(folder,
transform=transform,
target_transform=Categorical(num_ways),
num_classes_per_task=num_ways,
meta_val=True,
dataset_transform=dataset_transform)
meta_test_dataset = MiniImagenet(
folder,
transform=transform,
target_transform=Categorical(num_ways),
num_classes_per_task=num_ways,
meta_test=True,
dataset_transform=dataset_transform)
model = ModelConvMiniImagenet(num_ways, hidden_size=hidden_size)
loss_function = F.cross_entropy
elif name == 'doublenmnist':
from torchneuromorphic.doublenmnist_torchmeta.doublenmnist_dataloaders import DoubleNMNIST, Compose, ClassNMNISTDataset, CropDims, Downsample, ToCountFrame, ToTensor, ToEventSum, Repeat, toOneHot
from torchneuromorphic.utils import plot_frames_imshow
from matplotlib import pyplot as plt
from torchmeta.utils.data import CombinationMetaDataset
root = 'data/nmnist/n_mnist.hdf5'
chunk_size = 300
ds = 2
dt = 1000
transform = None
target_transform = None
size = [2, 32 // ds, 32 // ds]
transform = Compose([
CropDims(low_crop=[0, 0], high_crop=[32, 32], dims=[2, 3]),
Downsample(factor=[dt, 1, ds, ds]),
ToEventSum(T=chunk_size, size=size),
ToTensor()
])
if target_transform is None:
target_transform = Compose(
[Repeat(chunk_size), toOneHot(num_ways)])
loss_function = F.cross_entropy
meta_train_dataset = ClassSplitter(DoubleNMNIST(
root=root,
meta_train=True,
transform=transform,
target_transform=target_transform,
chunk_size=chunk_size,
num_classes_per_task=num_ways),
num_train_per_class=num_shots,
num_test_per_class=num_shots_test)
meta_val_dataset = ClassSplitter(DoubleNMNIST(
root=root,
meta_val=True,
transform=transform,
target_transform=target_transform,
chunk_size=chunk_size,
num_classes_per_task=num_ways),
num_train_per_class=num_shots,
num_test_per_class=num_shots_test)
meta_test_dataset = ClassSplitter(DoubleNMNIST(
root=root,
meta_test=True,
transform=transform,
target_transform=target_transform,
chunk_size=chunk_size,
num_classes_per_task=num_ways),
num_train_per_class=num_shots,
num_test_per_class=num_shots_test)
model = ModelConvDoubleNMNIST(num_ways, hidden_size=hidden_size)
elif name == 'doublenmnistsequence':
from torchneuromorphic.doublenmnist_torchmeta.doublenmnist_dataloaders import DoubleNMNIST, Compose, ClassNMNISTDataset, CropDims, Downsample, ToCountFrame, ToTensor, ToEventSum, Repeat, toOneHot
from torchneuromorphic.utils import plot_frames_imshow
from matplotlib import pyplot as plt
from torchmeta.utils.data import CombinationMetaDataset
root = 'data/nmnist/n_mnist.hdf5'
chunk_size = 300
ds = 2
dt = 1000
transform = None
target_transform = None
size = [2, 32 // ds, 32 // ds]
transform = Compose([
CropDims(low_crop=[0, 0], high_crop=[32, 32], dims=[2, 3]),
Downsample(factor=[dt, 1, ds, ds]),
ToCountFrame(T=chunk_size, size=size),
ToTensor()
])
if target_transform is None:
target_transform = Compose(
[Repeat(chunk_size), toOneHot(num_ways)])
loss_function = F.cross_entropy
meta_train_dataset = ClassSplitter(DoubleNMNIST(
root=root,
meta_train=True,
transform=transform,
target_transform=target_transform,
chunk_size=chunk_size,
num_classes_per_task=num_ways),
num_train_per_class=num_shots,
num_test_per_class=num_shots_test)
meta_val_dataset = ClassSplitter(DoubleNMNIST(
root=root,
meta_val=True,
transform=transform,
target_transform=target_transform,
chunk_size=chunk_size,
num_classes_per_task=num_ways),
num_train_per_class=num_shots,
num_test_per_class=num_shots_test)
meta_test_dataset = ClassSplitter(DoubleNMNIST(
root=root,
meta_test=True,
transform=transform,
target_transform=target_transform,
chunk_size=chunk_size,
num_classes_per_task=num_ways),
num_train_per_class=num_shots,
num_test_per_class=num_shots_test)
model = ModelDECOLLE(num_ways)
else:
raise NotImplementedError('Unknown dataset `{0}`.'.format(name))
return Benchmark(meta_train_dataset=meta_train_dataset,
meta_val_dataset=meta_val_dataset,
meta_test_dataset=meta_test_dataset,
model=model,
loss_function=loss_function)
def main(args):
with open(args.config, 'r') as f:
config = json.load(f)
if args.folder is not None:
config['folder'] = args.folder
if args.num_steps > 0:
config['num_steps'] = args.num_steps
if args.num_batches > 0:
config['num_batches'] = args.num_batches
device = torch.device(
'cuda' if args.use_cuda and torch.cuda.is_available() else 'cpu')
dataset_transform = ClassSplitter(
shuffle=True,
num_train_per_class=config['num_shots'],
num_test_per_class=config['num_shots_test'])
if config['dataset'] == 'sinusoid':
transform = ToTensor()
meta_test_dataset = Sinusoid(config['num_shots'] +
config['num_shots_test'],
num_tasks=1000000,
transform=transform,
target_transform=transform,
dataset_transform=dataset_transform)
model = ModelMLPSinusoid(hidden_sizes=[40, 40])
loss_function = F.mse_loss
elif config['dataset'] == 'omniglot':
transform = Compose([Resize(28), ToTensor()])
meta_test_dataset = Omniglot(config['folder'],
transform=transform,
target_transform=Categorical(
config['num_ways']),
num_classes_per_task=config['num_ways'],
meta_train=True,
dataset_transform=dataset_transform,
download=True)
model = ModelConvOmniglot(config['num_ways'],
hidden_size=config['hidden_size'])
loss_function = F.cross_entropy
elif config['dataset'] == 'miniimagenet':
transform = Compose([Resize(84), ToTensor()])
meta_test_dataset = MiniImagenet(
config['folder'],
transform=transform,
target_transform=Categorical(config['num_ways']),
num_classes_per_task=config['num_ways'],
meta_train=True,
dataset_transform=dataset_transform,
download=True)
model = ModelConvMiniImagenet(config['num_ways'],
hidden_size=config['hidden_size'])
loss_function = F.cross_entropy
else:
raise NotImplementedError('Unknown dataset `{0}`.'.format(
config['dataset']))
with open(config['model_path'], 'rb') as f:
model.load_state_dict(torch.load(f, map_location=device))
meta_test_dataloader = BatchMetaDataLoader(meta_test_dataset,
batch_size=config['batch_size'],
shuffle=True,
num_workers=args.num_workers,
pin_memory=True)
metalearner = ModelAgnosticMetaLearning(
model,
first_order=config['first_order'],
num_adaptation_steps=config['num_steps'],
step_size=config['step_size'],
loss_function=loss_function,
device=device)
results = metalearner.evaluate(meta_test_dataloader,
max_batches=config['num_batches'],
verbose=args.verbose,
desc='Test')
# Save results
dirname = os.path.dirname(config['model_path'])
with open(os.path.join(dirname, 'results.json'), 'w') as f:
json.dump(results, f)
def setup(self):
# called on every GPU
self.train = Sinusoid(num_samples_per_task=self.num_samples_per_task,
num_tasks=self.num_tasks)
self.test = Sinusoid(num_samples_per_task=self.shots,
num_tasks=self.num_tasks)
def get_benchmark_by_name(name,
folder,
num_ways,
num_shots,
num_shots_test,
hidden_size=None):
"""
Returns a namedtuple with the train/val/test split, model, and loss function
for the specified task.
Parameters
----------
name : str
Name of the dataset to use
folder : str
Folder where dataset is stored (or will download to this path if not found)
num_ways : int
Number of classes for each task
num_shots : int
Number of training examples provided per class
num_shots_test : int
Number of test examples provided per class (during adaptation)
"""
dataset_transform = ClassSplitter(shuffle=True,
num_train_per_class=num_shots,
num_test_per_class=num_shots_test)
if name == 'nmltoy2d':
model_hidden_sizes = [1024, 1024]
replay_pool_size = 100
clip_length = 100
from_beginning = False
# For validation and testing, we evaluate the outer loss on the entire dataset;
# for testing, we use smaller batches for efficiency
meta_train_dataset = NMLToy2D(replay_pool_size=replay_pool_size,
clip_length=clip_length,
from_beginning=from_beginning,
test_strategy='sample',
test_batch_size=10)
meta_val_dataset = NMLToy2D(replay_pool_size=replay_pool_size,
clip_length=clip_length,
from_beginning=from_beginning,
test_strategy='all')
meta_test_dataset = NMLToy2D(replay_pool_size=replay_pool_size,
clip_length=clip_length,
from_beginning=from_beginning,
test_strategy='all')
model = ModelMLPToy2D(model_hidden_sizes)
loss_function = F.cross_entropy
elif name == 'noisyduplicates':
model_hidden_sizes = [2048, 2048]
locations = [
([-2.5, 2.5], 1, 0), # Single visit (negative)
([2.5, 2.5], 10, 0), # Many visits
([-2.5, -2.5], 2, 15), # A few negatives, mostly positives
([2.5,
-2.5], 8, 15) # More negatives, but still majority positives
]
noise_std = 0
meta_train_dataset = NoisyDuplicatesProblem(locations,
noise_std=noise_std)
meta_val_dataset = NoisyDuplicatesProblem(locations,
noise_std=noise_std)
meta_test_dataset = NoisyDuplicatesProblem(locations,
noise_std=noise_std)
model = ModelMLPToy2D(model_hidden_sizes)
loss_function = F.cross_entropy
elif name == 'sinusoid':
transform = ToTensor1D()
meta_train_dataset = Sinusoid(num_shots + num_shots_test,
num_tasks=1000000,
transform=transform,
target_transform=transform,
dataset_transform=dataset_transform)
meta_val_dataset = Sinusoid(num_shots + num_shots_test,
num_tasks=1000000,
transform=transform,
target_transform=transform,
dataset_transform=dataset_transform)
meta_test_dataset = Sinusoid(num_shots + num_shots_test,
num_tasks=1000000,
transform=transform,
target_transform=transform,
dataset_transform=dataset_transform)
model = ModelMLPSinusoid(hidden_sizes=[40, 40])
loss_function = F.mse_loss
elif name == 'omniglot':
class_augmentations = [Rotation([90, 180, 270])]
transform = Compose([Resize(28), ToTensor()])
meta_train_dataset = Omniglot(folder,
transform=transform,
target_transform=Categorical(num_ways),
num_classes_per_task=num_ways,
meta_train=True,
class_augmentations=class_augmentations,
dataset_transform=dataset_transform,
download=True)
meta_val_dataset = Omniglot(folder,
transform=transform,
target_transform=Categorical(num_ways),
num_classes_per_task=num_ways,
meta_val=True,
class_augmentations=class_augmentations,
dataset_transform=dataset_transform)
meta_test_dataset = Omniglot(folder,
transform=transform,
target_transform=Categorical(num_ways),
num_classes_per_task=num_ways,
meta_test=True,
dataset_transform=dataset_transform)
model = ModelConvOmniglot(num_ways, hidden_size=hidden_size)
loss_function = F.cross_entropy
elif name == 'miniimagenet':
transform = Compose([Resize(84), ToTensor()])
meta_train_dataset = MiniImagenet(
folder,
transform=transform,
target_transform=Categorical(num_ways),
num_classes_per_task=num_ways,
meta_train=True,
dataset_transform=dataset_transform,
download=True)
meta_val_dataset = MiniImagenet(folder,
transform=transform,
target_transform=Categorical(num_ways),
num_classes_per_task=num_ways,
meta_val=True,
dataset_transform=dataset_transform)
meta_test_dataset = MiniImagenet(
folder,
transform=transform,
target_transform=Categorical(num_ways),
num_classes_per_task=num_ways,
meta_test=True,
dataset_transform=dataset_transform)
model = ModelConvMiniImagenet(num_ways, hidden_size=hidden_size)
loss_function = F.cross_entropy
else:
raise NotImplementedError('Unknown dataset `{0}`.'.format(name))
return Benchmark(meta_train_dataset=meta_train_dataset,
meta_val_dataset=meta_val_dataset,
meta_test_dataset=meta_test_dataset,
model=model,
loss_function=loss_function)