def benchmark(
num_evaluations: int,
optimizer_factory: Callable[[], Optimizer],
blackbox: Blackbox,
candidates: np.array,
num_seeds: int,
verbose: bool = False,
) -> Tuple[np.array]:
"""
For each seed, the optimizer is run 'num_evaluations'.
:param num_evaluations:
:param optimizer_factory:
:param blackbox:
:param candidates:
:param num_seeds:
:param verbose:
:return: two tensors of shape (num_seeds, num_evaluations, X) where X = [input_dim, output_dim]
"""
seeds = range(num_seeds)
#if verbose:
# seeds = tqdm(seeds)
seeds = tqdm(seeds)
Xs = np.empty((num_seeds, num_evaluations, blackbox.input_dim))
Xs[:] = np.nan
ys = np.empty((num_seeds, num_evaluations, blackbox.output_dim))
ys[:] = np.nan
for seed in seeds:
try:
set_seed(seed)
optimizer = optimizer_factory()
for i in range(num_evaluations):
x = optimizer.sample(candidates)
y = blackbox(x)
if verbose:
logging.info(f"criterion {y} for arguments {x}")
optimizer.observe(x=x, y=y)
Xs[seed, i] = x
ys[seed, i] = y
# memory leaks without gc, not sure why, perhaps a reference cycle
gc.collect()
del optimizer
except Exception:
print("seed evaluation failed")
traceback.print_exc(file=sys.stdout)
pass
return Xs, ys
def test_blackbox_works_with_optimization(blackbox: Blackbox):
logging.basicConfig(level=logging.INFO)
seed = 3
num_evaluations = 5
optimizer_cls = RS
set_seed(seed)
optimizer = optimizer_cls(
input_dim=blackbox.input_dim,
output_dim=blackbox.output_dim,
evaluations_other_tasks=Xy_train,
)
candidates = X_test
for i in range(num_evaluations):
x = optimizer.sample(candidates)
y = blackbox(x)
logging.info(f"criterion {y} for arguments {x}")
optimizer.observe(x=x, y=y)
parser = argparse.ArgumentParser()
parser.add_argument('--gpu', default='0', type=str)
parser.add_argument('--dataset', default='cifar10', type=str)
parser.add_argument('--arch', '-a', default='vgg16_bn', type=str)
parser.add_argument('--sparsity_level', '-s', default=0.2, type=float)
parser.add_argument('--lr', default=0.01, type=float)
parser.add_argument('--lambd', default=0.5, type=float)
parser.add_argument('--epochs', default=10, type=int)
parser.add_argument('--log_interval', default=100, type=int)
parser.add_argument('--train_batch_size', default=128, type=int)
parser.add_argument('--expanded_inchannel', '-e', default=80, type=int)
parser.add_argument('--seed', default=None, type=int)
args = parser.parse_args()
args.seed = misc.set_seed(args.seed)
args.num_classes = 10
args.device = 'cuda'
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
args.logdir = 'seed-%d/%s-%s/channel-%d-sparsity-%.2f' % (
args.seed, args.dataset, args.arch, args.expanded_inchannel,
args.sparsity_level)
misc.prepare_logging(args)
print('==> Preparing data..')
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
def main():
print(f'\n=== cuda is {"" if using_gpu() else "NOT"} available ===\n')
data_root = os.path.abspath(
os.path.join(os.path.expanduser('~'), 'datasets', 'mnist'))
set_seed(0)
# hyper-parameters:
BASIC_LR = 5e-4
MIN_LR = 0.
WEIGHT_DECAY = 1e-5
OP_MOMENTUM = 0.9
EPOCHS = 8
BATCH_SIZE = 64
DROP_OUT_RATE = 0.1
train_loader, test_loader = get_dataloaders(data_root=data_root,
batch_size=BATCH_SIZE)
ITERS = len(train_loader)
print(f'=== hyper-params ===\n'
f' epochs={EPOCHS}\n'
f' train iters={ITERS}\n'
f' batch size={BATCH_SIZE}\n'
f' cosine lr:{BASIC_LR} -> {MIN_LR}\n'
f' weight decay={WEIGHT_DECAY}\n'
f' momentum={OP_MOMENTUM}\n'
f' drop out={DROP_OUT_RATE}\n')
set_seed(0)
net = FCNet(input_dim=MNIST_img_ch * MNIST_img_size**2,
output_dim=MNIST_num_classes,
dropout_p=DROP_OUT_RATE)
init_params(net, verbose=True)
if using_gpu():
net = net.cuda()
print('=== start training from scratch ===\n')
train_accs, test_accs = [], []
train_losses, test_losses = [], []
lrs = []
recorders = (test_accs, test_losses, train_accs, train_losses, lrs)
set_seed(0)
optimizer = SGD(net.parameters(),
lr=BASIC_LR,
weight_decay=WEIGHT_DECAY,
momentum=OP_MOMENTUM)
scheduler = CosineAnnealingLR(optimizer,
T_max=EPOCHS * ITERS,
eta_min=MIN_LR)
components = (net, optimizer, scheduler)
set_seed(0)
for epoch in range(EPOCHS):
train_epoch(epoch, ITERS, EPOCHS, train_loader, test_loader,
components, recorders)
final_test_acc, _ = test(test_loader, net)
print(f'\n=== final test acc: {final_test_acc:.2f} ===\n')
plot_curves(train_accs, test_accs, train_losses, test_losses, lrs)
def _observe(self, x: np.array, y: np.array):
# remark, we could fit the GP there so that sampling several times avoid the cost of refitting the GP
self.X_observed = torch.cat(
(self.X_observed, torch.Tensor(x).unsqueeze(dim=0)), dim=0)
self.y_observed = torch.cat(
(self.y_observed, torch.Tensor(y).unsqueeze(dim=0)), dim=0)
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO)
num_evaluations = 10
Xy_train, X_test, y_test = artificial_task1(seed=0)
print(y_test[0])
set_seed(0)
blackbox = BlackboxOffline(
X=X_test,
y=y_test,
)
optimizer = GP(
input_dim=blackbox.input_dim,
output_dim=blackbox.output_dim,
)
candidates = X_test
for i in range(num_evaluations):
#x = optimizer.sample(candidates)