def testValues(self, value):
problem = problems.simple()
f = problem()
with self.test_session() as sess:
output = sess.run(f, feed_dict={"x:0": value})
self.assertEqual(output, value**2)
def testSecondDerivatives(self):
"""Tests second derivatives for simple problem."""
problem = problems.simple()
optimizer = meta.MetaOptimizer(
net=dict(net="CoordinateWiseDeepLSTM", net_options={"layers": ()}))
minimize_ops = optimizer.meta_minimize(problem,
3,
second_derivatives=True)
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
train(sess, minimize_ops, 1, 2)
def testSimple(self):
"""Tests L2L applied to simple problem."""
problem = problems.simple()
optimizer = meta.MetaOptimizer(net=dict(
net="CoordinateWiseDeepLSTM",
net_options={
"layers": (),
# Initializing the network to zeros makes learning more stable.
"initializer": "zeros"
}))
minimize_ops = optimizer.meta_minimize(problem, 20, learning_rate=1e-2)
# L2L should solve the simple problem is less than 500 epochs.
with self.test_session() as sess:
sess.run(tf.initialize_all_variables())
cost, _ = train(sess, minimize_ops, 500, 5)
self.assertLess(cost, 1e-5)
def testSaveAndLoad(self):
"""Tests saving and loading a meta-optimizer."""
layers = (2, 3)
net_options = {"layers": layers, "initializer": "zeros"}
num_unrolls = 2
num_epochs = 1
problem = problems.simple()
# Original optimizer.
with tf.Graph().as_default() as g1:
optimizer = meta.MetaOptimizer(net=dict(
net="CoordinateWiseDeepLSTM", net_options=net_options))
minimize_ops = optimizer.meta_minimize(problem, 3)
with self.test_session(graph=g1) as sess:
sess.run(tf.global_variables_initializer())
train(sess, minimize_ops, 1, 2)
# Save optimizer.
tmp_dir = tempfile.mkdtemp()
save_result = optimizer.save(sess, path=tmp_dir)
net_path = next(iter(save_result))
# Retrain original optimizer.
cost, x = train(sess, minimize_ops, num_unrolls, num_epochs)
# Load optimizer and retrain in a new session.
with tf.Graph().as_default() as g2:
optimizer = meta.MetaOptimizer(
net=dict(net="CoordinateWiseDeepLSTM",
net_options=net_options,
net_path=net_path))
minimize_ops = optimizer.meta_minimize(problem, 3)
with self.test_session(graph=g2) as sess:
sess.run(tf.global_variables_initializer())
cost_loaded, x_loaded = train(sess, minimize_ops, num_unrolls,
num_epochs)
# The last cost should be the same.
self.assertAlmostEqual(cost, cost_loaded, places=3)
self.assertAlmostEqual(x[0], x_loaded[0], places=3)
# Cleanup.
os.remove(net_path)
os.rmdir(tmp_dir)
def testResults(self):
"""Tests reproducibility of Torch results."""
problem = problems.simple()
optimizer = meta.MetaOptimizer(net=dict(net="CoordinateWiseDeepLSTM",
net_options={
"layers": (),
"initializer": "zeros"
}))
minimize_ops = optimizer.meta_minimize(problem, 5)
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
cost, final_x = train(sess, minimize_ops, 1, 2)
# Torch results
torch_cost = 0.7325327
torch_final_x = 0.8559
self.assertAlmostEqual(cost, torch_cost, places=4)
self.assertAlmostEqual(final_x[0], torch_final_x, places=4)
def get_config(problem_name, path=None):
"""Returns problem configuration."""
if problem_name == "simple":
problem = problems.simple()
net_config = {
"cw": {
"net": "CoordinateWiseDeepLSTM",
"net_options": {
"layers": (),
"initializer": "zeros"
},
"net_path": None
}
}
net_assignments = None
else:
raise ValueError("{} is not a valid problem".format(problem_name))
return problem, net_config, net_assignments
def get_config(problem_name, path=None):
"""Returns problem configuration."""
if problem_name == "simple":
problem = problems.simple()
net_config = {
"cw": {
"net": "CoordinateWiseDeepLSTM",
"net_options": {
"layers": (),
"initializer": "zeros"
},
"net_path": get_net_path("cw", path)
}
}
net_assignments = None
elif problem_name == "simple-multi":
problem = problems.simple_multi_optimizer()
net_config = {
"cw": {
"net": "CoordinateWiseDeepLSTM",
"net_options": {
"layers": (),
"initializer": "zeros"
},
"net_path": get_net_path("cw", path)
},
"adam": {
"net": "Adam",
"net_options": {
"learning_rate": 0.1
}
}
}
net_assignments = [("cw", ["x_0"]), ("adam", ["x_1"])]
elif problem_name == "quadratic":
problem = problems.quadratic(batch_size=128, num_dims=10)
net_config = {
"cw": {
"net": "CoordinateWiseDeepLSTM",
"net_options": {
"layers": (20, 20)
},
"net_path": get_net_path("cw", path)
}
}
net_assignments = None
elif problem_name == "mnist":
# mode = "train" if path is None else "test"
mode = "train"
problem = problems.mnist(layers=(20, ),
activation="sigmoid",
mode=mode)
# problem = problems.mnist(layers=(100,), activation="relu", mode=mode)
# problem = problems.mnist(layers=(20,), activation="relu", mode=mode)
net_config = {"cw": get_default_net_config("cw", path)}
net_assignments = None
elif problem_name == "cifar":
mode = "train" if path is None else "test"
problem = problems.cifar10("cifar10",
conv_channels=(16, 16, 16),
linear_layers=(32, ),
mode=mode)
net_config = {"cw": get_default_net_config("cw", path)}
net_assignments = None
elif problem_name == "cifar-multi":
mode = "train" if path is None else "test"
problem = problems.cifar10("cifar10",
conv_channels=(16, 16, 16),
linear_layers=(32, ),
mode=mode)
net_config = {
"conv": get_default_net_config("conv", path),
"fc": get_default_net_config("fc", path)
}
conv_vars = ["conv_net_2d/conv_2d_{}/w".format(i) for i in xrange(3)]
fc_vars = ["conv_net_2d/conv_2d_{}/b".format(i) for i in xrange(3)]
fc_vars += [
"conv_net_2d/batch_norm_{}/beta".format(i) for i in xrange(3)
]
fc_vars += ["mlp/linear_{}/w".format(i) for i in xrange(2)]
fc_vars += ["mlp/linear_{}/b".format(i) for i in xrange(2)]
fc_vars += ["mlp/batch_norm/beta"]
net_assignments = [("conv", conv_vars), ("fc", fc_vars)]
else:
raise ValueError("{} is not a valid problem".format(problem_name))
return problem, net_config, net_assignments
def get_config(problem_name,
path=None,
mode=None,
num_hidden_layer=None,
net_name=None):
"""Returns problem configuration."""
if problem_name == "simple":
problem = problems.simple()
net_config = {
"cw": {
"net": "CoordinateWiseDeepLSTM",
"net_options": {
"layers": (),
"initializer": "zeros"
},
"net_path": path
}
}
net_assignments = None
elif problem_name == "simple-multi":
problem = problems.simple_multi_optimizer()
net_config = {
"cw": {
"net": "CoordinateWiseDeepLSTM",
"net_options": {
"layers": (),
"initializer": "zeros"
},
"net_path": path
},
"adam": {
"net": "Adam",
"net_options": {
"learning_rate": 0.01
}
}
}
net_assignments = [("cw", ["x_0"]), ("adam", ["x_1"])]
elif problem_name == "quadratic":
problem = problems.quadratic(batch_size=128, num_dims=10)
net_config = {
"cw": {
"net": "CoordinateWiseDeepLSTM",
"net_options": {
"layers": (20, 20)
},
"net_path": path
}
}
net_assignments = None
### our tests
elif problem_name == "mnist":
if mode is None:
mode = "train" if path is None else "test"
problem = problems.mnist(layers=(20, ),
activation="sigmoid",
mode=mode)
net_config = {"cw": get_default_net_config(path)}
net_assignments = None
elif problem_name == "mnist_relu":
if mode is None:
mode = "train" if path is None else "test"
problem = problems.mnist(layers=(20, ), activation="relu", mode=mode)
net_config = {"cw": get_default_net_config(path)}
net_assignments = None
elif problem_name == "mnist_deeper":
if mode is None:
mode = "train" if path is None else "test"
num_hidden_layer = 2
problem = problems.mnist(layers=(20, ) * num_hidden_layer,
activation="sigmoid",
mode=mode)
net_config = {"cw": get_default_net_config(path)}
net_assignments = None
elif problem_name == "mnist_conv":
if mode is None:
mode = "train" if path is None else "test"
problem = problems.mnist_conv(mode=mode, batch_norm=True)
net_config = {"cw": get_default_net_config(path)}
net_assignments = None
elif problem_name == "cifar_conv":
if mode is None:
mode = "train" if path is None else "test"
problem = problems.cifar10("cifar10", mode=mode)
net_config = {"cw": get_default_net_config(path)}
net_assignments = None
elif problem_name == "lenet":
if mode is None:
mode = "train" if path is None else "test"
problem = problems.LeNet("cifar10",
conv_channels=(6, 16),
linear_layers=(120, 84),
mode=mode)
net_config = {"cw": get_default_net_config(path)}
net_assignments = None
elif problem_name == "nas":
if mode is None:
mode = "train" if path is None else "test"
problem = problems.NAS("cifar10", mode=mode)
net_config = {"cw": get_default_net_config(path)}
net_assignments = None
###
elif problem_name == "vgg16":
mode = "train" if path is None else "test"
problem = problems.vgg16_cifar10("cifar10", mode=mode)
net_config = {"cw": get_default_net_config(path)}
net_assignments = None
elif problem_name == "cifar-multi":
mode = "train" if path is None else "test"
problem = problems.cifar10("cifar10",
conv_channels=(16, 16, 16),
linear_layers=(32, ),
mode=mode)
net_config = {
"conv": get_default_net_config(path),
"fc": get_default_net_config(path)
}
conv_vars = ["conv_net_2d/conv_2d_{}/w".format(i) for i in xrange(3)]
fc_vars = ["conv_net_2d/conv_2d_{}/b".format(i) for i in xrange(3)]
fc_vars += [
"conv_net_2d/batch_norm_{}/beta".format(i) for i in xrange(3)
]
fc_vars += ["mlp/linear_{}/w".format(i) for i in xrange(2)]
fc_vars += ["mlp/linear_{}/b".format(i) for i in xrange(2)]
fc_vars += ["mlp/batch_norm/beta"]
net_assignments = [("conv", conv_vars), ("fc", fc_vars)]
elif problem_name == "confocal_microscopy_3d":
problem = problems.confocal_microscopy_3d(batch_size=32, num_points=5)
net_config = {
"cw": {
"net": "CoordinateWiseDeepLSTM",
"net_options": {
"layers": (20, 20)
},
"net_path": path
}
}
net_assignments = None
elif problem_name == "square_cos":
problem = problems.square_cos(batch_size=128, num_dims=2)
net_config = {
"cw": {
"net": "CoordinateWiseDeepLSTM",
"net_options": {
"layers": (20, 20)
},
"net_path": path
}
}
net_assignments = None
else:
raise ValueError("{} is not a valid problem".format(problem_name))
if net_name == "RNNprop":
default_config = {
"net": "RNNprop",
"net_options": {
"layers": (20, 20),
"preprocess_name": "fc",
"preprocess_options": {
"dim": 20
},
"scale": 0.01,
"tanh_output": True
},
"net_path": path
}
net_config = {"rp": default_config}
return problem, net_config, net_assignments
def testVariables(self):
problem = problems.simple()
problem()
variables = tf.trainable_variables()
self.assertEqual(len(variables), 1)
self.assertEqual(variables[0].get_shape().as_list(), [])
def testShape(self):
problem = problems.simple()
f = problem()
self.assertEqual(f.get_shape().as_list(), [])
def get_config(problem_name, path=None, problem_path=None):
"""Returns problem configuration."""
if problem_name == "simple":
problem = problems.simple()
net_config = {
"cw": {
"net": "CoordinateWiseDeepLSTM",
"net_options": {
"layers": (),
"initializer": "zeros"
},
"net_path": get_net_path("cw", path)
}
}
net_assignments = None
elif problem_name == "simple-multi":
problem = problems.simple_multi_optimizer()
net_config = {
"cw": {
"net": "CoordinateWiseDeepLSTM",
"net_options": {
"layers": (),
"initializer": "zeros"
},
"net_path": get_net_path("cw", path)
},
"adam": {
"net": "Adam",
"net_options": {
"learning_rate": 0.1
}
}
}
net_assignments = [("cw", ["x_0"]), ("adam", ["x_1"])]
elif problem_name == "quadratic":
if problem_path is not None:
npzfile = np.load(problem_path)
problems_w, problems_b = npzfile['arr_0'], npzfile['arr_1']
assert len(problems_w) == len(problems_b)
batch_size = len(problems_w)
problem = problems.quadratic(batch_size=batch_size,
num_dims=2,
problems_w=problems_w,
problems_b=problems_b)
else:
problem = problems.quadratic(batch_size=128, num_dims=2)
net_config = {
"cw": {
"net": "CoordinateWiseDeepLSTM",
"net_options": {
"layers": (20, 20)
},
"net_path": get_net_path("cw", path)
}
}
net_assignments = None
elif problem_name == "quadratic-wav":
if problem_path is not None:
npzfile = np.load(problem_path)
problems_w, problems_b = npzfile['arr_0'], npzfile['arr_1']
assert len(problems_w) == len(problems_b)
batch_size = len(problems_w)
problem = problems.quadratic(batch_size=batch_size,
num_dims=2,
problems_w=problems_w,
problems_b=problems_b)
else:
problem = problems.quadratic(batch_size=128, num_dims=2)
net_config = {
"cw-wav": {
"net": "CoordinateWiseWaveNet",
"net_options": {
"num_layers": 4
},
"net_path": get_net_path("cw-wav", path)
}
}
net_assignments = None
elif problem_name == "sin":
if problem_path is not None:
problems_sin = np.load(problem_path) # TODO
batch_size = len(problems_sin)
problem = problems.prob_sin(batch_size=batch_size,
problem_param=problems_sin)
else:
problem = problems.prob_sin(batch_size=128)
net_config = {
"cw": {
"net": "CoordinateWiseDeepLSTM",
"net_options": {
"layers": (20, 20)
},
"net_path": get_net_path("cw", path)
}
}
net_assignments = None
elif problem_name == "sin-wav":
if problem_path is not None:
problems_sin = np.load(problem_path) # TODO
batch_size = len(problems_sin)
problem = problems.prob_sin(batch_size=batch_size,
problem_param=problems_sin)
else:
problem = problems.prob_sin(batch_size=128)
net_config = {
"cw-wav": {
"net": "CoordinateWiseWaveNet",
"net_options": {
"num_layers": 4
},
"net_path": get_net_path("cw-wav", path)
}
}
net_assignments = None
elif problem_name == "mnist":
mode = "train" if path is None else "test"
problem = problems.mnist(layers=(20, ), mode=mode)
net_config = {"cw": get_default_net_config("cw", path)}
net_assignments = None
elif problem_name == "cifar":
mode = "train" if path is None else "test"
problem = problems.cifar10("cifar10",
conv_channels=(16, 16, 16),
linear_layers=(32, ),
mode=mode)
net_config = {"cw": get_default_net_config("cw", path)}
net_assignments = None
elif problem_name == "cifar-multi":
mode = "train" if path is None else "test"
problem = problems.cifar10("cifar10",
conv_channels=(16, 16, 16),
linear_layers=(32, ),
mode=mode)
net_config = {
"conv": get_default_net_config("conv", path),
"fc": get_default_net_config("fc", path)
}
conv_vars = ["conv_net_2d/conv_2d_{}/w".format(i) for i in xrange(3)]
fc_vars = ["conv_net_2d/conv_2d_{}/b".format(i) for i in xrange(3)]
fc_vars += [
"conv_net_2d/batch_norm_{}/beta".format(i) for i in xrange(3)
]
fc_vars += ["mlp/linear_{}/w".format(i) for i in xrange(2)]
fc_vars += ["mlp/linear_{}/b".format(i) for i in xrange(2)]
fc_vars += ["mlp/batch_norm/beta"]
net_assignments = [("conv", conv_vars), ("fc", fc_vars)]
else:
raise ValueError("{} is not a valid problem".format(problem_name))
return problem, net_config, net_assignments
def get_config(problem_name,
param=None,
path=None,
model_idx=0,
mode=None,
num_hidden_layer=None,
net_name=None,
num_linear_heads=1,
init="normal"):
"""Returns problem configuration."""
if problem_name == "simple":
problem = problems.simple()
net_config = {
"cw": {
"net": "CoordinateWiseDeepLSTM",
"net_options": {
"layers": (),
"initializer": "zeros"
},
"net_path": get_net_path("cw", path)
}
}
net_assignments = None
elif problem_name == "simple-multi":
problem = problems.simple_multi_optimizer()
net_config = {
"cw": {
"net": "CoordinateWiseDeepLSTM",
"net_options": {
"layers": (),
"initializer": "zeros"
},
"net_path": get_net_path("cw", path)
},
"adam": {
"net": "Adam",
"net_options": {
"learning_rate": 0.01
}
}
}
net_assignments = [("cw", ["x_0"]), ("adam", ["x_1"])]
elif problem_name == "quadratic":
problem = problems.quadratic(batch_size=128, num_dims=10)
net_config = {
"cw": {
"net": "CoordinateWiseDeepLSTM",
"net_options": {
"layers": (20, 20)
},
"net_path": get_net_path("cw", path)
}
}
net_assignments = None
elif problem_name == "lasso":
batch_size = param['bs']
num_dims_m = param['m']
num_dims_n = param['n']
problem = problems.lasso(batch_size=batch_size,
num_dims_m=num_dims_m,
num_dims_n=num_dims_n)
net_config = {
"cw": {
"net": "CoordinateWiseDeepLSTM",
"net_options": {
"layers": (20, 20)
},
"net_path": get_net_path("cw", path)
}
}
net_assignments = None
elif problem_name == "rastrigin":
batch_size = param['bs']
num_dims_n = param['n']
problem = problems.rastrigin(batch_size=batch_size,
num_dims=num_dims_n,
mode=mode)
net_config = {
"cw": {
"net": "CoordinateWiseDeepLSTM",
"net_options": {
"layers": (20, 20)
},
"net_path": get_net_path("cw", path)
}
}
net_assignments = None
elif problem_name == "mnist":
if mode is None:
mode = "train" if path is None else "test"
problem = problems.mnist(layers=(20, ),
activation="sigmoid",
mode=mode,
init="normal")
net_config = {"cw": get_default_net_config("cw", path, model_idx)}
net_assignments = None
elif problem_name == "mnist_relu":
mode = "test"
problem = problems.mnist(layers=(20, ), activation="relu", mode=mode)
net_config = {"cw": get_default_net_config("cw", path, model_idx)}
net_assignments = None
elif problem_name == "mnist_deeper":
mode = "test"
assert num_hidden_layer is not None
problem = problems.mnist(layers=(20, ) * num_hidden_layer,
activation="sigmoid",
mode=mode)
net_config = {"cw": get_default_net_config("cw", path, model_idx)}
net_assignments = None
elif problem_name == "mnist_conv":
mode = "test"
problem = problems.mnist_conv(mode=mode, batch_norm=True)
net_config = {"cw": get_default_net_config("cw", path, model_idx)}
net_assignments = None
elif problem_name == "cifar":
mode = "test"
problem = problems.cifar10("cifar10", mode=mode)
net_config = {"cw": get_default_net_config("cw", path, model_idx)}
net_assignments = None
elif problem_name == "vgg16":
mode = "train" if path is None else "test"
problem = problems.vgg16_cifar10("cifar10", mode=mode)
net_config = {"cw": get_default_net_config("cw", path)}
net_assignments = None
elif problem_name == "cifar-multi":
mode = "train" if path is None else "test"
problem = problems.cifar10("cifar10",
conv_channels=(16, 16, 16),
linear_layers=(32, ),
mode=mode)
net_config = {
"conv": get_default_net_config("conv", path),
"fc": get_default_net_config("fc", path)
}
conv_vars = ["conv_net_2d/conv_2d_{}/w".format(i) for i in xrange(3)]
fc_vars = ["conv_net_2d/conv_2d_{}/b".format(i) for i in xrange(3)]
fc_vars += [
"conv_net_2d/batch_norm_{}/beta".format(i) for i in xrange(3)
]
fc_vars += ["mlp/linear_{}/w".format(i) for i in xrange(2)]
fc_vars += ["mlp/linear_{}/b".format(i) for i in xrange(2)]
fc_vars += ["mlp/batch_norm/beta"]
net_assignments = [("conv", conv_vars), ("fc", fc_vars)]
else:
raise ValueError("{} is not a valid problem".format(problem_name))
if net_name == "RNNprop":
default_config = {
"net": "RNNprop",
"net_options": {
"layers": (20, 20),
"preprocess_name": "fc",
"preprocess_options": {
"dim": 20
},
"scale": 0.01,
"tanh_output": True,
"num_linear_heads": num_linear_heads
},
"net_path": get_net_path("rp", path, model_idx)
}
net_config = {"rp": default_config}
return problem, net_config, net_assignments
