def testValues(self, value):
problem = problems.simple_multi_optimizer(num_dims=1)
f = problem()
with self.test_session() as sess:
output = sess.run(f, feed_dict={"x_0:0": value})
self.assertEqual(output, value**2)
def testVariables(self):
num_dims = 3
problem = problems.simple_multi_optimizer(num_dims=num_dims)
problem()
variables = tf.trainable_variables()
self.assertEqual(len(variables), num_dims)
for v in variables:
self.assertEqual(v.get_shape().as_list(), [])
def testMultiOptimizer(self, net_assignments, net_config):
"""Tests different variable->net mappings in multi-optimizer problem."""
problem = problems.simple_multi_optimizer(num_dims=2)
optimizer = meta.MetaOptimizer(**net_config)
minimize_ops = optimizer.meta_minimize(problem,
3,
net_assignments=net_assignments)
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
train(sess, minimize_ops, 1, 2)
def get_config(problem_name, path=None):
"""Returns problem configuration."""
if problem_name == "simple":
problem = problems.simple()
net_config = {
"cw": {
"net": "CoordinateWiseDeepLSTM",
"net_options": {
"layers": (),
"initializer": "zeros"
},
"net_path": get_net_path("cw", path)
}
}
net_assignments = None
elif problem_name == "simple-multi":
problem = problems.simple_multi_optimizer()
net_config = {
"cw": {
"net": "CoordinateWiseDeepLSTM",
"net_options": {
"layers": (),
"initializer": "zeros"
},
"net_path": get_net_path("cw", path)
},
"adam": {
"net": "Adam",
"net_options": {
"learning_rate": 0.1
}
}
}
net_assignments = [("cw", ["x_0"]), ("adam", ["x_1"])]
elif problem_name == "quadratic":
problem = problems.quadratic(batch_size=128, num_dims=10)
net_config = {
"cw": {
"net": "CoordinateWiseDeepLSTM",
"net_options": {
"layers": (20, 20)
},
"net_path": get_net_path("cw", path)
}
}
net_assignments = None
elif problem_name == "mnist":
# mode = "train" if path is None else "test"
mode = "train"
problem = problems.mnist(layers=(20, ),
activation="sigmoid",
mode=mode)
# problem = problems.mnist(layers=(100,), activation="relu", mode=mode)
# problem = problems.mnist(layers=(20,), activation="relu", mode=mode)
net_config = {"cw": get_default_net_config("cw", path)}
net_assignments = None
elif problem_name == "cifar":
mode = "train" if path is None else "test"
problem = problems.cifar10("cifar10",
conv_channels=(16, 16, 16),
linear_layers=(32, ),
mode=mode)
net_config = {"cw": get_default_net_config("cw", path)}
net_assignments = None
elif problem_name == "cifar-multi":
mode = "train" if path is None else "test"
problem = problems.cifar10("cifar10",
conv_channels=(16, 16, 16),
linear_layers=(32, ),
mode=mode)
net_config = {
"conv": get_default_net_config("conv", path),
"fc": get_default_net_config("fc", path)
}
conv_vars = ["conv_net_2d/conv_2d_{}/w".format(i) for i in xrange(3)]
fc_vars = ["conv_net_2d/conv_2d_{}/b".format(i) for i in xrange(3)]
fc_vars += [
"conv_net_2d/batch_norm_{}/beta".format(i) for i in xrange(3)
]
fc_vars += ["mlp/linear_{}/w".format(i) for i in xrange(2)]
fc_vars += ["mlp/linear_{}/b".format(i) for i in xrange(2)]
fc_vars += ["mlp/batch_norm/beta"]
net_assignments = [("conv", conv_vars), ("fc", fc_vars)]
else:
raise ValueError("{} is not a valid problem".format(problem_name))
return problem, net_config, net_assignments
def get_config(problem_name,
path=None,
mode=None,
num_hidden_layer=None,
net_name=None):
"""Returns problem configuration."""
if problem_name == "simple":
problem = problems.simple()
net_config = {
"cw": {
"net": "CoordinateWiseDeepLSTM",
"net_options": {
"layers": (),
"initializer": "zeros"
},
"net_path": path
}
}
net_assignments = None
elif problem_name == "simple-multi":
problem = problems.simple_multi_optimizer()
net_config = {
"cw": {
"net": "CoordinateWiseDeepLSTM",
"net_options": {
"layers": (),
"initializer": "zeros"
},
"net_path": path
},
"adam": {
"net": "Adam",
"net_options": {
"learning_rate": 0.01
}
}
}
net_assignments = [("cw", ["x_0"]), ("adam", ["x_1"])]
elif problem_name == "quadratic":
problem = problems.quadratic(batch_size=128, num_dims=10)
net_config = {
"cw": {
"net": "CoordinateWiseDeepLSTM",
"net_options": {
"layers": (20, 20)
},
"net_path": path
}
}
net_assignments = None
### our tests
elif problem_name == "mnist":
if mode is None:
mode = "train" if path is None else "test"
problem = problems.mnist(layers=(20, ),
activation="sigmoid",
mode=mode)
net_config = {"cw": get_default_net_config(path)}
net_assignments = None
elif problem_name == "mnist_relu":
if mode is None:
mode = "train" if path is None else "test"
problem = problems.mnist(layers=(20, ), activation="relu", mode=mode)
net_config = {"cw": get_default_net_config(path)}
net_assignments = None
elif problem_name == "mnist_deeper":
if mode is None:
mode = "train" if path is None else "test"
num_hidden_layer = 2
problem = problems.mnist(layers=(20, ) * num_hidden_layer,
activation="sigmoid",
mode=mode)
net_config = {"cw": get_default_net_config(path)}
net_assignments = None
elif problem_name == "mnist_conv":
if mode is None:
mode = "train" if path is None else "test"
problem = problems.mnist_conv(mode=mode, batch_norm=True)
net_config = {"cw": get_default_net_config(path)}
net_assignments = None
elif problem_name == "cifar_conv":
if mode is None:
mode = "train" if path is None else "test"
problem = problems.cifar10("cifar10", mode=mode)
net_config = {"cw": get_default_net_config(path)}
net_assignments = None
elif problem_name == "lenet":
if mode is None:
mode = "train" if path is None else "test"
problem = problems.LeNet("cifar10",
conv_channels=(6, 16),
linear_layers=(120, 84),
mode=mode)
net_config = {"cw": get_default_net_config(path)}
net_assignments = None
elif problem_name == "nas":
if mode is None:
mode = "train" if path is None else "test"
problem = problems.NAS("cifar10", mode=mode)
net_config = {"cw": get_default_net_config(path)}
net_assignments = None
###
elif problem_name == "vgg16":
mode = "train" if path is None else "test"
problem = problems.vgg16_cifar10("cifar10", mode=mode)
net_config = {"cw": get_default_net_config(path)}
net_assignments = None
elif problem_name == "cifar-multi":
mode = "train" if path is None else "test"
problem = problems.cifar10("cifar10",
conv_channels=(16, 16, 16),
linear_layers=(32, ),
mode=mode)
net_config = {
"conv": get_default_net_config(path),
"fc": get_default_net_config(path)
}
conv_vars = ["conv_net_2d/conv_2d_{}/w".format(i) for i in xrange(3)]
fc_vars = ["conv_net_2d/conv_2d_{}/b".format(i) for i in xrange(3)]
fc_vars += [
"conv_net_2d/batch_norm_{}/beta".format(i) for i in xrange(3)
]
fc_vars += ["mlp/linear_{}/w".format(i) for i in xrange(2)]
fc_vars += ["mlp/linear_{}/b".format(i) for i in xrange(2)]
fc_vars += ["mlp/batch_norm/beta"]
net_assignments = [("conv", conv_vars), ("fc", fc_vars)]
elif problem_name == "confocal_microscopy_3d":
problem = problems.confocal_microscopy_3d(batch_size=32, num_points=5)
net_config = {
"cw": {
"net": "CoordinateWiseDeepLSTM",
"net_options": {
"layers": (20, 20)
},
"net_path": path
}
}
net_assignments = None
elif problem_name == "square_cos":
problem = problems.square_cos(batch_size=128, num_dims=2)
net_config = {
"cw": {
"net": "CoordinateWiseDeepLSTM",
"net_options": {
"layers": (20, 20)
},
"net_path": path
}
}
net_assignments = None
else:
raise ValueError("{} is not a valid problem".format(problem_name))
if net_name == "RNNprop":
default_config = {
"net": "RNNprop",
"net_options": {
"layers": (20, 20),
"preprocess_name": "fc",
"preprocess_options": {
"dim": 20
},
"scale": 0.01,
"tanh_output": True
},
"net_path": path
}
net_config = {"rp": default_config}
return problem, net_config, net_assignments
def testShape(self):
num_dims = 3
problem = problems.simple_multi_optimizer(num_dims=num_dims)
f = problem()
self.assertEqual(f.get_shape().as_list(), [])
def get_config(problem_name, path=None, problem_path=None):
"""Returns problem configuration."""
if problem_name == "simple":
problem = problems.simple()
net_config = {
"cw": {
"net": "CoordinateWiseDeepLSTM",
"net_options": {
"layers": (),
"initializer": "zeros"
},
"net_path": get_net_path("cw", path)
}
}
net_assignments = None
elif problem_name == "simple-multi":
problem = problems.simple_multi_optimizer()
net_config = {
"cw": {
"net": "CoordinateWiseDeepLSTM",
"net_options": {
"layers": (),
"initializer": "zeros"
},
"net_path": get_net_path("cw", path)
},
"adam": {
"net": "Adam",
"net_options": {
"learning_rate": 0.1
}
}
}
net_assignments = [("cw", ["x_0"]), ("adam", ["x_1"])]
elif problem_name == "quadratic":
if problem_path is not None:
npzfile = np.load(problem_path)
problems_w, problems_b = npzfile['arr_0'], npzfile['arr_1']
assert len(problems_w) == len(problems_b)
batch_size = len(problems_w)
problem = problems.quadratic(batch_size=batch_size,
num_dims=2,
problems_w=problems_w,
problems_b=problems_b)
else:
problem = problems.quadratic(batch_size=128, num_dims=2)
net_config = {
"cw": {
"net": "CoordinateWiseDeepLSTM",
"net_options": {
"layers": (20, 20)
},
"net_path": get_net_path("cw", path)
}
}
net_assignments = None
elif problem_name == "quadratic-wav":
if problem_path is not None:
npzfile = np.load(problem_path)
problems_w, problems_b = npzfile['arr_0'], npzfile['arr_1']
assert len(problems_w) == len(problems_b)
batch_size = len(problems_w)
problem = problems.quadratic(batch_size=batch_size,
num_dims=2,
problems_w=problems_w,
problems_b=problems_b)
else:
problem = problems.quadratic(batch_size=128, num_dims=2)
net_config = {
"cw-wav": {
"net": "CoordinateWiseWaveNet",
"net_options": {
"num_layers": 4
},
"net_path": get_net_path("cw-wav", path)
}
}
net_assignments = None
elif problem_name == "sin":
if problem_path is not None:
problems_sin = np.load(problem_path) # TODO
batch_size = len(problems_sin)
problem = problems.prob_sin(batch_size=batch_size,
problem_param=problems_sin)
else:
problem = problems.prob_sin(batch_size=128)
net_config = {
"cw": {
"net": "CoordinateWiseDeepLSTM",
"net_options": {
"layers": (20, 20)
},
"net_path": get_net_path("cw", path)
}
}
net_assignments = None
elif problem_name == "sin-wav":
if problem_path is not None:
problems_sin = np.load(problem_path) # TODO
batch_size = len(problems_sin)
problem = problems.prob_sin(batch_size=batch_size,
problem_param=problems_sin)
else:
problem = problems.prob_sin(batch_size=128)
net_config = {
"cw-wav": {
"net": "CoordinateWiseWaveNet",
"net_options": {
"num_layers": 4
},
"net_path": get_net_path("cw-wav", path)
}
}
net_assignments = None
elif problem_name == "mnist":
mode = "train" if path is None else "test"
problem = problems.mnist(layers=(20, ), mode=mode)
net_config = {"cw": get_default_net_config("cw", path)}
net_assignments = None
elif problem_name == "cifar":
mode = "train" if path is None else "test"
problem = problems.cifar10("cifar10",
conv_channels=(16, 16, 16),
linear_layers=(32, ),
mode=mode)
net_config = {"cw": get_default_net_config("cw", path)}
net_assignments = None
elif problem_name == "cifar-multi":
mode = "train" if path is None else "test"
problem = problems.cifar10("cifar10",
conv_channels=(16, 16, 16),
linear_layers=(32, ),
mode=mode)
net_config = {
"conv": get_default_net_config("conv", path),
"fc": get_default_net_config("fc", path)
}
conv_vars = ["conv_net_2d/conv_2d_{}/w".format(i) for i in xrange(3)]
fc_vars = ["conv_net_2d/conv_2d_{}/b".format(i) for i in xrange(3)]
fc_vars += [
"conv_net_2d/batch_norm_{}/beta".format(i) for i in xrange(3)
]
fc_vars += ["mlp/linear_{}/w".format(i) for i in xrange(2)]
fc_vars += ["mlp/linear_{}/b".format(i) for i in xrange(2)]
fc_vars += ["mlp/batch_norm/beta"]
net_assignments = [("conv", conv_vars), ("fc", fc_vars)]
else:
raise ValueError("{} is not a valid problem".format(problem_name))
return problem, net_config, net_assignments
def get_config(problem_name,
param=None,
path=None,
model_idx=0,
mode=None,
num_hidden_layer=None,
net_name=None,
num_linear_heads=1,
init="normal"):
"""Returns problem configuration."""
if problem_name == "simple":
problem = problems.simple()
net_config = {
"cw": {
"net": "CoordinateWiseDeepLSTM",
"net_options": {
"layers": (),
"initializer": "zeros"
},
"net_path": get_net_path("cw", path)
}
}
net_assignments = None
elif problem_name == "simple-multi":
problem = problems.simple_multi_optimizer()
net_config = {
"cw": {
"net": "CoordinateWiseDeepLSTM",
"net_options": {
"layers": (),
"initializer": "zeros"
},
"net_path": get_net_path("cw", path)
},
"adam": {
"net": "Adam",
"net_options": {
"learning_rate": 0.01
}
}
}
net_assignments = [("cw", ["x_0"]), ("adam", ["x_1"])]
elif problem_name == "quadratic":
problem = problems.quadratic(batch_size=128, num_dims=10)
net_config = {
"cw": {
"net": "CoordinateWiseDeepLSTM",
"net_options": {
"layers": (20, 20)
},
"net_path": get_net_path("cw", path)
}
}
net_assignments = None
elif problem_name == "lasso":
batch_size = param['bs']
num_dims_m = param['m']
num_dims_n = param['n']
problem = problems.lasso(batch_size=batch_size,
num_dims_m=num_dims_m,
num_dims_n=num_dims_n)
net_config = {
"cw": {
"net": "CoordinateWiseDeepLSTM",
"net_options": {
"layers": (20, 20)
},
"net_path": get_net_path("cw", path)
}
}
net_assignments = None
elif problem_name == "rastrigin":
batch_size = param['bs']
num_dims_n = param['n']
problem = problems.rastrigin(batch_size=batch_size,
num_dims=num_dims_n,
mode=mode)
net_config = {
"cw": {
"net": "CoordinateWiseDeepLSTM",
"net_options": {
"layers": (20, 20)
},
"net_path": get_net_path("cw", path)
}
}
net_assignments = None
elif problem_name == "mnist":
if mode is None:
mode = "train" if path is None else "test"
problem = problems.mnist(layers=(20, ),
activation="sigmoid",
mode=mode,
init="normal")
net_config = {"cw": get_default_net_config("cw", path, model_idx)}
net_assignments = None
elif problem_name == "mnist_relu":
mode = "test"
problem = problems.mnist(layers=(20, ), activation="relu", mode=mode)
net_config = {"cw": get_default_net_config("cw", path, model_idx)}
net_assignments = None
elif problem_name == "mnist_deeper":
mode = "test"
assert num_hidden_layer is not None
problem = problems.mnist(layers=(20, ) * num_hidden_layer,
activation="sigmoid",
mode=mode)
net_config = {"cw": get_default_net_config("cw", path, model_idx)}
net_assignments = None
elif problem_name == "mnist_conv":
mode = "test"
problem = problems.mnist_conv(mode=mode, batch_norm=True)
net_config = {"cw": get_default_net_config("cw", path, model_idx)}
net_assignments = None
elif problem_name == "cifar":
mode = "test"
problem = problems.cifar10("cifar10", mode=mode)
net_config = {"cw": get_default_net_config("cw", path, model_idx)}
net_assignments = None
elif problem_name == "vgg16":
mode = "train" if path is None else "test"
problem = problems.vgg16_cifar10("cifar10", mode=mode)
net_config = {"cw": get_default_net_config("cw", path)}
net_assignments = None
elif problem_name == "cifar-multi":
mode = "train" if path is None else "test"
problem = problems.cifar10("cifar10",
conv_channels=(16, 16, 16),
linear_layers=(32, ),
mode=mode)
net_config = {
"conv": get_default_net_config("conv", path),
"fc": get_default_net_config("fc", path)
}
conv_vars = ["conv_net_2d/conv_2d_{}/w".format(i) for i in xrange(3)]
fc_vars = ["conv_net_2d/conv_2d_{}/b".format(i) for i in xrange(3)]
fc_vars += [
"conv_net_2d/batch_norm_{}/beta".format(i) for i in xrange(3)
]
fc_vars += ["mlp/linear_{}/w".format(i) for i in xrange(2)]
fc_vars += ["mlp/linear_{}/b".format(i) for i in xrange(2)]
fc_vars += ["mlp/batch_norm/beta"]
net_assignments = [("conv", conv_vars), ("fc", fc_vars)]
else:
raise ValueError("{} is not a valid problem".format(problem_name))
if net_name == "RNNprop":
default_config = {
"net": "RNNprop",
"net_options": {
"layers": (20, 20),
"preprocess_name": "fc",
"preprocess_options": {
"dim": 20
},
"scale": 0.01,
"tanh_output": True,
"num_linear_heads": num_linear_heads
},
"net_path": get_net_path("rp", path, model_idx)
}
net_config = {"rp": default_config}
return problem, net_config, net_assignments
