def testValues(self, value): problem = problems.simple() f = problem() with self.test_session() as sess: output = sess.run(f, feed_dict={"x:0": value}) self.assertEqual(output, value**2)
def testSecondDerivatives(self): """Tests second derivatives for simple problem.""" problem = problems.simple() optimizer = meta.MetaOptimizer( net=dict(net="CoordinateWiseDeepLSTM", net_options={"layers": ()})) minimize_ops = optimizer.meta_minimize(problem, 3, second_derivatives=True) with self.test_session() as sess: sess.run(tf.global_variables_initializer()) train(sess, minimize_ops, 1, 2)
def testSimple(self): """Tests L2L applied to simple problem.""" problem = problems.simple() optimizer = meta.MetaOptimizer(net=dict( net="CoordinateWiseDeepLSTM", net_options={ "layers": (), # Initializing the network to zeros makes learning more stable. "initializer": "zeros" })) minimize_ops = optimizer.meta_minimize(problem, 20, learning_rate=1e-2) # L2L should solve the simple problem is less than 500 epochs. with self.test_session() as sess: sess.run(tf.initialize_all_variables()) cost, _ = train(sess, minimize_ops, 500, 5) self.assertLess(cost, 1e-5)
def testSaveAndLoad(self): """Tests saving and loading a meta-optimizer.""" layers = (2, 3) net_options = {"layers": layers, "initializer": "zeros"} num_unrolls = 2 num_epochs = 1 problem = problems.simple() # Original optimizer. with tf.Graph().as_default() as g1: optimizer = meta.MetaOptimizer(net=dict( net="CoordinateWiseDeepLSTM", net_options=net_options)) minimize_ops = optimizer.meta_minimize(problem, 3) with self.test_session(graph=g1) as sess: sess.run(tf.global_variables_initializer()) train(sess, minimize_ops, 1, 2) # Save optimizer. tmp_dir = tempfile.mkdtemp() save_result = optimizer.save(sess, path=tmp_dir) net_path = next(iter(save_result)) # Retrain original optimizer. cost, x = train(sess, minimize_ops, num_unrolls, num_epochs) # Load optimizer and retrain in a new session. with tf.Graph().as_default() as g2: optimizer = meta.MetaOptimizer( net=dict(net="CoordinateWiseDeepLSTM", net_options=net_options, net_path=net_path)) minimize_ops = optimizer.meta_minimize(problem, 3) with self.test_session(graph=g2) as sess: sess.run(tf.global_variables_initializer()) cost_loaded, x_loaded = train(sess, minimize_ops, num_unrolls, num_epochs) # The last cost should be the same. self.assertAlmostEqual(cost, cost_loaded, places=3) self.assertAlmostEqual(x[0], x_loaded[0], places=3) # Cleanup. os.remove(net_path) os.rmdir(tmp_dir)
def testResults(self): """Tests reproducibility of Torch results.""" problem = problems.simple() optimizer = meta.MetaOptimizer(net=dict(net="CoordinateWiseDeepLSTM", net_options={ "layers": (), "initializer": "zeros" })) minimize_ops = optimizer.meta_minimize(problem, 5) with self.test_session() as sess: sess.run(tf.global_variables_initializer()) cost, final_x = train(sess, minimize_ops, 1, 2) # Torch results torch_cost = 0.7325327 torch_final_x = 0.8559 self.assertAlmostEqual(cost, torch_cost, places=4) self.assertAlmostEqual(final_x[0], torch_final_x, places=4)
def get_config(problem_name, path=None): """Returns problem configuration.""" if problem_name == "simple": problem = problems.simple() net_config = { "cw": { "net": "CoordinateWiseDeepLSTM", "net_options": { "layers": (), "initializer": "zeros" }, "net_path": None } } net_assignments = None else: raise ValueError("{} is not a valid problem".format(problem_name)) return problem, net_config, net_assignments
def get_config(problem_name, path=None): """Returns problem configuration.""" if problem_name == "simple": problem = problems.simple() net_config = { "cw": { "net": "CoordinateWiseDeepLSTM", "net_options": { "layers": (), "initializer": "zeros" }, "net_path": get_net_path("cw", path) } } net_assignments = None elif problem_name == "simple-multi": problem = problems.simple_multi_optimizer() net_config = { "cw": { "net": "CoordinateWiseDeepLSTM", "net_options": { "layers": (), "initializer": "zeros" }, "net_path": get_net_path("cw", path) }, "adam": { "net": "Adam", "net_options": { "learning_rate": 0.1 } } } net_assignments = [("cw", ["x_0"]), ("adam", ["x_1"])] elif problem_name == "quadratic": problem = problems.quadratic(batch_size=128, num_dims=10) net_config = { "cw": { "net": "CoordinateWiseDeepLSTM", "net_options": { "layers": (20, 20) }, "net_path": get_net_path("cw", path) } } net_assignments = None elif problem_name == "mnist": # mode = "train" if path is None else "test" mode = "train" problem = problems.mnist(layers=(20, ), activation="sigmoid", mode=mode) # problem = problems.mnist(layers=(100,), activation="relu", mode=mode) # problem = problems.mnist(layers=(20,), activation="relu", mode=mode) net_config = {"cw": get_default_net_config("cw", path)} net_assignments = None elif problem_name == "cifar": mode = "train" if path is None else "test" problem = problems.cifar10("cifar10", conv_channels=(16, 16, 16), linear_layers=(32, ), mode=mode) net_config = {"cw": get_default_net_config("cw", path)} net_assignments = None elif problem_name == "cifar-multi": mode = "train" if path is None else "test" problem = problems.cifar10("cifar10", conv_channels=(16, 16, 16), linear_layers=(32, ), mode=mode) net_config = { "conv": get_default_net_config("conv", path), "fc": get_default_net_config("fc", path) } conv_vars = ["conv_net_2d/conv_2d_{}/w".format(i) for i in xrange(3)] fc_vars = ["conv_net_2d/conv_2d_{}/b".format(i) for i in xrange(3)] fc_vars += [ "conv_net_2d/batch_norm_{}/beta".format(i) for i in xrange(3) ] fc_vars += ["mlp/linear_{}/w".format(i) for i in xrange(2)] fc_vars += ["mlp/linear_{}/b".format(i) for i in xrange(2)] fc_vars += ["mlp/batch_norm/beta"] net_assignments = [("conv", conv_vars), ("fc", fc_vars)] else: raise ValueError("{} is not a valid problem".format(problem_name)) return problem, net_config, net_assignments
def get_config(problem_name, path=None, mode=None, num_hidden_layer=None, net_name=None): """Returns problem configuration.""" if problem_name == "simple": problem = problems.simple() net_config = { "cw": { "net": "CoordinateWiseDeepLSTM", "net_options": { "layers": (), "initializer": "zeros" }, "net_path": path } } net_assignments = None elif problem_name == "simple-multi": problem = problems.simple_multi_optimizer() net_config = { "cw": { "net": "CoordinateWiseDeepLSTM", "net_options": { "layers": (), "initializer": "zeros" }, "net_path": path }, "adam": { "net": "Adam", "net_options": { "learning_rate": 0.01 } } } net_assignments = [("cw", ["x_0"]), ("adam", ["x_1"])] elif problem_name == "quadratic": problem = problems.quadratic(batch_size=128, num_dims=10) net_config = { "cw": { "net": "CoordinateWiseDeepLSTM", "net_options": { "layers": (20, 20) }, "net_path": path } } net_assignments = None ### our tests elif problem_name == "mnist": if mode is None: mode = "train" if path is None else "test" problem = problems.mnist(layers=(20, ), activation="sigmoid", mode=mode) net_config = {"cw": get_default_net_config(path)} net_assignments = None elif problem_name == "mnist_relu": if mode is None: mode = "train" if path is None else "test" problem = problems.mnist(layers=(20, ), activation="relu", mode=mode) net_config = {"cw": get_default_net_config(path)} net_assignments = None elif problem_name == "mnist_deeper": if mode is None: mode = "train" if path is None else "test" num_hidden_layer = 2 problem = problems.mnist(layers=(20, ) * num_hidden_layer, activation="sigmoid", mode=mode) net_config = {"cw": get_default_net_config(path)} net_assignments = None elif problem_name == "mnist_conv": if mode is None: mode = "train" if path is None else "test" problem = problems.mnist_conv(mode=mode, batch_norm=True) net_config = {"cw": get_default_net_config(path)} net_assignments = None elif problem_name == "cifar_conv": if mode is None: mode = "train" if path is None else "test" problem = problems.cifar10("cifar10", mode=mode) net_config = {"cw": get_default_net_config(path)} net_assignments = None elif problem_name == "lenet": if mode is None: mode = "train" if path is None else "test" problem = problems.LeNet("cifar10", conv_channels=(6, 16), linear_layers=(120, 84), mode=mode) net_config = {"cw": get_default_net_config(path)} net_assignments = None elif problem_name == "nas": if mode is None: mode = "train" if path is None else "test" problem = problems.NAS("cifar10", mode=mode) net_config = {"cw": get_default_net_config(path)} net_assignments = None ### elif problem_name == "vgg16": mode = "train" if path is None else "test" problem = problems.vgg16_cifar10("cifar10", mode=mode) net_config = {"cw": get_default_net_config(path)} net_assignments = None elif problem_name == "cifar-multi": mode = "train" if path is None else "test" problem = problems.cifar10("cifar10", conv_channels=(16, 16, 16), linear_layers=(32, ), mode=mode) net_config = { "conv": get_default_net_config(path), "fc": get_default_net_config(path) } conv_vars = ["conv_net_2d/conv_2d_{}/w".format(i) for i in xrange(3)] fc_vars = ["conv_net_2d/conv_2d_{}/b".format(i) for i in xrange(3)] fc_vars += [ "conv_net_2d/batch_norm_{}/beta".format(i) for i in xrange(3) ] fc_vars += ["mlp/linear_{}/w".format(i) for i in xrange(2)] fc_vars += ["mlp/linear_{}/b".format(i) for i in xrange(2)] fc_vars += ["mlp/batch_norm/beta"] net_assignments = [("conv", conv_vars), ("fc", fc_vars)] elif problem_name == "confocal_microscopy_3d": problem = problems.confocal_microscopy_3d(batch_size=32, num_points=5) net_config = { "cw": { "net": "CoordinateWiseDeepLSTM", "net_options": { "layers": (20, 20) }, "net_path": path } } net_assignments = None elif problem_name == "square_cos": problem = problems.square_cos(batch_size=128, num_dims=2) net_config = { "cw": { "net": "CoordinateWiseDeepLSTM", "net_options": { "layers": (20, 20) }, "net_path": path } } net_assignments = None else: raise ValueError("{} is not a valid problem".format(problem_name)) if net_name == "RNNprop": default_config = { "net": "RNNprop", "net_options": { "layers": (20, 20), "preprocess_name": "fc", "preprocess_options": { "dim": 20 }, "scale": 0.01, "tanh_output": True }, "net_path": path } net_config = {"rp": default_config} return problem, net_config, net_assignments
def testVariables(self): problem = problems.simple() problem() variables = tf.trainable_variables() self.assertEqual(len(variables), 1) self.assertEqual(variables[0].get_shape().as_list(), [])
def testShape(self): problem = problems.simple() f = problem() self.assertEqual(f.get_shape().as_list(), [])
def get_config(problem_name, path=None, problem_path=None): """Returns problem configuration.""" if problem_name == "simple": problem = problems.simple() net_config = { "cw": { "net": "CoordinateWiseDeepLSTM", "net_options": { "layers": (), "initializer": "zeros" }, "net_path": get_net_path("cw", path) } } net_assignments = None elif problem_name == "simple-multi": problem = problems.simple_multi_optimizer() net_config = { "cw": { "net": "CoordinateWiseDeepLSTM", "net_options": { "layers": (), "initializer": "zeros" }, "net_path": get_net_path("cw", path) }, "adam": { "net": "Adam", "net_options": { "learning_rate": 0.1 } } } net_assignments = [("cw", ["x_0"]), ("adam", ["x_1"])] elif problem_name == "quadratic": if problem_path is not None: npzfile = np.load(problem_path) problems_w, problems_b = npzfile['arr_0'], npzfile['arr_1'] assert len(problems_w) == len(problems_b) batch_size = len(problems_w) problem = problems.quadratic(batch_size=batch_size, num_dims=2, problems_w=problems_w, problems_b=problems_b) else: problem = problems.quadratic(batch_size=128, num_dims=2) net_config = { "cw": { "net": "CoordinateWiseDeepLSTM", "net_options": { "layers": (20, 20) }, "net_path": get_net_path("cw", path) } } net_assignments = None elif problem_name == "quadratic-wav": if problem_path is not None: npzfile = np.load(problem_path) problems_w, problems_b = npzfile['arr_0'], npzfile['arr_1'] assert len(problems_w) == len(problems_b) batch_size = len(problems_w) problem = problems.quadratic(batch_size=batch_size, num_dims=2, problems_w=problems_w, problems_b=problems_b) else: problem = problems.quadratic(batch_size=128, num_dims=2) net_config = { "cw-wav": { "net": "CoordinateWiseWaveNet", "net_options": { "num_layers": 4 }, "net_path": get_net_path("cw-wav", path) } } net_assignments = None elif problem_name == "sin": if problem_path is not None: problems_sin = np.load(problem_path) # TODO batch_size = len(problems_sin) problem = problems.prob_sin(batch_size=batch_size, problem_param=problems_sin) else: problem = problems.prob_sin(batch_size=128) net_config = { "cw": { "net": "CoordinateWiseDeepLSTM", "net_options": { "layers": (20, 20) }, "net_path": get_net_path("cw", path) } } net_assignments = None elif problem_name == "sin-wav": if problem_path is not None: problems_sin = np.load(problem_path) # TODO batch_size = len(problems_sin) problem = problems.prob_sin(batch_size=batch_size, problem_param=problems_sin) else: problem = problems.prob_sin(batch_size=128) net_config = { "cw-wav": { "net": "CoordinateWiseWaveNet", "net_options": { "num_layers": 4 }, "net_path": get_net_path("cw-wav", path) } } net_assignments = None elif problem_name == "mnist": mode = "train" if path is None else "test" problem = problems.mnist(layers=(20, ), mode=mode) net_config = {"cw": get_default_net_config("cw", path)} net_assignments = None elif problem_name == "cifar": mode = "train" if path is None else "test" problem = problems.cifar10("cifar10", conv_channels=(16, 16, 16), linear_layers=(32, ), mode=mode) net_config = {"cw": get_default_net_config("cw", path)} net_assignments = None elif problem_name == "cifar-multi": mode = "train" if path is None else "test" problem = problems.cifar10("cifar10", conv_channels=(16, 16, 16), linear_layers=(32, ), mode=mode) net_config = { "conv": get_default_net_config("conv", path), "fc": get_default_net_config("fc", path) } conv_vars = ["conv_net_2d/conv_2d_{}/w".format(i) for i in xrange(3)] fc_vars = ["conv_net_2d/conv_2d_{}/b".format(i) for i in xrange(3)] fc_vars += [ "conv_net_2d/batch_norm_{}/beta".format(i) for i in xrange(3) ] fc_vars += ["mlp/linear_{}/w".format(i) for i in xrange(2)] fc_vars += ["mlp/linear_{}/b".format(i) for i in xrange(2)] fc_vars += ["mlp/batch_norm/beta"] net_assignments = [("conv", conv_vars), ("fc", fc_vars)] else: raise ValueError("{} is not a valid problem".format(problem_name)) return problem, net_config, net_assignments
def get_config(problem_name, param=None, path=None, model_idx=0, mode=None, num_hidden_layer=None, net_name=None, num_linear_heads=1, init="normal"): """Returns problem configuration.""" if problem_name == "simple": problem = problems.simple() net_config = { "cw": { "net": "CoordinateWiseDeepLSTM", "net_options": { "layers": (), "initializer": "zeros" }, "net_path": get_net_path("cw", path) } } net_assignments = None elif problem_name == "simple-multi": problem = problems.simple_multi_optimizer() net_config = { "cw": { "net": "CoordinateWiseDeepLSTM", "net_options": { "layers": (), "initializer": "zeros" }, "net_path": get_net_path("cw", path) }, "adam": { "net": "Adam", "net_options": { "learning_rate": 0.01 } } } net_assignments = [("cw", ["x_0"]), ("adam", ["x_1"])] elif problem_name == "quadratic": problem = problems.quadratic(batch_size=128, num_dims=10) net_config = { "cw": { "net": "CoordinateWiseDeepLSTM", "net_options": { "layers": (20, 20) }, "net_path": get_net_path("cw", path) } } net_assignments = None elif problem_name == "lasso": batch_size = param['bs'] num_dims_m = param['m'] num_dims_n = param['n'] problem = problems.lasso(batch_size=batch_size, num_dims_m=num_dims_m, num_dims_n=num_dims_n) net_config = { "cw": { "net": "CoordinateWiseDeepLSTM", "net_options": { "layers": (20, 20) }, "net_path": get_net_path("cw", path) } } net_assignments = None elif problem_name == "rastrigin": batch_size = param['bs'] num_dims_n = param['n'] problem = problems.rastrigin(batch_size=batch_size, num_dims=num_dims_n, mode=mode) net_config = { "cw": { "net": "CoordinateWiseDeepLSTM", "net_options": { "layers": (20, 20) }, "net_path": get_net_path("cw", path) } } net_assignments = None elif problem_name == "mnist": if mode is None: mode = "train" if path is None else "test" problem = problems.mnist(layers=(20, ), activation="sigmoid", mode=mode, init="normal") net_config = {"cw": get_default_net_config("cw", path, model_idx)} net_assignments = None elif problem_name == "mnist_relu": mode = "test" problem = problems.mnist(layers=(20, ), activation="relu", mode=mode) net_config = {"cw": get_default_net_config("cw", path, model_idx)} net_assignments = None elif problem_name == "mnist_deeper": mode = "test" assert num_hidden_layer is not None problem = problems.mnist(layers=(20, ) * num_hidden_layer, activation="sigmoid", mode=mode) net_config = {"cw": get_default_net_config("cw", path, model_idx)} net_assignments = None elif problem_name == "mnist_conv": mode = "test" problem = problems.mnist_conv(mode=mode, batch_norm=True) net_config = {"cw": get_default_net_config("cw", path, model_idx)} net_assignments = None elif problem_name == "cifar": mode = "test" problem = problems.cifar10("cifar10", mode=mode) net_config = {"cw": get_default_net_config("cw", path, model_idx)} net_assignments = None elif problem_name == "vgg16": mode = "train" if path is None else "test" problem = problems.vgg16_cifar10("cifar10", mode=mode) net_config = {"cw": get_default_net_config("cw", path)} net_assignments = None elif problem_name == "cifar-multi": mode = "train" if path is None else "test" problem = problems.cifar10("cifar10", conv_channels=(16, 16, 16), linear_layers=(32, ), mode=mode) net_config = { "conv": get_default_net_config("conv", path), "fc": get_default_net_config("fc", path) } conv_vars = ["conv_net_2d/conv_2d_{}/w".format(i) for i in xrange(3)] fc_vars = ["conv_net_2d/conv_2d_{}/b".format(i) for i in xrange(3)] fc_vars += [ "conv_net_2d/batch_norm_{}/beta".format(i) for i in xrange(3) ] fc_vars += ["mlp/linear_{}/w".format(i) for i in xrange(2)] fc_vars += ["mlp/linear_{}/b".format(i) for i in xrange(2)] fc_vars += ["mlp/batch_norm/beta"] net_assignments = [("conv", conv_vars), ("fc", fc_vars)] else: raise ValueError("{} is not a valid problem".format(problem_name)) if net_name == "RNNprop": default_config = { "net": "RNNprop", "net_options": { "layers": (20, 20), "preprocess_name": "fc", "preprocess_options": { "dim": 20 }, "scale": 0.01, "tanh_output": True, "num_linear_heads": num_linear_heads }, "net_path": get_net_path("rp", path, model_idx) } net_config = {"rp": default_config} return problem, net_config, net_assignments