def test_create_model(self):
flags = get_default_flags()
topology = Topology()
model = CrocuBotModel(topology, flags, is_training=True)
n_connections_between_layers = DEFAULT_HIDDEN_LAYERS + 1
self.assertEqual(model.number_of_layers, n_connections_between_layers)
self.assertEqual(model.topology, topology)
with self.test_session() as session:
for layer_number in range(model.number_of_layers):
for variable_name in model.layer_variables_list:
self.assertRaises(ValueError, model.get_variable,
layer_number, variable_name)
with self.test_session() as session:
model.build_layers_variables()
session.run(tf.global_variables_initializer())
self.assertEqual(
len(session.run(tf.report_uninitialized_variables())), 0)
for layer_number in range(model.number_of_layers):
for variable_name in model.layer_variables_list:
variable = model.get_variable(layer_number, variable_name)
self.assertIsInstance(variable, tf.Variable)
self.assertIsNotNone(variable.eval())
def run_timed_benchmark_mnist(series_name, do_training):
topology = load_default_topology(series_name)
batch_options = BatchOptions(batch_size=200,
batch_number=0,
train=do_training,
dtype=D_TYPE)
data_source = data_source_generator.make_data_source(series_name)
_, labels = io.load_batch(batch_options, data_source)
start_time = timer()
execution_time = datetime.datetime.now()
if do_training:
crocubot_train.train(topology, series_name, execution_time)
else:
tf.reset_default_graph()
model = CrocuBotModel(topology)
model.build_layers_variables()
mid_time = timer()
train_time = mid_time - start_time
print("Training complete.")
metrics = evaluate_network(topology, series_name, bin_dist=None)
eval_time = timer() - mid_time
print('Metrics:')
accuracy = print_MNIST_accuracy(metrics)
print_time_info(train_time, eval_time)
return accuracy, metrics
def setUp(self):
self._flags = get_default_flags()
tf.reset_default_graph()
topology = Topology()
self.crocubot_model = CrocuBotModel(topology,
self._flags,
is_training=True)
def run_timed_benchmark_time_series(series_name, flags, do_training=True):
topology = load_default_topology(series_name)
# First need to establish bin edges using full training set
template_sample_size = np.minimum(flags.n_training_samples_benchmark,
10000)
batch_options = BatchOptions(batch_size=template_sample_size,
batch_number=0,
train=do_training,
dtype=D_TYPE)
data_source = data_source_generator.make_data_source(series_name)
_, labels = io.load_batch(batch_options, data_source)
bin_dist = BinDistribution(labels, topology.n_classification_bins)
start_time = timer()
execution_time = datetime.datetime.now()
if do_training:
crocubot_train.train(topology,
series_name,
execution_time,
bin_edges=bin_dist.bin_edges)
else:
tf.reset_default_graph()
model = CrocuBotModel(topology)
model.build_layers_variables()
mid_time = timer()
train_time = mid_time - start_time
print("Training complete.")
evaluate_network(topology, series_name, bin_dist)
eval_time = timer() - mid_time
print('Metrics:')
print_time_info(train_time, eval_time)
def _do_training():
execution_time = datetime.datetime.now()
if do_training:
data_provider = TrainDataProviderForDataSource(
series_name, D_TYPE, n_train_samples, batch_size, True,
bin_distribution.bin_edges)
train_x = data_provider.get_batch(0)
raw_train_data = TrainDataProvider(train_x, train_y,
tf_flags.batch_size)
tensorflow_path = TensorflowPath(save_path,
tf_flags.model_save_path)
tensorboard_options = TensorboardOptions(
tf_flags.tensorboard_log_path, tf_flags.learning_rate,
batch_size, execution_time)
crocubot_train.train(topology, data_provider, tensorflow_path,
tensorboard_options, tf_flags)
else:
tf.reset_default_graph()
model = CrocuBotModel(topology)
model.build_layers_variables()
class TestEstimator(tf.test.TestCase):
def setUp(self):
self._flags = get_default_flags()
tf.reset_default_graph()
topology = Topology()
self.crocubot_model = CrocuBotModel(topology,
self._flags,
is_training=True)
def test_forward_pass(self):
estimator = Estimator(self.crocubot_model, self._flags)
self.crocubot_model.build_layers_variables()
data = np.ones(shape=(DEFAULT_BATCH_SIZE, DEFAULT_DEPTH,
DEFAULT_N_SERIES, DEFAULT_TIMESTEPS),
dtype=np.float32)
output_signal = estimator.forward_pass(data)
with self.test_session() as session:
session.run(tf.global_variables_initializer())
self.assertIsInstance(output_signal, tf.Tensor)
value = output_signal.eval()
self.assertEqual((DEFAULT_BATCH_SIZE, DEFAULT_DEPTH,
DEFAULT_N_FORECASTS, DEFAULT_BINS), value.shape)
def test_collate_multiple_passes(self):
estimator = Estimator(self.crocubot_model, self._flags)
self.crocubot_model.build_layers_variables()
data = np.ones(shape=(DEFAULT_BATCH_SIZE, DEFAULT_DEPTH,
DEFAULT_N_SERIES, DEFAULT_TIMESTEPS),
dtype=np.float32)
number_of_passes = 3
stacked_output = estimator.collate_multiple_passes(
data, number_of_passes)
with self.test_session() as session:
session.run(tf.global_variables_initializer())
self.assertIsInstance(stacked_output, tf.Tensor)
value = stacked_output.eval()
expected_shape = (number_of_passes, DEFAULT_BATCH_SIZE,
DEFAULT_DEPTH, DEFAULT_N_FORECASTS, DEFAULT_BINS)
self.assertEqual(expected_shape, value.shape)
def test_average_multiple_passes(self):
estimator = Estimator(self.crocubot_model, self._flags)
self.crocubot_model.build_layers_variables()
data = np.ones(shape=(DEFAULT_BATCH_SIZE, DEFAULT_DEPTH,
DEFAULT_N_SERIES, DEFAULT_TIMESTEPS),
dtype=np.float32)
number_of_passes = 3
mean = estimator.average_multiple_passes(data, number_of_passes)
with self.test_session() as session:
session.run(tf.global_variables_initializer())
self.assertIsInstance(mean, tf.Tensor)
mean_value = mean.eval()
expected_shape = (DEFAULT_BATCH_SIZE, DEFAULT_DEPTH,
DEFAULT_N_FORECASTS, DEFAULT_BINS)
self.assertEqual(expected_shape, mean_value.shape)
def test_init(self):
flags = get_default_flags()
layer_number = [
{"activation_func": "relu", "trainable": False, "height": 20, "width": 10, "cell_height": 1},
{"activation_func": "relu", "trainable": False, "height": 20, "width": 10, "cell_height": 1},
{"activation_func": "linear", "trainable": False, "height": 20, "width": 10, "cell_height": 1}
]
topology = Topology(layer_number)
self.model = CrocuBotModel(topology, flags, is_training=True)
# case1 no error thrown
use_double_gaussian_weights_prior = True
slab_std_dvn = 1.2
spike_std_dvn = 0.05
spike_slab_weighting = 0.5
BayesianCost(self.model, use_double_gaussian_weights_prior, slab_std_dvn, spike_std_dvn, spike_slab_weighting)
# case2 slab_std_dvn < 0
slab_std_dvn = -1.
self.assertRaises(ValueError, BayesianCost, model=self.model,
use_double_gaussian_weights_prior=use_double_gaussian_weights_prior,
slab_std_dvn=slab_std_dvn, spike_std_dvn=spike_std_dvn,
spike_slab_weighting=spike_slab_weighting)
# case3 slab_std_dvn > 100
slab_std_dvn = 101.
self.assertRaises(ValueError, BayesianCost, model=self.model,
use_double_gaussian_weights_prior=use_double_gaussian_weights_prior,
slab_std_dvn=slab_std_dvn, spike_std_dvn=spike_std_dvn,
spike_slab_weighting=spike_slab_weighting)
# case4 spike_std_dvn < 0
spike_std_dvn = -1.
self.assertRaises(ValueError, BayesianCost, model=self.model,
use_double_gaussian_weights_prior=use_double_gaussian_weights_prior,
slab_std_dvn=slab_std_dvn, spike_std_dvn=spike_std_dvn,
spike_slab_weighting=spike_slab_weighting)
# case5 spike_std_dvn > 100
spike_std_dvn = 101.
self.assertRaises(ValueError, BayesianCost, model=self.model,
use_double_gaussian_weights_prior=use_double_gaussian_weights_prior,
slab_std_dvn=slab_std_dvn, spike_std_dvn=spike_std_dvn,
spike_slab_weighting=spike_slab_weighting)
# case6 spike_std_dvn > slab_std_dvn
spike_std_dvn = 5.
slab_std_dvn = 1.
self.assertRaises(ValueError, BayesianCost, model=self.model,
use_double_gaussian_weights_prior=use_double_gaussian_weights_prior,
slab_std_dvn=slab_std_dvn, spike_std_dvn=spike_std_dvn,
spike_slab_weighting=spike_slab_weighting)
# case7 spike_slab_weighting < 0
spike_slab_weighting = -1.
self.assertRaises(ValueError, BayesianCost, model=self.model,
use_double_gaussian_weights_prior=use_double_gaussian_weights_prior,
slab_std_dvn=slab_std_dvn, spike_std_dvn=spike_std_dvn,
spike_slab_weighting=spike_slab_weighting)
# case8 spike_slab_weighting > 1.
spike_slab_weighting = 2.
self.assertRaises(ValueError, BayesianCost, model=self.model,
use_double_gaussian_weights_prior=use_double_gaussian_weights_prior,
slab_std_dvn=slab_std_dvn, spike_std_dvn=spike_std_dvn,
spike_slab_weighting=spike_slab_weighting)
class TestBayesianCost(tf.test.TestCase):
def test_calc_log_q_prior(self):
parameters = [
(1., 0., 1., -1.4813652),
(2., -1., -1., -45.614418)
]
parameters = np.array(parameters, dtype=np.float32)
with self.test_session():
for test in parameters:
theta, mu, rho, expected_result = test
actual_result = BayesianCost.calculate_log_q_prior(theta, mu, rho).eval()
self.assertAlmostEqual(
actual_result,
expected_result,
places=4
)
def test_init(self):
flags = get_default_flags()
layer_number = [
{"activation_func": "relu", "trainable": False, "height": 20, "width": 10, "cell_height": 1},
{"activation_func": "relu", "trainable": False, "height": 20, "width": 10, "cell_height": 1},
{"activation_func": "linear", "trainable": False, "height": 20, "width": 10, "cell_height": 1}
]
topology = Topology(layer_number)
self.model = CrocuBotModel(topology, flags, is_training=True)
# case1 no error thrown
use_double_gaussian_weights_prior = True
slab_std_dvn = 1.2
spike_std_dvn = 0.05
spike_slab_weighting = 0.5
BayesianCost(self.model, use_double_gaussian_weights_prior, slab_std_dvn, spike_std_dvn, spike_slab_weighting)
# case2 slab_std_dvn < 0
slab_std_dvn = -1.
self.assertRaises(ValueError, BayesianCost, model=self.model,
use_double_gaussian_weights_prior=use_double_gaussian_weights_prior,
slab_std_dvn=slab_std_dvn, spike_std_dvn=spike_std_dvn,
spike_slab_weighting=spike_slab_weighting)
# case3 slab_std_dvn > 100
slab_std_dvn = 101.
self.assertRaises(ValueError, BayesianCost, model=self.model,
use_double_gaussian_weights_prior=use_double_gaussian_weights_prior,
slab_std_dvn=slab_std_dvn, spike_std_dvn=spike_std_dvn,
spike_slab_weighting=spike_slab_weighting)
# case4 spike_std_dvn < 0
spike_std_dvn = -1.
self.assertRaises(ValueError, BayesianCost, model=self.model,
use_double_gaussian_weights_prior=use_double_gaussian_weights_prior,
slab_std_dvn=slab_std_dvn, spike_std_dvn=spike_std_dvn,
spike_slab_weighting=spike_slab_weighting)
# case5 spike_std_dvn > 100
spike_std_dvn = 101.
self.assertRaises(ValueError, BayesianCost, model=self.model,
use_double_gaussian_weights_prior=use_double_gaussian_weights_prior,
slab_std_dvn=slab_std_dvn, spike_std_dvn=spike_std_dvn,
spike_slab_weighting=spike_slab_weighting)
# case6 spike_std_dvn > slab_std_dvn
spike_std_dvn = 5.
slab_std_dvn = 1.
self.assertRaises(ValueError, BayesianCost, model=self.model,
use_double_gaussian_weights_prior=use_double_gaussian_weights_prior,
slab_std_dvn=slab_std_dvn, spike_std_dvn=spike_std_dvn,
spike_slab_weighting=spike_slab_weighting)
# case7 spike_slab_weighting < 0
spike_slab_weighting = -1.
self.assertRaises(ValueError, BayesianCost, model=self.model,
use_double_gaussian_weights_prior=use_double_gaussian_weights_prior,
slab_std_dvn=slab_std_dvn, spike_std_dvn=spike_std_dvn,
spike_slab_weighting=spike_slab_weighting)
# case8 spike_slab_weighting > 1.
spike_slab_weighting = 2.
self.assertRaises(ValueError, BayesianCost, model=self.model,
use_double_gaussian_weights_prior=use_double_gaussian_weights_prior,
slab_std_dvn=slab_std_dvn, spike_std_dvn=spike_std_dvn,
spike_slab_weighting=spike_slab_weighting)
def test_calculate_log_weight_prior(self):
self.test_init()
with self.test_session():
layer = 0
layer_name = str(layer)
alpha_value = 0.2
log_alpha_value = np.log(alpha_value).astype(np.float32)
init_log_alpha = tf.constant_initializer(log_alpha_value)
with tf.variable_scope(layer_name):
log_alpha = tf.get_variable('log_alpha', shape=(), initializer=init_log_alpha)
log_alpha.initializer.run()
self.assertEqual(log_alpha.eval(), log_alpha_value)
log_alpha_retrieved = self.model.get_variable(0, self.model.VAR_LOG_ALPHA)
# TODO this a now a test get_layer_variable()
self.assertEqual(log_alpha_retrieved.eval(), log_alpha_value)
# case 1 slab prior
use_double_gaussian_weights_prior = True
slab_std_dvn = 1.
spike_std_dvn = 0.05
spike_slab_weighting = 1.
bayes_cost = BayesianCost(self.model, use_double_gaussian_weights_prior, slab_std_dvn, spike_std_dvn,
spike_slab_weighting)
weights = np.random.normal(size=5)
weights = weights.astype(np.float32)
log_prior_value_computed = bayes_cost.calculate_log_weight_prior(weights, layer)
log_prior_value_expected = np.sum(norm.logpdf(weights))
self.assertAlmostEqual(log_prior_value_computed.eval(), log_prior_value_expected, places=5)
# case 2 spike prior
slab_std_dvn = 2. # note that we have condition that slab_std_dvn >= spike_std_dvn
spike_std_dvn = 1.
spike_slab_weighting = 0.
bayes_cost = BayesianCost(self.model, use_double_gaussian_weights_prior, slab_std_dvn, spike_std_dvn,
spike_slab_weighting)
log_prior_value_computed = bayes_cost.calculate_log_weight_prior(weights, layer)
self.assertAlmostEqual(log_prior_value_computed.eval(), log_prior_value_expected, places=5)
# case 3 50/50 spike/slab
slab_std_dvn = 2.
spike_std_dvn = 1.
spike_slab_weighting = 0.5
bayes_cost = BayesianCost(self.model, use_double_gaussian_weights_prior, slab_std_dvn, spike_std_dvn,
spike_slab_weighting)
log_prior_value_computed = bayes_cost.calculate_log_weight_prior(weights, layer)
log_prior_value_expected = np.sum(np.log(spike_slab_weighting * norm.pdf(weights / slab_std_dvn) /
slab_std_dvn + (1. - spike_slab_weighting) *
norm.pdf(weights / spike_std_dvn) / spike_std_dvn))
self.assertAlmostEqual(log_prior_value_computed.eval(), log_prior_value_expected, places=5)
# case 4 no double Gaussian prior
use_double_gaussian_weights_prior = False
bayes_cost = BayesianCost(self.model, use_double_gaussian_weights_prior, slab_std_dvn, spike_std_dvn,
spike_slab_weighting)
log_prior_value_computed = bayes_cost.calculate_log_weight_prior(weights, layer)
log_prior_value_expected = np.sum(norm.logpdf(weights, scale=alpha_value))
self.assertAlmostEqual(log_prior_value_computed.eval(), log_prior_value_expected, places=5)
def test_calculate_log_bias_prior(self):
# FIXME the only difference between this and the previous one is the function
# FIXME calculate_log_weight_prior() changes to calculate_log_weight_prior().
# FIXME Otherwise, they are identical.
self.test_init()
with self.test_session():
layer = 0
layer_name = str(layer)
alpha_value = 0.2
log_alpha_value = np.log(alpha_value).astype(np.float32)
init_log_alpha = tf.constant_initializer(log_alpha_value)
with tf.variable_scope(layer_name):
log_alpha = tf.get_variable('log_alpha', shape=(), initializer=init_log_alpha)
log_alpha.initializer.run()
self.assertEqual(log_alpha.eval(), log_alpha_value)
log_alpha_retrieved = self.model.get_variable(0, self.model.VAR_LOG_ALPHA)
# TODO this a now a test get_layer_variable()
self.assertEqual(log_alpha_retrieved.eval(), log_alpha_value)
# case 1 slab prior
use_double_gaussian_weights_prior = True
slab_std_dvn = 1.
spike_std_dvn = 0.05
spike_slab_weighting = 1.
bayes_cost = BayesianCost(self.model, use_double_gaussian_weights_prior, slab_std_dvn, spike_std_dvn,
spike_slab_weighting)
weights = np.random.normal(size=5)
weights = weights.astype(np.float32)
log_prior_value_computed = bayes_cost.calculate_log_bias_prior(weights, layer)
log_prior_value_expected = np.sum(norm.logpdf(weights))
self.assertAlmostEqual(log_prior_value_computed.eval(), log_prior_value_expected, places=5)
# case 2 spike prior
slab_std_dvn = 2. # note that we have condition that slab_std_dvn >= spike_std_dvn
spike_std_dvn = 1.
spike_slab_weighting = 0.
bayes_cost = BayesianCost(self.model, use_double_gaussian_weights_prior, slab_std_dvn, spike_std_dvn,
spike_slab_weighting)
log_prior_value_computed = bayes_cost.calculate_log_weight_prior(weights, layer)
self.assertAlmostEqual(log_prior_value_computed.eval(), log_prior_value_expected, places=5)
# case 3 50/50 spike/slab
slab_std_dvn = 2.
spike_std_dvn = 1.
spike_slab_weighting = 0.5
bayes_cost = BayesianCost(self.model, use_double_gaussian_weights_prior, slab_std_dvn, spike_std_dvn,
spike_slab_weighting)
log_prior_value_computed = bayes_cost.calculate_log_bias_prior(weights, layer)
log_prior_value_expected = np.sum(np.log(spike_slab_weighting * norm.pdf(weights / slab_std_dvn) /
slab_std_dvn + (1. - spike_slab_weighting) *
norm.pdf(weights / spike_std_dvn) / spike_std_dvn))
self.assertAlmostEqual(log_prior_value_computed.eval(), log_prior_value_expected, places=5)
# case 4 no double Gaussian prior
use_double_gaussian_weights_prior = False
bayes_cost = BayesianCost(self.model, use_double_gaussian_weights_prior, slab_std_dvn, spike_std_dvn,
spike_slab_weighting)
log_prior_value_computed = bayes_cost.calculate_log_bias_prior(weights, layer)
log_prior_value_expected = np.sum(norm.logpdf(weights, scale=alpha_value))
self.assertAlmostEqual(log_prior_value_computed.eval(), log_prior_value_expected, places=5)
def test_calculate_log_hyperprior(self):
self.test_init()
with self.test_session():
layer = 0
layer_name = str(layer)
alpha_value = 0.2
log_alpha_value = np.log(alpha_value).astype(np.float32)
init_log_alpha = tf.constant_initializer(log_alpha_value)
with tf.variable_scope(layer_name):
log_alpha = tf.get_variable(self.model.VAR_LOG_ALPHA, shape=(), initializer=init_log_alpha)
log_alpha.initializer.run()
self.assertEqual(log_alpha.eval(), log_alpha_value)
log_alpha_retrieved = self.model.get_variable(0, self.model.VAR_LOG_ALPHA)
# TODO this a now a test get_variable()
self.assertEqual(log_alpha_retrieved.eval(), log_alpha_value)
# case 1 test the hyper prior
use_double_gaussian_weights_prior = True
slab_std_dvn = 1.
spike_std_dvn = 0.05
spike_slab_weighting = 1.
bayes_cost = BayesianCost(self.model, use_double_gaussian_weights_prior, slab_std_dvn, spike_std_dvn,
spike_slab_weighting)
log_prior_value_computed = bayes_cost.calculate_log_hyperprior(layer)
log_prior_value_expected = - log_alpha_value
self.assertAlmostEqual(log_prior_value_computed.eval(), log_prior_value_expected, places=5)