def main(argv=None):
# custom parse of flags for list input
compression_flags.custom_parse_flags(FLAGS)
# set random seeds
np.random.seed(FLAGS.random_seed)
tf.set_random_seed(FLAGS.random_seed)
# load dataset
input_filepath = FLAGS.dataset_filepath
data = dataset_loaders.risk_dataset_loader(
input_filepath,
shuffle=True,
train_split=.9,
debug_size=FLAGS.debug_size,
timesteps=FLAGS.timesteps,
num_target_bins=FLAGS.num_target_bins,
balanced_class_loss=FLAGS.balanced_class_loss,
target_index=FLAGS.target_index)
if FLAGS.use_priority:
d = priority_dataset.PrioritizedDataset(data, FLAGS)
else:
if FLAGS.balanced_class_loss:
d = dataset.WeightedDataset(data, FLAGS)
else:
d = dataset.Dataset(data, FLAGS)
print('means:\n{}\n{}'.format(np.mean(d.data['y_train'], axis=0),
np.mean(d.data['y_val'], axis=0)))
y = copy.deepcopy(d.data['y_val'])
y[y == 0.] = 1e-8
y[y == 1.] = 1 - 1e-8
compression_metrics.regression_score(y, np.mean(y, axis=0), 'baseline')
compression_metrics.regression_score(y, y, 'correct')
# fit the model
with tf.Session(config=tf.ConfigProto(
log_device_placement=False)) as session:
# if the timestep dimension is > 1, use recurrent network
if FLAGS.timesteps > 1:
network = rnn.RecurrentNeuralNetwork(session, FLAGS)
else:
if FLAGS.task_type == 'classification':
if FLAGS.balanced_class_loss:
network = ffnn.WeightedClassificationFeedForwardNeuralNetwork(
session, FLAGS)
else:
network = ffnn.ClassificationFeedForwardNeuralNetwork(
session, FLAGS)
else:
network = ffnn.FeedForwardNeuralNetwork(session, FLAGS)
network.fit(d)
# save weights to a julia-compatible weight file
neural_networks.utils.save_trainable_variables(
FLAGS.julia_weights_filepath, session, data)
# evaluate the fit
compression_metrics.evaluate_fit(network, data, FLAGS)
def test_fit_complex(self):
tf.set_random_seed(1)
np.random.seed(1)
flags = testing_flags.FLAGS
flags.input_dim = 1
flags.hidden_dim = 32
flags.num_hidden_layers = 2
flags.output_dim = 1
flags.batch_size = 32
flags.num_epochs = 8
flags.learning_rate = .002
flags.l2_reg = 0.0
flags.verbose = False
flags.save_weights_every = 100000
flags.snapshot_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, 'data',
'snapshots', 'test'))
x = np.random.randn(1000).reshape(-1, 1)
y = np.ones(x.shape)
y[x < 0] = 0
data = {'x_train': x, 'y_train': y, 'x_val': x, 'y_val': y}
d = dataset.Dataset(data, flags)
with tf.Session() as session:
network = ffnn.FeedForwardNeuralNetwork(session, flags)
network.fit(d)
actual = network.predict(x)
actual[actual < .5] = 0
actual[actual >= .5] = 1
np.testing.assert_array_almost_equal(y, actual, 8)
def test_fit_basic(self):
"""
Description:
- overfit a debug dataset using the 'fit' method
"""
tf.set_random_seed(1)
np.random.seed(1)
flags = testing_flags.FLAGS
flags.input_dim = 3
flags.hidden_dim = 32
flags.num_hidden_layers = 2
flags.output_dim = 2
flags.batch_size = 16
flags.num_epochs = 200
flags.learning_rate = .05
flags.l2_reg = 0.0
flags.save_weights_every = 100000
flags.snapshot_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, 'data',
'snapshots', 'test'))
x = np.vstack((np.ones(
(flags.batch_size // 2, flags.input_dim)), -1 * np.ones(
(flags.batch_size // 2, flags.input_dim))))
y = np.vstack((np.zeros((flags.batch_size // 2, flags.output_dim)),
np.ones((flags.batch_size // 2, flags.output_dim))))
data = {'x_train': x, 'y_train': y, 'x_val': x, 'y_val': y}
d = dataset.Dataset(data, flags)
with tf.Session() as session:
network = ffnn.FeedForwardNeuralNetwork(session, flags)
network.fit(d)
actual = network.predict(x)
np.testing.assert_array_almost_equal(y, actual, 8)
def main(argv=None):
# use the flags from regular compression
FLAGS = compression.run_compression.FLAGS
# set random seeds
np.random.seed(FLAGS.random_seed)
tf.set_random_seed(FLAGS.random_seed)
# set up training constants
basedir = os.path.split(FLAGS.dataset_filepath)[0]
if not os.path.exists(basedir):
os.mkdir(basedir)
dataset_filepath_template = os.path.join(basedir, 'iter_{}.h5')
basedir = os.path.split(FLAGS.julia_weights_filepath)[0]
if not os.path.exists(basedir):
os.mkdir(basedir)
network_filepath_template = os.path.join(basedir, 'iter_{}.weights')
# need some way to convey to the dataset collector the seed at which it
# should begin collection. Accomplish this by tracking the value in flags
FLAGS.initial_seed = 1
# create the model then repeatedly collect and fit datasets
with tf.Session() as session:
# build the network to use throughout training
network = ffnn.FeedForwardNeuralNetwork(session, FLAGS)
# none signifies non bootstrap dataset
network_filepath = FLAGS.initial_network_filepath
for bootstrap_iter in range(FLAGS.bootstrap_iterations):
# generate a dataset
dataset_filepath = dataset_filepath_template.format(bootstrap_iter)
generate_dataset(FLAGS, dataset_filepath, network_filepath)
# load in the dataset
data = dataset_loaders.risk_dataset_loader(
dataset_filepath,
normalize=True,
shuffle=True,
train_split=.9,
debug_size=FLAGS.debug_size)
d = dataset.Dataset(data, FLAGS)
# fit the network to the dataset
network.fit(d)
# save weights to a julia-compatible weight file for next iteration
network_filepath = network_filepath_template.format(bootstrap_iter)
neural_networks.utils.save_trainable_variables(
network_filepath, session, data)
# increment the initial seed by the number of scenarios simulated
# during each collection iteration
FLAGS.initial_seed += FLAGS.num_scenarios
def test_build_network(self):
flags = testing_flags.FLAGS
flags.input_dim = 3
flags.hidden_dim = 6
flags.num_hidden_layers = 2
flags.output_dim = 2
flags.batch_size = 6
flags.save_weights_every = 100000
with tf.Session() as session:
network = ffnn.FeedForwardNeuralNetwork(session, flags)
# check scores for zero weights
testing_utils.assign_trainable_variables_to_constant(constant=0)
inputs = np.zeros((flags.batch_size, flags.input_dim))
actual_outputs = session.run(network._scores,
feed_dict={
network._input_ph: inputs,
network._dropout_ph: 1.,
network._lr_ph:
flags.learning_rate
})
expected_outputs = np.zeros((flags.batch_size, flags.output_dim))
np.testing.assert_array_equal(expected_outputs, actual_outputs)
# check for scores for ones weights, zero input
testing_utils.assign_trainable_variables_to_constant(constant=1)
actual_outputs = session.run(network._scores,
feed_dict={
network._input_ph: inputs,
network._dropout_ph: 1.,
network._lr_ph:
flags.learning_rate
})
output_value = 43
expected_outputs = np.ones(
(flags.batch_size, flags.output_dim)) * output_value
np.testing.assert_array_equal(expected_outputs, actual_outputs)
# check for scores for ones weights, negative one input
testing_utils.assign_trainable_variables_to_constant(constant=1)
inputs = np.ones((flags.batch_size, flags.input_dim)) * -1
actual_outputs = session.run(network._scores,
feed_dict={
network._input_ph: inputs,
network._dropout_ph: 1.,
network._lr_ph:
flags.learning_rate
})
output_value = 7
expected_outputs = np.ones(
(flags.batch_size, flags.output_dim)) * output_value
np.testing.assert_array_equal(expected_outputs, actual_outputs)
def main(argv=None):
compression_flags.custom_parse_flags(FLAGS)
np.random.seed(FLAGS.random_seed)
tf.set_random_seed(FLAGS.random_seed)
data = dataset_loaders.risk_dataset_loader(
FLAGS.dataset_filepath,
shuffle=True,
train_split=.9,
debug_size=FLAGS.debug_size,
timesteps=FLAGS.timesteps,
num_target_bins=FLAGS.num_target_bins,
balanced_class_loss=FLAGS.balanced_class_loss,
target_index=FLAGS.target_index)
x = data['x_train']
y = data['y_train']
eps = 1e-8
y[y == 0.] = eps
y[y == 1.] = 1 - eps
base_dir = '/home/sisl/blake/risk_prediction/data/snapshots'
snapshot_dir_names = ['mc_{}'.format(c) for c in [1, 2, 4, 8, 16, 32]]
snapshot_dirs = [
os.path.join(base_dir, name) for name in snapshot_dir_names
]
print(snapshot_dirs)
r2s = []
with tf.Session() as session:
network = ffnn.FeedForwardNeuralNetwork(session, FLAGS)
for snapshot_dir in snapshot_dirs:
FLAGS.snapshot_dir = snapshot_dir
network.load()
y_pred = network.predict(x)
y_pred[y_pred < eps] = eps
y_pred[y_pred > 1 - eps] = 1 - eps
ll = np.sum(y * np.log(y_pred)) + np.sum(
(1 - y) * np.log(1 - y_pred))
y_null = np.mean(y, axis=0, keepdims=True)
y_null[y_null < eps] = eps
y_null[y_null > 1 - eps] = 1 - eps
ll_null = np.sum(y * np.log(y_null)) + -np.sum(
(1 - y) * np.log(1 - y_null))
mcfadden_r2 = 1 - ll / ll_null
mse = np.sum((y_pred - y)**2)
mean = np.mean(y_pred)
r2s.append((mcfadden_r2, mse, mean))
for (r2, snapshot_dir) in zip(r2s, snapshot_dirs):
print(snapshot_dir)
print(r2)
print()
def test_ability_to_overfit_debug_dataset(self):
"""
Description:
- Test network's ability to overfit a small dataset independent
of whether 'fit' is correctly implemented.
"""
tf.set_random_seed(1)
np.random.seed(1)
flags = testing_flags.FLAGS
flags.input_dim = 3
flags.hidden_dim = 16
flags.num_hidden_layers = 2
flags.output_dim = 2
flags.batch_size = 12
flags.num_epochs = 200
flags.learning_rate = .05
flags.save_weights_every = 100000
with tf.Session() as session:
network = ffnn.FeedForwardNeuralNetwork(session, flags)
# data set maps 1->0 and 0->1
x = np.vstack((np.ones((flags.batch_size // 2, flags.input_dim)),
np.zeros((flags.batch_size // 2, flags.input_dim))))
y = np.vstack((np.zeros((flags.batch_size // 2, flags.output_dim)),
np.ones((flags.batch_size // 2, flags.output_dim))))
# training epochs
for epoch in range(flags.num_epochs):
loss, probs, scores, _ = session.run(
[
network._loss, network._probs, network._scores,
network._train_op
],
feed_dict={
network._input_ph: x,
network._target_ph: y,
network._dropout_ph: network.flags.dropout_keep_prob,
network._lr_ph: flags.learning_rate
})
probs = session.run(network._probs,
feed_dict={
network._input_ph: x,
network._target_ph: y,
network._dropout_ph: 1.,
network._lr_ph: flags.learning_rate
})
self.assertAlmostEqual(loss, 0, 4)
np.testing.assert_array_almost_equal(probs, y, 4)
def test_fit_mnist(self):
# set flags
tf.set_random_seed(1)
np.random.seed(1)
flags = testing_flags.FLAGS
flags.input_dim = 28 * 28
flags.hidden_dim = 256
flags.num_hidden_layers = 3
flags.output_dim = 10
flags.batch_size = 8
flags.num_epochs = 20
flags.learning_rate = .001
flags.l2_reg = 0.0
flags.dropout_keep_prob = .5
flags.verbose = False
flags.save_weights_every = 100000
# load data
data = testing_utils.load_mnist(debug_size=1000)
d = dataset.Dataset(data, flags)
# build network
with tf.Session() as session:
network = ffnn.FeedForwardNeuralNetwork(session, flags)
network.fit(d)
y_pred = network.predict(data['x_val'])
y_pred = np.argmax(y_pred, axis=1)
y = np.argmax(data['y_val'], axis=1)
acc = len(np.where(y_pred == y)[0]) / float(len(y_pred))
# check that validation accuracy is above 90%
self.assertTrue(acc > .9)
# if run solo, then display some images and predictions
if __name__ == '__main__':
for num_samp in range(1):
x = data['x_train'][num_samp].reshape(1, -1)
print(np.argmax(network.predict(x)[0], axis=0))
print(np.argmax(data['y_train'][num_samp], axis=0))
plt.imshow(data['x_train'][num_samp].reshape(28,28))
plt.show()
print('\n')
x = data['x_val'][num_samp].reshape(1, -1)
print(np.argmax(network.predict(x)[0], axis=0))
print(np.argmax(data['y_val'][num_samp], axis=0))
plt.imshow(data['x_val'][num_samp].reshape(28,28))
plt.show()
def test_predict(self):
flags = testing_flags.FLAGS
flags.input_dim = 3
flags.hidden_dim = 6
flags.num_hidden_layers = 2
flags.output_dim = 2
flags.batch_size = 6
flags.save_weights_every = 100000
x = np.vstack((np.zeros(
(flags.batch_size // 2, flags.input_dim)), -1 * np.ones(
(flags.batch_size // 2, flags.input_dim))))
with tf.Session() as session:
network = ffnn.FeedForwardNeuralNetwork(session, flags)
testing_utils.assign_trainable_variables_to_constant(constant=1)
actual = network.predict(x)
expected = np.ones((flags.batch_size, flags.output_dim))
expected[:flags.batch_size // 2] *= testing_utils.sigmoid(43)
expected[flags.batch_size // 2:] *= testing_utils.sigmoid(7)
np.testing.assert_array_almost_equal(expected, actual, 4)
def test_init(self):
flags = testing_flags.FLAGS
with tf.Session() as session:
ffnn.FeedForwardNeuralNetwork(session, flags)
def main(argv=None):
# custom parse of flags for list input
custom_parse_flags(FLAGS)
# set random seeds
np.random.seed(FLAGS.random_seed)
tf.set_random_seed(FLAGS.random_seed)
# load dataset
input_filepath = FLAGS.dataset_filepath
data = dataset_loaders.risk_dataset_loader(
input_filepath,
shuffle=True,
train_split=.9,
debug_size=FLAGS.debug_size,
timesteps=FLAGS.timesteps,
num_target_bins=FLAGS.num_target_bins,
balanced_class_loss=FLAGS.balanced_class_loss,
target_index=FLAGS.target_index)
if FLAGS.use_priority:
d = priority_dataset.PrioritizedDataset(data, FLAGS)
else:
if FLAGS.balanced_class_loss:
d = dataset.WeightedDataset(data, FLAGS)
else:
d = dataset.Dataset(data, FLAGS)
print(np.mean(d.data['y_train'], axis=0))
print(np.mean(d.data['y_val'], axis=0))
y = copy.deepcopy(d.data['y_val'])
y[y == 0.] = 1e-8
y[y == 1.] = 1 - 1e-8
baseline = np.mean(y, axis=0)
ce = -np.sum(y * np.log(baseline)) + -np.sum(
(1 - y) * np.log(1 - baseline))
mse = np.sum((y - baseline)**2)
r2 = 1 - ((y - baseline)**2).sum() / ((y - y.mean(axis=0))**2).sum()
num_samples = len(y)
print("cross entropy from outputting validation mean: {}".format(
ce / num_samples))
print("mse from outputting validation mean: {}".format(mse / num_samples))
print("r2 from outputting validation mean: {}".format(r2))
ce = -np.sum(y * np.log(y)) + -np.sum((1 - y) * np.log(1 - y))
print("cross entropy from outputting correct values: {}".format(
ce / num_samples))
try:
ce = -np.sum(y[:, 3] * np.log(y[:, 3])) + -np.sum(
(1 - y[:, 3]) * np.log(1 - y[:, 3]))
print("hard brake cross entropy from outputting correct values: {}".
format(ce / num_samples))
except:
pass
# fit the model
with tf.Session(config=tf.ConfigProto(
log_device_placement=False)) as session:
# if the timestep dimension is > 1, use recurrent network
if FLAGS.timesteps > 1:
network = rnn.RecurrentNeuralNetwork(session, FLAGS)
else:
if FLAGS.task_type == 'classification':
if FLAGS.balanced_class_loss:
network = ffnn.WeightedClassificationFeedForwardNeuralNetwork(
session, FLAGS)
else:
network = ffnn.ClassificationFeedForwardNeuralNetwork(
session, FLAGS)
else:
network = ffnn.FeedForwardNeuralNetwork(session, FLAGS)
network.fit(d)
# save weights to a julia-compatible weight file
neural_networks.utils.save_trainable_variables(
FLAGS.julia_weights_filepath, session, data)
y_idxs = np.where(np.sum(data['y_val'][:10000], axis=1) > 1e-4)[0]
y_idxs = np.random.permutation(y_idxs)[:10]
y_pred = network.predict(data['x_val'][y_idxs])
for y_pred_s, y_s in zip(y_pred, data['y_val'][y_idxs]):
print(y_pred_s)
print(y_s)
print()
# determine the function used for assessment based on the task
score = (regression_score
if FLAGS.task_type == 'regression' else classification_score)
# final train loss
y_pred = network.predict(data['x_train'])
y = data['y_train']
y_null = np.mean(y, axis=0)
score(y, y_pred, 'train', y_null=y_null)
# final validation loss
y_pred = network.predict(data['x_val'])
y = data['y_val']
y_null = np.mean(y, axis=0)
score(y, y_pred, 'val', y_null=y_null)
# only makes sense to run this with regression
if FLAGS.task_type == 'regression':
# final validation loss, hard braking
y_pred = network.predict(data['x_val'])
y_pred = y_pred[:, 3]
y = data['y_val'][:, 3]
y_null = np.mean(y, axis=0)
score(y, y_pred, 'hard brake', y_null=y_null)
# score again the unshuffled data
if FLAGS.task_type == 'regression':
data = dataset_loaders.risk_dataset_loader(
input_filepath,
shuffle=False,
train_split=1.,
debug_size=FLAGS.debug_size)
unnorm_data = dataset_loaders.risk_dataset_loader(
input_filepath,
shuffle=False,
train_split=1.,
debug_size=FLAGS.debug_size,
normalize=False)
y_pred = network.predict(data['x_train'])
y = data['y_train']
score(y, y_pred, 'unshuffled', data, unnorm_data)
# save weights to a julia-compatible weight file
neural_networks.utils.save_trainable_variables(
FLAGS.julia_weights_filepath, session, data)
def main(argv=None):
# custom parse of flags for list input
compression.compression_flags.custom_parse_flags(FLAGS)
# set random seeds
np.random.seed(FLAGS.random_seed)
tf.set_random_seed(FLAGS.random_seed)
# load dataset
input_filepath = FLAGS.dataset_filepath
data = dataset_loaders.risk_dataset_loader(input_filepath,
shuffle=True,
train_split=.8,
debug_size=FLAGS.debug_size,
normalize=True,
timesteps=1)
# set training sizes
num_runs = 12
train_scores = []
val_scores = []
sizes = [
int(v) for v in np.logspace(
np.log2(1000), np.log2(len(data['x_train'])), num_runs, base=2.0)
]
print(sizes)
eps = 1e-12
# for each size, fit for a set number of epochs and then compute loss
for i in range(num_runs):
# train for longer with more data
FLAGS.num_epochs += 10
# create run-specific dataset
cur_data = {
'x_train': data['x_train'][:sizes[i]],
'y_train': data['y_train'][:sizes[i]],
'x_val': data['x_val'],
'y_val': data['y_val']
}
d = dataset.Dataset(cur_data, FLAGS)
with tf.Session() as session:
network = ffnn.FeedForwardNeuralNetwork(session, FLAGS)
network.fit(d)
# final train loss
y_pred = network.predict(cur_data['x_train']).astype(np.float128)
y_pred[y_pred < eps] = eps
y_pred[y_pred > (1 - eps)] = 1 - eps
y = cur_data['y_train']
ce = (-np.sum(y * np.log(y_pred)) + -np.sum(
(1 - y) * np.log(1 - y_pred))) / len(y)
mse = np.mean((y - y_pred)**2)
train_scores.append((ce, mse))
# final validation loss
y_pred = network.predict(cur_data['x_val']).astype(np.float128)
y_pred[y_pred < eps] = eps
y_pred[y_pred > (1 - eps)] = 1 - eps
y = cur_data['y_val']
ce = (-np.sum(y * np.log(y_pred)) + -np.sum(
(1 - y) * np.log(1 - y_pred))) / len(y)
mse = np.mean((y - y_pred)**2)
np.savez('../../data/scratch.npz', y_pred=y_pred)
val_scores.append((ce, mse))
print('size: {}\ttrain: {}\tval: {}'.format(
sizes[i], train_scores[i], val_scores[i]))
# reset graph after each run
tf.python.reset_default_graph()
output_filepath = os.path.join(
'../../data/learning_curves/',
os.path.split(input_filepath)[-1].replace('.h5', '.npz'))
np.savez(output_filepath,
sizes=sizes,
train_scores=train_scores,
val_scores=val_scores)