def main(argv=None): # custom parse of flags for list input compression_flags.custom_parse_flags(FLAGS) # set random seeds np.random.seed(FLAGS.random_seed) tf.set_random_seed(FLAGS.random_seed) # load dataset input_filepath = FLAGS.dataset_filepath data = dataset_loaders.risk_dataset_loader( input_filepath, shuffle=True, train_split=.9, debug_size=FLAGS.debug_size, timesteps=FLAGS.timesteps, num_target_bins=FLAGS.num_target_bins, balanced_class_loss=FLAGS.balanced_class_loss, target_index=FLAGS.target_index) if FLAGS.use_priority: d = priority_dataset.PrioritizedDataset(data, FLAGS) else: if FLAGS.balanced_class_loss: d = dataset.WeightedDataset(data, FLAGS) else: d = dataset.Dataset(data, FLAGS) print('means:\n{}\n{}'.format(np.mean(d.data['y_train'], axis=0), np.mean(d.data['y_val'], axis=0))) y = copy.deepcopy(d.data['y_val']) y[y == 0.] = 1e-8 y[y == 1.] = 1 - 1e-8 compression_metrics.regression_score(y, np.mean(y, axis=0), 'baseline') compression_metrics.regression_score(y, y, 'correct') # fit the model with tf.Session(config=tf.ConfigProto( log_device_placement=False)) as session: # if the timestep dimension is > 1, use recurrent network if FLAGS.timesteps > 1: network = rnn.RecurrentNeuralNetwork(session, FLAGS) else: if FLAGS.task_type == 'classification': if FLAGS.balanced_class_loss: network = ffnn.WeightedClassificationFeedForwardNeuralNetwork( session, FLAGS) else: network = ffnn.ClassificationFeedForwardNeuralNetwork( session, FLAGS) else: network = ffnn.FeedForwardNeuralNetwork(session, FLAGS) network.fit(d) # save weights to a julia-compatible weight file neural_networks.utils.save_trainable_variables( FLAGS.julia_weights_filepath, session, data) # evaluate the fit compression_metrics.evaluate_fit(network, data, FLAGS)
def test_fit_complex(self): tf.set_random_seed(1) np.random.seed(1) flags = testing_flags.FLAGS flags.input_dim = 1 flags.hidden_dim = 32 flags.num_hidden_layers = 2 flags.output_dim = 1 flags.batch_size = 32 flags.num_epochs = 8 flags.learning_rate = .002 flags.l2_reg = 0.0 flags.verbose = False flags.save_weights_every = 100000 flags.snapshot_dir = os.path.abspath( os.path.join(os.path.dirname(__file__), os.pardir, 'data', 'snapshots', 'test')) x = np.random.randn(1000).reshape(-1, 1) y = np.ones(x.shape) y[x < 0] = 0 data = {'x_train': x, 'y_train': y, 'x_val': x, 'y_val': y} d = dataset.Dataset(data, flags) with tf.Session() as session: network = ffnn.FeedForwardNeuralNetwork(session, flags) network.fit(d) actual = network.predict(x) actual[actual < .5] = 0 actual[actual >= .5] = 1 np.testing.assert_array_almost_equal(y, actual, 8)
def test_fit_basic(self): """ Description: - overfit a debug dataset using the 'fit' method """ tf.set_random_seed(1) np.random.seed(1) flags = testing_flags.FLAGS flags.input_dim = 3 flags.hidden_dim = 32 flags.num_hidden_layers = 2 flags.output_dim = 2 flags.batch_size = 16 flags.num_epochs = 200 flags.learning_rate = .05 flags.l2_reg = 0.0 flags.save_weights_every = 100000 flags.snapshot_dir = os.path.abspath( os.path.join(os.path.dirname(__file__), os.pardir, 'data', 'snapshots', 'test')) x = np.vstack((np.ones( (flags.batch_size // 2, flags.input_dim)), -1 * np.ones( (flags.batch_size // 2, flags.input_dim)))) y = np.vstack((np.zeros((flags.batch_size // 2, flags.output_dim)), np.ones((flags.batch_size // 2, flags.output_dim)))) data = {'x_train': x, 'y_train': y, 'x_val': x, 'y_val': y} d = dataset.Dataset(data, flags) with tf.Session() as session: network = ffnn.FeedForwardNeuralNetwork(session, flags) network.fit(d) actual = network.predict(x) np.testing.assert_array_almost_equal(y, actual, 8)
def main(argv=None): # use the flags from regular compression FLAGS = compression.run_compression.FLAGS # set random seeds np.random.seed(FLAGS.random_seed) tf.set_random_seed(FLAGS.random_seed) # set up training constants basedir = os.path.split(FLAGS.dataset_filepath)[0] if not os.path.exists(basedir): os.mkdir(basedir) dataset_filepath_template = os.path.join(basedir, 'iter_{}.h5') basedir = os.path.split(FLAGS.julia_weights_filepath)[0] if not os.path.exists(basedir): os.mkdir(basedir) network_filepath_template = os.path.join(basedir, 'iter_{}.weights') # need some way to convey to the dataset collector the seed at which it # should begin collection. Accomplish this by tracking the value in flags FLAGS.initial_seed = 1 # create the model then repeatedly collect and fit datasets with tf.Session() as session: # build the network to use throughout training network = ffnn.FeedForwardNeuralNetwork(session, FLAGS) # none signifies non bootstrap dataset network_filepath = FLAGS.initial_network_filepath for bootstrap_iter in range(FLAGS.bootstrap_iterations): # generate a dataset dataset_filepath = dataset_filepath_template.format(bootstrap_iter) generate_dataset(FLAGS, dataset_filepath, network_filepath) # load in the dataset data = dataset_loaders.risk_dataset_loader( dataset_filepath, normalize=True, shuffle=True, train_split=.9, debug_size=FLAGS.debug_size) d = dataset.Dataset(data, FLAGS) # fit the network to the dataset network.fit(d) # save weights to a julia-compatible weight file for next iteration network_filepath = network_filepath_template.format(bootstrap_iter) neural_networks.utils.save_trainable_variables( network_filepath, session, data) # increment the initial seed by the number of scenarios simulated # during each collection iteration FLAGS.initial_seed += FLAGS.num_scenarios
def test_build_network(self): flags = testing_flags.FLAGS flags.input_dim = 3 flags.hidden_dim = 6 flags.num_hidden_layers = 2 flags.output_dim = 2 flags.batch_size = 6 flags.save_weights_every = 100000 with tf.Session() as session: network = ffnn.FeedForwardNeuralNetwork(session, flags) # check scores for zero weights testing_utils.assign_trainable_variables_to_constant(constant=0) inputs = np.zeros((flags.batch_size, flags.input_dim)) actual_outputs = session.run(network._scores, feed_dict={ network._input_ph: inputs, network._dropout_ph: 1., network._lr_ph: flags.learning_rate }) expected_outputs = np.zeros((flags.batch_size, flags.output_dim)) np.testing.assert_array_equal(expected_outputs, actual_outputs) # check for scores for ones weights, zero input testing_utils.assign_trainable_variables_to_constant(constant=1) actual_outputs = session.run(network._scores, feed_dict={ network._input_ph: inputs, network._dropout_ph: 1., network._lr_ph: flags.learning_rate }) output_value = 43 expected_outputs = np.ones( (flags.batch_size, flags.output_dim)) * output_value np.testing.assert_array_equal(expected_outputs, actual_outputs) # check for scores for ones weights, negative one input testing_utils.assign_trainable_variables_to_constant(constant=1) inputs = np.ones((flags.batch_size, flags.input_dim)) * -1 actual_outputs = session.run(network._scores, feed_dict={ network._input_ph: inputs, network._dropout_ph: 1., network._lr_ph: flags.learning_rate }) output_value = 7 expected_outputs = np.ones( (flags.batch_size, flags.output_dim)) * output_value np.testing.assert_array_equal(expected_outputs, actual_outputs)
def main(argv=None): compression_flags.custom_parse_flags(FLAGS) np.random.seed(FLAGS.random_seed) tf.set_random_seed(FLAGS.random_seed) data = dataset_loaders.risk_dataset_loader( FLAGS.dataset_filepath, shuffle=True, train_split=.9, debug_size=FLAGS.debug_size, timesteps=FLAGS.timesteps, num_target_bins=FLAGS.num_target_bins, balanced_class_loss=FLAGS.balanced_class_loss, target_index=FLAGS.target_index) x = data['x_train'] y = data['y_train'] eps = 1e-8 y[y == 0.] = eps y[y == 1.] = 1 - eps base_dir = '/home/sisl/blake/risk_prediction/data/snapshots' snapshot_dir_names = ['mc_{}'.format(c) for c in [1, 2, 4, 8, 16, 32]] snapshot_dirs = [ os.path.join(base_dir, name) for name in snapshot_dir_names ] print(snapshot_dirs) r2s = [] with tf.Session() as session: network = ffnn.FeedForwardNeuralNetwork(session, FLAGS) for snapshot_dir in snapshot_dirs: FLAGS.snapshot_dir = snapshot_dir network.load() y_pred = network.predict(x) y_pred[y_pred < eps] = eps y_pred[y_pred > 1 - eps] = 1 - eps ll = np.sum(y * np.log(y_pred)) + np.sum( (1 - y) * np.log(1 - y_pred)) y_null = np.mean(y, axis=0, keepdims=True) y_null[y_null < eps] = eps y_null[y_null > 1 - eps] = 1 - eps ll_null = np.sum(y * np.log(y_null)) + -np.sum( (1 - y) * np.log(1 - y_null)) mcfadden_r2 = 1 - ll / ll_null mse = np.sum((y_pred - y)**2) mean = np.mean(y_pred) r2s.append((mcfadden_r2, mse, mean)) for (r2, snapshot_dir) in zip(r2s, snapshot_dirs): print(snapshot_dir) print(r2) print()
def test_ability_to_overfit_debug_dataset(self): """ Description: - Test network's ability to overfit a small dataset independent of whether 'fit' is correctly implemented. """ tf.set_random_seed(1) np.random.seed(1) flags = testing_flags.FLAGS flags.input_dim = 3 flags.hidden_dim = 16 flags.num_hidden_layers = 2 flags.output_dim = 2 flags.batch_size = 12 flags.num_epochs = 200 flags.learning_rate = .05 flags.save_weights_every = 100000 with tf.Session() as session: network = ffnn.FeedForwardNeuralNetwork(session, flags) # data set maps 1->0 and 0->1 x = np.vstack((np.ones((flags.batch_size // 2, flags.input_dim)), np.zeros((flags.batch_size // 2, flags.input_dim)))) y = np.vstack((np.zeros((flags.batch_size // 2, flags.output_dim)), np.ones((flags.batch_size // 2, flags.output_dim)))) # training epochs for epoch in range(flags.num_epochs): loss, probs, scores, _ = session.run( [ network._loss, network._probs, network._scores, network._train_op ], feed_dict={ network._input_ph: x, network._target_ph: y, network._dropout_ph: network.flags.dropout_keep_prob, network._lr_ph: flags.learning_rate }) probs = session.run(network._probs, feed_dict={ network._input_ph: x, network._target_ph: y, network._dropout_ph: 1., network._lr_ph: flags.learning_rate }) self.assertAlmostEqual(loss, 0, 4) np.testing.assert_array_almost_equal(probs, y, 4)
def test_fit_mnist(self): # set flags tf.set_random_seed(1) np.random.seed(1) flags = testing_flags.FLAGS flags.input_dim = 28 * 28 flags.hidden_dim = 256 flags.num_hidden_layers = 3 flags.output_dim = 10 flags.batch_size = 8 flags.num_epochs = 20 flags.learning_rate = .001 flags.l2_reg = 0.0 flags.dropout_keep_prob = .5 flags.verbose = False flags.save_weights_every = 100000 # load data data = testing_utils.load_mnist(debug_size=1000) d = dataset.Dataset(data, flags) # build network with tf.Session() as session: network = ffnn.FeedForwardNeuralNetwork(session, flags) network.fit(d) y_pred = network.predict(data['x_val']) y_pred = np.argmax(y_pred, axis=1) y = np.argmax(data['y_val'], axis=1) acc = len(np.where(y_pred == y)[0]) / float(len(y_pred)) # check that validation accuracy is above 90% self.assertTrue(acc > .9) # if run solo, then display some images and predictions if __name__ == '__main__': for num_samp in range(1): x = data['x_train'][num_samp].reshape(1, -1) print(np.argmax(network.predict(x)[0], axis=0)) print(np.argmax(data['y_train'][num_samp], axis=0)) plt.imshow(data['x_train'][num_samp].reshape(28,28)) plt.show() print('\n') x = data['x_val'][num_samp].reshape(1, -1) print(np.argmax(network.predict(x)[0], axis=0)) print(np.argmax(data['y_val'][num_samp], axis=0)) plt.imshow(data['x_val'][num_samp].reshape(28,28)) plt.show()
def test_predict(self): flags = testing_flags.FLAGS flags.input_dim = 3 flags.hidden_dim = 6 flags.num_hidden_layers = 2 flags.output_dim = 2 flags.batch_size = 6 flags.save_weights_every = 100000 x = np.vstack((np.zeros( (flags.batch_size // 2, flags.input_dim)), -1 * np.ones( (flags.batch_size // 2, flags.input_dim)))) with tf.Session() as session: network = ffnn.FeedForwardNeuralNetwork(session, flags) testing_utils.assign_trainable_variables_to_constant(constant=1) actual = network.predict(x) expected = np.ones((flags.batch_size, flags.output_dim)) expected[:flags.batch_size // 2] *= testing_utils.sigmoid(43) expected[flags.batch_size // 2:] *= testing_utils.sigmoid(7) np.testing.assert_array_almost_equal(expected, actual, 4)
def test_init(self): flags = testing_flags.FLAGS with tf.Session() as session: ffnn.FeedForwardNeuralNetwork(session, flags)
def main(argv=None): # custom parse of flags for list input custom_parse_flags(FLAGS) # set random seeds np.random.seed(FLAGS.random_seed) tf.set_random_seed(FLAGS.random_seed) # load dataset input_filepath = FLAGS.dataset_filepath data = dataset_loaders.risk_dataset_loader( input_filepath, shuffle=True, train_split=.9, debug_size=FLAGS.debug_size, timesteps=FLAGS.timesteps, num_target_bins=FLAGS.num_target_bins, balanced_class_loss=FLAGS.balanced_class_loss, target_index=FLAGS.target_index) if FLAGS.use_priority: d = priority_dataset.PrioritizedDataset(data, FLAGS) else: if FLAGS.balanced_class_loss: d = dataset.WeightedDataset(data, FLAGS) else: d = dataset.Dataset(data, FLAGS) print(np.mean(d.data['y_train'], axis=0)) print(np.mean(d.data['y_val'], axis=0)) y = copy.deepcopy(d.data['y_val']) y[y == 0.] = 1e-8 y[y == 1.] = 1 - 1e-8 baseline = np.mean(y, axis=0) ce = -np.sum(y * np.log(baseline)) + -np.sum( (1 - y) * np.log(1 - baseline)) mse = np.sum((y - baseline)**2) r2 = 1 - ((y - baseline)**2).sum() / ((y - y.mean(axis=0))**2).sum() num_samples = len(y) print("cross entropy from outputting validation mean: {}".format( ce / num_samples)) print("mse from outputting validation mean: {}".format(mse / num_samples)) print("r2 from outputting validation mean: {}".format(r2)) ce = -np.sum(y * np.log(y)) + -np.sum((1 - y) * np.log(1 - y)) print("cross entropy from outputting correct values: {}".format( ce / num_samples)) try: ce = -np.sum(y[:, 3] * np.log(y[:, 3])) + -np.sum( (1 - y[:, 3]) * np.log(1 - y[:, 3])) print("hard brake cross entropy from outputting correct values: {}". format(ce / num_samples)) except: pass # fit the model with tf.Session(config=tf.ConfigProto( log_device_placement=False)) as session: # if the timestep dimension is > 1, use recurrent network if FLAGS.timesteps > 1: network = rnn.RecurrentNeuralNetwork(session, FLAGS) else: if FLAGS.task_type == 'classification': if FLAGS.balanced_class_loss: network = ffnn.WeightedClassificationFeedForwardNeuralNetwork( session, FLAGS) else: network = ffnn.ClassificationFeedForwardNeuralNetwork( session, FLAGS) else: network = ffnn.FeedForwardNeuralNetwork(session, FLAGS) network.fit(d) # save weights to a julia-compatible weight file neural_networks.utils.save_trainable_variables( FLAGS.julia_weights_filepath, session, data) y_idxs = np.where(np.sum(data['y_val'][:10000], axis=1) > 1e-4)[0] y_idxs = np.random.permutation(y_idxs)[:10] y_pred = network.predict(data['x_val'][y_idxs]) for y_pred_s, y_s in zip(y_pred, data['y_val'][y_idxs]): print(y_pred_s) print(y_s) print() # determine the function used for assessment based on the task score = (regression_score if FLAGS.task_type == 'regression' else classification_score) # final train loss y_pred = network.predict(data['x_train']) y = data['y_train'] y_null = np.mean(y, axis=0) score(y, y_pred, 'train', y_null=y_null) # final validation loss y_pred = network.predict(data['x_val']) y = data['y_val'] y_null = np.mean(y, axis=0) score(y, y_pred, 'val', y_null=y_null) # only makes sense to run this with regression if FLAGS.task_type == 'regression': # final validation loss, hard braking y_pred = network.predict(data['x_val']) y_pred = y_pred[:, 3] y = data['y_val'][:, 3] y_null = np.mean(y, axis=0) score(y, y_pred, 'hard brake', y_null=y_null) # score again the unshuffled data if FLAGS.task_type == 'regression': data = dataset_loaders.risk_dataset_loader( input_filepath, shuffle=False, train_split=1., debug_size=FLAGS.debug_size) unnorm_data = dataset_loaders.risk_dataset_loader( input_filepath, shuffle=False, train_split=1., debug_size=FLAGS.debug_size, normalize=False) y_pred = network.predict(data['x_train']) y = data['y_train'] score(y, y_pred, 'unshuffled', data, unnorm_data) # save weights to a julia-compatible weight file neural_networks.utils.save_trainable_variables( FLAGS.julia_weights_filepath, session, data)
def main(argv=None): # custom parse of flags for list input compression.compression_flags.custom_parse_flags(FLAGS) # set random seeds np.random.seed(FLAGS.random_seed) tf.set_random_seed(FLAGS.random_seed) # load dataset input_filepath = FLAGS.dataset_filepath data = dataset_loaders.risk_dataset_loader(input_filepath, shuffle=True, train_split=.8, debug_size=FLAGS.debug_size, normalize=True, timesteps=1) # set training sizes num_runs = 12 train_scores = [] val_scores = [] sizes = [ int(v) for v in np.logspace( np.log2(1000), np.log2(len(data['x_train'])), num_runs, base=2.0) ] print(sizes) eps = 1e-12 # for each size, fit for a set number of epochs and then compute loss for i in range(num_runs): # train for longer with more data FLAGS.num_epochs += 10 # create run-specific dataset cur_data = { 'x_train': data['x_train'][:sizes[i]], 'y_train': data['y_train'][:sizes[i]], 'x_val': data['x_val'], 'y_val': data['y_val'] } d = dataset.Dataset(cur_data, FLAGS) with tf.Session() as session: network = ffnn.FeedForwardNeuralNetwork(session, FLAGS) network.fit(d) # final train loss y_pred = network.predict(cur_data['x_train']).astype(np.float128) y_pred[y_pred < eps] = eps y_pred[y_pred > (1 - eps)] = 1 - eps y = cur_data['y_train'] ce = (-np.sum(y * np.log(y_pred)) + -np.sum( (1 - y) * np.log(1 - y_pred))) / len(y) mse = np.mean((y - y_pred)**2) train_scores.append((ce, mse)) # final validation loss y_pred = network.predict(cur_data['x_val']).astype(np.float128) y_pred[y_pred < eps] = eps y_pred[y_pred > (1 - eps)] = 1 - eps y = cur_data['y_val'] ce = (-np.sum(y * np.log(y_pred)) + -np.sum( (1 - y) * np.log(1 - y_pred))) / len(y) mse = np.mean((y - y_pred)**2) np.savez('../../data/scratch.npz', y_pred=y_pred) val_scores.append((ce, mse)) print('size: {}\ttrain: {}\tval: {}'.format( sizes[i], train_scores[i], val_scores[i])) # reset graph after each run tf.python.reset_default_graph() output_filepath = os.path.join( '../../data/learning_curves/', os.path.split(input_filepath)[-1].replace('.h5', '.npz')) np.savez(output_filepath, sizes=sizes, train_scores=train_scores, val_scores=val_scores)