def main(csv_path, target_name, task='classification', model_name='tabnet', tb_log_location='./tflog', categorical_features=[], val_frac=0.25, test_frac=0.25, emb_size=1, feature_dim=128, output_dim=64, batch_size=512, virtual_batch_size=512, batch_momentum=0.7, gamma=1.5, n_steps=6, max_steps=25, lr=0.02, decay_every=500, lambda_sparsity=0.0001): all_data = pd.read_csv(csv_path) trainval_df, test_df = train_test_split(all_data, test_size=test_frac, stratify=all_data[target_name]) val_frac_after_test_split = val_frac / (1 - test_frac) train_df, val_df = train_test_split(trainval_df, test_size=val_frac_after_test_split) dataset_info = prepare_dataset(all_data, categorical_features, target_name, task, embedding_dim=emb_size) # TabNet model tabnet = tabnet_model.TabNet(columns=dataset_info['feature_columns'], num_features=dataset_info['num_features'], feature_dim=feature_dim, output_dim=output_dim, num_decision_steps=n_steps, relaxation_factor=gamma, batch_momentum=batch_momentum, virtual_batch_size=virtual_batch_size, num_classes=dataset_info['num_classes']) label_column = target_name # Training parameters max_steps = max_steps display_step = 5 val_step = 5 save_step = 5 init_localearning_rate = lr decay_every = decay_every decay_rate = 0.95 batch_size = batch_size sparsity_loss_weight = lambda_sparsity gradient_thresh = 2000.0 # Input sampling train_batch = pandas_input_fn(train_df, label_column, dataset_info, num_epochs=100000, shuffle=True, batch_size=batch_size, n_buffer=1) val_batch = pandas_input_fn(val_df, label_column, dataset_info, num_epochs=10000, shuffle=False, batch_size=batch_size, n_buffer=1) test_batch = pandas_input_fn(test_df, label_column, dataset_info, num_epochs=10000, shuffle=False, batch_size=batch_size, n_buffer=1) train_iter = train_batch.make_initializable_iterator() val_iter = val_batch.make_initializable_iterator() test_iter = test_batch.make_initializable_iterator() feature_train_batch, label_train_batch = train_iter.get_next() feature_val_batch, label_val_batch = val_iter.get_next() feature_test_batch, label_test_batch = test_iter.get_next() # Define the model and losses encoded_train_batch, total_entropy = tabnet.encoder(feature_train_batch, is_training=True) if task == 'classification': logits_orig_batch, _ = tabnet.classify(encoded_train_batch) softmax_orig_key_op = tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits( logits=logits_orig_batch, labels=label_train_batch)) train_loss_op = softmax_orig_key_op + sparsity_loss_weight * total_entropy else: predictions = tabnet.regress(encoded_train_batch) #l2_loss = tf.reduce_mean( # tf.nn.l2_loss(t = tf.subtract(predictions, label_train_batch)) / tf.to_float(tf.size(predictions)) #) l2_loss = tf.reduce_mean( tf.square(tf.subtract(predictions, label_train_batch))) train_loss_op = l2_loss + sparsity_loss_weight * total_entropy tf.compat.v1.summary.scalar("Total loss", train_loss_op) # Optimization step global_step = tf.compat.v1.train.get_or_create_global_step() learning_rate = tf.compat.v1.train.exponential_decay( init_localearning_rate, global_step=global_step, decay_steps=decay_every, decay_rate=decay_rate) optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate=learning_rate) update_ops = tf.compat.v1.get_collection(tf.compat.v1.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): gvs = optimizer.compute_gradients(train_loss_op) capped_gvs = [(tf.clip_by_value(grad, -gradient_thresh, gradient_thresh), var) for grad, var in gvs] train_op = optimizer.apply_gradients(capped_gvs, global_step=global_step) # Model evaluation # Validation performance encoded_val_batch, _ = tabnet.encoder(feature_val_batch, is_training=True) val_op = None if task == 'classification': _, prediction_val = tabnet.classify(encoded_val_batch) predicted_labels = tf.cast(tf.argmax(prediction_val, 1), dtype=tf.int32) val_eq_op = tf.equal(predicted_labels, label_val_batch) val_acc_op = tf.reduce_mean(tf.cast(val_eq_op, dtype=tf.float32)) tf.compat.v1.summary.scalar("Val accuracy", val_acc_op) #val_op=tf.compat.v1.metrics.auc(label_val_batch, prediction_val[:,0]) #val_op = val_acc_op val_op = result( tabnet.compile('sgd', loss='mse', metrics=[tf.keras.metrics.AUC()])) else: predictions = tabnet.regress(encoded_val_batch) val_loss_op = tf.reduce_mean( tf.square(tf.subtract(predictions, label_train_batch))) val_op = val_loss_op tf.compat.v1.summary.scalar("Validation loss", val_loss_op) # Test performance encoded_test_batch, _ = tabnet.encoder(feature_test_batch, is_training=True) test_op = None if task == 'classification': _, prediction_test = tabnet.classify(encoded_test_batch) predicted_labels = tf.cast(tf.argmax(prediction_test, 1), dtype=tf.int32) # test_eq_op = tf.equal(predicted_labels, label_test_batch) # test_acc_op = tf.reduce_mean(tf.cast(test_eq_op, dtype=tf.float32)) # tf.compat.v1.summary.scalar("Test accuracy", test_acc_op) # fpr, tpr, _ = roc_curve(label_test_batch, prediction_test) #test_op=tf.compat.v1.metrics.auc(label_test_batch, prediction_test[:,0]) # test_op = test_acc_op test_op = result( tabnet.compile('sgd', loss='mse', metrics=[tf.keras.metrics.AUC()])) else: predictions = tabnet.regress(encoded_test_batch) test_loss_op = tf.reduce_mean( tf.square(tf.subtract(predictions, label_test_batch))) tf.compat.v1.summary.scalar("Test loss", test_loss_op) test_op = test_loss_op # Training setup init = tf.initialize_all_variables() init_local = tf.compat.v1.local_variables_initializer() init_table = tf.compat.v1.tables_initializer(name="Initialize_all_tables") saver = tf.compat.v1.train.Saver() summaries = tf.compat.v1.summary.merge_all() with tf.compat.v1.Session() as sess: summary_writer = tf.compat.v1.summary.FileWriter( f'{tb_log_location}/{model_name}', sess.graph) sess.run(init) sess.run(init_local) sess.run(init_table) sess.run(train_iter.initializer) sess.run(val_iter.initializer) sess.run(test_iter.initializer) early_stop_steps = 25 best_val_acc = -1 for step in range(1, max_steps + 1): if step % display_step == 0: _, train_loss, merged_summary = sess.run( [train_op, train_loss_op, summaries]) summary_writer.add_summary(merged_summary, step) print("Step " + str(step) + " , Training Loss = " + "{:.4f}".format(train_loss)) else: _ = sess.run(train_op) if step % val_step == 0: feed_arr = [ vars()["summaries"], vars()[f"val_op"], vars()[f"test_op"] ] val_arr = sess.run(feed_arr) merged_summary = val_arr[0] val_acc = val_arr[1] if val_acc > best_val_acc: best_val_acc = val_acc best_val_step = step if (step - best_val_step) > early_stop_steps: break print("Step " + str(step) + " , Val Metric = " + "{:.4f}".format(val_acc)) summary_writer.add_summary(merged_summary, step) if step % save_step == 0: saver.save(sess, "./checkpoints/" + model_name + ".ckpt") print(f'Best validation accuracy: {best_val_acc}')
def train_and_evaluate(params, batch_size, virtual_batch_size, max_steps, lr, decay_every, target_name, dataset_info, train_df, val_df): tf.compat.v1.reset_default_graph() print(params) # TabNet model tabnet = tabnet_model.TabNet( columns=dataset_info['feature_columns'], num_features=dataset_info['num_features'], feature_dim=int(params['n_a']), output_dim=int(params['n_a']), # Same dims for feature and output num_decision_steps=int(params['n_steps']), relaxation_factor=params['gamma'], batch_momentum=params['batch_momentum'], virtual_batch_size=virtual_batch_size, num_classes=dataset_info['num_classes']) label_column = target_name # Training parameters max_steps = max_steps display_step = 5 val_step = 5 init_localearning_rate = lr decay_every = decay_every decay_rate = 0.95 batch_size = batch_size sparsity_loss_weight = params['lambda'] gradient_thresh = 2000.0 # Input sampling train_batch = pandas_input_fn( train_df, label_column, dataset_info, num_epochs=100000, shuffle=True, batch_size=batch_size, n_buffer=1) val_batch = pandas_input_fn( val_df, label_column, dataset_info, num_epochs=10000, shuffle=False, batch_size=batch_size, n_buffer=1) train_iter = train_batch.make_initializable_iterator() val_iter = val_batch.make_initializable_iterator() feature_train_batch, label_train_batch = train_iter.get_next() feature_val_batch, label_val_batch = val_iter.get_next() # Define the model and losses encoded_train_batch, total_entropy = tabnet.encoder( feature_train_batch, is_training=True) logits_orig_batch, _ = tabnet.classify( encoded_train_batch) softmax_orig_key_op = tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits( logits=logits_orig_batch, labels=label_train_batch)) train_loss_op = softmax_orig_key_op + sparsity_loss_weight * total_entropy # Optimization step global_step = tf.compat.v1.train.get_or_create_global_step() learning_rate = tf.compat.v1.train.exponential_decay( init_localearning_rate, global_step=global_step, decay_steps=decay_every, decay_rate=decay_rate) optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate=learning_rate) update_ops = tf.compat.v1.get_collection(tf.compat.v1.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): gvs = optimizer.compute_gradients(train_loss_op) capped_gvs = [(tf.clip_by_value(grad, -gradient_thresh, gradient_thresh), var) for grad, var in gvs] train_op = optimizer.apply_gradients(capped_gvs, global_step=global_step) # Model evaluation # Validation performance encoded_val_batch, _ = tabnet.encoder( feature_val_batch, is_training=True) val_op = None _, prediction_val = tabnet.classify( encoded_val_batch) predicted_labels = tf.cast(tf.argmax(prediction_val, 1), dtype=tf.int32) val_eq_op = tf.equal(predicted_labels, label_val_batch) val_acc_op = tf.reduce_mean(tf.cast(val_eq_op, dtype=tf.float32)) val_op = val_acc_op # Training setup init = tf.initialize_all_variables() init_local = tf.compat.v1.local_variables_initializer() init_table = tf.compat.v1.tables_initializer(name="Initialize_all_tables") summaries = tf.compat.v1.summary.merge_all() with tf.compat.v1.Session() as sess: sess.run(init) sess.run(init_local) sess.run(init_table) sess.run(train_iter.initializer) sess.run(val_iter.initializer) early_stop_steps = 25 best_val_acc = -1 for step in range(1, max_steps + 1): if step % display_step == 0: _, train_loss, merged_summary = sess.run( [train_op, train_loss_op, summaries]) else: _ = sess.run(train_op) if step % val_step == 0: feed_arr = [ vars()["summaries"], vars()[f"val_op"], ] val_arr = sess.run(feed_arr) merged_summary = val_arr[0] val_acc = val_arr[1] if val_acc > best_val_acc: best_val_acc = val_acc best_val_step = step if (step - best_val_step) > early_stop_steps: break print(f'Best validation accuracy: {best_val_acc}') return -1*best_val_acc
def main(unused_argv): # Load training and eval data. train_file = "data/train.csv" val_file = "data/val.csv" test_file = "data/test.csv" # TabNet model tabnet_forest_covertype = tabnet_model.TabNet( columns=data_helper_covertype.get_columns(), num_features=data_helper_covertype.num_features, feature_dim=128, output_dim=64, num_decision_steps=6, relaxation_factor=1.5, batch_momentum=0.7, virtual_batch_size=512, num_classes=data_helper_covertype.num_classes) column_names = sorted(data_helper_covertype.feature_columns) print( "Ordered column names, corresponding to the indexing in Tensorboard visualization" ) for fi in range(len(column_names)): print(str(fi) + " : " + column_names[fi]) # Training parameters max_steps = 10 display_step = 5 val_step = 5 save_step = 5 init_localearning_rate = 0.02 decay_every = 500 decay_rate = 0.95 batch_size = 512 sparsity_loss_weight = 0.0001 gradient_thresh = 2000.0 # Input sampling train_batch = data_helper_covertype.input_fn(train_file, num_epochs=100000, shuffle=True, batch_size=batch_size, n_buffer=1, n_parallel=1) val_batch = data_helper_covertype.input_fn(val_file, num_epochs=10000, shuffle=False, batch_size=batch_size, n_buffer=1, n_parallel=1) test_batch = data_helper_covertype.input_fn(test_file, num_epochs=10000, shuffle=False, batch_size=batch_size, n_buffer=1, n_parallel=1) train_iter = train_batch.make_initializable_iterator() val_iter = val_batch.make_initializable_iterator() test_iter = test_batch.make_initializable_iterator() feature_train_batch, label_train_batch = train_iter.get_next() feature_val_batch, label_val_batch = val_iter.get_next() feature_test_batch, label_test_batch = test_iter.get_next() # Define the model and losses encoded_train_batch, total_entropy = tabnet_forest_covertype.encoder( feature_train_batch, reuse=False, is_training=True) logits_orig_batch, _ = tabnet_forest_covertype.classify( encoded_train_batch, reuse=False) softmax_orig_key_op = tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits( logits=logits_orig_batch, labels=label_train_batch)) train_loss_op = softmax_orig_key_op + sparsity_loss_weight * total_entropy tf.summary.scalar("Total loss", train_loss_op) # Optimization step global_step = tf.train.get_or_create_global_step() learning_rate = tf.train.exponential_decay(init_localearning_rate, global_step=global_step, decay_steps=decay_every, decay_rate=decay_rate) optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate) update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): gvs = optimizer.compute_gradients(train_loss_op) capped_gvs = [(tf.clip_by_value(grad, -gradient_thresh, gradient_thresh), var) for grad, var in gvs] train_op = optimizer.apply_gradients(capped_gvs, global_step=global_step) # Model evaluation # Validation performance encoded_val_batch, _ = tabnet_forest_covertype.encoder(feature_val_batch, reuse=True, is_training=True) _, prediction_val = tabnet_forest_covertype.classify(encoded_val_batch, reuse=True) predicted_labels = tf.cast(tf.argmax(prediction_val, 1), dtype=tf.int32) val_eq_op = tf.equal(predicted_labels, label_val_batch) val_acc_op = tf.reduce_mean(tf.cast(val_eq_op, dtype=tf.float32)) tf.summary.scalar("Val accuracy", val_acc_op) # Test performance encoded_test_batch, _ = tabnet_forest_covertype.encoder(feature_test_batch, reuse=True, is_training=True) _, prediction_test = tabnet_forest_covertype.classify(encoded_test_batch, reuse=True) predicted_labels = tf.cast(tf.argmax(prediction_test, 1), dtype=tf.int32) test_eq_op = tf.equal(predicted_labels, label_test_batch) test_acc_op = tf.reduce_mean(tf.cast(test_eq_op, dtype=tf.float32)) tf.summary.scalar("Test accuracy", test_acc_op) # Training setup model_name = "tabnet_forest_covertype_model" init = tf.initialize_all_variables() init_local = tf.local_variables_initializer() init_table = tf.tables_initializer(name="Initialize_all_tables") saver = tf.train.Saver() summaries = tf.summary.merge_all() with tf.Session() as sess: summary_writer = tf.summary.FileWriter("./tflog/" + model_name, sess.graph) sess.run(init) sess.run(init_local) sess.run(init_table) sess.run(train_iter.initializer) sess.run(val_iter.initializer) sess.run(test_iter.initializer) for step in range(1, max_steps + 1): if step % display_step == 0: _, train_loss, merged_summary = sess.run( [train_op, train_loss_op, summaries]) summary_writer.add_summary(merged_summary, step) print("Step " + str(step) + " , Training Loss = " + "{:.4f}".format(train_loss)) else: _ = sess.run(train_op) if step % val_step == 0: feed_arr = [ vars()["summaries"], vars()["val_acc_op"], vars()["test_acc_op"] ] val_arr = sess.run(feed_arr) merged_summary = val_arr[0] val_acc = val_arr[1] print("Step " + str(step) + " , Val Accuracy = " + "{:.4f}".format(val_acc)) summary_writer.add_summary(merged_summary, step) if step % save_step == 0: saver.save(sess, "./checkpoints/" + model_name + ".ckpt")
def main(unused_argv): # Fix random seeds tf.set_random_seed(SEED) np.random.seed(SEED) # Define the TabNet model tabnet_forest_covertype = tabnet_model.TabNet( columns=data_helper_covertype.get_columns(), num_features=data_helper_covertype.NUM_FEATURES, feature_dim=4, output_dim=2, num_decision_steps=6, relaxation_factor=1.5, batch_momentum=0.7, virtual_batch_size=4, num_classes=data_helper_covertype.NUM_CLASSES) column_names = sorted(data_helper_covertype.FEATURE_COLUMNS) print( "Ordered column names, corresponding to the indexing in Tensorboard visualization" ) for fi in range(len(column_names)): print(str(fi) + " : " + column_names[fi]) # Input sampling train_batch = data_helper_covertype.input_fn(TRAIN_FILE, num_epochs=100000, shuffle=True, batch_size=BATCH_SIZE) val_batch = data_helper_covertype.input_fn( VAL_FILE, num_epochs=10000, shuffle=False, batch_size=data_helper_covertype.N_VAL_SAMPLES) test_batch = data_helper_covertype.input_fn( TEST_FILE, num_epochs=10000, shuffle=False, batch_size=data_helper_covertype.N_TEST_SAMPLES) train_iter = train_batch.make_initializable_iterator() val_iter = val_batch.make_initializable_iterator() test_iter = test_batch.make_initializable_iterator() feature_train_batch, label_train_batch = train_iter.get_next() feature_val_batch, label_val_batch = val_iter.get_next() feature_test_batch, label_test_batch = test_iter.get_next() # Define the model and losses encoded_train_batch, total_entropy = tabnet_forest_covertype.encoder( feature_train_batch, reuse=False, is_training=True) logits_orig_batch, _ = tabnet_forest_covertype.classify( encoded_train_batch, reuse=False) softmax_orig_key_op = tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits( logits=logits_orig_batch, labels=label_train_batch)) train_loss_op = softmax_orig_key_op + SPARSITY_LOSS_WEIGHT * total_entropy tf.summary.scalar("Total loss", train_loss_op) # Optimization step global_step = tf.train.get_or_create_global_step() learning_rate = tf.train.exponential_decay(INIT_LEARNING_RATE, global_step=global_step, decay_steps=DECAY_EVERY, decay_rate=DECAY_RATE) optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate) update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): gvs = optimizer.compute_gradients(train_loss_op) capped_gvs = [(tf.clip_by_value(grad, -GRADIENT_THRESH, GRADIENT_THRESH), var) for grad, var in gvs] train_op = optimizer.apply_gradients(capped_gvs, global_step=global_step) # Model evaluation # Validation performance encoded_val_batch, _ = tabnet_forest_covertype.encoder(feature_val_batch, reuse=True, is_training=False) _, prediction_val = tabnet_forest_covertype.classify(encoded_val_batch, reuse=True) predicted_labels = tf.cast(tf.argmax(prediction_val, 1), dtype=tf.int32) val_eq_op = tf.equal(predicted_labels, label_val_batch) val_acc_op = tf.reduce_mean(tf.cast(val_eq_op, dtype=tf.float32)) tf.summary.scalar("Val accuracy", val_acc_op) # Test performance encoded_test_batch, _ = tabnet_forest_covertype.encoder(feature_test_batch, reuse=True, is_training=False) _, prediction_test = tabnet_forest_covertype.classify(encoded_test_batch, reuse=True) predicted_labels = tf.cast(tf.argmax(prediction_test, 1), dtype=tf.int32) test_eq_op = tf.equal(predicted_labels, label_test_batch) test_acc_op = tf.reduce_mean(tf.cast(test_eq_op, dtype=tf.float32)) tf.summary.scalar("Test accuracy", test_acc_op) # Training setup model_name = "tabnet_forest_covertype_model" init = tf.initialize_all_variables() init_local = tf.local_variables_initializer() init_table = tf.tables_initializer(name="Initialize_all_tables") saver = tf.train.Saver() summaries = tf.summary.merge_all() with tf.Session() as sess: summary_writer = tf.summary.FileWriter("./tflog/" + model_name, sess.graph) sess.run(init) sess.run(init_local) sess.run(init_table) sess.run(train_iter.initializer) sess.run(val_iter.initializer) sess.run(test_iter.initializer) for step in range(1, MAX_STEPS + 1): if step % DISPLAY_STEP == 0: _, train_loss, merged_summary = sess.run( [train_op, train_loss_op, summaries]) summary_writer.add_summary(merged_summary, step) print("Step " + str(step) + " , Training Loss = " + "{:.4f}".format(train_loss)) else: _ = sess.run(train_op) if step % VAL_STEP == 0: feed_arr = [ vars()["summaries"], vars()["val_acc_op"], vars()["test_acc_op"] ] val_arr = sess.run(feed_arr) merged_summary = val_arr[0] val_acc = val_arr[1] print("Step " + str(step) + " , Val Accuracy = " + "{:.4f}".format(val_acc)) summary_writer.add_summary(merged_summary, step) if step % SAVE_STEP == 0: saver.save(sess, "./checkpoints/" + model_name + ".ckpt")
def main(csv_path, target_name, task='classification', model_name='tabnet', tb_log_location='./tflog', categorical_suffix=None, val_frac=0.25, test_frac=0.25, emb_size=1, feature_dim=128, output_dim=64, batch_size=512, virtual_batch_size=512, batch_momentum=0.7, gamma=1.5, n_steps=6, max_steps=25, lr=0.02, decay_every=500, lambda_sparsity=0.0001): all_data = pd.read_csv(csv_path) trainval_df, test_df = train_test_split(all_data, test_size=test_frac, stratify=all_data[target_name]) val_frac_after_test_split = val_frac / (1 - test_frac) train_df, val_df = train_test_split(trainval_df, test_size=val_frac_after_test_split) print(f'Datasets sizes:') print(f'Train: {train_df.shape}:') print(f'Val: {val_df.shape}:') print(f'Test: {test_df.shape}:') # save data sets data_dir = os.path.dirname(csv_path) for subset, label in zip((train_df, val_df, test_df), ('train', 'val', 'test')): subset.to_csv(os.path.join(data_dir, f'{label}_set.csv'), index=False) dataset_info = prepare_dataset(all_data, categorical_suffix, target_name, task, embedding_dim=emb_size) # TabNet model tabnet = tabnet_model.TabNet( columns=dataset_info['feature_columns'], num_features=dataset_info['num_features'], feature_dim=feature_dim, output_dim=output_dim, num_decision_steps=n_steps, relaxation_factor=gamma, batch_momentum=batch_momentum, virtual_batch_size=virtual_batch_size, num_classes=dataset_info['num_classes']) label_column = target_name # Training parameters max_steps = max_steps display_step = 5 val_step = 100 save_step = 100 init_localearning_rate = lr decay_every = decay_every decay_rate = 0.95 batch_size = batch_size sparsity_loss_weight = lambda_sparsity gradient_thresh = 2000.0 # Input sampling train_batch = pandas_input_fn( train_df, label_column, dataset_info, num_epochs=100000, shuffle=True, batch_size=batch_size, n_buffer=1) val_batch = pandas_input_fn( val_df, label_column, dataset_info, num_epochs=10000, shuffle=False, batch_size=batch_size, n_buffer=1) test_batch = pandas_input_fn( test_df, label_column, dataset_info, num_epochs=10000, shuffle=False, batch_size=batch_size, n_buffer=1) train_iter = train_batch.make_initializable_iterator() val_iter = val_batch.make_initializable_iterator() test_iter = test_batch.make_initializable_iterator() feature_train_batch, label_train_batch = train_iter.get_next() feature_val_batch, label_val_batch = val_iter.get_next() feature_test_batch, label_test_batch = test_iter.get_next() # Define the model and losses # for training encoded_train_batch, total_entropy = tabnet.encoder( feature_train_batch, is_training=True) if task == 'classification': logits_orig_batch, predictions_train = tabnet.classify( encoded_train_batch) softmax_orig_key_op = tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits( logits=logits_orig_batch, labels=label_train_batch)) train_loss_op = softmax_orig_key_op + sparsity_loss_weight * total_entropy else: predictions = tabnet.regress( encoded_train_batch ) # l2_loss = tf.reduce_mean( # tf.nn.l2_loss(t = tf.subtract(predictions, label_train_batch)) / tf.to_float(tf.size(predictions)) # ) l2_loss = tf.reduce_mean(tf.square(tf.subtract(predictions, label_train_batch))) train_loss_op = l2_loss + sparsity_loss_weight * total_entropy tf.compat.v1.summary.scalar("Total loss", train_loss_op) # Optimization step global_step = tf.compat.v1.train.get_or_create_global_step() learning_rate = tf.compat.v1.train.exponential_decay( init_localearning_rate, global_step=global_step, decay_steps=decay_every, decay_rate=decay_rate) optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate=learning_rate) update_ops = tf.compat.v1.get_collection(tf.compat.v1.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): gvs = optimizer.compute_gradients(train_loss_op) capped_gvs = [(tf.clip_by_value(grad, -gradient_thresh, gradient_thresh), var) for grad, var in gvs] train_op = optimizer.apply_gradients(capped_gvs, global_step=global_step) # Model evaluation # Validation performance encoded_val_batch, _ = tabnet.encoder( feature_val_batch, is_training=True) val_op = None if task == 'classification': _, predictions_val = tabnet.classify( encoded_val_batch) predicted_labels = tf.cast(tf.argmax(predictions_val, 1), dtype=tf.int32) val_eq_op = tf.equal(predicted_labels, label_val_batch) val_acc_op = tf.reduce_mean(tf.cast(val_eq_op, dtype=tf.float32)) tf.compat.v1.summary.scalar("Val accuracy", val_acc_op) val_op = val_acc_op else: predictions = tabnet.regress( encoded_val_batch ) val_loss_op = tf.reduce_mean(tf.square(tf.subtract(predictions, label_train_batch))) val_op = val_loss_op tf.compat.v1.summary.scalar("Validation loss", val_loss_op) # Test performance encoded_test_batch, _ = tabnet.encoder( feature_test_batch, is_training=True) test_op = None if task == 'classification': logits_test, predictions_test = tabnet.classify( encoded_test_batch) predicted_labels = tf.cast(tf.argmax(predictions_test, 1), dtype=tf.int32) test_eq_op = tf.equal(predicted_labels, label_test_batch) test_acc_op = -tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits( logits=logits_test, labels=label_test_batch)) tf.compat.v1.summary.scalar("Test accuracy", test_acc_op) test_op = test_acc_op else: predictions = tabnet.regress( encoded_test_batch ) test_loss_op = tf.reduce_mean(tf.square(tf.subtract(predictions, label_test_batch))) tf.compat.v1.summary.scalar("Test loss", test_loss_op) test_op = test_loss_op # Training setup init = tf.initialize_all_variables() init_local = tf.compat.v1.local_variables_initializer() init_table = tf.compat.v1.tables_initializer(name="Initialize_all_tables") saver = tf.compat.v1.train.Saver() summaries = tf.compat.v1.summary.merge_all() with tf.compat.v1.Session() as sess: summary_writer = tf.compat.v1.summary.FileWriter(f'{tb_log_location}/{model_name}', sess.graph) sess.run(init) sess.run(init_local) sess.run(init_table) sess.run(train_iter.initializer) sess.run(val_iter.initializer) sess.run(test_iter.initializer) early_stop_steps = 100 best_val_acc = -1000.0 for step in range(1, max_steps + 1): if step % display_step == 0: _, train_loss, merged_summary = sess.run( [train_op, train_loss_op, summaries]) summary_writer.add_summary(merged_summary, step) print("Step " + str(step) + " , Training Loss = " + "{:.4f}".format(train_loss)) else: _ = sess.run(train_op) if step % val_step == 0: feed_arr = [ vars()["summaries"], vars()[f"val_op"], vars()[f"test_op"] ] val_arr = sess.run(feed_arr) merged_summary = val_arr[0] # this computes acc on one batch, we want it on all batches # val_acc = val_arr[1] test_acc_list = [] for i in range(len(test_df) // batch_size): test_acc_list.append(sess.run(test_acc_op)) val_acc = sum(test_acc_list) / len(test_acc_list) if val_acc > best_val_acc: best_val_acc = val_acc best_val_step = step if (step - best_val_step) > early_stop_steps: break print("Step " + str(step) + " , Val Metric = " + "{:.4f}".format(val_acc)) summary_writer.add_summary(merged_summary, step) if step % save_step == 0: saver.save(sess, "./checkpoints/" + model_name + ".ckpt") # validate on test and dump results print('Validating on test set ...') predictions_test_list = [] labels_test_list = [] test_acc_list = [] for i in range(len(test_df) // batch_size): pr_test, label_test, test_acc = sess.run([predictions_test, label_test_batch, test_acc_op]) predictions_test_list.append(pr_test) labels_test_list.append(label_test) test_acc_list.append(test_acc) predictions_test_concat = np.concatenate(predictions_test_list) labels_test_concat = np.concatenate(labels_test_list) avg_acc = sum(test_acc_list) / len(test_acc_list) print(f'Test predictions size: {predictions_test_concat.shape}') print(f'Test set size: {len(test_df)}') print(f'Did not preform inference on last {len(test_df) - predictions_test_concat.shape[0]} samples') print(f'Avg. accuracy on test set: {round(avg_acc, 5)}') print('Saving test predictions ...') pd.DataFrame(predictions_test_concat).to_csv(os.path.join(data_dir, 'test_pred.csv'), index=False) pd.DataFrame(labels_test_concat).to_csv(os.path.join(data_dir, 'test_labels.csv'), index=False) print(f'Best validation accuracy: {best_val_acc}')
def main(unused_argv): # Fix random seeds tf.set_random_seed(SEED) np.random.seed(SEED) # Define the TabNet model tabnet_forest_covertype = tabnet_model.TabNet( columns=data_helper_covertype.get_columns(), num_features=data_helper_covertype.NUM_FEATURES, feature_dim=4, output_dim=2, num_decision_steps=6, relaxation_factor=1.5, batch_momentum=0.7, virtual_batch_size=4, num_classes=data_helper_covertype.NUM_CLASSES) column_names = sorted(data_helper_covertype.FEATURE_COLUMNS) print( "Ordered column names, corresponding to the indexing in Tensorboard visualization" ) for fi in range(len(column_names)): print(str(fi) + " : " + column_names[fi]) # Input sampling/ Разбитие на несколько выборок. Для тренировки, для обучения, валидации train_batch = data_helper_covertype.input_fn(TRAIN_FILE, num_epochs=100000, shuffle=True, batch_size=BATCH_SIZE) val_batch = data_helper_covertype.input_fn( VAL_FILE, num_epochs=10000, shuffle=False, batch_size=data_helper_covertype.N_VAL_SAMPLES) test_batch = data_helper_covertype.input_fn( TEST_FILE, num_epochs=10000, shuffle=False, batch_size=data_helper_covertype.N_TEST_SAMPLES) # Создание итератора для последующего сохранения состояний. train_iter = train_batch.make_initializable_iterator() val_iter = val_batch.make_initializable_iterator() test_iter = test_batch.make_initializable_iterator() # Иициализация и запуск итератора под своими именами. feature_train_batch, label_train_batch = train_iter.get_next() feature_val_batch, label_val_batch = val_iter.get_next() feature_test_batch, label_test_batch = test_iter.get_next() # Define the model and losses # Прогнать обучающую выборку и результирующую по модели в прямом ходе encoded_train_batch, total_entropy = tabnet_forest_covertype.encoder( feature_train_batch, reuse=False, is_training=True) # Классифицировать результат на выходе logits_orig_batch, _ = tabnet_forest_covertype.classify( encoded_train_batch, reuse=False) # Эта функция вычисляет разреженную кросс-энтропию softmax между логитами и метками. # Другими словами, она измеряет вероятность ошибки в дискретных задачах классификации, # в которых классы являются взаимоисключающими. # Это означает, что каждый элемент данных принадлежит только одному классу. # Сверху все это усредняем. softmax_orig_key_op = tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits( logits=logits_orig_batch, labels=label_train_batch)) # Рассчитать ошибку на обучающих данных train_loss_op = softmax_orig_key_op + SPARSITY_LOSS_WEIGHT * total_entropy # Рассчитать сумму средних ошибок. tf.summary.scalar("Total loss", train_loss_op) # =======================Optimization step global_step = tf.train.get_or_create_global_step( ) # Returns and create (if necessary) the global step tensor. # При обучении модели часто рекомендуется снижать скорость обучения по мере продвижения обучения. # Эта функция применяет экспоненциальную функцию затухания к заданной начальной скорости обучения. learning_rate = tf.train.exponential_decay(INIT_LEARNING_RATE, global_step=global_step, decay_steps=DECAY_EVERY, decay_rate=DECAY_RATE) # Adam-алгоритм градиентной оптимизации # стохастических целевых функций первого порядка, основанный на адаптивных оценках моментов более низкого порядка optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate) # The standard library uses various well-known names to collect and retrieve values associated with a graph update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) # иногда полезно знать какая версия значения переменной была использована в любой конкретной точке времени, # для того чтобы принудительно заставлять перечитывать значение переменной после того как что-то произошло with tf.control_dependencies(update_ops): # Вычислить градиенты gvs = optimizer.compute_gradients(train_loss_op) # Обрезает значения тензора до заданных значений min и max. capped_gvs = [(tf.clip_by_value(grad, -GRADIENT_THRESH, GRADIENT_THRESH), var) for grad, var in gvs] # Применение обработанныx градиентoв train_op = optimizer.apply_gradients(capped_gvs, global_step=global_step) # ===================Model evaluation \ Оценка модели # ===================Validation performance # Прогнать валидационную выборку, указать, что это переиспользование и что не обучение. encoded_val_batch, _ = tabnet_forest_covertype.encoder(feature_val_batch, reuse=True, is_training=False) # Классификация результирующего набора. _, prediction_val = tabnet_forest_covertype.classify(encoded_val_batch, reuse=True) # Возвращает индекс и приводит тензор к новому типу. predicted_labels = tf.cast(tf.argmax(prediction_val, 1), dtype=tf.int32) # Сверка значений. val_eq_op = tf.equal(predicted_labels, label_val_batch) # Вычисляет среднее значение элементов по измерениям тензора. val_acc_op = tf.reduce_mean(tf.cast(val_eq_op, dtype=tf.float32)) # Сумма всех проверенных значений. tf.summary.scalar("Val accuracy", val_acc_op) # ===================Test performance # Прогнать тестовую выборку, указать, что не переиспользуется и что не обучение. encoded_test_batch, _ = tabnet_forest_covertype.encoder(feature_test_batch, reuse=True, is_training=False) # Классификация результирующего набора. _, prediction_test = tabnet_forest_covertype.classify(encoded_test_batch, reuse=True) # Возвращает индекс и приводит тензор к новому типу. predicted_labels = tf.cast(tf.argmax(prediction_test, 1), dtype=tf.int32) # Сверка значений test_eq_op = tf.equal(predicted_labels, label_test_batch) # Вычисляет среднее значение элементов по измерениям тензора. test_acc_op = tf.reduce_mean(tf.cast(test_eq_op, dtype=tf.float32)) # Сумма всех проверенных значений. tf.summary.scalar("Test accuracy", test_acc_op) # ===================Training setup # Наименование модели. model_name = "tabnet_forest_covertype_model" # Задание глобальных и локальных параметров. init = tf.initialize_all_variables() init_local = tf.local_variables_initializer() # Возвращает операцию, которая инициализирует все таблицы графика по умолчанию. init_table = tf.tables_initializer(name="Initialize_all_tables") # Сохранение всех значений переменных. saver = tf.train.Saver() # Объединяет все информацию, собранную по графу по умолчанию summaries = tf.summary.merge_all() # запись логов в заданную директорию with tf.Session() as sess: summary_writer = tf.summary.FileWriter("./tflog/" + model_name, sess.graph) # Этот метод выполняет один "шаг" вычисления TensorFlow, запуская необходимый фрагмент графа для выполнения # каждой операции и оценки каждого тензора в выборках, # подставляя значения в feed_dict для соответствующих входных значений. sess.run(init) sess.run(init_local) sess.run(init_table) # Учимся, Валидируем, Проверяем sess.run(train_iter.initializer) sess.run(val_iter.initializer) sess.run(test_iter.initializer) # Запуск записи результатов # for step in range(1, MAX_STEPS + 1): if step % DISPLAY_STEP == 0: _, train_loss, merged_summary = sess.run( [train_op, train_loss_op, summaries]) summary_writer.add_summary( merged_summary, step ) # Запись результатов работы на каждом шаге кратный 0. Шаг, величина ошибки. print("Step " + str(step) + " , Training Loss = " + "{:.4f}".format(train_loss)) else: _ = sess.run( train_op ) # Запуск расчета работы с оптимизированными градиентами. if step % VAL_STEP == 0: feed_arr = [ vars()["summaries"], vars()["val_acc_op"], vars()["test_acc_op"] ] # Запись валидационных результатов и результатов на тестовой выборке. val_arr = sess.run(feed_arr) merged_summary = val_arr[0] val_acc = val_arr[1] print("Step " + str(step) + " , Val Accuracy = " + "{:.4f}".format(val_acc)) summary_writer.add_summary(merged_summary, step) # сохранение точек и параметров. if step % SAVE_STEP == 0: saver.save(sess, "./checkpoints/" + model_name + ".ckpt")