tf.flags.DEFINE_boolean("allow_soft_placement", True, "Allow device soft device placement")
tf.flags.DEFINE_boolean("log_device_placement", False, "Log placement of ops on devices")
tf.flags.DEFINE_boolean("verbose_for_debugging", True, "Allow info to be printed to understand the behaviour of the network")
tf.flags.DEFINE_boolean("verbose_for_experiments", True, "Print only the perplexity")
FLAGS = tf.flags.FLAGS
# Prepare data
# Load data
print("Load vocabulary list \n")
vocab, generated_embeddings = preprocess_helper.load_frequent_words_and_embeddings(FLAGS.vocab_with_emb_path)
print("Loading and preprocessing test dataset \n")
x_test, y_test = preprocess_helper.load_and_process_data(FLAGS.data_file_path,
vocab,
FLAGS.sentence_length,
pad_sentence=False)
## EVALUATION ##
checkpoint_file = tf.train.latest_checkpoint(FLAGS.checkpoint_dir)
graph = tf.Graph()
with graph.as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
sess = tf.Session(config=session_conf)
with sess.as_default():
# Load the saved meta graph and restore variables
saver = tf.train.import_meta_graph("{}.meta".format(checkpoint_file))
saver.restore(sess, checkpoint_file)
"inter_op_parallelism_threads available in each process.")
tf.flags.DEFINE_integer(
"intra_op_parallelism_threads", 0,
"The execution of an individual op (for some op types) can be parallelized"
" on a pool of intra_op_parallelism_threads.")
FLAGS = tf.flags.FLAGS
# Prepare the data
print("Load vocabulary list \n")
vocab, generated_embeddings = preprocess_helper.load_frequent_words_and_embeddings(
FLAGS.vocab_with_emb_path)
print("Loading and preprocessing training and validation datasets \n")
data, labels = preprocess_helper.load_and_process_data(FLAGS.data_file_path,
vocab,
FLAGS.sentence_length)
# Randomly shuffle data
np.random.seed(10)
shuffled_indices = np.random.permutation(len(labels))
data = data[shuffled_indices]
labels = labels[shuffled_indices]
# Split train/dev sets
val_sample_index = -1 * int(FLAGS.val_sample_percentage * float(len(labels)))
x_train, x_val = data[:val_sample_index], data[val_sample_index:]
y_train, y_val = labels[:val_sample_index], labels[val_sample_index:]
# Summary of the loaded data
print('Loaded: ', len(x_train), ' samples for training')