def build_eval_graph(self, words, pos, chunks, capitals):
# convert to tensor of strings
split_sentence = tf.string_split(words, " ")
split_pos = tf.string_split(pos, ' ')
split_chunks = tf.string_split(chunks, ' ')
split_capitals = tf.string_split(capitals, ' ')
# convert sparse to dense
dense_words = tf.sparse_tensor_to_dense(split_sentence, default_value="")
dense_pos = tf.sparse_tensor_to_dense(split_pos, default_value="")
dense_chunks = tf.sparse_tensor_to_dense(split_chunks, default_value="")
dense_capitals = tf.sparse_tensor_to_dense(split_capitals, default_value="")
# do table lookup
table_words = self.table.lookup(dense_words)
table_pos = self.table_pos.lookup(dense_pos)
table_chunks = self.table_chunk.lookup(dense_chunks)
table_capitals = tf.string_to_number(dense_capitals, out_type=tf.int64)
return table_words, table_pos, table_chunks, table_capitals
def build_eval_graph(self, x):
# convert to tensor of strings
split_example = tf.string_split(x, " ")
# convert sparse to dense
tensor_entry = tf.sparse_tensor_to_dense(split_example,
default_value="")
tensor_entry = self.table.lookup(tensor_entry)
return tensor_entry
def build_eval_graph(self, words, pos, chunks, capitals):
# convert to tensor of strings
split_sentence = tf.string_split(words, " ")
split_pos = tf.string_split(pos, ' ')
split_chunks = tf.string_split(chunks, ' ')
split_capitals = tf.string_split(capitals, ' ')
# convert sparse to dense
dense_words = tf.sparse_tensor_to_dense(split_sentence,
default_value="")
dense_pos = tf.sparse_tensor_to_dense(split_pos, default_value="")
dense_chunks = tf.sparse_tensor_to_dense(split_chunks,
default_value="")
dense_capitals = tf.sparse_tensor_to_dense(split_capitals,
default_value="")
# do table lookup
table_words = self.table.lookup(dense_words)
table_pos = self.table_pos.lookup(dense_pos)
table_chunks = self.table_chunk.lookup(dense_chunks)
table_capitals = tf.string_to_number(dense_capitals, out_type=tf.int64)
return table_words, table_pos, table_chunks, table_capitals
def __load_data(self, file_names, record_defaults, data_column, bucket_boundaries, field_delim=__DEFAULT_DELIM,
skip_header_lines=0,
num_epochs=None, shuffle=True):
original_file_names = file_names[:]
file_names = self.__generate_preprocessed_files(file_names, data_column, field_delim=field_delim)
filename_queue = tf.train.string_input_producer(
file_names, num_epochs=num_epochs, shuffle=shuffle
)
sentence, pos, chunks, capitals, entities = self._read_file(filename_queue, record_defaults, field_delim,
skip_header_lines)
voca_path, voca_suffix = BaseDataLoader._split_file_to_path_and_name(
original_file_names[0]) # TODO: will be break with multiple filenames
voca_name = ConllPreprocessor.VOCABULARY_PREFIX + voca_suffix
self.__vocabulary_file = voca_path + voca_name
# load look up tables that maps words to ids
if self.table is None:
print('vocabulary table is None => creating it')
main_voca_file = voca_path + voca_name
if self._use_pretrained_emb:
self.pretrained_emb_matrix, vocabulary = self.preload_embeddings(embed_dim=self._embed_dim,
file_name=self._pretrained_emb_file,
train_vocabulary=main_voca_file,
other_vocabularies=self._other_voca_files)
tensor_vocabulary = tf.constant(vocabulary)
self.table = tf.contrib.lookup.index_table_from_tensor(tensor_vocabulary,
default_value=ConllPreprocessor.UNK_TOKEN_ID,
num_oov_buckets=0)
else:
self.table = tf.contrib.lookup.index_table_from_file(vocabulary_file=main_voca_file,
default_value=ConllPreprocessor.UNK_TOKEN_ID,
num_oov_buckets=0)
if self.table_pos is None:
print('vocabulary table_pos is None => creating it')
self.table_pos = tf.contrib.lookup.index_table_from_file(
vocabulary_file=voca_path + self._TABLE_POS + voca_suffix,
num_oov_buckets=0)
if self.table_chunk is None:
print('vocabulary table_chunk is None => creating it')
self.table_chunk = tf.contrib.lookup.index_table_from_file(
vocabulary_file=voca_path + self._TABLE_CHUNK + voca_suffix,
num_oov_buckets=0)
if self.table_entity is None:
print('vocabulary table_entity is None => creating it')
self.table_entity = tf.contrib.lookup.index_table_from_file(
vocabulary_file=voca_path + self._TABLE_ENTITY + voca_suffix,
num_oov_buckets=0)
if self._used_for_test_data:
print('Reverse vocabulary is needed => creating it')
self.reverse_table = tf.contrib.lookup.index_to_string_table_from_file(
vocabulary_file=voca_path + voca_name)
print('Reverse entity vocabulary is needed => creating it')
self.reverse_table_entity = tf.contrib.lookup.index_to_string_table_from_file(
vocabulary_file=voca_path + self._TABLE_ENTITY + voca_suffix)
# convert to tensor of strings
split_sentence = tf.string_split([sentence], " ")
split_pos = tf.string_split([pos], ' ')
split_chunks = tf.string_split([chunks], ' ')
split_capitals = tf.string_split([capitals], ' ')
split_entities = tf.string_split([entities], ' ')
# determine lengths of sequences
line_number = split_sentence.indices[:, 0]
line_position = split_sentence.indices[:, 1]
lengths = (tf.segment_max(data=line_position,
segment_ids=line_number) + 1).sg_cast(dtype=tf.int32)
# convert sparse to dense
dense_sent = tf.sparse_tensor_to_dense(split_sentence, default_value="")
dense_sent = self.table.lookup(dense_sent)
dense_pos = tf.sparse_tensor_to_dense(split_pos, default_value="")
dense_pos = self.table_pos.lookup(dense_pos)
dense_chunks = tf.sparse_tensor_to_dense(split_chunks, default_value="")
dense_chunks = self.table_chunk.lookup(dense_chunks)
dense_capitals = tf.sparse_tensor_to_dense(split_capitals, default_value="")
dense_capitals = tf.string_to_number(dense_capitals, out_type=tf.int64)
dense_entities = tf.sparse_tensor_to_dense(split_entities, default_value="")
dense_entities = self.table_entity.lookup(dense_entities)
# get the enqueue op to pass to a coordinator to be run
self.enqueue_op = self.shuffle_queue.enqueue(
[dense_sent, dense_pos, dense_chunks, dense_capitals, dense_entities])
dense_sent, dense_pos, dense_chunks, dense_capitals, dense_entities = self.shuffle_queue.dequeue()
# add queue to queue runner
self.qr = tf.train.QueueRunner(self.shuffle_queue, [self.enqueue_op] * self.num_threads)
tf.train.queue_runner.add_queue_runner(self.qr)
# reshape from <unknown> shape into proper form after dequeue from random shuffle queue
# this is needed so next queue can automatically infer the shape properly
dense_sent = dense_sent.sg_reshape(shape=[1, -1])
dense_pos = dense_pos.sg_reshape(shape=[1, -1])
dense_chunks = dense_chunks.sg_reshape(shape=[1, -1])
dense_capitals = dense_capitals.sg_reshape(shape=[1, -1])
dense_entities = dense_entities.sg_reshape(shape=[1, -1])
_, (padded_sent, padded_pos, padded_chunk, padded_capitals, padded_entities) = \
tf.contrib.training.bucket_by_sequence_length(lengths,
[dense_sent, dense_pos, dense_chunks, dense_capitals,
dense_entities],
batch_size=self._batch_size,
bucket_boundaries=bucket_boundaries,
dynamic_pad=True,
capacity=self._capacity,
num_threads=self.num_threads, name='bucket_queue')
# reshape shape into proper form after dequeue from bucket queue
padded_sent = padded_sent.sg_reshape(shape=[self._batch_size, -1])
padded_pos = padded_pos.sg_reshape(shape=[self._batch_size, -1])
padded_chunk = padded_chunk.sg_reshape(shape=[self._batch_size, -1])
padded_capitals = padded_capitals.sg_reshape(shape=[self._batch_size, -1, 1])
padded_entities = padded_entities.sg_reshape(shape=[self._batch_size, -1])
return padded_sent, padded_pos, padded_chunk, padded_capitals, padded_entities
initial=0,
desc='test',
ncols=70,
unit='b',
leave=False)
# batch loop
loss_avg = 0.
for _ in iterator:
# run session
batch_loss = None
#batch_loss = sess.run(loss)
adv, diff, orig_x, preds, target, predsx, filenames = sess.run(
[adv_x, diff_x, x, preds_adv, y, preds_x, filenames_t])
preds = tf.sparse_tensor_to_dense(preds, default_value=-1).eval()
predsx = tf.sparse_tensor_to_dense(predsx, default_value=-1).eval()
for p, px, t, filename in zip(preds, predsx, target, filenames):
p = [(int(ch) + 1) for ch in p if ch != -1]
str_p = index2str(p)
t = [ch for ch in t if ch != 0]
str_t = index2str(t)
px = [(int(ch) + 1) for ch in px if ch != -1]
str_px = index2str(px)
if px != p: correct = "DIFF"
else: correct = "SAME"
def __load_data(self,
file_names,
record_defaults,
data_column,
bucket_boundaries,
field_delim=__DEFAULT_DELIM,
skip_header_lines=0,
num_epochs=None,
shuffle=True):
original_file_names = file_names[:]
file_names = self.__generate_preprocessed_files(
file_names, data_column, field_delim=field_delim)
filename_queue = tf.train.string_input_producer(file_names,
num_epochs=num_epochs,
shuffle=shuffle)
sentence, pos, chunks, capitals, entities = self._read_file(
filename_queue, record_defaults, field_delim, skip_header_lines)
voca_path, voca_suffix = BaseDataLoader._split_file_to_path_and_name(
original_file_names[0]
) # TODO: will be break with multiple filenames
voca_name = ConllPreprocessor.VOCABULARY_PREFIX + voca_suffix
self.__vocabulary_file = voca_path + voca_name
# load look up tables that maps words to ids
if self.table is None:
print('vocabulary table is None => creating it')
main_voca_file = voca_path + voca_name
if self._use_pretrained_emb:
self.pretrained_emb_matrix, vocabulary = self.preload_embeddings(
embed_dim=self._embed_dim,
file_name=self._pretrained_emb_file,
train_vocabulary=main_voca_file,
other_vocabularies=self._other_voca_files)
tensor_vocabulary = tf.constant(vocabulary)
self.table = tf.contrib.lookup.index_table_from_tensor(
tensor_vocabulary,
default_value=ConllPreprocessor.UNK_TOKEN_ID,
num_oov_buckets=0)
else:
self.table = tf.contrib.lookup.index_table_from_file(
vocabulary_file=main_voca_file,
default_value=ConllPreprocessor.UNK_TOKEN_ID,
num_oov_buckets=0)
if self.table_pos is None:
print('vocabulary table_pos is None => creating it')
self.table_pos = tf.contrib.lookup.index_table_from_file(
vocabulary_file=voca_path + self._TABLE_POS + voca_suffix,
num_oov_buckets=0)
if self.table_chunk is None:
print('vocabulary table_chunk is None => creating it')
self.table_chunk = tf.contrib.lookup.index_table_from_file(
vocabulary_file=voca_path + self._TABLE_CHUNK + voca_suffix,
num_oov_buckets=0)
if self.table_entity is None:
print('vocabulary table_entity is None => creating it')
self.table_entity = tf.contrib.lookup.index_table_from_file(
vocabulary_file=voca_path + self._TABLE_ENTITY + voca_suffix,
num_oov_buckets=0)
if self._used_for_test_data:
print('Reverse vocabulary is needed => creating it')
self.reverse_table = tf.contrib.lookup.index_to_string_table_from_file(
vocabulary_file=voca_path + voca_name)
print('Reverse entity vocabulary is needed => creating it')
self.reverse_table_entity = tf.contrib.lookup.index_to_string_table_from_file(
vocabulary_file=voca_path + self._TABLE_ENTITY + voca_suffix)
# convert to tensor of strings
split_sentence = tf.string_split([sentence], " ")
split_pos = tf.string_split([pos], ' ')
split_chunks = tf.string_split([chunks], ' ')
split_capitals = tf.string_split([capitals], ' ')
split_entities = tf.string_split([entities], ' ')
# determine lengths of sequences
line_number = split_sentence.indices[:, 0]
line_position = split_sentence.indices[:, 1]
lengths = (
tf.segment_max(data=line_position, segment_ids=line_number) +
1).sg_cast(dtype=tf.int32)
# convert sparse to dense
dense_sent = tf.sparse_tensor_to_dense(split_sentence,
default_value="")
dense_sent = self.table.lookup(dense_sent)
dense_pos = tf.sparse_tensor_to_dense(split_pos, default_value="")
dense_pos = self.table_pos.lookup(dense_pos)
dense_chunks = tf.sparse_tensor_to_dense(split_chunks,
default_value="")
dense_chunks = self.table_chunk.lookup(dense_chunks)
dense_capitals = tf.sparse_tensor_to_dense(split_capitals,
default_value="")
dense_capitals = tf.string_to_number(dense_capitals, out_type=tf.int64)
dense_entities = tf.sparse_tensor_to_dense(split_entities,
default_value="")
dense_entities = self.table_entity.lookup(dense_entities)
# get the enqueue op to pass to a coordinator to be run
self.enqueue_op = self.shuffle_queue.enqueue([
dense_sent, dense_pos, dense_chunks, dense_capitals, dense_entities
])
dense_sent, dense_pos, dense_chunks, dense_capitals, dense_entities = self.shuffle_queue.dequeue(
)
# add queue to queue runner
self.qr = tf.train.QueueRunner(self.shuffle_queue,
[self.enqueue_op] * self.num_threads)
tf.train.queue_runner.add_queue_runner(self.qr)
# reshape from <unknown> shape into proper form after dequeue from random shuffle queue
# this is needed so next queue can automatically infer the shape properly
dense_sent = dense_sent.sg_reshape(shape=[1, -1])
dense_pos = dense_pos.sg_reshape(shape=[1, -1])
dense_chunks = dense_chunks.sg_reshape(shape=[1, -1])
dense_capitals = dense_capitals.sg_reshape(shape=[1, -1])
dense_entities = dense_entities.sg_reshape(shape=[1, -1])
_, (padded_sent, padded_pos, padded_chunk, padded_capitals, padded_entities) = \
tf.contrib.training.bucket_by_sequence_length(lengths,
[dense_sent, dense_pos, dense_chunks, dense_capitals,
dense_entities],
batch_size=self._batch_size,
bucket_boundaries=bucket_boundaries,
dynamic_pad=True,
capacity=self._capacity,
num_threads=self.num_threads, name='bucket_queue')
# reshape shape into proper form after dequeue from bucket queue
padded_sent = padded_sent.sg_reshape(shape=[self._batch_size, -1])
padded_pos = padded_pos.sg_reshape(shape=[self._batch_size, -1])
padded_chunk = padded_chunk.sg_reshape(shape=[self._batch_size, -1])
padded_capitals = padded_capitals.sg_reshape(
shape=[self._batch_size, -1, 1])
padded_entities = padded_entities.sg_reshape(
shape=[self._batch_size, -1])
return padded_sent, padded_pos, padded_chunk, padded_capitals, padded_entities
def __load_batch(self,
file_names,
record_defaults,
data_column,
bucket_boundaries,
field_delim=_CSV_DELIM,
skip_header_lines=0,
num_epochs=None,
shuffle=True):
original_file_names = file_names[:]
file_names = self.__generate_preprocessed_files(
file_names,
data_column,
bucket_boundaries,
field_delim=field_delim)
filename_queue = tf.train.string_input_producer(file_names,
num_epochs=num_epochs,
shuffle=shuffle)
example, label = self._read_file(filename_queue, record_defaults,
field_delim, skip_header_lines)
voca_path, voca_name = BaseDataLoader._split_file_to_path_and_name(
original_file_names[0]
) # TODO: will be break with multiple filenames
voca_name = KagglePreprocessor.VOCABULARY_PREFIX + voca_name
self.__vocabulary_file = voca_path + voca_name
# load look up table that maps words to ids
self.table = tf.contrib.lookup.index_table_from_file(
vocabulary_file=voca_path + voca_name,
default_value=KagglePreprocessor.UNK_TOKEN_ID,
num_oov_buckets=0)
# convert to tensor of strings
split_example = tf.string_split([example], " ")
# determine lengths of sequences
line_number = split_example.indices[:, 0]
line_position = split_example.indices[:, 1]
lengths = (
tf.segment_max(data=line_position, segment_ids=line_number) +
1).sg_cast(dtype=tf.int32)
# convert sparse to dense
dense_example = tf.sparse_tensor_to_dense(split_example,
default_value="")
dense_example = self.table.lookup(dense_example)
# get the enqueue op to pass to a coordintor to be run
self.enqueue_op = self.shuffle_queue.enqueue([dense_example, label])
dense_example, label = self.shuffle_queue.dequeue()
# add queue to queue runner
self.qr = tf.train.QueueRunner(self.shuffle_queue,
[self.enqueue_op] * self.num_threads)
tf.train.queue_runner.add_queue_runner(self.qr)
# reshape from <unknown> shape into proper form after dequeue from random shuffle queue
# this is needed so next queue can automatically infer the shape properly
dense_example = dense_example.sg_reshape(shape=[1, -1])
label = label.sg_reshape(shape=[1])
_, (padded_examples,
label_examples) = tf.contrib.training.bucket_by_sequence_length(
lengths, [dense_example, label],
batch_size=self._batch_size,
bucket_boundaries=bucket_boundaries,
dynamic_pad=True,
capacity=self._capacity,
num_threads=self._num_threads)
# reshape shape into proper form after dequeue from bucket queue
padded_examples = padded_examples.sg_reshape(
shape=[self._batch_size, -1])
label_examples = label_examples.sg_reshape(shape=[self._batch_size])
return padded_examples, label_examples
def train_loop():
with tf.device("/cpu:0"):
# Launch the graph
with tf.Session(graph=graph, config=config) as sess:
print("Starting Tensorboard...")
initstart = time.time()
train_writer = tf.summary.FileWriter(logs_path + '/TRAIN',
graph=sess.graph)
test_writer = tf.summary.FileWriter(logs_path + '/TEST',
graph=sess.graph)
run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE,
output_partition_graphs=True)
run_metadata = tf.RunMetadata()
tf.global_variables_initializer().run()
saver = tf.train.Saver()
#Load paths
for curr_epoch in range(num_epochs):
print('>>>', time.strftime('[%H:%M:%S]'), 'Epoch',
curr_epoch + 1, '/', num_epochs)
train_cost = train_ler = 0
start = t_time = time.time()
index_list = range(0, datasetsize)
for batch in range(num_batches_per_epoch):
# Getting the index
indexes = random.sample(index_list, batchsize)
index_list = [x for x in index_list if x not in indexes]
train_inputs = next_miniBatch(indexes, dr[0])
train_targets = next_target_miniBatch(indexes, dr[1])
#train_inputs,train_targets = fake_data(num_examples,num_mfccs,num_classes-1)
newindex = [i % num_examples for i in range(batchsize)]
random.shuffle(newindex)
batch_train_inputs = train_inputs[newindex]
# Padding input to max_time_step of this batch
batch_train_inputs, batch_train_seq_len = pad_sequences(
batch_train_inputs)
#for x in range(batchsize):
# print('>>>'+str(x)+': ',train_targets[newindex][x].size,batch_train_seq_len[x],dr[0][x])
# print(decode_to_chars(train_targets[newindex][x]))
#if train_targets[newindex][x].size > batch_train_seq_len[x]:
# Converting to sparse representation so as to to feed SparseTensor input
batch_train_targets = sparse_tuple_from(
train_targets[newindex])
#saveImg(batch_train_inputs)
feed = {
inputs: batch_train_inputs,
targets: batch_train_targets,
seq_len: batch_train_seq_len
}
batch_cost, _, l = sess.run(
[cost, train_optimizer, ler],
feed,
options=run_options) #,run_metadata = run_metadata)
train_cost += batch_cost * batchsize
train_ler += l * batchsize
print('[' + str(curr_epoch) + ']', ' >>>',
time.strftime('[%H:%M:%S]'), 'Batch', batch + 1, '/',
num_batches_per_epoch, '@Cost', batch_cost,
'Time Elapsed',
time.time() - t_time, 's')
t_time = time.time()
if (batch % 16 == 0):
summary = sess.run(
merged, feed_dict=feed,
options=run_options) #,run_metadata=run_metadata)
train_writer.add_summary(
summary,
int(batch + (curr_epoch * num_batches_per_epoch)))
#train_writer.add_run_metadata(run_metadata, 'step%03d' % int(batch+(curr_epoch*num_batches_per_epoch)))
train_writer.flush()
# Metrics mean
train_cost /= num_examples
train_ler /= num_examples
#Testing
print('>>>', time.strftime('[%H:%M:%S]'),
'Evaluating Test Accuracy...')
t_index = random.sample(range(0, testsetsize), testbatchsize)
test_inputs = next_miniBatch(t_index, t_dr[0], test=True)
test_targets = next_target_miniBatch(t_index, t_dr[1])
newindex = [i % testbatchsize for i in range(testbatchsize)]
batch_test_inputs = test_inputs[newindex]
batch_test_inputs, batch_test_seq_len = pad_sequences(
batch_test_inputs, test=True)
batch_test_targets = sparse_tuple_from(test_targets[newindex])
t_feed = {
inputs: batch_test_inputs,
targets: batch_test_targets,
seq_len: batch_test_seq_len
}
test_ler, d = sess.run(
(ler, decoded[0]), feed_dict=t_feed,
options=run_options) #,run_metadata = run_metadata)
dense_decoded = tf.sparse_tensor_to_dense(
d, default_value=-1).eval(session=sess)
for i, seq in enumerate(dense_decoded):
seq = [s for s in seq if s != -1]
tmp_o = decode_to_chars(test_targets[i])
tmp_d = decode_to_chars(seq)
print('Sequence %d' % i)
print('\t Original:\n%s' % tmp_o)
print('\t Decoded:\n%s' % tmp_d)
#print('\t Corrected:\n%s' % tmp_corr)
print('Done!')
log = "Epoch {}/{} | Batch Cost : {:.3f} | Train Accuracy : {:.3f}% | Test Accuracy : {:.3f}% | Time Elapsed : {:.3f}s"
print(
log.format(curr_epoch + 1, num_epochs, train_cost,
100 - (train_ler * 100), 100 - (test_ler * 100),
time.time() - start))
t_summary = sess.run(
merged, feed_dict=t_feed,
options=run_options) #, run_metadata=run_metadata)
test_writer.add_summary(
t_summary,
int(batch + (curr_epoch * num_batches_per_epoch)))
#test_writer.add_run_metadata(run_metadata, 'step%03d' % int(batch+(curr_epoch*num_batches_per_epoch)))
test_writer.flush()
save_path = saver.save(sess, savepath + '/model')
print(">>> Model saved succesfully")
print('Total Training Time: ' + str(time.time() - initstart) + 's')
