def generate_data(self):
"""Generate data for offline training."""
text, label = load_seq_label_raw_data(
paths=self.paths,
mode=self.mode,
infer_no_label=self.infer_no_label)
text_placeholder = tf.placeholder(tf.string, name="text")
label_placeholder = tf.placeholder(tf.string, name="label")
self.init_feed_dict[text_placeholder] = text
self.init_feed_dict[label_placeholder] = label
text_ds = self.load_text_dataset(text_placeholder)
if self.infer_without_label:
data_set = text_ds
else:
label_ds = load_multi_label_dataset(label_placeholder, self.config)
data_set = tf.data.Dataset.zip((text_ds, label_ds))
self.config['data']['vocab_size'] = get_vocab_size(
self.text_vocab_file_path)
self.config['data']['{}_data_size'.format(self.mode)] = len(text)
return data_set
def generate_data(self):
"""Generate data for offline training."""
text, label = load_seq_label_raw_data(
paths=self.paths,
mode=self.mode,
infer_no_label=self.infer_no_label)
text_ds = self.load_text_dataset(text)
if self.infer_without_label:
data_set = text_ds
else:
label_ds = load_multi_label_dataset(label, self.config)
data_set = tf.data.Dataset.zip((text_ds, label_ds))
self.config['data']['vocab_size'] = get_vocab_size(
self.text_vocab_file_path)
self.config['data']['{}_data_size'.format(self.mode)] = len(text)
return data_set
def test_load_seq_label_raw_data(self): mode = utils.TRAIN paths = self.config["data"]["train"]["paths"] text, label = load_seq_label_raw_data(paths, mode) self.assertEqual(text[0], "I feel good .") self.assertEqual(label[0], "O O O O")