def feature_to_torch(self, all_data):
for i, data in enumerate(all_data):
if "example_id" not in data:
data["example_id"] = ''.join([
random.choice(string.ascii_letters + string.digits)
for n in range(8)
])
data["snt_id"] = seq_to_id(self.dicts["word_dict"], data["tok"])[0]
data["lemma_id"] = seq_to_id(self.dicts["lemma_dict"],
data["lem"])[0]
data["pos_id"] = seq_to_id(self.dicts["pos_dict"], data["pos"])[0]
data["ner_id"] = seq_to_id(self.dicts["ner_dict"], data["ner"])[0]
# when using all_data, batch_size is all
if self.opt.bert_model:
data_iterator = BertDataIterator([],
self.opt,
self.dicts["ucca_rel_dict"],
all_data=all_data)
order, idsBatch, srcBatch, src_charBatch, sourceBatch, srcBertBatch = data_iterator[
0]
else:
data_iterator = DataIterator([],
self.opt,
self.dicts["ucca_rel_dict"],
all_data=all_data)
order, idsBatch, srcBatch, src_charBatch, sourceBatch = data_iterator[
0]
srcBertBatch = None
return order, idsBatch, srcBatch, src_charBatch, sourceBatch, srcBertBatch, data_iterator
def feature_to_torch(self, all_data):
for i, data in enumerate(all_data):
data["snt_id"] = seq_to_id(self.dicts["word_dict"], data["tok"])[0]
data["lemma_id"] = seq_to_id(self.dicts["lemma_dict"],
data["lem"])[0]
data["pos_id"] = seq_to_id(self.dicts["pos_dict"], data["pos"])[0]
data["ner_id"] = seq_to_id(self.dicts["ner_dict"], data["ner"])[0]
data_iterator = DataIterator([],
self.opt,
self.dicts["rel_dict"],
volatile=True,
all_data=all_data)
order, srcBatch, src_sourceBatch = data_iterator[0]
return order, srcBatch, src_sourceBatch, data_iterator