def conll_data_reader(self, filePath):
data = BBNDataReader.readFile(filePath=filePath)
return data
'json_filePath':
'/datastore/liu121/nosqldb2/cadec/json/draw.json',
'max_len': 200
}
else:
data_config = {
'Conll_filePath':
'/datastore/liu121/nosqldb2/cadec/Conll/data_test_eval',
'table_filePath':
'/datastore/liu121/nosqldb2/bbn_data/table.pkl',
'iteration': 5,
'json_filePath':
'/datastore/liu121/nosqldb2/cadec/json/eval.json',
'max_len': 200
}
data = BBNDataReader.readFile(filePath=data_config['Conll_filePath'])
ds = DataSplit(data_config)
t2sentenceID = ds.mct_gen(data)
samples = {}
for i in range(data_config['iteration']):
for j in [1, 2, 4, 8, 16]:
sample = ds.split(t2sentenceID, True, k=j)
if str(j) not in samples:
samples[str(j)] = {str(i): sample}
else:
samples[str(j)][str(i)] = sample
# print('{}, {}, {}'.format(str(j),str(i),str(len(sample))))
json_filePath = data_config['json_filePath']
with open(json_filePath, 'w+') as f:
json.dump(samples, f, indent=4, sort_keys=False)
import sys
sys.path.append('/home/liu121/dlnlp')
from nerd.data.util.readers.BBNDataReader import BBNDataReader
data = BBNDataReader.readFile(
filePath='/datastore/liu121/nosqldb2/bbn_data/data_test_draw')
print(data.text[2])
print(data.labels[2])
def conll_data_reader(self):
data = BBNDataReader.readFile(filePath=self.data_config['file_path'])
return data