def populate_new_parses(auto_parsed_root):
file_names = glob.glob('%s/*' % auto_parsed_root)
procs = [Process(target=populate_parses_from_files, args=(chunk,)) for chunk in slice_list(file_names, 8)]
with Timer('Populate the database with parses'):
for p in procs:
p.start()
for p in procs:
p.join()
if not isinstance(coref_chain_list, list):
coref_chain_list = [coref_chain_list]
for coref_chain in coref_chain_list:
mentions = []
for mention in coref_chain['mention']:
mentions.append((mention['text'], int(mention['sentence']), int(mention['head']), \
int(mention['start']), int(mention['end'])))
new_coref_chain_list.append(mentions)
return new_coref_chain_list
def convert_files(file_names):
for file_name in file_names:
print file_name
parse_dict = xml2json(open(file_name).read())
json_file_name = file_name.rsplit('.')[0] + '.json'
new_file = open(json_file_name, mode='w')
new_file.write(json.dumps(parse_dict))
new_file.close()
if __name__ == '__main__':
procs = [
Process(target=convert_files, args=(chunk, ))
for chunk in slice_list(sys.argv[1:], 12)
]
for p in procs:
p.start()
for p in procs:
p.join()
for mention in coref_chain["mention"]:
mentions.append(
(
mention["text"],
int(mention["sentence"]),
int(mention["head"]),
int(mention["start"]),
int(mention["end"]),
)
)
new_coref_chain_list.append(mentions)
return new_coref_chain_list
def convert_files(file_names):
for file_name in file_names:
print file_name
parse_dict = xml2json(open(file_name).read())
json_file_name = file_name.rsplit(".")[0] + ".json"
new_file = open(json_file_name, mode="w")
new_file.write(json.dumps(parse_dict))
new_file.close()
if __name__ == "__main__":
procs = [Process(target=convert_files, args=(chunk,)) for chunk in slice_list(sys.argv[1:], 12)]
for p in procs:
p.start()
for p in procs:
p.join()