def setUp(self):
"""
:return:
"""
self.wikipedia_abstract_xml = APP_ROOT + "/Data/jawiki-20160901-abstract_part.xml"
self.wiki_pedia_xml_to_json = WikiPediaXmlToJson(
self.wikipedia_abstract_xml)
self.answer_data = APP_ROOT + "/Data/jawiki-20160901-abstract_part0.json"
self.correct_data = APP_ROOT + "/Data/answer.json"
def setUp(self):
"""
:return:
"""
self.wikipedia_abstract_xml = APP_ROOT + "/../../Data/wiki_image/enwiki-20080103-abstract_part.xml"
self.wiki_pedia_xml_to_json = WikiPediaXmlToJson(
self.wikipedia_abstract_xml)
self.answer_data = APP_ROOT + "/../../Data/wiki_image/enwiki-20080103-abstract_part0.json"
self.correct_data = APP_ROOT + "/../../Data/wiki_image/answer.json"
default='jawiki-20160901-abstract_part.xml',
help='set xml file')
parser.add_argument('--img_flag',
'-img',
default="False",
help='set image Flag')
parser.add_argument('--set_worker',
'-work',
default=2,
help='set image Flag')
args = parser.parse_args()
Image_Flag = ast.literal_eval(args.img_flag)
if Image_Flag is True:
wikipedia_abstract_xml = APP_ROOT + "/../Data/wiki_image/" + args.xml_file
else:
wikipedia_abstract_xml = APP_ROOT + "/../Data/jawiki-20160901-abstract_dir/" + args.xml_file
wiki_pedia_xml_to_json = WikiPediaXmlToJson(wikipedia_abstract_xml)
wiki_pedia_xml_to_json.input(image_Flag=Image_Flag)
# Multi Process
with ProcessPoolExecutor() as executor:
# executor.map(wiki_pedia_xml_to_json.extract_contents, wiki_pedia_xml_to_json.xml_data)
all_process = []
for xml_name in wiki_pedia_xml_to_json.xml_data:
process = executor.submit(wiki_pedia_xml_to_json.extract_contents,
xml_name)
all_process.append(process)
for process in as_completed(all_process):
print(process.result())
