def fuse_items(): # Merge categories from wikipedias. if flags.arg.merge_categories: log.info("Merge wikipedia categories") wf = wiki.WikiWorkflow("category-merging") wf.merge_wikipedia_categories() workflow.run(wf.wf) # Invert categories. if flags.arg.invert_categories: log.info("Invert categories") wf = wiki.WikiWorkflow("category-inversion") wf.invert_wikipedia_categories() workflow.run(wf.wf) # Compute item popularity. if flags.arg.compute_item_popularity: log.info("Compute item popularity") wf = wiki.WikiWorkflow("item-popularity") wf.compute_item_popularity() workflow.run(wf.wf) # Fuse items. if flags.arg.fuse_items: log.info("Fuse items") wf = wiki.WikiWorkflow("fuse-items") wf.fuse_items() workflow.run(wf.wf)
def build_knowledge_base(): # Build knowledge base repository. if flags.arg.build_kb: log.info("Build knowledge base repository") wf = wiki.WikiWorkflow("knowledge-base") wf.build_knowledge_base() workflow.run(wf.wf) # Extract item names from wikidata and wikipedia. if flags.arg.extract_names: for language in flags.arg.languages: log.info("Extract " + language + " names") wf = wiki.WikiWorkflow(language + "-name-extraction") wf.extract_names(language=language) workflow.run(wf.wf) # Build name table. if flags.arg.build_nametab: for language in flags.arg.languages: log.info("Build " + language + " name table") wf = wiki.WikiWorkflow(language + "-name-table") wf.build_name_table(language=language) workflow.run(wf.wf) # Build phrase table. if flags.arg.build_phrasetab: for language in flags.arg.languages: log.info("Build " + language + " phrase table") wf = wiki.WikiWorkflow(language + "-phrase-table") wf.build_phrase_table(language=language) workflow.run(wf.wf)
def build_knowledge_base(): # Merge categories from wikipedias. if flags.arg.merge_categories: log.info("Merge wikipedia categories") wf = wiki.WikiWorkflow("category-merging") wf.merge_wikipedia_categories() workflow.run(wf.wf) # Invert categories. if flags.arg.invert_categories: log.info("Invert categories") wf = wiki.WikiWorkflow("category-inversion") wf.invert_wikipedia_categories() workflow.run(wf.wf) # Extract link graph. if flags.arg.extract_wikilinks: log.info("Extract link graph") wf = wiki.WikiWorkflow("link-graph") wf.extract_links() workflow.run(wf.wf) # Fuse items. if flags.arg.fuse_items: log.info("Fuse items") wf = wiki.WikiWorkflow("fuse-items") wf.fuse_items() workflow.run(wf.wf) # Build knowledge base repository. if flags.arg.build_kb: log.info("Build knowledge base repository") wf = wiki.WikiWorkflow("knowledge-base") wf.build_knowledge_base() workflow.run(wf.wf)
def parse_wikipedia(): # Convert wikipedia pages to SLING documents. if flags.arg.parse_wikipedia: for language in flags.arg.languages: log.info("Parse " + language + " wikipedia") wf = wiki.WikiWorkflow(language + "-wikipedia-parsing") wf.parse_wikipedia(language=language) workflow.run(wf.wf)
def fuse_items(): # Merge categories from wikipedias. if flags.arg.merge_categories: log.info("Merge wikipedia categories") wf = wiki.WikiWorkflow("category-merging") wf.merge_wikipedia_categories() run_workflow(wf) # Invert categories. if flags.arg.invert_categories: log.info("Invert categories") wf = wiki.WikiWorkflow("category-inversion") wf.invert_wikipedia_categories() run_workflow(wf) # Fuse items. if flags.arg.fuse_items: log.info("Fuse items") wf = wiki.WikiWorkflow("fuse-items") wf.fuse_items() run_workflow(wf)
def import_wiki(): if flags.arg.import_wikidata or flags.arg.import_wikipedia: wf = wiki.WikiWorkflow("wiki-import") # Import wikidata. if flags.arg.import_wikidata: log.info("Import wikidata") wf.wikidata() # Import wikipedia(s). if flags.arg.import_wikipedia: for language in flags.arg.languages: log.info("Import " + language + " wikipedia") wf.wikipedia(language=language) workflow.run(wf.wf)
def build_alias_tables(): # Extract item names from wikidata and wikipedia. if flags.arg.extract_names: for language in flags.arg.languages: log.info("Extract " + language + " names") wf = wiki.WikiWorkflow(language + "-name-extraction") wf.extract_names(language=language) workflow.run(wf.wf) # Build name table. if flags.arg.build_nametab: for language in flags.arg.languages: log.info("Build " + language + " name table") wf = wiki.WikiWorkflow(language + "-name-table") wf.build_name_table(language=language) workflow.run(wf.wf) # Build phrase table. if flags.arg.build_phrasetab: for language in flags.arg.languages: log.info("Build " + language + " phrase table") wf = wiki.WikiWorkflow(language + "-phrase-table") wf.build_phrase_table(language=language) workflow.run(wf.wf)