wf = entity.EntityWorkflow("ner-knowledge-base")
wf.build_knowledge_base()
workflow.run(wf.wf)
# Run NER labeling of Wikipedia documents.
if flags.arg.label_wiki:
for language in flags.arg.languages:
log.info("Label " + language + " wikipedia")
wf = entity.EntityWorkflow(language + "-wiki-label")
wf.label_documents(language=language)
workflow.run(wf.wf)
if __name__ == '__main__':
# Parse command-line arguments.
flags.parse()
if flags.arg.build_wiki:
flags.arg.import_wikidata = True
flags.arg.import_wikipedia = True
flags.arg.parse_wikipedia = True
flags.arg.merge_categories = True
flags.arg.invert_categories = True
flags.arg.compute_item_popularity = True
flags.arg.fuse_items = True
flags.arg.build_kb = True
flags.arg.extract_names = True
flags.arg.build_nametab = True
flags.arg.build_phrasetab = True
# Run workflows.
def main():
# Parse command-line arguments. Load modules for commands before parsing
# flags to allow each of these to register more flags.
for arg in sys.argv:
if arg.startswith("-"): continue
for cmd in commands:
if arg == cmd.name:
if cmd.package is not None:
importlib.import_module(cmd.package)
if cmd.load is not None:
for pkg in cmd.load:
importlib.import_module(pkg)
break
flags.parse()
# Output version information.
if flags.arg.version:
sling.which()
sys.exit(0)
# List commands.
if flags.arg.list:
print("commands:")
for cmd in commands:
if not cmd.internal:
print(" %-30s %s" % (cmd.name, cmd.help))
sys.exit(0)
# Run command in background if requested.
if flags.arg.spawn:
# Build command.
cmd = []
for arg in sys.argv:
if arg != "--spawn": cmd.append(arg)
cmd.append("--flushlog")
# Output to log file.
logfn = flags.arg.logdir + "/" + time.strftime(
"%Y%m%d-%H%M%S") + ".log"
logfile = open(logfn, "w")
# Start background job.
process = subprocess.Popen(cmd,
stdin=None,
stdout=logfile,
stderr=subprocess.STDOUT,
bufsize=1,
shell=False,
close_fds=True)
print("Running process", process.pid, "in background logging to",
logfn)
sys.exit(0)
# Start up workflow system.
workflow.startup()
# Run commands.
for cmd in commands:
if cmd.name not in flags.arg.COMMAND: continue
if cmd.package:
# Load module with command.
module = importlib.import_module(cmd.package)
# Run command.
if cmd.function is not None:
log.info("Execute command " + cmd.name)
getattr(module, cmd.function)()
# Add triggered commands.
if cmd.triggers is not None:
for trigger in cmd.triggers:
flags.arg.COMMAND.append(trigger)
# Done.
workflow.shutdown()
log.info("Done")
