def define_query_command_parser(parser=None):
"""Define an argument parser that can parse KGTK query command-style definitions.
"""
parser = parser or KGTKArgumentParser()
# duplicate some of the logic in cli_entry which doesn't have a command parser factory:
parser.add_argument('--debug',
dest='_debug',
action='store_true',
help='enable debug mode')
parser.add_argument('--expert',
dest='_expert',
action='store_true',
help='enable expert mode')
# add standard query argument definitions:
cliquery.add_arguments_extended(parser, None)
# define some additional API-only command options:
parser.add_argument('--name',
default=None,
action='store',
dest='name',
help="name to use for this query")
parser.add_argument('--documentation',
'--doc',
default=None,
action='store',
dest='docstring',
help="documentation string for this query")
parser.add_argument(
'--max-cache',
'--maxcache',
default=None,
action='store',
type=int,
dest='maxcache',
help="maximal number of results to cache, 0 disables caching")
parser.add_argument('--log-level',
'--loglevel',
default=None,
action='store',
type=int,
dest='loglevel',
help="log level to use for this query")
return parser
def cli_entry(*args):
"""
Usage:
kgtk <command> [options]
"""
global ret_code
# Capture the initial time for timing measurements.
start_time: float = time.time()
process_start_time: float = time.process_time()
# get all arguments
if not args:
args = tuple(sys.argv)
args = args[1:]
# base parser for shared arguments
base_parser = KGTKArgumentParser(add_help=False)
base_parser.add_argument('-V',
'--version',
action='version',
version='KGTK %s' % __version__,
help='show KGTK version number and exit.')
base_parser.add_argument(
'--check-deps',
action=CheckDepsAction,
help='check dependencies',
)
shared_args = base_parser.add_argument_group('shared optional arguments')
shared_args.add_argument('--debug',
dest='_debug',
action='store_true',
default=False,
help='enable debug mode')
shared_args.add_argument('--expert',
dest='_expert',
action='store_true',
default=False,
help='enable expert mode')
shared_args.add_argument('--pipedebug',
dest='_pipedebug',
action='store_true',
default=False,
help='enable pipe debug mode')
shared_args.add_argument('--progress',
dest='_progress',
action='store_true',
default=False,
help='enable progress monitoring')
shared_args.add_argument('--progress-tty',
dest='_progress_tty',
action='store',
default="/dev/tty",
help='progress monitoring output tty')
shared_args.add_argument('--timing',
dest='_timing',
action='store_true',
default=False,
help='enable timing measurements')
add_shared_arguments(shared_args)
# parse shared arguments
parsed_shared_args, rest_args = base_parser.parse_known_args(args)
shared_args = tuple(filter(lambda a: a not in rest_args, args))
args = tuple(rest_args)
# complete parser, load sub-parser of each module
parser = KGTKArgumentParser(
parents=[base_parser],
prog='kgtk',
description='kgtk --- Knowledge Graph Toolkit',
)
sub_parsers = parser.add_subparsers(metavar='command', dest='cmd')
subparser_lookup = {}
sub_parsers.required = True
for h in handlers:
mod = importlib.import_module('.{}'.format(h), 'kgtk.cli')
subp = mod.parser()
# only create sub-parser with sub-command name and defer full build
cmd: str = h.replace("_", "-")
sub_parser = sub_parsers.add_parser(cmd, **subp)
subparser_lookup[cmd] = (mod, sub_parser)
if 'aliases' in subp:
for alias in subp['aliases']:
subparser_lookup[alias] = (mod, sub_parser)
# add root level usage after sub-parsers are created
# this won't pollute help info in sub-parsers
parser.usage = '%(prog)s [options] command [ / command]*'
# parse internal pipe
pipe = [
list(y)
for x, y in itertools.groupby(args, lambda a: a == pipe_delimiter)
if not x
]
if len(pipe) == 0:
parser.print_usage()
parser.exit(KGTKArgumentParseException.return_code)
elif len(pipe) == 1: # single command
cmd_args = pipe[0]
cmd_name = cmd_args[0].replace("_", "-")
cmd_args[0] = cmd_name
# build sub-parser
if cmd_name in subparser_lookup:
mod, sub_parser = subparser_lookup[cmd_name]
add_default_arguments(
sub_parser) # call this before adding other arguments
if hasattr(mod, 'add_arguments_extended'):
mod.add_arguments_extended(sub_parser, parsed_shared_args)
else:
mod.add_arguments(sub_parser)
parsed_args = parser.parse_args(cmd_args)
# load module
kwargs = {}
func = None
if parsed_args.cmd:
h = parsed_args.cmd
func = mod.run
# remove sub-command name
kwargs = vars(parsed_args)
del kwargs['cmd']
# set shared arguments
for sa in vars(parsed_shared_args):
if sa not in sub_parsers.choices[h].shared_arguments:
del kwargs[sa]
else:
kwargs[sa] = getattr(parsed_shared_args, sa)
global _save_progress
_save_progress = parsed_shared_args._progress
global _save_progress_tty
_save_progress_tty = parsed_shared_args._progress_tty
if parsed_shared_args._progress:
if hasattr(mod, 'custom_progress') and mod.custom_progress():
pass
else:
progress_startup()
# run module
try:
kgtk_exception_handler = KGTKExceptionHandler(
debug=parsed_shared_args._debug)
ret_code = kgtk_exception_handler(func, **kwargs)
except KeyboardInterrupt as e:
print("\nKeyboard interrupt in %s." % " ".join(args),
file=sys.stderr,
flush=True)
progress_shutdown()
if hasattr(mod, 'keyboard_interrupt'):
mod.keyboard_interrupt()
# Silently exit instead of re-raising the KeyboardInterrupt.
# raise
else: # piped commands
if parsed_shared_args._pipedebug:
print("Building a KGTK pipe. pid=%d" % (os.getpid()),
file=sys.stderr,
flush=True)
processes = []
try:
for idx, cmd_args in enumerate(pipe):
# add shared arguments
full_args = []
for shared_arg in shared_args:
if idx == 0 or str(shared_arg) != "--progress":
full_args.append(shared_arg)
full_args.extend(cmd_args)
kwargs = {
"_bg_exc": False,
"_done": cmd_done,
"_err": sys.stderr,
"_bg": True,
"_internal_bufsize": 1,
}
# add specific arguments
if idx == 0: # The first commamd reads from our STDIN.
kwargs["_in"] = sys.stdin
if idx + 1 < len(pipe):
# All commands but the last pipe their output to the next command.
kwargs["_piped"] = True
else:
# The last command writes to our STDOUT.
kwargs["_out"] = sys.stdout
if parsed_shared_args._pipedebug:
cmd_str = " ".join(full_args)
for key in kwargs:
cmd_str += " " + key + "=" + str(kwargs[key])
print("pipe[%d]: kgtk %s" % (idx, cmd_str),
file=sys.stderr,
flush=True)
if idx == 0:
processes.append(sh.kgtk(*full_args, **kwargs))
else:
processes.append(
sh.kgtk(processes[idx - 1], *full_args, **kwargs))
for idx in range(len(pipe)):
processes[idx].wait()
except sh.SignalException_SIGPIPE:
if parsed_shared_args._pipedebug:
print("\npipe: sh.SignalException_SIGPIPE",
file=sys.stderr,
flush=True)
except sh.SignalException_SIGTERM:
if parsed_shared_args._pipedebug:
print("\npipe: sh.SignalException_SIGTERM",
file=sys.stderr,
flush=True)
raise
except KeyboardInterrupt as e:
if parsed_shared_args._pipedebug:
print("\npipe: KeyboardInterrupt", file=sys.stderr, flush=True)
if len(processes) > 0:
for idx in range(len(processes)):
process = processes[idx]
pgid = process.pgid
print("Killing cmd %d process group %d" % (idx, pgid),
file=sys.stderr,
flush=True)
process.signal_group(signal.SIGINT)
except sh.ErrorReturnCode as e:
if parsed_shared_args._pipedebug:
print("\npipe: sh.ErrorReturnCode",
file=sys.stderr,
flush=True)
# mimic parser exit
parser.exit(KGTKArgumentParseException.return_code,
e.stderr.decode('utf-8'))
if parsed_shared_args._timing:
end_time: float = time.time()
elapsed_seconds: float = end_time - start_time
process_end_time: float = time.process_time()
process_elapsed_seconds: float = process_end_time - process_start_time
cpu_ratio: float = process_elapsed_seconds / elapsed_seconds
print("Timing: elapsed=%s CPU=%s (%5.1f%%): %s" %
(str(datetime.timedelta(seconds=elapsed_seconds)),
str(datetime.timedelta(seconds=process_elapsed_seconds)),
cpu_ratio * 100.0, " ".join(args)),
file=sys.stderr,
flush=True)
return ret_code
def cli_entry(*args):
"""
Usage:
kgtk <command> [options]
"""
global ret_code
# get all arguments
if not args:
args = tuple(sys.argv)
args = args[1:]
# base parser for shared arguments
base_parser = KGTKArgumentParser(add_help=False)
base_parser.add_argument('-V',
'--version',
action='version',
version='KGTK %s' % __version__,
help='show KGTK version number and exit.')
shared_args = base_parser.add_argument_group('shared optional arguments')
shared_args.add_argument('--debug',
dest='_debug',
action='store_true',
default=False,
help='enable debug mode')
shared_args.add_argument('--expert',
dest='_expert',
action='store_true',
default=False,
help='enable expert mode')
shared_args.add_argument('--pipedebug',
dest='_pipedebug',
action='store_true',
default=False,
help='enable pipe debug mode')
add_shared_arguments(shared_args)
# parse shared arguments
parsed_shared_args, rest_args = base_parser.parse_known_args(args)
shared_args = tuple(filter(lambda a: a not in rest_args, args))
args = tuple(rest_args)
# complete parser, load sub-parser of each module
parser = KGTKArgumentParser(
parents=[base_parser],
prog='kgtk',
description='kgtk --- Knowledge Graph Toolkit',
)
sub_parsers = parser.add_subparsers(metavar='command', dest='cmd')
subparser_lookup = {}
sub_parsers.required = True
for h in handlers:
mod = importlib.import_module('.{}'.format(h), 'kgtk.cli')
subp = mod.parser()
# only create sub-parser with sub-command name and defer full build
cmd: str = h.replace("_", "-")
sub_parser = sub_parsers.add_parser(cmd, **subp)
subparser_lookup[cmd] = (mod, sub_parser)
if 'aliases' in subp:
for alias in subp['aliases']:
subparser_lookup[alias] = (mod, sub_parser)
# add root level usage after sub-parsers are created
# this won't pollute help info in sub-parsers
parser.usage = '%(prog)s [options] command [ / command]*'
# parse internal pipe
pipe = [
list(y)
for x, y in itertools.groupby(args, lambda a: a == pipe_delimiter)
if not x
]
if len(pipe) == 0:
parser.print_usage()
parser.exit(KGTKArgumentParseException.return_code)
elif len(pipe) == 1: # single command
cmd_args = pipe[0]
cmd_name = cmd_args[0].replace("_", "-")
cmd_args[0] = cmd_name
# build sub-parser
if cmd_name in subparser_lookup:
mod, sub_parser = subparser_lookup[cmd_name]
add_default_arguments(
sub_parser) # call this before adding other arguments
if hasattr(mod, 'add_arguments_extended'):
mod.add_arguments_extended(sub_parser, parsed_shared_args)
else:
mod.add_arguments(sub_parser)
parsed_args = parser.parse_args(cmd_args)
# load module
kwargs = {}
func = None
if parsed_args.cmd:
h = parsed_args.cmd
func = mod.run
# remove sub-command name
kwargs = vars(parsed_args)
del kwargs['cmd']
# set shared arguments
for sa in vars(parsed_shared_args):
if sa not in sub_parsers.choices[h].shared_arguments:
del kwargs[sa]
else:
kwargs[sa] = getattr(parsed_shared_args, sa)
# run module
kgtk_exception_handler = KGTKExceptionHandler(
debug=parsed_shared_args._debug)
ret_code = kgtk_exception_handler(func, **kwargs)
else: # piped commands
concat_cmd_str = None
for idx, cmd_args in enumerate(pipe):
# add shared arguments
cmd_str = ', '.join(['"{}"'.format(a) for a in shared_args])
if cmd_str:
cmd_str += ', '
# parse command and options
cmd_str += ', '.join(['"{}"'.format(a) for a in cmd_args])
# add other common arguments
cmd_str += ', _bg_exc=False, _done=cmd_done' # , _err=sys.stdout
# add specific arguments
if idx == 0: # first command
concat_cmd_str = 'sh.kgtk({}, _in=sys.stdin, _piped=True, _err=sys.stderr)'.format(
cmd_str)
elif idx + 1 == len(pipe): # last command
concat_cmd_str = 'sh.kgtk({}, {}, _out=sys.stdout, _err=sys.stderr)'.format(
concat_cmd_str, cmd_str)
else:
concat_cmd_str = 'sh.kgtk({}, {}, _piped=True, _err=sys.stderr)'.format(
concat_cmd_str, cmd_str)
try:
if parsed_shared_args._pipedebug:
print("eval: %s" % concat_cmd_str)
process = eval(concat_cmd_str)
process.wait()
except sh.SignalException_SIGPIPE:
pass
except sh.ErrorReturnCode as e:
# mimic parser exit
parser.exit(KGTKArgumentParseException.return_code,
e.stderr.decode('utf-8'))
return ret_code
def cli_entry(*args):
"""
Usage:
kgtk <command> [options]
"""
global ret_code
# Capture the initial time for timing measurements.
start_time: float = time.time()
process_start_time: float = time.process_time()
# get all arguments
if not args:
args = tuple(sys.argv)
args = args[1:]
# base parser for shared arguments
base_parser = KGTKArgumentParser(add_help=False)
base_parser.add_argument(
'-V', '--version',
action='version',
version='KGTK %s' % __version__,
help='show KGTK version number and exit.'
)
base_parser.add_argument(
'--check-deps',
action=CheckDepsAction,
help='check dependencies',
)
shared_args = base_parser.add_argument_group('shared optional arguments')
shared_args.add_argument('--debug', dest='_debug', action='store_true',
default=os.getenv('KGTK_OPTION_DEBUG', 'False').lower() in ['y', 'yes', 'true'],
help='enable debug mode')
shared_args.add_argument('--expert', dest='_expert', action='store_true',
default=os.getenv('KGTK_OPTION_EXPERT', 'False').lower() in ['y', 'yes', 'true'],
help='enable expert mode')
shared_args.add_argument('--pipedebug', dest='_pipedebug', action='store_true',
default=os.getenv('KGTK_OPTION_PIPEDEBUG', 'False').lower() in ['y', 'yes', 'true'],
help='enable pipe debug mode')
shared_args.add_argument('--progress', dest='_progress', action='store_true',
default=os.getenv('KGTK_OPTION_PROGRESS', 'False').lower() in ['y', 'yes', 'true'],
help='enable progress monitoring')
shared_args.add_argument('--progress-tty', dest='_progress_tty', action='store',
default=os.getenv('KGTK_OPTION_PROGRESS_TTY', "/dev/tty"),
help='progress monitoring output tty')
shared_args.add_argument('--timing', dest='_timing', action='store_true',
default=os.getenv('KGTK_OPTION_TIMING', 'False').lower() in ['y', 'yes', 'true'],
help='enable timing measurements')
add_shared_arguments(shared_args)
# parse shared arguments
parsed_shared_args, rest_args = base_parser.parse_known_args(args)
shared_args = tuple(filter(lambda a: a not in rest_args, args))
args = tuple(rest_args)
# complete parser, load sub-parser of each module
parser = KGTKArgumentParser(
parents=[base_parser], prog='kgtk',
description='kgtk --- Knowledge Graph Toolkit',
)
sub_parsers = parser.add_subparsers(
metavar='command',
dest='cmd'
)
subparsers_built = set()
subparser_lookup = {}
sub_parsers.required = True
for h in handlers:
hname, hpath = h
mod = importlib.import_module('.{}'.format(hname), hpath)
subp = mod.parser()
# only create sub-parser with sub-command name and defer full build
cmd: str = hname.replace("_", "-")
sub_parser = sub_parsers.add_parser(cmd, **subp)
subparser_lookup[cmd] = (mod, sub_parser)
if 'aliases' in subp:
for alias in subp['aliases']:
subparser_lookup[alias] = (mod, sub_parser)
# add root level usage after sub-parsers are created
# this won't pollute help info in sub-parsers
parser.usage = '%(prog)s [options] command [ / command]*'
ret_code = cli_entry_sequential_commands(args, parsed_shared_args, shared_args, parser, sub_parsers, subparser_lookup, subparsers_built)
if parsed_shared_args._timing:
end_time: float = time.time()
elapsed_seconds: float = end_time - start_time
process_end_time: float = time.process_time()
process_elapsed_seconds: float = process_end_time - process_start_time
cpu_ratio: float = process_elapsed_seconds / elapsed_seconds
print("Timing: elapsed=%s CPU=%s (%5.1f%%): %s" % (str(datetime.timedelta(seconds=elapsed_seconds)),
str(datetime.timedelta(seconds=process_elapsed_seconds)),
cpu_ratio * 100.0,
" ".join(args)), file=sys.stderr, flush=True)
return ret_code
