def cli_entry(*args): """ Usage: kgtk <command> [options] """ global ret_code # Capture the initial time for timing measurements. start_time: float = time.time() process_start_time: float = time.process_time() # get all arguments if not args: args = tuple(sys.argv) args = args[1:] # base parser for shared arguments base_parser = KGTKArgumentParser(add_help=False) base_parser.add_argument('-V', '--version', action='version', version='KGTK %s' % __version__, help='show KGTK version number and exit.') base_parser.add_argument( '--check-deps', action=CheckDepsAction, help='check dependencies', ) shared_args = base_parser.add_argument_group('shared optional arguments') shared_args.add_argument('--debug', dest='_debug', action='store_true', default=False, help='enable debug mode') shared_args.add_argument('--expert', dest='_expert', action='store_true', default=False, help='enable expert mode') shared_args.add_argument('--pipedebug', dest='_pipedebug', action='store_true', default=False, help='enable pipe debug mode') shared_args.add_argument('--progress', dest='_progress', action='store_true', default=False, help='enable progress monitoring') shared_args.add_argument('--progress-tty', dest='_progress_tty', action='store', default="/dev/tty", help='progress monitoring output tty') shared_args.add_argument('--timing', dest='_timing', action='store_true', default=False, help='enable timing measurements') add_shared_arguments(shared_args) # parse shared arguments parsed_shared_args, rest_args = base_parser.parse_known_args(args) shared_args = tuple(filter(lambda a: a not in rest_args, args)) args = tuple(rest_args) # complete parser, load sub-parser of each module parser = KGTKArgumentParser( parents=[base_parser], prog='kgtk', description='kgtk --- Knowledge Graph Toolkit', ) sub_parsers = parser.add_subparsers(metavar='command', dest='cmd') subparser_lookup = {} sub_parsers.required = True for h in handlers: mod = importlib.import_module('.{}'.format(h), 'kgtk.cli') subp = mod.parser() # only create sub-parser with sub-command name and defer full build cmd: str = h.replace("_", "-") sub_parser = sub_parsers.add_parser(cmd, **subp) subparser_lookup[cmd] = (mod, sub_parser) if 'aliases' in subp: for alias in subp['aliases']: subparser_lookup[alias] = (mod, sub_parser) # add root level usage after sub-parsers are created # this won't pollute help info in sub-parsers parser.usage = '%(prog)s [options] command [ / command]*' # parse internal pipe pipe = [ list(y) for x, y in itertools.groupby(args, lambda a: a == pipe_delimiter) if not x ] if len(pipe) == 0: parser.print_usage() parser.exit(KGTKArgumentParseException.return_code) elif len(pipe) == 1: # single command cmd_args = pipe[0] cmd_name = cmd_args[0].replace("_", "-") cmd_args[0] = cmd_name # build sub-parser if cmd_name in subparser_lookup: mod, sub_parser = subparser_lookup[cmd_name] add_default_arguments( sub_parser) # call this before adding other arguments if hasattr(mod, 'add_arguments_extended'): mod.add_arguments_extended(sub_parser, parsed_shared_args) else: mod.add_arguments(sub_parser) parsed_args = parser.parse_args(cmd_args) # load module kwargs = {} func = None if parsed_args.cmd: h = parsed_args.cmd func = mod.run # remove sub-command name kwargs = vars(parsed_args) del kwargs['cmd'] # set shared arguments for sa in vars(parsed_shared_args): if sa not in sub_parsers.choices[h].shared_arguments: del kwargs[sa] else: kwargs[sa] = getattr(parsed_shared_args, sa) global _save_progress _save_progress = parsed_shared_args._progress global _save_progress_tty _save_progress_tty = parsed_shared_args._progress_tty if parsed_shared_args._progress: if hasattr(mod, 'custom_progress') and mod.custom_progress(): pass else: progress_startup() # run module try: kgtk_exception_handler = KGTKExceptionHandler( debug=parsed_shared_args._debug) ret_code = kgtk_exception_handler(func, **kwargs) except KeyboardInterrupt as e: print("\nKeyboard interrupt in %s." % " ".join(args), file=sys.stderr, flush=True) progress_shutdown() if hasattr(mod, 'keyboard_interrupt'): mod.keyboard_interrupt() # Silently exit instead of re-raising the KeyboardInterrupt. # raise else: # piped commands if parsed_shared_args._pipedebug: print("Building a KGTK pipe. pid=%d" % (os.getpid()), file=sys.stderr, flush=True) processes = [] try: for idx, cmd_args in enumerate(pipe): # add shared arguments full_args = [] for shared_arg in shared_args: if idx == 0 or str(shared_arg) != "--progress": full_args.append(shared_arg) full_args.extend(cmd_args) kwargs = { "_bg_exc": False, "_done": cmd_done, "_err": sys.stderr, "_bg": True, "_internal_bufsize": 1, } # add specific arguments if idx == 0: # The first commamd reads from our STDIN. kwargs["_in"] = sys.stdin if idx + 1 < len(pipe): # All commands but the last pipe their output to the next command. kwargs["_piped"] = True else: # The last command writes to our STDOUT. kwargs["_out"] = sys.stdout if parsed_shared_args._pipedebug: cmd_str = " ".join(full_args) for key in kwargs: cmd_str += " " + key + "=" + str(kwargs[key]) print("pipe[%d]: kgtk %s" % (idx, cmd_str), file=sys.stderr, flush=True) if idx == 0: processes.append(sh.kgtk(*full_args, **kwargs)) else: processes.append( sh.kgtk(processes[idx - 1], *full_args, **kwargs)) for idx in range(len(pipe)): processes[idx].wait() except sh.SignalException_SIGPIPE: if parsed_shared_args._pipedebug: print("\npipe: sh.SignalException_SIGPIPE", file=sys.stderr, flush=True) except sh.SignalException_SIGTERM: if parsed_shared_args._pipedebug: print("\npipe: sh.SignalException_SIGTERM", file=sys.stderr, flush=True) raise except KeyboardInterrupt as e: if parsed_shared_args._pipedebug: print("\npipe: KeyboardInterrupt", file=sys.stderr, flush=True) if len(processes) > 0: for idx in range(len(processes)): process = processes[idx] pgid = process.pgid print("Killing cmd %d process group %d" % (idx, pgid), file=sys.stderr, flush=True) process.signal_group(signal.SIGINT) except sh.ErrorReturnCode as e: if parsed_shared_args._pipedebug: print("\npipe: sh.ErrorReturnCode", file=sys.stderr, flush=True) # mimic parser exit parser.exit(KGTKArgumentParseException.return_code, e.stderr.decode('utf-8')) if parsed_shared_args._timing: end_time: float = time.time() elapsed_seconds: float = end_time - start_time process_end_time: float = time.process_time() process_elapsed_seconds: float = process_end_time - process_start_time cpu_ratio: float = process_elapsed_seconds / elapsed_seconds print("Timing: elapsed=%s CPU=%s (%5.1f%%): %s" % (str(datetime.timedelta(seconds=elapsed_seconds)), str(datetime.timedelta(seconds=process_elapsed_seconds)), cpu_ratio * 100.0, " ".join(args)), file=sys.stderr, flush=True) return ret_code
def cli_entry_pipe(args, parsed_shared_args, shared_args, parser, sub_parsers, subparser_lookup, subparsers_built)->int: # parse internal pipe import copy ret_code: int = 0 pipe = [list(y) for x, y in itertools.groupby(args, lambda a: a == pipe_delimiter) if not x] if len(pipe) == 0: parser.print_usage() parser.exit(KGTKArgumentParseException.return_code) elif len(pipe) == 1: # single command cmd_args = pipe[0] cmd_name = cmd_args[0].replace("_", "-") cmd_args[0] = cmd_name # build sub-parser if cmd_name in subparser_lookup: mod, sub_parser = subparser_lookup[cmd_name] add_default_arguments(sub_parser) # call this before adding other arguments if cmd_name not in subparsers_built: if hasattr(mod, 'add_arguments_extended'): psa = copy.deepcopy(parsed_shared_args) psa._command = cmd_name mod.add_arguments_extended(sub_parser, psa) else: mod.add_arguments(sub_parser) subparsers_built.add(cmd_name) parsed_args = parser.parse_args(cmd_args) # load module kwargs = {} func = None if parsed_args.cmd: h = parsed_args.cmd func = mod.run # remove sub-command name kwargs = vars(parsed_args) del kwargs['cmd'] # set shared arguments for sa in vars(parsed_shared_args): if sa not in sub_parsers.choices[h].shared_arguments: del kwargs[sa] else: kwargs[sa] = getattr(parsed_shared_args, sa) global _save_progress _save_progress = parsed_shared_args._progress global _save_progress_tty _save_progress_tty = parsed_shared_args._progress_tty if parsed_shared_args._progress: if hasattr(mod, 'custom_progress') and mod.custom_progress(): pass else: progress_startup() # run module try: kgtk_exception_handler = KGTKExceptionHandler(debug=parsed_shared_args._debug) ret_code = kgtk_exception_handler(func, **kwargs) except KeyboardInterrupt as e: print("\nKeyboard interrupt in %s." % " ".join(args), file=sys.stderr, flush=True) progress_shutdown() if hasattr(mod, 'keyboard_interrupt'): mod.keyboard_interrupt() # Silently exit instead of re-raising the KeyboardInterrupt. # raise else: # piped commands if parsed_shared_args._pipedebug: print("Building a KGTK pipe. pid=%d" % (os.getpid()), file=sys.stderr, flush=True) processes = [ ] try: for idx, cmd_args in enumerate(pipe): # add shared arguments full_args = [ ] for shared_arg in shared_args: if idx == 0 or str(shared_arg) != "--progress": full_args.append(shared_arg) full_args.extend(cmd_args) kwargs = { "_bg_exc": False, "_done": cmd_done, "_err": sys.stderr, "_bg": True, "_internal_bufsize": 1, } # add specific arguments if idx == 0: # The first commamd reads from our STDIN. kwargs["_in"] = sys.stdin if idx + 1 < len(pipe): # All commands but the last pipe their output to the next command. kwargs["_piped"] = True else: # The last command writes to our STDOUT. kwargs["_out"] = sys.stdout if parsed_shared_args._pipedebug: cmd_str = " ".join(full_args) for key in kwargs: cmd_str += " " + key + "=" + str(kwargs[key]) print("pipe[%d]: kgtk %s" % (idx, cmd_str), file=sys.stderr, flush=True) if idx == 0: processes.append(sh.kgtk(*full_args, **kwargs)) else: processes.append(sh.kgtk(processes[idx - 1], *full_args, **kwargs)) for idx in range(len(pipe)): processes[idx].wait() except sh.SignalException_SIGPIPE: if parsed_shared_args._pipedebug: print("\npipe: sh.SignalException_SIGPIPE", file=sys.stderr, flush=True) except sh.SignalException_SIGTERM: if parsed_shared_args._pipedebug: print("\npipe: sh.SignalException_SIGTERM", file=sys.stderr, flush=True) raise except KeyboardInterrupt as e: if parsed_shared_args._pipedebug: print("\npipe: KeyboardInterrupt", file=sys.stderr, flush=True) if len(processes) > 0: for idx in range(len(processes)): process = processes[idx] pgid = process.pgid print("Killing cmd %d process group %d" % (idx, pgid), file=sys.stderr, flush=True) process.signal_group(signal.SIGINT) except sh.ErrorReturnCode as e: if parsed_shared_args._pipedebug: print("\npipe: sh.ErrorReturnCode", file=sys.stderr, flush=True) # mimic parser exit parser.exit(KGTKArgumentParseException.return_code, e.stderr.decode('utf-8')) return ret_code
def cli_entry(*args): """ Usage: kgtk <command> [options] """ global ret_code # get all arguments if not args: args = tuple(sys.argv) args = args[1:] # base parser for shared arguments base_parser = KGTKArgumentParser(add_help=False) base_parser.add_argument('-V', '--version', action='version', version='KGTK %s' % __version__, help='show KGTK version number and exit.') shared_args = base_parser.add_argument_group('shared optional arguments') shared_args.add_argument('--debug', dest='_debug', action='store_true', default=False, help='enable debug mode') shared_args.add_argument('--expert', dest='_expert', action='store_true', default=False, help='enable expert mode') shared_args.add_argument('--pipedebug', dest='_pipedebug', action='store_true', default=False, help='enable pipe debug mode') add_shared_arguments(shared_args) # parse shared arguments parsed_shared_args, rest_args = base_parser.parse_known_args(args) shared_args = tuple(filter(lambda a: a not in rest_args, args)) args = tuple(rest_args) # complete parser, load sub-parser of each module parser = KGTKArgumentParser( parents=[base_parser], prog='kgtk', description='kgtk --- Knowledge Graph Toolkit', ) sub_parsers = parser.add_subparsers(metavar='command', dest='cmd') subparser_lookup = {} sub_parsers.required = True for h in handlers: mod = importlib.import_module('.{}'.format(h), 'kgtk.cli') subp = mod.parser() # only create sub-parser with sub-command name and defer full build cmd: str = h.replace("_", "-") sub_parser = sub_parsers.add_parser(cmd, **subp) subparser_lookup[cmd] = (mod, sub_parser) if 'aliases' in subp: for alias in subp['aliases']: subparser_lookup[alias] = (mod, sub_parser) # add root level usage after sub-parsers are created # this won't pollute help info in sub-parsers parser.usage = '%(prog)s [options] command [ / command]*' # parse internal pipe pipe = [ list(y) for x, y in itertools.groupby(args, lambda a: a == pipe_delimiter) if not x ] if len(pipe) == 0: parser.print_usage() parser.exit(KGTKArgumentParseException.return_code) elif len(pipe) == 1: # single command cmd_args = pipe[0] cmd_name = cmd_args[0].replace("_", "-") cmd_args[0] = cmd_name # build sub-parser if cmd_name in subparser_lookup: mod, sub_parser = subparser_lookup[cmd_name] add_default_arguments( sub_parser) # call this before adding other arguments if hasattr(mod, 'add_arguments_extended'): mod.add_arguments_extended(sub_parser, parsed_shared_args) else: mod.add_arguments(sub_parser) parsed_args = parser.parse_args(cmd_args) # load module kwargs = {} func = None if parsed_args.cmd: h = parsed_args.cmd func = mod.run # remove sub-command name kwargs = vars(parsed_args) del kwargs['cmd'] # set shared arguments for sa in vars(parsed_shared_args): if sa not in sub_parsers.choices[h].shared_arguments: del kwargs[sa] else: kwargs[sa] = getattr(parsed_shared_args, sa) # run module kgtk_exception_handler = KGTKExceptionHandler( debug=parsed_shared_args._debug) ret_code = kgtk_exception_handler(func, **kwargs) else: # piped commands concat_cmd_str = None for idx, cmd_args in enumerate(pipe): # add shared arguments cmd_str = ', '.join(['"{}"'.format(a) for a in shared_args]) if cmd_str: cmd_str += ', ' # parse command and options cmd_str += ', '.join(['"{}"'.format(a) for a in cmd_args]) # add other common arguments cmd_str += ', _bg_exc=False, _done=cmd_done' # , _err=sys.stdout # add specific arguments if idx == 0: # first command concat_cmd_str = 'sh.kgtk({}, _in=sys.stdin, _piped=True, _err=sys.stderr)'.format( cmd_str) elif idx + 1 == len(pipe): # last command concat_cmd_str = 'sh.kgtk({}, {}, _out=sys.stdout, _err=sys.stderr)'.format( concat_cmd_str, cmd_str) else: concat_cmd_str = 'sh.kgtk({}, {}, _piped=True, _err=sys.stderr)'.format( concat_cmd_str, cmd_str) try: if parsed_shared_args._pipedebug: print("eval: %s" % concat_cmd_str) process = eval(concat_cmd_str) process.wait() except sh.SignalException_SIGPIPE: pass except sh.ErrorReturnCode as e: # mimic parser exit parser.exit(KGTKArgumentParseException.return_code, e.stderr.decode('utf-8')) return ret_code
def cli_single_command(args, parsed_shared_args, shared_args, parser, sub_parsers, subparser_lookup, subparsers_built) -> int: import copy if parsed_shared_args._pipedebug: print("pid %d: Executing a single command: %s" % (os.getpid(), repr(args)), file=sys.stderr, flush=True) ret_code: int = 0 cmd_args = copy.deepcopy(args) cmd_name = cmd_args[0].replace("_", "-") cmd_args[0] = cmd_name # build sub-parser if cmd_name in subparser_lookup: mod, sub_parser = subparser_lookup[cmd_name] add_default_arguments( sub_parser) # call this before adding other arguments if cmd_name not in subparsers_built: if hasattr(mod, 'add_arguments_extended'): psa = copy.deepcopy(parsed_shared_args) psa._command = cmd_name mod.add_arguments_extended(sub_parser, psa) else: mod.add_arguments(sub_parser) subparsers_built.add(cmd_name) parsed_args = parser.parse_args(cmd_args) # load module kwargs = {} func = None if parsed_args.cmd: h = parsed_args.cmd func = mod.run # remove sub-command name kwargs = vars(parsed_args) del kwargs['cmd'] # set shared arguments for sa in vars(parsed_shared_args): # Shared arguments that have been accepted by the subcommand # (see KGTKArgumentParser.accept_shared_argument(...)) # will be passed as keyword arguments to the subcommand. if sa not in sub_parsers.choices[h].shared_arguments: del kwargs[sa] else: kwargs[sa] = getattr(parsed_shared_args, sa) global _save_progress _save_progress = parsed_shared_args._progress global _save_progress_debug _save_progress_debug = parsed_shared_args._progress_debug global _save_progress_tty _save_progress_tty = parsed_shared_args._progress_tty if parsed_shared_args._progress: if hasattr(mod, 'custom_progress') and mod.custom_progress(): pass else: progress_startup() # run module try: kgtk_exception_handler = KGTKExceptionHandler( debug=parsed_shared_args._debug) ret_code = kgtk_exception_handler(func, **kwargs) except KeyboardInterrupt as e: print("\nKeyboard interrupt in %s." % " ".join(args), file=sys.stderr, flush=True) progress_shutdown() if hasattr(mod, 'keyboard_interrupt'): mod.keyboard_interrupt() # Silently exit instead of re-raising the KeyboardInterrupt. # raise return ret_code