def add_arguments_extended(parser: KGTKArgumentParser,
parsed_shared_args: Namespace):
"""
Parse arguments
Args:
parser (argparse.ArgumentParser)
"""
_expert: bool = parsed_shared_args._expert
# This helper function makes it easy to suppress options from
# The help message. The options are still there, and initialize
# what they need to initialize.
def h(msg: str) -> str:
if _expert:
return msg
else:
return SUPPRESS
parser.add_argument("input_file_path",
help="The KGTK input file. (default=%(default)s).",
type=Path,
default="-")
parser.add_argument("-o",
"--output-file",
dest="output_file_path",
help="The KGTK file to write (default=%(default)s).",
type=Path,
default="-")
parser.add_argument("--output-format",
dest="output_format",
help=h("The file format (default=kgtk)"),
type=str)
parser.add_argument(
"--output-columns",
dest="output_column_names",
metavar="NEW_COLUMN_NAME",
help="The list of new column names when renaming all columns.",
type=str,
nargs='+')
parser.add_argument(
"--old-columns",
dest="old_column_names",
metavar="OLD_COLUMN_NAME",
help="The list of old column names for selective renaming.",
type=str,
nargs='+')
parser.add_argument(
"--new-columns",
dest="new_column_names",
metavar="NEW_COLUMN_NAME",
help="The list of new column names for selective renaming.",
type=str,
nargs='+')
KgtkReader.add_debug_arguments(parser, expert=_expert)
KgtkReaderOptions.add_arguments(parser, mode_options=True, expert=_expert)
KgtkValueOptions.add_arguments(parser, expert=_expert)
def main():
"""
Test the KGTK node file reader.
"""
parser = ArgumentParser()
parser.add_argument(dest="kgtk_file", help="The KGTK edge file to read", type=Path, nargs="?")
KgtkReader.add_debug_arguments(parser, expert=True)
KgtkReaderOptions.add_arguments(parser, validate_by_default=True, expert=True)
KgtkValueOptions.add_arguments(parser, expert=True)
args = parser.parse_args()
error_file: typing.TextIO = sys.stdout if args.errors_to_stdout else sys.stderr
# Build the option structures.
reader_options: KgtkReaderOptions = KgtkReaderOptions.from_args(args, mode=KgtkReaderMode.NODE)
value_options: KgtkValueOptions = KgtkValueOptions.from_args(args)
if args.show_options:
reader_options.show(out=error_file)
value_options.show(out=error_file)
print("=======", file=error_file, flush=True)
nr: NodeReader = NodeReader.open_node_file(args.kgtk_file,
error_file=error_file,
options=reader_options,
value_options=value_options,
verbose=args.verbose, very_verbose=args.very_verbose)
line_count: int = 0
row: typing.List[str]
for row in nr:
line_count += 1
print("Read %d lines" % line_count)
def add_arguments_extended(parser: KGTKArgumentParser, parsed_shared_args: Namespace):
"""
Parse arguments
Args:
parser (argparse.ArgumentParser)
"""
from kgtk.io.kgtkreader import KgtkReader, KgtkReaderOptions, KgtkReaderMode
from kgtk.value.kgtkvalueoptions import KgtkValueOptions
_expert: bool = parsed_shared_args._expert
# This helper function makes it easy to suppress options from
# The help message. The options are still there, and initialize
# what they need to initialize.
def h(msg: str)->str:
if _expert:
return msg
else:
return SUPPRESS
parser.add_input_file(who="The KGTK file to convert to an HTML table.", positional=True)
parser.add_output_file(who="The GitHub markdown file to write.")
parser.add_argument( "--output-format", dest="output_format", help=h("The file format (default=%(default)s)"), type=str, default="table")
KgtkReader.add_debug_arguments(parser, expert=_expert)
KgtkReaderOptions.add_arguments(parser, mode_options=True, default_mode=KgtkReaderMode.NONE, expert=_expert)
KgtkValueOptions.add_arguments(parser, expert=_expert)
def add_arguments_extended(parser: KGTKArgumentParser,
parsed_shared_args: Namespace):
"""
Parse arguments
Args:
parser (argparse.ArgumentParser)
"""
from kgtk.io.kgtkreader import KgtkReader, KgtkReaderOptions
from kgtk.utils.argparsehelpers import optional_bool
from kgtk.value.kgtkvalueoptions import KgtkValueOptions
_expert: bool = parsed_shared_args._expert
parser.add_input_file(positional=True, optional=False)
parser.add_output_file(
who=
"Graph tool file to dump the graph too - if empty, it will not be saved.",
optional=True)
parser.add_argument(
'--undirected',
dest="undirected",
help="When True, the graph is undirected. (default=%(default)s)",
type=optional_bool,
nargs='?',
const=True,
default=False,
metavar="True|False")
KgtkReader.add_debug_arguments(parser, expert=_expert)
KgtkReaderOptions.add_arguments(parser, mode_options=True, expert=_expert)
KgtkValueOptions.add_arguments(parser, expert=_expert)
def add_arguments_extended(parser: KGTKArgumentParser,
parsed_shared_args: Namespace):
"""
Parse arguments
Args:
parser (argparse.ArgumentParser)
"""
from kgtk.io.kgtkreader import KgtkReader, KgtkReaderOptions
from kgtk.value.kgtkvalueoptions import KgtkValueOptions
_expert: bool = parsed_shared_args._expert
parser.add_input_file(positional=True)
parser.add_output_file()
parser.add_argument(
"--columns",
dest="key_column_names",
help=
"The key columns will not be expanded. They will be repeated on each output record. "
+ "(default=id for node files, (node1, label, node2) for edge files).",
nargs='+',
default=[])
KgtkReader.add_debug_arguments(parser, expert=_expert)
KgtkReaderOptions.add_arguments(parser, mode_options=True, expert=_expert)
KgtkValueOptions.add_arguments(parser, expert=_expert)
def add_arguments_extended(parser: KGTKArgumentParser,
parsed_shared_args: Namespace):
"""
Parse arguments
Args:
parser (argparse.ArgumentParser)
"""
_expert: bool = parsed_shared_args._expert
parser.add_argument(
"kgtk_files",
nargs="*",
help="The KGTK file(s) to validate. May be omitted or '-' for stdin.",
type=Path)
parser.add_argument(
"--header-only",
dest="header_only",
help=
"Process the only the header of the input file (default=%(default)s).",
type=optional_bool,
nargs='?',
const=True,
default=False)
KgtkReader.add_debug_arguments(parser, expert=_expert)
KgtkReaderOptions.add_arguments(parser,
mode_options=True,
validate_by_default=True,
expert=_expert)
KgtkValueOptions.add_arguments(parser, expert=_expert)
def add_arguments_extended(parser: KGTKArgumentParser, parsed_shared_args: Namespace):
"""
Parse arguments
Args:
parser (argparse.ArgumentParser)
"""
_expert: bool = parsed_shared_args._expert
parser.add_input_file(positional=True)
parser.add_output_file()
parser.add_argument( "--columns", dest="filter_column_names",
help="The columns in the file being filtered (Required).", nargs='+', required=True)
parser.add_argument( "--count", dest="only_count", metavar="True|False",
help="Only count the records, do not copy them. (default=%(default)s).",
type=optional_bool, nargs='?', const=True, default=False)
parser.add_argument( "--all", dest="all_are", metavar="True|False",
help="False: Test if any are not empty, True: test if all are not empty (default=%(default)s).",
type=optional_bool, nargs='?', const=True, default=False)
KgtkReader.add_debug_arguments(parser, expert=_expert)
KgtkReaderOptions.add_arguments(parser, mode_options=True, expert=_expert)
KgtkValueOptions.add_arguments(parser, expert=_expert)
def add_arguments_extended(parser: KGTKArgumentParser,
parsed_shared_args: Namespace):
"""
Parse arguments
Args:
parser (argparse.ArgumentParser)
"""
# import modules locally
from kgtk.io.kgtkreader import KgtkReader, KgtkReaderOptions
from kgtk.value.kgtkvalueoptions import KgtkValueOptions
_expert: bool = parsed_shared_args._expert
parser.add_input_file(positional=True)
parser.add_output_file(positional=True)
parser.add_output_file(who="Reject file",
dest="reject_file",
options=["--reject-file"],
metavar="REJECT_FILE",
optional=True)
KgtkReader.add_debug_arguments(parser, expert=_expert)
KgtkReaderOptions.add_arguments(parser,
mode_options=True,
validate_by_default=True,
expert=_expert)
KgtkValueOptions.add_arguments(parser, expert=_expert)
def add_arguments_extended(parser: KGTKArgumentParser, parsed_shared_args: Namespace):
"""
Parse arguments
Args:
parser (argparse.ArgumentParser)
"""
_expert: bool = parsed_shared_args._expert
# This helper function makes it easy to suppress options from
# The help message. The options are still there, and initialize
# what they need to initialize.
def h(msg: str)->str:
if _expert:
return msg
else:
return SUPPRESS
parser.add_argument( "input_kgtk_file", nargs="?", type=Path, default="-",
help="The KGTK file to filter. May be omitted or '-' for stdin (default=%(default)s).")
parser.add_argument("-o", "--output-file", dest="output_kgtk_file", help="The KGTK file to write (default=%(default)s).", type=Path, default="-")
KgtkIdBuilderOptions.add_arguments(parser, expert=True) # Show all the options.
KgtkReader.add_debug_arguments(parser, expert=_expert)
KgtkReaderOptions.add_arguments(parser, mode_options=True, expert=_expert)
KgtkValueOptions.add_arguments(parser, expert=_expert)
def add_arguments_extended(parser: KGTKArgumentParser, parsed_shared_args: Namespace):
"""
Parse arguments
Args:
parser (argparse.ArgumentParser)
"""
from kgtk.io.kgtkreader import KgtkReader, KgtkReaderOptions, KgtkReaderMode
from kgtk.utils.argparsehelpers import optional_bool
from kgtk.value.kgtkvalueoptions import KgtkValueOptions
_expert: bool = parsed_shared_args._expert
parser.add_input_file(positional=True)
parser.add_output_file()
parser.add_argument('-c', "--columns", action="store", type=str, dest="columns", nargs='+', required=True,
help="Columns to remove as a comma- or space-separated strings, e.g., id,docid or id docid")
parser.add_argument( "--split-on-commas", dest="split_on_commas", help="Parse the list of columns, splitting on commas. (default=%(default)s).",
type=optional_bool, nargs='?', const=True, default=True)
parser.add_argument( "--split-on-spaces", dest="split_on_spaces", help="Parse the list of columns, splitting on spaces. (default=%(default)s).",
type=optional_bool, nargs='?', const=True, default=False)
parser.add_argument( "--strip-spaces", dest="strip_spaces", help="Parse the list of columns, stripping whitespace. (default=%(default)s).",
type=optional_bool, nargs='?', const=True, default=True)
KgtkReader.add_debug_arguments(parser, expert=_expert)
KgtkReaderOptions.add_arguments(parser, mode_options=True, default_mode=KgtkReaderMode.NONE, expert=_expert)
KgtkValueOptions.add_arguments(parser, expert=_expert)
def add_arguments_extended(parser: KGTKArgumentParser,
parsed_shared_args: Namespace):
"""
Parse arguments
Args:
parser (argparse.ArgumentParser)
"""
from kgtk.io.kgtkreader import KgtkReader, KgtkReaderOptions
from kgtk.utils.argparsehelpers import optional_bool
from kgtk.value.kgtkvalueoptions import KgtkValueOptions
_expert: bool = parsed_shared_args._expert
parser.add_input_file(who="The KGTK file(s) to validate.",
dest="input_files",
options=["-i", "--input-files"],
allow_list=True,
positional=True)
parser.add_argument(
"--header-only",
dest="header_only",
help=
"Process the only the header of the input file (default=%(default)s).",
type=optional_bool,
nargs='?',
const=True,
default=False)
KgtkReader.add_debug_arguments(parser, expert=_expert)
KgtkReaderOptions.add_arguments(parser,
mode_options=True,
validate_by_default=True,
expert=_expert)
KgtkValueOptions.add_arguments(parser, expert=_expert)
def add_arguments_extended(parser: KGTKArgumentParser,
parsed_shared_args: Namespace):
"""
Parse arguments
Args:
parser (argparse.ArgumentParser)
"""
# import modules locally
from kgtk.io.kgtkreader import KgtkReader, KgtkReaderOptions
from kgtk.reshape.kgtkidbuilder import KgtkIdBuilder, KgtkIdBuilderOptions
from kgtk.value.kgtkvalueoptions import KgtkValueOptions
_expert: bool = parsed_shared_args._expert
# This helper function makes it easy to suppress options from
# The help message. The options are still there, and initialize
# what they need to initialize.
def h(msg: str) -> str:
if _expert:
return msg
else:
return SUPPRESS
parser.add_input_file(positional=True)
parser.add_output_file()
KgtkIdBuilderOptions.add_arguments(parser,
expert=True) # Show all the options.
KgtkReader.add_debug_arguments(parser, expert=_expert)
KgtkReaderOptions.add_arguments(parser, mode_options=True, expert=_expert)
KgtkValueOptions.add_arguments(parser, expert=_expert)
def add_arguments_extended(parser: KGTKArgumentParser, parsed_shared_args: Namespace):
"""
Parse arguments
Args:
parser (argparse.ArgumentParser)
"""
_expert: bool = parsed_shared_args._expert
parser.add_input_file(positional=True, who="The KGTK file to find connected components in.")
parser.add_output_file()
parser.add_argument("--no-header", action="store_true", dest="no_header",
help="Specify if the input file does not have a header, default FALSE")
parser.add_argument("--properties", action="store", type=str, dest="properties",
help=' A comma separated list of properties to traverse while finding connected components, '
'by default all properties will be considered',
default='')
parser.add_argument('--undirected', action='store_true', dest="undirected",
help="Specify if the input graph is undirected, default FALSE")
parser.add_argument('--strong', action='store_true', dest="strong",
help="Treat graph as directed or not, independent of its actual directionality.")
# CMR: The folowing options aren't used. Is the intent to support them, or
# was this cut-and-paste overkill?
KgtkReader.add_debug_arguments(parser, expert=_expert)
KgtkReaderOptions.add_arguments(parser, mode_options=True, expert=_expert)
KgtkReaderOptions.add_arguments(parser, mode_options=True, who="input", expert=_expert, defaults=False)
def add_arguments_extended(parser: KGTKArgumentParser, parsed_shared_args: Namespace):
"""
Parse arguments
Args:
parser (argparse.ArgumentParser)
"""
from kgtk.value.kgtkvalueoptions import KgtkValueOptions
from kgtk.io.kgtkreader import KgtkReader, KgtkReaderOptions
_expert: bool = parsed_shared_args._expert
parser.add_input_file()
parser.add_argument('--output-path', action='store', type=str, dest="output_path", required=True,
help="Path of an existing folder where the split files will be written")
parser.add_argument('--file-prefix', action='store', type=str, default='split_', dest='file_prefix', required=False,
help="file name prefix, will be appended to output file names before a number")
parser.add_argument('--split-by-qnode', default=False, action="store_true", dest='split_by_qnode',
help="If specified, all edges for a qnode will be written to a separate file, "
"qnode will be added to the file name. WARNING: If there are millions of Qnodes, "
"this option will create millions of file."
" Default [FALSE]")
parser.add_argument('--gzipped-output', default=False, action="store_true", dest='gzipped_output',
help="If specified, the output split files will be gzipped. Default FALSE")
parser.add_argument('--lines', action='store', dest='lines', type=int, default=1000000, required=False,
help="number of lines in each split file. The actual number of lines will exceed this number, "
"since Qnode boundaries are preserved.")
KgtkReader.add_debug_arguments(parser, expert=_expert)
KgtkReaderOptions.add_arguments(parser, mode_options=True, expert=_expert)
KgtkValueOptions.add_arguments(parser, expert=_expert)
def add_arguments_extended(parser: KGTKArgumentParser, parsed_shared_args: Namespace):
"""
Parse arguments
Args:
parser (argparse.ArgumentParser)
"""
_expert: bool = parsed_shared_args._expert
# This helper function makes it easy to suppress options from
# The help message. The options are still there, and initialize
# what they need to initialize.
def h(msg: str)->str:
if _expert:
return msg
else:
return SUPPRESS
parser.add_argument( "input_file_path", help="The KGTK file to convert to a GitHub markdown table.", type=Path, nargs='?', default=Path("-"))
parser.add_argument("-o", "--output-file", dest="output_file_path", help="The KGTK file to write (default=%(default)s).", type=Path, default="-")
parser.add_argument( "--output-format", dest="output_format", help=h("The file format (default=%(default)s)"), type=str, default="md")
KgtkReader.add_debug_arguments(parser, expert=_expert)
KgtkReaderOptions.add_arguments(parser, mode_options=True, expert=_expert)
KgtkValueOptions.add_arguments(parser, expert=_expert)
def add_arguments_extended(parser: KGTKArgumentParser,
parsed_shared_args: Namespace):
"""
Parse arguments
Args:
parser (argparse.ArgumentParser)
"""
# import modules locally
from kgtk.io.kgtkreader import KgtkReader, KgtkReaderOptions
from kgtk.utils.argparsehelpers import optional_bool
from kgtk.value.kgtkvalueoptions import KgtkValueOptions
_expert: bool = parsed_shared_args._expert
# This helper function makes it easy to suppress options from
# The help message. The options are still there, and initialize
# what they need to initialize.
def h(msg: str) -> str:
if _expert:
return msg
else:
return SUPPRESS
parser.add_input_file()
parser.add_output_file()
parser.add_argument("--output-format",
dest="output_format",
help=h("The file format (default=kgtk)"),
type=str)
parser.add_argument(
'-c',
"--columns",
dest="column_names",
required=True,
nargs='+',
metavar="COLUMN_NAME",
help=
"The list of source column names, optionally containing '..' for column ranges "
+ "and '...' for column names not explicitly mentioned.")
parser.add_argument(
"--into",
dest="into_column_name",
help="The name of the column to receive the result of the calculation.",
required=True)
parser.add_argument("--do",
dest="operation",
help="The name of the operation.",
required=True,
choices=["percentage"])
parser.add_argument("--format",
dest="format_string",
help="The format string for the calculation.")
KgtkReader.add_debug_arguments(parser, expert=_expert)
KgtkReaderOptions.add_arguments(parser, mode_options=True, expert=_expert)
KgtkValueOptions.add_arguments(parser, expert=_expert)
def add_arguments_extended(parser: KGTKArgumentParser,
parsed_shared_args: Namespace):
"""
Parse arguments
Args:
parser (argparse.ArgumentParser)
"""
from kgtk.io.kgtkreader import KgtkReader, KgtkReaderOptions, KgtkReaderMode
from kgtk.utils.argparsehelpers import optional_bool
from kgtk.value.kgtkvalueoptions import KgtkValueOptions
_expert: bool = parsed_shared_args._expert
parser.add_input_file(positional=True)
parser.add_output_file()
parser.add_output_file(
who="The KGTK file for input records that fail the filter.",
dest="reject_file",
options=["--reject-file"],
metavar="REJECT_FILE",
optional=True)
parser.add_argument(
"--columns",
dest="filter_column_names",
help="The columns in the file being filtered (Required).",
nargs='+',
required=True)
parser.add_argument(
"--count",
dest="only_count",
metavar="True|False",
help="Only count the records, do not copy them. (default=%(default)s).",
type=optional_bool,
nargs='?',
const=True,
default=False)
parser.add_argument(
"--all",
dest="all_are",
metavar="True|False",
help=
"False: Test if any are empty, True: test if all are empty (default=%(default)s).",
type=optional_bool,
nargs='?',
const=True,
default=False)
KgtkReader.add_debug_arguments(parser, expert=_expert)
KgtkReaderOptions.add_arguments(
parser,
mode_options=True,
default_mode=KgtkReaderMode[parsed_shared_args._mode],
expert=_expert)
KgtkValueOptions.add_arguments(parser, expert=_expert)
def add_arguments_extended(parser: KGTKArgumentParser,
parsed_shared_args: Namespace):
"""
Parse arguments
Args:
parser (argparse.ArgumentParser)
"""
_expert: bool = parsed_shared_args._expert
# This helper function makes it easy to suppress options from
# The help message. The options are still there, and initialize
# what they need to initialize.
def h(msg: str) -> str:
if _expert:
return msg
else:
return SUPPRESS
parser.add_input_file(who="KGTK input files",
dest="input_files",
options=["-i", "--input-files"],
allow_list=True,
positional=True)
parser.add_output_file()
parser.add_argument("--output-format",
dest="output_format",
help="The file format (default=kgtk)",
type=str,
choices=KgtkWriter.OUTPUT_FORMAT_CHOICES)
parser.add_argument(
"--output-columns",
dest="output_column_names",
metavar="NEW_COLUMN_NAME",
help=h("The list of new column names when renaming all columns."),
type=str,
nargs='+')
parser.add_argument(
"--old-columns",
dest="old_column_names",
metavar="OLD_COLUMN_NAME",
help=h("The list of old column names for selective renaming."),
type=str,
nargs='+')
parser.add_argument(
"--new-columns",
dest="new_column_names",
metavar="NEW_COLUMN_NAME",
help=h("The list of new column names for selective renaming."),
type=str,
nargs='+')
KgtkReader.add_debug_arguments(parser, expert=_expert)
KgtkReaderOptions.add_arguments(parser, mode_options=True, expert=_expert)
KgtkValueOptions.add_arguments(parser, expert=_expert)
def add_arguments_extended(parser: KGTKArgumentParser,
parsed_shared_args: Namespace):
"""
Parse arguments
Args:
parser (argparse.ArgumentParser)
"""
# import modules locally
from kgtk.io.kgtkreader import KgtkReader, KgtkReaderOptions
from kgtk.utils.argparsehelpers import optional_bool
from kgtk.value.kgtkvalueoptions import KgtkValueOptions
_expert: bool = parsed_shared_args._expert
_command: str = parsed_shared_args._command
# This helper function makes it easy to suppress options from
# The help message. The options are still there, and initialize
# what they need to initialize.
def h(msg: str) -> str:
if _expert:
return msg
else:
return SUPPRESS
parser.add_input_file()
parser.add_output_file()
parser.add_argument("--output-format",
dest="output_format",
help=h("The file format (default=kgtk)"),
type=str)
parser.add_argument(
'-c',
"--columns",
dest="column_names",
required=True,
nargs='+',
metavar="COLUMN_NAME",
help=
"The list of reordered column names, optionally containing '...' for column names not explicitly mentioned."
)
parser.add_argument(
"--trim",
dest="omit_remaining_columns",
help="If true, omit unmentioned columns. (default=%(default)s).",
metavar="True|False",
type=optional_bool,
nargs='?',
const=True,
default=(_command == SELECT_COLUMNS_COMMAND))
KgtkReader.add_debug_arguments(parser, expert=_expert)
KgtkReaderOptions.add_arguments(parser, mode_options=True, expert=_expert)
KgtkValueOptions.add_arguments(parser, expert=_expert)
def add_arguments_extended(parser: KGTKArgumentParser, parsed_shared_args: Namespace):
"""
Parse arguments
Args:
parser (argparse.ArgumentParser)
"""
_expert: bool = parsed_shared_args._expert
# This helper function makes it easy to suppress options from
# The help message. The options are still there, and initialize
# what they need to initialize.
def h(msg: str)->str:
if _expert:
return msg
else:
return SUPPRESS
parser.add_argument( "left_file_path", help="The left-side KGTK file to join (required).", type=Path, default=None)
parser.add_argument( "right_file_path", help="The right-side KGTK file to join (required).", type=Path, default=None)
parser.add_argument( "--join-on-label", dest="join_on_label",
help="If both input files are edge files, include the label column in the join (default=%(default)s).",
type=optional_bool, nargs='?', const=True, default=False)
parser.add_argument( "--join-on-node2", dest="join_on_node2",
help="If both input files are edge files, include the node2 column in the join (default=%(default)s).",
type=optional_bool, nargs='?', const=True, default=False)
parser.add_argument( "--left-file-join-columns", dest="left_join_columns", help="Left file join columns (default=None).", nargs='+')
parser.add_argument( "--left-join", dest="left_join", help="Perform a left outer join (default=%(default)s).",
type=optional_bool, nargs='?', const=True, default=False)
parser.add_argument("-o", "--output-file", dest="output_file_path", help="The KGTK file to write (default=%(default)s).", type=Path, default="-")
parser.add_argument( "--prefix", dest="prefix",
help="An optional prefix applied to right file column names in the output file (default=None).")
parser.add_argument( "--right-file-join-columns", dest="right_join_columns", help="Right file join columns (default=None).", nargs='+')
parser.add_argument( "--right-join", dest="right_join", help="Perform a right outer join (default=%(default)s).",
type=optional_bool, nargs='?', const=True, default=False)
parser.add_argument( "--field-separator", dest="field_separator",
help=h("Separator for multifield keys (default=%(default)s)")
, default=KgtkJoiner.FIELD_SEPARATOR_DEFAULT)
# Build the command arguments. File arguments can be set for individual
# files, or for all files.
KgtkReader.add_debug_arguments(parser, expert=_expert)
KgtkReaderOptions.add_arguments(parser, mode_options=True, expert=_expert)
KgtkReaderOptions.add_arguments(parser, mode_options=True, who="left", expert=_expert, defaults=False)
KgtkReaderOptions.add_arguments(parser, mode_options=True, who="right", expert=_expert, defaults=False)
KgtkValueOptions.add_arguments(parser, expert=_expert)
def add_arguments_extended(parser: KGTKArgumentParser,
parsed_shared_args: Namespace):
"""
Parse arguments
Args:
parser (argparse.ArgumentParser)
"""
from kgtk.imports.kgtkntriples import KgtkNtriples
from kgtk.io.kgtkreader import KgtkReader, KgtkReaderOptions, KgtkReaderMode
from kgtk.reshape.kgtkidbuilder import KgtkIdBuilder, KgtkIdBuilderOptions
from kgtk.utils.argparsehelpers import optional_bool
from kgtk.value.kgtkvalueoptions import KgtkValueOptions
_expert: bool = parsed_shared_args._expert
# This helper function makes it easy to suppress options from
# The help message. The options are still there, and initialize
# what they need to initialize.
def h(msg: str) -> str:
if _expert:
return msg
else:
return SUPPRESS
parser.add_input_file(who="The ntriples file(s) to import.",
allow_list=True)
parser.add_output_file()
parser.add_output_file(
who="The ntriples output file for records that are rejected.",
dest="reject_file",
options=["--reject-file"],
metavar="REJECT_FILE",
optional=True)
parser.add_input_file(who="The KGTK input file with known namespaces.",
dest="namespace_file",
options=["--namespace-file"],
metavar="NAMESPACE_FILE",
optional=True)
parser.add_output_file(who="The KGTK output file with updated namespaces.",
dest="updated_namespace_file",
options=["--updated-namespace-file"],
metavar="NAMESPACE_FILE",
optional=True)
KgtkNtriples.add_arguments(parser)
KgtkIdBuilderOptions.add_arguments(parser)
KgtkReader.add_debug_arguments(parser, expert=_expert)
KgtkReaderOptions.add_arguments(
parser,
mode_options=True,
default_mode=KgtkReaderMode[parsed_shared_args._mode],
expert=_expert)
KgtkValueOptions.add_arguments(parser)
def add_arguments_extended(parser: KGTKArgumentParser,
parsed_shared_args: Namespace):
"""
Parse arguments
Args:
parser (argparse.ArgumentParser)
"""
from kgtk.exceptions import KGTKException
from kgtk.lift.kgtklift import KgtkLift
from kgtk.io.kgtkreader import KgtkReader, KgtkReaderOptions
from kgtk.utils.argparsehelpers import optional_bool
from kgtk.value.kgtkvalueoptions import KgtkValueOptions
_expert: bool = parsed_shared_args._expert
_command: str = parsed_shared_args._command
# This helper function makes it easy to suppress options from
# The help message. The options are still there, and initialize
# what they need to initialize.
def h(msg: str) -> str:
if _expert:
return msg
else:
return SUPPRESS
parser.add_input_file()
parser.add_output_file()
# The default value for this option depends upon the command used.
parser.add_argument(
'-l',
'--lines',
dest="count_records",
metavar="True/False",
help="If true, count records and print a single number to stdout. " +
"If false, count non-empty values per column and produce a simple KGTK output file. (default=%(default)s).",
type=optional_bool,
nargs='?',
const=True,
default=DEFAULT_COUNT_RECORDS_WC
if _command == WC_COMMAND else DEFAULT_COUNT_RECORDS)
# This is an expert option. It will not show up on `--help` without `--expert`:
parser.add_argument(
"--count-property",
dest="count_property",
help=
h("The property used for column count output edges. (default=%(default)s)."
),
default=DEFAULT_COUNT_PROPERTY)
KgtkReader.add_debug_arguments(parser, expert=_expert)
KgtkReaderOptions.add_arguments(parser, mode_options=True, expert=_expert)
KgtkValueOptions.add_arguments(parser, expert=_expert)
def add_arguments_extended(parser: KGTKArgumentParser, parsed_shared_args: Namespace):
"""
Parse arguments
Args:
parser (argparse.ArgumentParser)
"""
from kgtk.io.kgtkreader import KgtkReader, KgtkReaderOptions, KgtkReaderMode
from kgtk.unreify.kgtkunreifyvalues import KgtkUnreifyValues
from kgtk.utils.argparsehelpers import optional_bool
from kgtk.value.kgtkvalueoptions import KgtkValueOptions
_expert: bool = parsed_shared_args._expert
# This helper function makes it easy to suppress options from
# The help message. The options are still there, and initialize
# what they need to initialize.
def h(msg: str)->str:
if _expert:
return msg
else:
return SUPPRESS
parser.add_input_file(who="The KGTK input file with the reified data. " +
"It must have node1, label, and node2 columns, or their aliases. " +
"It may have an ID column; if it does not, one will be appended to the output file. " +
"It may not have any additional columns. ")
parser.add_output_file(who="The KGTK file to write output records with unreified data. " +
"This file may differ in shape from the input file by the addition of an ID column. " +
"The records in the output file will not, generally, be in the same order as they appeared in the input file. ")
parser.add_output_file(who="A KGTK output file that will contain only the reified RDF statements.",
dest="reified_file",
options=["--reified-file"],
metavar="REIFIED_FILE",
optional=True)
parser.add_output_file(who="A KGTK output file that will contain only the unreified RDF statements.",
dest="unreified_file",
options=["--unreified-file"],
metavar="UNREIFIED_FILE",
optional=True)
parser.add_output_file(who="A KGTK output file that will contain only the uninvolved input.",
dest="uninvolved_file",
options=["--uninvolved-file"],
metavar="UNINVOLVED_FILE",
optional=True)
KgtkUnreifyValues.add_arguments(parser)
KgtkReader.add_debug_arguments(parser, expert=_expert)
KgtkReaderOptions.add_arguments(parser,
mode_options=True,
default_mode=KgtkReaderMode[parsed_shared_args._mode],
expert=_expert)
KgtkValueOptions.add_arguments(parser)
def add_arguments_extended(parser: KGTKArgumentParser, parsed_shared_args: Namespace):
"""
Parse arguments
Args:
parser (argparse.ArgumentParser)
"""
_expert: bool = parsed_shared_args._expert
# This helper function makes it easy to suppress options from
# The help message. The options are still there, and initialize
# what they need to initialize.
def h(msg: str)->str:
if _expert:
return msg
else:
return SUPPRESS
parser.add_argument( "input_kgtk_file", nargs="?", type=Path, default="-",
help="The KGTK file to filter. May be omitted or '-' for stdin (default=%(default)s).")
parser.add_argument("-o", "--output-file", dest="output_kgtk_file", help="The KGTK file to write (default=%(default)s).", type=Path, default="-")
parser.add_argument( "--column", dest="column_name", help="The name of the column to explode. (default=%(default)s).", default=KgtkFormat.NODE2)
fgroup: _MutuallyExclusiveGroup = parser.add_mutually_exclusive_group()
fgroup.add_argument( "--types", dest="type_names", nargs='*',
help="The KGTK data types for which fields should be exploded. (default=%(default)s).",
choices=KgtkFormat.DataType.choices(),
default=KgtkFormat.DataType.choices())
fgroup.add_argument( "--fields", dest="field_names", nargs='*',
help=h("The names of the fields to extract (overrides --types). (default=%(default)s)."),
choices=KgtkValueFields.FIELD_NAMES)
parser.add_argument( "--prefix", dest="prefix", help="The prefix for exploded column names. (default=%(default)s).",
default=KgtkFormat.NODE2 + ";" + KgtkFormat.KGTK_NAMESPACE)
parser.add_argument( "--overwrite", dest="overwrite_columns",
help="Indicate that it is OK to overwrite existing columns. (default=%(default)s).",
type=optional_bool, nargs='?', const=True, default=False)
parser.add_argument( "--expand", dest="expand_list",
help="Expand the source column if it contains a list, else fail. (default=%(default)s).",
type=optional_bool, nargs='?', const=True, default=False)
parser.add_argument( "--show-data-types", dest="show_data_types",
help="Print the list of data types and exit. (default=%(default)s).",
type=optional_bool, nargs='?', const=True, default=False)
KgtkReader.add_debug_arguments(parser, expert=_expert)
KgtkReaderOptions.add_arguments(parser, mode_options=True, expert=_expert)
KgtkValueOptions.add_arguments(parser, expert=_expert)
def add_arguments_extended(parser: KGTKArgumentParser, parsed_shared_args: Namespace ):
from kgtk.io.kgtkreader import KgtkReader, KgtkReaderOptions, KgtkReaderMode
from kgtk.utils.argparsehelpers import optional_bool
from kgtk.value.kgtkvalueoptions import KgtkValueOptions
_expert: bool = parsed_shared_args._expert
parser.add_input_file()
parser.add_input_file(who="The entity label file(s)",
dest="entity_label_files",
options=['--entity-label-file'],
metavar="ENTITY_LABEL_FILE",
optional=True,
allow_list=True,
default_stdin=False)
parser.add_output_file()
parser.add_argument("--label-properties", dest="label_properties", nargs="*",
help="The label properties. (default=%s)" % repr(DEFAULT_LABEL_PROPERTIES))
parser.add_argument("--description-properties", dest="description_properties", nargs="*",
help="The description properties. (default=%s)" % repr(DEFAULT_DESCRIPTION_PROPERTIES))
parser.add_argument("--isa-properties", dest="isa_properties", nargs="*",
help="The isa properties. (default=%s)" % repr(DEFAULT_ISA_PROPERTIES))
parser.add_argument("--has-properties", dest="has_properties", nargs="*",
help="The has properties. (default=%s)" % repr(DEFAULT_HAS_PROPERTIES))
parser.add_argument("--property-values", dest="property_values", nargs="*",
help="The property values. (default=%s)" % repr(DEFAULT_PROPERTY_VALUES))
parser.add_argument('--sentence-label', action='store', type=str, dest='sentence_label', default=DEFAULT_SENTENCE_LABEL,
help="The relationship to write in the output file. (default=%(default)s)")
parser.add_argument("--explain", dest="explain", metavar="True|False",
help="When true, include an explanation column that tells how the sentence was constructed. (default=%(default)s).",
type=optional_bool, nargs='?', const=True, default=False)
parser.add_argument("--presorted", dest="presorted", metavar="True|False",
help="When true, the input file is presorted on node1. (default=%(default)s).",
type=optional_bool, nargs='?', const=True, default=False)
parser.add_argument("--add-entity-labels-from-input", dest="add_entity_labels_from_input", metavar="True|False",
help="When true, extract entity labels from the unsorted input file. (default=%(default)s).",
type=optional_bool, nargs='?', const=True, default=False)
KgtkReader.add_debug_arguments(parser, expert=False)
KgtkReaderOptions.add_arguments(parser,
mode_options=True,
default_mode=KgtkReaderMode[parsed_shared_args._mode],
expert=_expert)
KgtkValueOptions.add_arguments(parser, expert=False)
def main():
"""
Update the documentation files.
"""
parser: ArgumentParser = ArgumentParser()
parser.add_argument("--md-files", dest="md_files", help="The .md files to be updated.", type=Path, nargs='+')
parser.add_argument("--kgtk-command", dest="kgtk_command", help="The kgtk command (default %(default)s.", type=str, default="kgtk")
parser.add_argument("--process-usage", dest="process_usage", metavar="optional True|False",
help="Process the ## Usage section (default=%(default)s).",
type=optional_bool, nargs='?', const=True, default=True)
parser.add_argument("--update-usage", dest="update_usage", metavar="optional True|False",
help="Update the ## Usage section (default=%(default)s).",
type=optional_bool, nargs='?', const=True, default=True)
parser.add_argument("--process-examples", dest="process_examples", metavar="optional True|False",
help="Process the ## Examples section (default=%(default)s).",
type=optional_bool, nargs='?', const=True, default=True)
parser.add_argument("--update-examples", dest="update_examples", metavar="optional True|False",
help="Update the ## Examples section (default=%(default)s).",
type=optional_bool, nargs='?', const=True, default=True)
KgtkReader.add_debug_arguments(parser)
args: Namespace = parser.parse_args()
error_file: typing.TextIO = sys.stdout if args.errors_to_stdout else sys.stderr
if args.show_options:
print("--md-files %s" % " ".join([repr(str(x)) for x in args.md_files]), file=error_file, flush=True)
print("--kgtk-command=%s" % repr(args.kgtk_command), file=error_file, flush=True)
print("--process-usage=%s" % repr(args.process_usage), file=error_file, flush=True)
print("--update-usage=%s" % repr(args.update_usage), file=error_file, flush=True)
print("--process-examples=%s" % repr(args.process_examples), file=error_file, flush=True)
print("--update-examples=%s" % repr(args.update_examples), file=error_file, flush=True)
print("--verbose=%s" % repr(args.verbose), file=error_file, flush=True)
print("--very-verbose=%s" % repr(args.very_verbose), file=error_file, flush=True)
du: DocUpdater = DocUpdater(kgtk_command=args.kgtk_command,
process_usage=args.process_usage,
update_usage=args.update_usage,
process_examples=args.process_examples,
update_examples=args.update_examples,
error_file=error_file,
verbose=args.verbose,
very_verbose=args.very_verbose)
md_file: Path
for md_file in args.md_files:
du.process(md_file)
def add_arguments_extended(parser: KGTKArgumentParser,
parsed_shared_args: Namespace):
"""
Parse arguments
Args:
parser (argparse.ArgumentParser)
"""
_expert: bool = parsed_shared_args._expert
parser.add_argument(
"input_kgtk_file",
nargs="?",
help="The KGTK file to filter. May be omitted or '-' for stdin.",
type=Path)
parser.add_argument(
"--columns",
dest="filter_column_names",
help="The columns in the file being filtered (Required).",
nargs='+',
required=True)
parser.add_argument(
"--count",
dest="only_count",
help="Only count the records, do not copy them. (default=%(default)s).",
type=optional_bool,
nargs='?',
const=True,
default=False)
parser.add_argument("-o",
"--output-file",
dest="output_kgtk_file",
help="The KGTK file to write (default=%(default)s).",
type=Path,
default="-")
parser.add_argument(
"--all",
dest="all_are",
help=
"False: Test if any are, True: test if all are (default=%(default)s).",
type=optional_bool,
nargs='?',
const=True,
default=False)
KgtkReader.add_debug_arguments(parser, expert=_expert)
KgtkReaderOptions.add_arguments(parser, mode_options=True, expert=_expert)
KgtkValueOptions.add_arguments(parser, expert=_expert)
def add_arguments_extended(parser: KGTKArgumentParser, parsed_shared_args: Namespace):
"""
Parse arguments
Args:
parser (argparse.ArgumentParser)
"""
from kgtk.io.kgtkreader import KgtkReader, KgtkReaderOptions
from kgtk.join.unique import Unique
from kgtk.utils.argparsehelpers import optional_bool
from kgtk.value.kgtkvalueoptions import KgtkValueOptions
_expert: bool = parsed_shared_args._expert
# This helper function makes it easy to suppress options from
# The help message. The options are still there, and initialize
# what they need to initialize.
def h(msg: str)->str:
if _expert:
return msg
else:
return SUPPRESS
parser.add_input_file(positional=True)
parser.add_output_file()
parser.add_argument( "--column", dest="column_name",
help="The column to count unique values (required).", required=True)
parser.add_argument( "--empty", dest="empty_value", help="A value to substitute for empty values (default=%(default)s).", default="")
parser.add_argument( "--label", dest="label_value", help="The output file label column value (default=%(default)s).", default="count")
# TODO: use an emum
parser.add_argument( "--format", dest="output_format", help=h("The output file format and mode (default=%(default)s)."),
default=Unique.DEFAULT_FORMAT, choices=Unique.OUTPUT_FORMATS)
parser.add_argument( "--prefix", dest="prefix", help=h("The value prefix (default=%(default)s)."), default="")
parser.add_argument( "--where", dest="where_column_name",
help="The name of a column for a record selection test. (default=%(default)s).", default=None)
parser.add_argument( "--in", dest="where_values", nargs="+",
help="The list of values for a record selection test. (default=%(default)s).", default=None)
parser.add_argument( "--presorted", dest="presorted", metavar="True|False",
help="When True, the input file is presorted. (default=%(default)s).",
type=optional_bool, nargs='?', const=True, default=False)
KgtkReader.add_debug_arguments(parser, expert=_expert)
KgtkReaderOptions.add_arguments(parser, mode_options=True, expert=_expert)
KgtkValueOptions.add_arguments(parser, expert=_expert)
def process(self):
# Open the input file.
if self.verbose:
if self.input_file_path is not None:
print("Opening the input file: %s" % self.input_file_path,
file=self.error_file,
flush=True)
else:
print("Reading the input data from stdin",
file=self.error_file,
flush=True)
ikr: KgtkReader = KgtkReader.open(
self.input_file_path,
error_file=self.error_file,
options=self.reader_options,
value_options=self.value_options,
verbose=self.verbose,
very_verbose=self.very_verbose,
)
# If supplied, open the label file.
lkr: typing.Optional[KgtkReader] = None
if self.label_file_path is not None:
if self.verbose:
if self.input_file_path is not None:
print("Opening the label file: %s" % self.label_file_path,
file=self.error_file,
flush=True)
else:
print("Reading the label data from stdin",
file=self.error_file,
flush=True)
lkr = KgtkReader.open(
self.label_file_path,
error_file=self.error_file,
options=self.reader_options,
value_options=self.value_options,
verbose=self.verbose,
very_verbose=self.very_verbose,
)
if self.lift_column_names is not None and len(self.lift_column_names) == 1 and \
not self.suppress_empty_columns and \
self.input_is_presorted and \
self.labels_are_presorted and \
lkr is not None:
self.process_as_merge(ikr, lkr)
else:
self.process_in_memory(ikr, lkr)
def add_arguments_extended(parser: KGTKArgumentParser,
parsed_shared_args: Namespace):
"""
Parse arguments
Args:
parser (argparse.ArgumentParser)
"""
from kgtk.io.kgtkreader import KgtkReader, KgtkReaderOptions, KgtkReaderMode
from kgtk.io.kgtkwriter import KgtkWriter
from kgtk.utils.argparsehelpers import optional_bool
from kgtk.value.kgtkvalueoptions import KgtkValueOptions
_expert: bool = parsed_shared_args._expert
# This helper function makes it easy to suppress options from
# The help message. The options are still there, and initialize
# what they need to initialize.
def h(msg: str) -> str:
if _expert:
return msg
else:
return SUPPRESS
parser.add_input_file(who="The KGTK file to convert to an HTML table.",
positional=True)
parser.add_output_file(who="The GitHub markdown file to write.")
parser.add_argument(
"--pp",
"--readable",
dest="readable",
help=
"If true, use a human-readable output format. (default=%(default)s).",
metavar="True/False",
type=optional_bool,
nargs='?',
const=True,
default=False)
parser.add_argument("--output-format",
dest="output_format",
type=str,
help=h("The file format (default=%(default)s)"),
default=KgtkWriter.OUTPUT_FORMAT_HTML_COMPACT)
KgtkReader.add_debug_arguments(parser, expert=_expert)
KgtkReaderOptions.add_arguments(parser,
mode_options=True,
default_mode=KgtkReaderMode.NONE,
expert=_expert)
KgtkValueOptions.add_arguments(parser, expert=_expert)
