def add_arguments(parser: KGTKArgumentParser): """ Parse arguments Args: parser (kgtk.cli_argparse.KGTKArgumentParser) """ parser.add_argument(action="store", type=str, metavar="name", dest="name") parser.add_argument("-i", "--info", action="store", type=str, dest="info") parser.add_argument("-e", "--error", action="store_true", help="raise an error") parser.accept_shared_argument('_debug')
def add_arguments_extended(parser: KGTKArgumentParser, parsed_shared_args): parser.accept_shared_argument('_debug') parser.accept_shared_argument('_expert') parser.add_input_file(options=["-i", "--input-files"], dest='input_files', allow_list=True, default_stdin=False, allow_stdin=True, who="One or more input files to query (maybe compressed).") parser.add_argument('--as', metavar='NAME', default={}, action=InputOptionAction, dest='alias', help="alias name to be used for preceding input") # future extension: #parser.add_argument('--in-memory', default=False, type=bool, nargs=0, action=InputOptionAction, dest='in_memory', # help="load the preceding input into a temporary in-memory table only") parser.add_argument('--query', default=None, action='store', dest='query', help="complete Kypher query combining all clauses," + " if supplied, all other specialized clause arguments will be ignored") parser.add_argument('--match', metavar='PATTERN', default='()', action='store', dest='match', help="MATCH pattern of a Kypher query, defaults to universal node pattern `()'") parser.add_argument('--where', metavar='CLAUSE', default=None, action='store', dest='where', help="WHERE clause of a Kypher query") parser.add_argument('--return', metavar='CLAUSE', default='*', action='store', dest='return_', help="RETURN clause of a Kypher query (defaults to *)") parser.add_argument('--order-by', metavar='CLAUSE', default=None, action='store', dest='order', help="ORDER BY clause of a Kypher query") parser.add_argument('--skip', metavar='CLAUSE', default=None, action='store', dest='skip', help="SKIP clause of a Kypher query") parser.add_argument('--limit', metavar='CLAUSE', default=None, action='store', dest='limit', help="LIMIT clause of a Kypher query") parser.add_argument('--para', metavar='NAME=VAL', action='append', dest='regular_paras', help="zero or more named value parameters to be passed to the query") parser.add_argument('--spara', metavar='NAME=VAL', action='append', dest='string_paras', help="zero or more named string parameters to be passed to the query") parser.add_argument('--lqpara', metavar='NAME=VAL', action='append', dest='lqstring_paras', help="zero or more named LQ-string parameters to be passed to the query") parser.add_argument('--no-header', action='store_true', dest='no_header', help="do not generate a header row with column names") parser.add_argument('--index', metavar='MODE', nargs='?', action='store', dest='index', choices=INDEX_MODES, const=INDEX_MODES[0], default=INDEX_MODES[0], help="control column index creation according to MODE" + " (%(choices)s, default: %(const)s)") parser.add_argument('--explain', metavar='MODE', nargs='?', action='store', dest='explain', choices=EXPLAIN_MODES, const=EXPLAIN_MODES[0], help="explain the query execution and indexing plan according to MODE" + " (%(choices)s, default: %(const)s)." + " This will not actually run or create anything.") parser.add_argument('--graph-cache', default=DEFAULT_GRAPH_CACHE_FILE, action='store', dest='graph_cache_file', help="database cache where graphs will be imported before they are queried" + " (defaults to per-user temporary file)") parser.add_argument('-o', '--out', default='-', action='store', dest='output', help="output file to write to, if `-' (the default) output goes to stdout." + " Files with extensions .gz, .bz2 or .xz will be appropriately compressed.")
def add_arguments_extended(parser: KGTKArgumentParser, parsed_shared_args): parser.accept_shared_argument('_debug') parser.accept_shared_argument('_expert') parser.add_input_file(options=["-i", "--input-files"], dest='input_files', # default_stdin is what makes it not required which is what we need for --show-cache: allow_list=True, default_stdin=True, allow_stdin=True, who="One or more input files to query, maybe compressed") parser.add_argument('--as', metavar='NAME', default={}, action=InputOptionAction, dest='alias', help="alias name to be used for preceding input") parser.add_argument('--comment', default=None, action=InputOptionAction, dest='comment', help="comment string to store for the preceding input (displayed by --show-cache)") # future extension: #parser.add_argument('--in-memory', default=False, type=bool, nargs=0, action=InputOptionAction, dest='in_memory', # help="load the preceding input into a temporary in-memory table only") parser.add_argument('--query', default=None, action='store', dest='query', help="complete Kypher query combining all clauses," + " if supplied, all other specialized clause arguments will be ignored") parser.add_argument('--match', metavar='PATTERN', default='()', action=MatchOptionAction, dest='match', help="MATCH pattern of a Kypher query, defaults to universal node pattern `()'") parser.add_argument('--where', metavar='CLAUSE', default=None, action=MatchOptionAction, dest='where', help="WHERE clause to a preceding --match, --opt or --with clause") parser.add_argument('--opt', '--optional', metavar='PATTERN', default=None, action=MatchOptionAction, help="OPTIONAL MATCH pattern(s) of a Kypher query (zero or more)") parser.add_argument('--with', metavar='CLAUSE', default='*', action=MatchOptionAction, dest='with', help="WITH clause of a Kypher query (only 'WITH * ...' is currently supported)") parser.add_argument('--where:', metavar='CLAUSE', default=None, action='store', dest='with_where', help="final global WHERE clause, shorthand for 'WITH * WHERE ...'") parser.add_argument('--return', metavar='CLAUSE', default='*', action='store', dest='return', help="RETURN clause of a Kypher query (defaults to *)") parser.add_argument('--order-by', metavar='CLAUSE', default=None, action='store', dest='order', help="ORDER BY clause of a Kypher query") parser.add_argument('--skip', metavar='CLAUSE', default=None, action='store', dest='skip', help="SKIP clause of a Kypher query") parser.add_argument('--limit', metavar='CLAUSE', default=None, action='store', dest='limit', help="LIMIT clause of a Kypher query") parser.add_argument('--para', metavar='NAME=VAL', action='append', dest='regular_paras', help="zero or more named value parameters to be passed to the query") parser.add_argument('--spara', metavar='NAME=VAL', action='append', dest='string_paras', help="zero or more named string parameters to be passed to the query") parser.add_argument('--lqpara', metavar='NAME=VAL', action='append', dest='lqstring_paras', help="zero or more named LQ-string parameters to be passed to the query") parser.add_argument('--no-header', action='store_true', dest='no_header', help="do not generate a header row with column names") parser.add_argument('--force', action='store_true', dest='force', help="force problematic queries to run against advice") parser.add_argument('--index', '--index-mode', metavar='MODE', nargs='+', action='store', dest='index_mode', default=[INDEX_MODES[0]], help="default index creation MODE for all inputs" + f" (default: {INDEX_MODES[0]});" + " can be overridden with --idx for specific inputs") parser.add_argument('--idx', '--input-index', metavar='SPEC', nargs='+', default=None, action=InputOptionAction, dest='index_specs', help="create index(es) according to SPEC for the preceding input only") parser.add_argument('--explain', metavar='MODE', nargs='?', action='store', dest='explain', choices=EXPLAIN_MODES, const=EXPLAIN_MODES[0], help="explain the query execution and indexing plan according to MODE" + " (%(choices)s, default: %(const)s)." + " This will not actually run or create anything.") parser.add_argument('--graph-cache', '--gc', action='store', dest='graph_cache_file', help="database cache where graphs will be imported before they are queried" + " (defaults to per-user temporary file)") parser.add_argument('--show-cache', '--sc', action='store_true', dest='show_cache', help="describe the current content of the graph cache and exit" + " (does not actually run a query or import data)") parser.add_argument('--read-only', '--ro', action='store_true', dest='readonly', help="do not create or update the graph cache in any way" + ", only run queries against already imported and indexed data") parser.add_argument('--import', metavar='MODULE_LIST', default=None, action='store', dest='import', help="Python modules needed to define user extensions to built-in functions") parser.add_argument('-o', '--out', default='-', action='store', dest='output', help="output file to write to, if `-' (the default) output goes to stdout." + " Files with extensions .gz, .bz2 or .xz will be appropriately compressed.")
def add_arguments(parser: KGTKArgumentParser): from kgtk.utils.argparsehelpers import optional_bool parser.accept_shared_argument('_debug') # input file # parser.add_argument('input_file', nargs='?', type=argparse.FileType('r'), default=sys.stdin) parser.add_input_file(positional=True) # model name all_models_names = ALL_EMBEDDING_MODELS_NAMES parser.add_argument('-m', '--model', action='store', nargs='+', dest='all_models_names', default="bert-base-nli-cls-token", choices=all_models_names, help="the model to used for embedding") # parser.add_argument('-i', '--input', action='store', nargs='+', dest='input_uris', # help="input path", ) parser.add_argument( '-f', '--input-data-format', action='store', dest='data_format', choices=("test_format", "kgtk_format"), default="kgtk_format", help= "the input file format, could either be `test_format` or `kgtk_format`, default is `kgtk_format`", ) parser.add_argument( '-p', '--property-labels-file', action='store', nargs='+', dest='property_labels_file_uri', help="the path to the property labels file.", ) # This should probably default to "--label-properties" if not specified. parser.add_argument( '--property-labels-filter', action='store', nargs='+', dest='property_labels_filter', default=["label"], help= "The label columns value(s) of the edges to process in the property labels file. Default is [\"label\"]." ) # properties (only valid for kgtk format input/output data) parser.add_argument( '--label-properties', action='store', nargs='+', dest='label_properties', default=["label"], help= """The names of the edges for label properties, Default is ["label"]. \n This argument is only valid for input in kgtk format.""" ) parser.add_argument( '--description-properties', action='store', nargs='+', dest='description_properties', default=["description"], help= """The names of the edges for description properties, Default is ["description"].\n This argument is only valid for input in kgtk format.""" ) parser.add_argument( '--isa-properties', action='store', nargs='+', dest='isa_properties', default=["P31"], help= """The names of the edges for `isa` properties, Default is ["P31"] (the `instance of` node in wikidata).""") parser.add_argument( '--has-properties', action='store', nargs='+', dest='has_properties', default=[], help= """The names of the edges for `has` properties, Default is ["all"] (will automatically append all properties found for each node).""") parser.add_argument( '--property-value', action='store', nargs='+', dest='property_values', default=[], help= """For those edges found in `has` properties, the nodes specified here will display with corresponding edge(property) values. instead of edge name. """ ) parser.add_argument( '--property-value-file', action='store', dest='property_values_file', help= """Read the properties for --property-value option from an KGTK edge file""" ) parser.add_argument( '--output-property', action='store', dest='output_properties', default="text_embedding", help= """The output property name used to record the embedding. Default is `output_properties`. \n This argument is only valid for output in kgtk format.""" ) # output parser.add_argument( '--save-embedding-sentence', action='store_true', dest='save_embedding_sentence', help="if set, will also save the embedding sentences to output.") parser.add_argument( '-o', '--embedding-projector-metadata-path', action='store', dest='output_uri', default="", help= "output path for the metadata file, default will be current user's home directory" ) parser.add_argument( '--output-data-format', action='store', dest='output_data_format', default="kgtk_format", choices=("tsv_format", "kgtk_format"), help= "output format, can either be `tsv_format` or `kgtk_format`. \nIf choose `tsv_format`, the output " "will be a tsv file, with each row contains only the vector representation of a node. Each " "dimension is separated by a tab") parser.add_argument( '--embedding-projector-metadata', action='store', nargs='+', dest='metadata_properties', default=[], help= """list of properties used to construct a metadata file for use in the Google Embedding Projector: http://projector.tensorflow.org. \n Default: the label and description of each node.""" ) # black list file parser.add_argument( '-b', '--black-list', nargs='+', action='store', dest='black_list_files', default=[], help= "the black list file, contains the Q nodes which should not consider as candidates." ) # dimensional reduction relate parser.add_argument( "--dimensional-reduction", nargs='?', action='store', default="none", dest="dimensional_reduction", choices=("pca", "tsne", "none"), help= 'whether to run dimensional reduction algorithm or not after the embedding, default is None (not ' 'run). ') parser.add_argument( "--dimension", type=int, nargs='?', action='store', default=2, dest="dimension_val", help= 'How many dimension should remained after reductions, only valid when set to run dimensional ' 'reduction, default value is 2 ') parser.add_argument( "--parallel", nargs='?', action='store', default="1", dest="parallel_count", help="How many processes to be run in same time, default is 1.") # cache config parser.add_argument( "--use-cache", type=optional_bool, nargs='?', action='store', default=False, dest="use_cache", help= "whether to use cache to get some embedding vectors quicker, default is False" ) parser.add_argument( "--cache-host", nargs='?', action='store', default="dsbox01.isi.edu", dest="cache_host", help="cache host address, default is `dsbox01.isi.edu`") parser.add_argument("--cache-port", nargs='?', action='store', default="6379", dest="cache_port", help="cache server port, default is `6379`") # query server parser.add_argument( "--query-server", nargs='?', action='store', default="", dest="query_server", help= "sparql query endpoint used for test_format input files, default is " "https://query.wikidata.org/sparql ") KgtkReader.add_debug_arguments(parser, expert=False) KgtkReaderOptions.add_arguments(parser, mode_options=True, expert=False) KgtkValueOptions.add_arguments(parser, expert=False)