def parse_arguments():
"""
Creat parser instance and parse command-line arguments passed to the pipeline
:return argparse.Namespace: parsed arguments namespace
"""
parser = VersionInHelpParser(prog="PEPATAC_collator",
description='PEPATAC collator',
version=__version__)
parser = pypiper.add_pypiper_args(parser, groups=['pypiper', 'looper'])
parser.add_argument("-n",
"--name",
help="Name of the project to use.",
type=str)
parser.add_argument("-r",
"--results",
help="Output results sub directory path.",
type=str)
parser.add_argument("--skip-consensus",
action='store_true',
dest="skip_consensus",
default=False,
help="Do not calculate consensus peaks.")
parser.add_argument("--skip-table",
action='store_true',
dest="skip_table",
default=False,
help="Do not calculate peak counts table.")
args = parser.parse_args()
return args
def build_parser():
"""
Building argument parser
:return argparse.ArgumentParser
"""
banner = "%(prog)s - API for sequence collections"
additional_description = "For subcommand-specific options, type: '%(prog)s <subcommand> -h'"
additional_description += "\nhttps://github.com/regenie/seqcolapi"
parser = VersionInHelpParser(prog=PKG_NAME,
description=banner,
epilog=additional_description)
parser.add_argument("-V",
"--version",
action="version",
version="%(prog)s {v}".format(v=seqcolapi_version))
msg_by_cmd = {"serve": "run the server"}
subparsers = parser.add_subparsers(dest="command")
def add_subparser(cmd, description):
return subparsers.add_parser(cmd,
description=description,
help=description)
sps = {}
# add arguments that are common for all subparsers
for cmd, desc in msg_by_cmd.items():
sps[cmd] = add_subparser(cmd, desc)
sps[cmd].add_argument('-c',
'--config',
required=True,
dest="config",
help="Path to the seqcolapi config file (YAML).")
sps[cmd].add_argument("-d",
"--dbg",
action="store_true",
dest="debug",
help="Set logger verbosity to debug")
# add subparser-specific arguments
sps["serve"].add_argument("-p",
"--port",
dest="port",
type=int,
help="The port the webserver should be run on.",
default=None)
return parser
def build_argparser():
# TODO: add cli hooks for ^
"""
Builds argument parser.
:return argparse.ArgumentParser
"""
banner = "%(prog)s - Keeper of druids: " \
"a python interface to Decomposable Recursive UIDs"
additional_description = "\n..."
parser = VersionInHelpParser(version=__version__,
description=banner,
epilog=additional_description)
parser.add_argument("-V",
"--version",
action="version",
version="%(prog)s {v}".format(v=__version__))
parser.add_argument("-i",
"--input",
required=True,
help="File path to input file.")
parser.add_argument("-p",
"--parameter",
type=int,
default=0,
help="Some parameter.")
return parser
def build_argparser():
banner = "%(prog)s - Interact with PEPs"
additional_description = "\nhttp://eido.databio.org/"
parser = VersionInHelpParser(
prog=PKG_NAME,
description=banner,
epilog=additional_description,
version=__version__)
subparsers = parser.add_subparsers(dest="command")
parser.add_argument(
"--verbosity", dest="verbosity",
type=int, choices=range(len(_LEVEL_BY_VERBOSITY)),
help="Choose level of verbosity (default: %(default)s)")
parser.add_argument(
"--logging-level", dest="logging_level",
help="logging level")
parser.add_argument(
"--dbg", dest="dbg", action="store_true",
help="Turn on debug mode (default: %(default)s)")
sps = {}
SUBPARSER_MSGS = {
"convert": "Convert a PEP using an available filter",
"list": "List available filters"
}
for cmd, desc in SUBPARSER_MSGS.items():
sps[cmd] = subparsers.add_parser(cmd, description=desc, help=desc)
sps["convert"].add_argument('pep', metavar="PEP",
help="Path to a PEP configuration "
"file in yaml format.")
sps["convert"].add_argument(
"-f", "--format", required=True, default="yaml",
help="Path to a PEP schema file in yaml format.")
sps["convert"].add_argument(
"-n", "--sample-name", required=False, nargs="+",
help="Name of the samples to inspect.")
return parser
def build_parser():
"""
Building argument parser
:return argparse.ArgumentParser
"""
banner = "%(prog)s - PEP web server"
additional_description = (
"For subcommand-specific options, type: '%(prog)s <subcommand> -h'")
additional_description += "\nhttps://github.com/pepkit/pepserver"
parser = VersionInHelpParser(prog=PKG_NAME,
description=banner,
epilog=additional_description)
parser.add_argument("-V",
"--version",
action="version",
version="%(prog)s {v}".format(v=v))
msg_by_cmd = {"serve": "run the server"}
subparsers = parser.add_subparsers(dest="command")
def add_subparser(cmd, description):
return subparsers.add_parser(cmd,
description=description,
help=description)
sps = {}
# add arguments that are common for both subparsers
for cmd, desc in msg_by_cmd.items():
sps[cmd] = add_subparser(cmd, desc)
sps[cmd].add_argument(
"-c",
"--config",
required=False,
dest="config",
help=f"A path to the pepserver config file",
),
sps[cmd].add_argument(
"-d",
"--dbg",
action="store_true",
dest="debug",
help="Set logger verbosity to debug",
)
sps["serve"].add_argument(
"-p",
"--port",
dest="port",
type=int,
help="The port the webserver should be run on.",
default=DEFAULT_PORT,
)
sps["serve"].add_argument("-r",
"--reload",
action="store_true",
default=False,
help="Live reloading with uvicorn")
return parser
def build_parser():
"""
Building argument parser.
:return argparse.ArgumentParser
"""
# Main looper program help text messages
banner = "%(prog)s - A project job submission engine and project manager."
additional_description = "For subcommand-specific options, " \
"type: '%(prog)s <subcommand> -h'"
additional_description += "\nhttps://github.com/pepkit/looper"
parser = VersionInHelpParser(
prog="looper", description=banner, epilog=additional_description,
version=__version__)
aux_parser = VersionInHelpParser(
prog="looper", description=banner, epilog=additional_description,
version=__version__)
result = []
for parser in [parser, aux_parser]:
# Logging control
parser.add_argument(
"--logfile", help="Optional output file for looper logs "
"(default: %(default)s)")
parser.add_argument(
"--verbosity", type=int, choices=range(len(_LEVEL_BY_VERBOSITY)),
help="Choose level of verbosity (default: %(default)s)")
parser.add_argument(
"--logging-level", help=argparse.SUPPRESS)
parser.add_argument(
"--dbg", action="store_true",
help="Turn on debug mode (default: %(default)s)")
# Individual subcommands
msg_by_cmd = {
"run": "Run or submit sample jobs.",
"rerun": "Resubmit sample jobs with failed flags.",
"runp": "Run or submit project jobs.",
"table": "Write summary stats table for project samples.",
"report": "Create browsable HTML report of project results.",
"destroy": "Remove output files of the project.",
"check": "Check flag status of current runs.",
"clean": "Run clean scripts of already processed jobs.",
"inspect": "Print information about a project.",
"init": "Initialize looper dotfile."
}
subparsers = parser.add_subparsers(dest="command")
def add_subparser(cmd):
message = msg_by_cmd[cmd]
return subparsers.add_parser(cmd, description=message, help=message,
formatter_class=lambda prog: argparse.HelpFormatter(
prog, max_help_position=37, width=90))
# Run and rerun command
run_subparser = add_subparser("run")
rerun_subparser = add_subparser("rerun")
collate_subparser = add_subparser("runp")
table_subparser = add_subparser("table")
report_subparser = add_subparser("report")
destroy_subparser = add_subparser("destroy")
check_subparser = add_subparser("check")
clean_subparser = add_subparser("clean")
inspect_subparser = add_subparser("inspect")
init_subparser = add_subparser("init")
# Flag arguments
####################################################################
for subparser in [run_subparser, rerun_subparser, collate_subparser]:
subparser.add_argument(
"-i", "--ignore-flags", default=False,
action=_StoreBoolActionType, type=html_checkbox(checked=False),
help="Ignore run status flags? Default=False")
for subparser in [run_subparser, rerun_subparser, destroy_subparser,
clean_subparser, collate_subparser]:
subparser.add_argument(
"-d", "--dry-run",
action=_StoreBoolActionType, default=False,
type=html_checkbox(checked=False),
help="Don't actually submit the jobs. Default=False")
# Parameter arguments
####################################################################
for subparser in [run_subparser, rerun_subparser, collate_subparser]:
subparser.add_argument(
"-t", "--time-delay", metavar="S",
type=html_range(min_val=0, max_val=30, value=0), default=0,
help="Time delay in seconds between job submissions")
subparser.add_argument(
"-l", "--limit", default=None, metavar="N",
type=html_range(min_val=1, max_val="num_samples",
value="num_samples"),
help="Limit to n samples")
subparser.add_argument(
"-x", "--command-extra", default="",
metavar="S", help="String to append to every command")
subparser.add_argument(
"-y", "--command-extra-override", metavar="S", default="",
help="Same as command-extra, but overrides values in PEP")
subparser.add_argument(
"-f", "--skip-file-checks",
action=_StoreBoolActionType, default=False,
type=html_checkbox(checked=False),
help="Do not perform input file checks")
divvy_group = \
subparser.add_argument_group(
"divvy arguments",
"Configure divvy to change computing settings")
divvy_group.add_argument(
"--divvy", default=None, metavar="DIVCFG",
help="Path to divvy configuration file. Default=$DIVCFG env "
"variable. Currently: {}".format(os.getenv('DIVCFG', None)
or "not set"))
divvy_group.add_argument(
"-p", "--package", metavar="P",
help="Name of computing resource package to use")
divvy_group.add_argument(
"-s", "--settings", default="", metavar="S",
help="Path to a YAML settings file with compute settings")
divvy_group.add_argument(
"-c", "--compute", metavar="K", nargs="+",
help="List of key-value pairs (k1=v1)")
for subparser in [run_subparser, rerun_subparser]:
subparser.add_argument(
"-u", "--lump", default=None, metavar="X",
type=html_range(min_val=0, max_val=100, step=0.1, value=0),
help="Total input file size (GB) to batch into one job")
subparser.add_argument(
"-n", "--lumpn", default=None, metavar="N",
type=html_range(min_val=1, max_val="num_samples", value=1),
help="Number of commands to batch into one job")
inspect_subparser.add_argument(
"-n", "--snames", required=False, nargs="+", metavar="S",
help="Name of the samples to inspect")
inspect_subparser.add_argument(
"-l", "--attr-limit", required=False, type=int, default=10,
metavar="L", help="Number of sample attributes to display")
check_subparser.add_argument(
"-A", "--all-folders", action=_StoreBoolActionType,
default=False, type=html_checkbox(checked=False),
help="Check status for all output folders, not just for "
"samples specified in the config. Default=False")
check_subparser.add_argument(
"-f", "--flags", nargs='*', default=FLAGS,
type=html_select(choices=FLAGS), metavar="F",
help="Check on only these flags/status values")
for subparser in [destroy_subparser, clean_subparser]:
subparser.add_argument(
"--force-yes", action=_StoreBoolActionType, default=False,
type=html_checkbox(checked=False),
help="Provide upfront confirmation of destruction intent, "
"to skip console query. Default=False")
init_subparser.add_argument("config_file", help="Project configuration "
"file (YAML)")
init_subparser.add_argument("-f", "--force", help="Force overwrite",
action="store_true", default=False)
# Common arguments
for subparser in [run_subparser, rerun_subparser, table_subparser,
report_subparser, destroy_subparser, check_subparser,
clean_subparser, collate_subparser, inspect_subparser]:
subparser.add_argument("config_file", nargs="?", default=None,
help="Project configuration file (YAML)")
# help="Path to the output directory"
subparser.add_argument("-o", "--output-dir", metavar="DIR",
help=argparse.SUPPRESS)
# "Submission subdirectory name"
subparser.add_argument("--submission-subdir", metavar="DIR",
help=argparse.SUPPRESS)
# "Results subdirectory name"
subparser.add_argument("--results-subdir", metavar="DIR",
help=argparse.SUPPRESS)
# "Sample attribute for pipeline interface sources"
subparser.add_argument("--pipeline-interfaces-key", metavar="K",
help=argparse.SUPPRESS)
# "Paths to pipeline interface files"
subparser.add_argument("--pipeline-interfaces", metavar="P",
nargs="+", action="append",
help=argparse.SUPPRESS)
for subparser in [run_subparser, rerun_subparser, table_subparser,
report_subparser, destroy_subparser, check_subparser,
clean_subparser, collate_subparser, inspect_subparser]:
fetch_samples_group = \
subparser.add_argument_group(
"sample selection arguments",
"Specify samples to include or exclude based on sample attribute values")
fetch_samples_group.add_argument(
"-g", "--toggle-key", metavar="K",
help="Sample attribute specifying toggle. Default: toggle")
fetch_samples_group.add_argument(
"--sel-attr", default="toggle", metavar="ATTR",
help="Attribute for sample exclusion OR inclusion")
protocols = fetch_samples_group.add_mutually_exclusive_group()
protocols.add_argument(
"--sel-excl", nargs='*', metavar="E",
help="Exclude samples with these values")
protocols.add_argument(
"--sel-incl", nargs='*', metavar="I",
help="Include only samples with these values")
subparser.add_argument(
"-a", "--amend", nargs="+", metavar="A",
help="List of amendments to activate")
result.append(parser)
return result
def build_parser():
"""
Example argument parser, needed solely for testing purposes.
This example parser was copied from looper/__init__.py
:return argparse.ArgumentParser
"""
banner = "%(prog)s - Loop through samples and submit pipelines."
additional_description = "For subcommand-specific options, " \
"type: '%(prog)s <subcommand> -h'"
additional_description += "\nhttps://github.com/pepkit/looper"
parser = VersionInHelpParser(prog="looper",
description=banner,
epilog=additional_description,
version=0.1)
# Logging control
parser.add_argument(
"--logfile",
dest="logfile",
help="Optional output file for looper logs (default: %(default)s)")
parser.add_argument(
"--verbosity",
dest="verbosity",
type=int,
choices=range(4),
help="Choose level of verbosity (default: %(default)s)")
parser.add_argument("--logging-level",
dest="logging_level",
help=argparse.SUPPRESS)
parser.add_argument("--dbg",
dest="dbg",
action="store_true",
help="Turn on debug mode (default: %(default)s)")
parser.add_argument(
"--env",
dest="env",
default=None,
help="Environment variable that points to the DIVCFG file. "
"(default: DIVCFG)")
parser.add_argument("--dotfile-template",
action="store_true",
help="Print out a looper dotfile template and exit")
# Individual subcommands
msg_by_cmd = {
"run":
"Main Looper function: Submit jobs for samples.",
"rerun":
"Resubmit jobs with failed flags.",
"runp":
"Submit jobs for a project.",
"summarize":
"Summarize statistics of project samples.",
"destroy":
"Remove all files of the project.",
"check":
"Checks flag status of current runs.",
"clean":
"Runs clean scripts to remove intermediate "
"files of already processed jobs."
}
subparsers = parser.add_subparsers(dest="command")
def add_subparser(cmd):
message = msg_by_cmd[cmd]
return subparsers.add_parser(cmd, description=message, help=message)
# Run and rerun command
run_subparser = add_subparser("run")
rerun_subparser = add_subparser("rerun")
collate_subparser = add_subparser("runp")
for subparser in [run_subparser, rerun_subparser, collate_subparser]:
subparser.add_argument(
"--ignore-flags",
dest="ignore_flags",
default=False,
action="store_true",
help="Ignore run status flags? Default: False. "
"By default, pipelines will not be submitted if a pypiper "
"flag file exists marking the run (e.g. as "
"'running' or 'failed'). Set this option to ignore flags "
"and submit the runs anyway. Default=False")
subparser.add_argument(
"-t",
"--time-delay",
dest="time_delay",
type=int,
default=0,
help="Time delay in seconds between job submissions.")
subparser.add_argument(
"-p",
"--package",
help="Name of computing resource package to use")
subparser.add_argument(
"--compute",
help="Specification of individual computing resource settings; "
"separate setting name/key from value with equals sign, "
"and separate key-value pairs from each other by comma")
subparser.add_argument("--limit",
dest="limit",
default=None,
type=int,
help="Limit to n samples.")
subparser.add_argument("-a",
"--pipeline-args",
dest="pipeline_args",
default="",
help="arguments to pass to a pipline")
subparser.add_argument(
"-s",
"--settings",
dest="settings",
default="",
help="path to a YAML-formatted settings file used to populate "
"the command template")
for subparser in [run_subparser, rerun_subparser]:
# Note that defaults for otherwise numeric lump parameters are
# set to
# null by default so that the logic that parses their values may
# distinguish between explicit 0 and lack of specification.
subparser.add_argument(
"--lump",
default=None,
type=float,
help="Maximum total input file size for a lump/batch of "
"commands in a single job (in GB)")
subparser.add_argument(
"--lumpn",
default=None,
type=int,
help="Number of individual scripts grouped into "
"single submission")
# Other commands
summarize_subparser = add_subparser("summarize")
destroy_subparser = add_subparser("destroy")
check_subparser = add_subparser("check")
clean_subparser = add_subparser("clean")
check_subparser.add_argument(
"-A",
"--all-folders",
action="store_true",
default=False,
help="Check status for all project's output folders, not just "
"those for samples specified in the config file used. "
"Default=False")
for subparser in [destroy_subparser, clean_subparser]:
subparser.add_argument(
"--force-yes",
action="store_true",
default=False,
help="Provide upfront confirmation of destruction intent, "
"to skip console query. Default=False")
# Common arguments
for subparser in [
run_subparser, rerun_subparser, summarize_subparser,
destroy_subparser, check_subparser, clean_subparser,
collate_subparser
]:
subparser.add_argument("config_file",
nargs="?",
help="Project configuration file (YAML).")
# subparser.add_argument(
# "-c", "--config", required=False, default=None,
# dest="looper_config", help="Looper configuration file (
# YAML).")
subparser.add_argument("--pipeline-interfaces",
dest="pifaces",
nargs="+",
action='append',
help="Path to a pipeline interface file")
subparser.add_argument("--file-checks",
dest="file_checks",
action="store_true",
help="Perform input file checks. Default=True.")
subparser.add_argument(
"-d",
"--dry-run",
dest="dry_run",
action="store_true",
help="Don't actually submit the jobs. Default=False")
fetch_samples_group = subparser.add_argument_group(
"select samples",
"This group of arguments lets you specify samples to use by "
"exclusion OR inclusion of the samples attribute values.")
fetch_samples_group.add_argument(
"--selector-attribute",
dest="selector_attribute",
default="toggle",
help="Specify the attribute for samples exclusion OR inclusion")
protocols = fetch_samples_group.add_mutually_exclusive_group()
protocols.add_argument(
"--selector-exclude",
nargs='*',
dest="selector_exclude",
help="Operate only on samples that either lack this attribute "
"value or for which this value is not in this collection.")
protocols.add_argument(
"--selector-include",
nargs='*',
dest="selector_include",
help="Operate only on samples associated with these attribute "
"values; if not provided, all samples are used.")
subparser.add_argument(
"--amendments",
dest="amendments",
nargs="+",
help="Name of amendment(s) to use, as designated in the "
"project's configuration file")
return parser, msg_by_cmd
def build_parser():
"""
Building argument parser
:return argparse.ArgumentParser
"""
env_var_val = get_first_env_var(CFG_ENV_VARS)[1] if get_first_env_var(
CFG_ENV_VARS) is not None else "not set"
banner = "%(prog)s - refgenie web server utilities"
additional_description = "For subcommand-specific options, type: '%(prog)s <subcommand> -h'"
additional_description += "\nhttps://github.com/databio/refgenieserver"
parser = VersionInHelpParser(prog=PKG_NAME,
description=banner,
epilog=additional_description)
parser.add_argument("-V",
"--version",
action="version",
version="%(prog)s {v}".format(v=v))
msg_by_cmd = {
"serve": "run the server",
"archive": "prepare servable archives"
}
subparsers = parser.add_subparsers(dest="command")
def add_subparser(cmd, description):
return subparsers.add_parser(cmd,
description=description,
help=description)
sps = {}
# add arguments that are common for both subparsers
for cmd, desc in msg_by_cmd.items():
sps[cmd] = add_subparser(cmd, desc)
sps[cmd].add_argument(
'-c',
'--config',
required=False,
dest="config",
help=
"A path to the refgenie config file (YAML). If not provided, the first available environment variable "
"among: \'{}\' will be used if set. Currently: {}".format(
", ".join(CFG_ENV_VARS), env_var_val))
sps[cmd].add_argument("-d",
"--dbg",
action="store_true",
dest="debug",
help="Set logger verbosity to debug")
# add subparser-specific arguments
sps["serve"].add_argument("-p",
"--port",
dest="port",
type=int,
help="The port the webserver should be run on.",
default=DEFAULT_PORT)
sps["archive"].add_argument(
"--genomes-desc",
dest="genomes_desc",
type=str,
default=None,
help=
"Path to a CSV file with genomes descriptions. Format: genome_name, genome description"
)
sps["archive"].add_argument(
"-f",
"--force",
action="store_true",
dest="force",
help="whether the server file tree should be rebuilt even if exists")
sps["archive"].add_argument(
"-r",
"--remove",
action="store_true",
dest="remove",
help="Remove selected genome, genome/asset or genome/asset:tag")
sps["archive"].add_argument(
"asset_registry_paths",
metavar="asset-registry-paths",
type=str,
nargs='*',
help=
"One or more registry path strings that identify assets, e.g. hg38/fasta:tag"
)
return parser