def test_each_option_gis_functional(parser, opt): """ Each added CLI opt can be used as expected. """ add_logging_options(parser) for a in parser._actions: if opt in a.option_strings: use = get_act_use(parser, a) break else: pytest.fail("Parser lacks action with name: {}; available: {}".format( opt, _get_optnames(parser))) try: parser.parse_args(use) except Exception as e: pytest.fail("Use of option '{}' ({}) failed: {}".format(opt, use, e))
def main(): global sc global rgc global _LOGGER parser = build_parser() parser = logmuse.add_logging_options(parser) args = parser.parse_args() if not args.command: parser.print_help() print("No subcommand given") sys.exit(1) _LOGGER = logmuse.logger_via_cli(args, make_root=True) _LOGGER.info("Welcome to the SeqCol API app") # demo_filepath="/home/nsheff/code/seqcolapi/seqcolapi/seqcolapi_config_demo.yaml" scc = SeqColConf(filepath=args.config) _LOGGER.info(f"Connecting to database... {scc.database.host}") pgdb = RDBDict(scc.database.name, scc.database.user, scc.database.password, scc.database.host, scc.database.port) rgc = refget.RefGetClient(scc.refget_provider_apis, pgdb) sc = SeqColClient(database=pgdb, api_url_base=scc.refget_provider_apis, schemas=scc.schemas) seqcolapi_port = args.port if args.port else scc.server.port _LOGGER.info("Running on port {}".format(seqcolapi_port)) uvicorn.run(app, host=scc.server.host, port=seqcolapi_port)
def _parse_cmdl(cmdl): """ Define and parse command-line interface. """ parser = argparse.ArgumentParser( description="Read count as template for ParaReadProcessor " "implementation", formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("readsfile", help="Path to sequencing reads file.") parser.add_argument("-O", "--outfile", required=True, help="Path to output file.") parser.add_argument("-C", "--cores", required=False, default=1, help="Number of cores.") parser.add_argument('-t', '--limit', dest='limit', help="Limit to these chromosomes", nargs="+", default=None) parser = logmuse.add_logging_options(parser) return parser.parse_args(cmdl)
def main(): """ Primary workflow """ from inspect import getdoc parser = logmuse.add_logging_options( build_argparser(getdoc(PipestatManager))) args = parser.parse_args() if args.command is None: parser.print_help(sys.stderr) sys.exit(1) global _LOGGER _LOGGER = logmuse.logger_via_cli(args, make_root=True) _LOGGER.debug("Args namespace:\n{}".format(args)) if args.database_config and not args.schema: parser.error("the following arguments are required: -s/--schema") psm = PipestatManager( name=args.namespace, schema_path=args.schema, results_file=args.results_file, database_config=args.database_config ) if args.command == REPORT_CMD: value = args.value result_metadata = psm.schema[args.result_identifier] if result_metadata[SCHEMA_TYPE_KEY] in ["object", "image", "file"] \ and os.path.exists(expandpath(value)): from json import load _LOGGER.info(f"Reading JSON file with object type value: " f"{expandpath(value)}") with open(expandpath(value), "r") as json_file: value = load(json_file) psm.report( result_identifier=args.result_identifier, record_identifier=args.record_identifier, value=value, force_overwrite=args.overwrite, strict_type=not args.try_convert ) sys.exit(0) if args.command == INSPECT_CMD: print("\n") print(psm) if args.data: print("\nData:") print(psm.data) sys.exit(0) if args.command == REMOVE_CMD: psm.remove( result_identifier=args.result_identifier, record_identifier=args.record_identifier ) sys.exit(0) if args.command == RETRIEVE_CMD: print(psm.retrieve( result_identifier=args.result_identifier, record_identifier=args.record_identifier )) sys.exit(0)
def main(): """ Primary workflow """ parser = logmuse.add_logging_options(build_argparser()) args = parser.parse_args() global _LOGGER _LOGGER = logmuse.logger_via_cli(args, make_root=True) msg = "Input: {input}; Parameter: {parameter}" _LOGGER.info(msg.format(input=args.input, parameter=args.parameter))
def parse_args(cmdl): parser = ArgumentParser(description='--Produce bamQC File--') parser.add_argument('-i', '--infile', dest='infile', help="Path to input file (in BAM format).", required=True) parser.add_argument('-o', '--outfile', dest='outfile', help="Output file name.") parser.add_argument('-c', '--cores', dest='cores', default=20, type=int, help="Number of processors to use. Default=20") parser = logmuse.add_logging_options(parser) return parser.parse_args(cmdl)
def main(): """ Primary workflow """ parser = logmuse.add_logging_options(arguments.build_argparser()) args, remaining_args = parser.parse_known_args() global _LOGGER _LOGGER = logmuse.logger_via_cli(args) _LOGGER.info("Welcome to bedshift version {}".format(__version__)) _LOGGER.info("Shifting file: '{}'".format(args.bedfile)) if not args.bedfile: parser.print_help() _LOGGER.error("No BED file given") sys.exit(1) if args.chrom_lengths: pass elif args.genome: try: import refgenconf rgc = refgenconf.RefGenConf(refgenconf.select_genome_config()) args.chrom_lengths = rgc.seek(args.genome, "fasta", None, "chrom_sizes") except ModuleNotFoundError: _LOGGER.error( "You must have package refgenconf installed to use a refgenie genome" ) sys.exit(1) msg = arguments.param_msg if args.repeat < 1: _LOGGER.error("repeats specified is less than 1") sys.exit(1) if args.outputfile: outfile_base = args.outputfile else: outfile_base = "bedshifted_{}".format(os.path.basename(args.bedfile)) _LOGGER.info( msg.format( bedfile=args.bedfile, chromsizes=args.chrom_lengths, droprate=args.droprate, dropfile=args.dropfile, addrate=args.addrate, addmean=args.addmean, addstdev=args.addstdev, addfile=args.addfile, valid_regions=args.valid_regions, shiftrate=args.shiftrate, shiftmean=args.shiftmean, shiftstdev=args.shiftstdev, shiftfile=args.shiftfile, cutrate=args.cutrate, mergerate=args.mergerate, outputfile=outfile_base, repeat=args.repeat, yaml_config=args.yaml_config, )) bedshifter = Bedshift(args.bedfile, args.chrom_lengths) _LOGGER.info(f"Generating {args.repeat} repetitions...") pct_reports = [int(x * args.repeat / 100) for x in [5, 25, 50, 75, 100]] for i in range(args.repeat): n = bedshifter.all_perturbations( args.addrate, args.addmean, args.addstdev, args.addfile, args.valid_regions, args.shiftrate, args.shiftmean, args.shiftstdev, args.shiftfile, args.cutrate, args.mergerate, args.droprate, args.dropfile, args.yaml_config, ) if args.repeat == 1: bedshifter.to_bed(outfile_base) _LOGGER.info( "REGION COUNT | original: {}\tnew: {}\tchanged: {}\t\noutput file: {}" .format( bedshifter.original_num_regions, bedshifter.bed.shape[0], str(n), outfile_base, )) else: basename, ext = os.path.splitext(os.path.basename(outfile_base)) dirname = os.path.dirname(outfile_base) digits = int(math.log10(args.repeat)) + 1 rep = str(i + 1).zfill(digits) modified_outfile_path = os.path.join(dirname, f"{basename}_rep{rep}{ext}") bedshifter.to_bed(modified_outfile_path) pct_finished = int((100 * (i + 1)) / args.repeat) if i + 1 in pct_reports: _LOGGER.info( f"Rep {i+1}. Finished: {pct_finished}%. Output file: {modified_outfile_path}" ) bedshifter.reset_bed()
def main(): """Primary workflow""" from inspect import getdoc parser = logmuse.add_logging_options( build_argparser(getdoc(PipestatManager))) args = parser.parse_args() if args.command is None: parser.print_help(sys.stderr) sys.exit(1) global _LOGGER _LOGGER = logmuse.logger_via_cli(args, make_root=True) _LOGGER.debug("Args namespace:\n{}".format(args)) if args.config and not args.schema and args.command != STATUS_CMD: parser.error("the following arguments are required: -s/--schema") psm = PipestatManager( namespace=args.namespace, schema_path=args.schema, results_file_path=args.results_file, config=args.config, database_only=args.database_only, status_schema_path=args.status_schema, flag_file_dir=args.flag_dir, ) if args.command == REPORT_CMD: value = args.value if psm.schema is None: raise SchemaNotFoundError(msg="report", cli=True) result_metadata = psm.schema[args.result_identifier] if (result_metadata[SCHEMA_TYPE_KEY] in [ "object", "image", "file", ] and os.path.exists(expandpath(value))): from json import load _LOGGER.info( f"Reading JSON file with object type value: {expandpath(value)}" ) with open(expandpath(value), "r") as json_file: value = load(json_file) psm.report( record_identifier=args.record_identifier, values={args.result_identifier: value}, force_overwrite=args.overwrite, strict_type=args.skip_convert, ) if args.command == INSPECT_CMD: print("\n") print(psm) if args.data and not args.database_only: print("\nData:") print(psm.data) if args.command == REMOVE_CMD: psm.remove( result_identifier=args.result_identifier, record_identifier=args.record_identifier, ) if args.command == RETRIEVE_CMD: print( psm.retrieve( result_identifier=args.result_identifier, record_identifier=args.record_identifier, )) if args.command == STATUS_CMD: if args.subcommand == STATUS_GET_CMD: print(psm.get_status(record_identifier=args.record_identifier)) if args.subcommand == STATUS_SET_CMD: psm.set_status( status_identifier=args.status_identifier, record_identifier=args.record_identifier, ) sys.exit(0)
def parse_args(cmdl): parser = ArgumentParser(description='Bam processor') parser.add_argument('-i', '--infile', dest='infile', help="Input file (in bam or sam format)", required=True) parser.add_argument('-c', '--chrom-sizes-file', help="Chromosome sizes file", required=True) parser.add_argument( '-v', '--variable-step', default=False, action='store_true', help="Use variableStep wiggle format. Default: fixedStep") parser.add_argument('-s', '--scale', dest='scale', default=1, help="Scale read count by this value. Default: 1") parser.add_argument('-o', '--exactbw', dest='exactbw', default=None, help="Output filename for exact bigwig. Default: None") parser.add_argument( '-w', '--smoothbw', dest='smoothbw', default=None, help="Output filename for smooth bigwig. Default: None") parser.add_argument('-r', '--step-size', default=5, help="Step size for smooth tracks. Default: 5") parser.add_argument('-b', '--bedout', default=None, help="Output filename for bed file. Default: None") parser.add_argument('-l', '--smooth-length', help="Smooth length for bed file", default=25, type=int) parser.add_argument( '-d', '--tail-edge', action='store_true', default=False, help="Output the 3' end of the sequence read. Default: False") parser.add_argument( '-m', '--mode', dest='mode', default=None, choices=MODES, help="Turn on DNase or ATAC mode (this adjusts the shift parameters)") parser.add_argument('-t', '--limit', dest='limit', help="Limit to these chromosomes", nargs="+", default=None) parser.add_argument('-p', '--cores', dest='cores', help="Number of cores to use", default=2, type=int) parser.add_argument( '-e', '--temp-parent', default="", #os.getcwd(), help="Temporary file location. By default it will use the working" " directory, but you can place this elsewhere if you'd like." " The actual folder will be based on the exactbw filename.") parser.add_argument('--retain-temp', action='store_true', default=False, help="Retain temporary files? Default: False") parser = logmuse.add_logging_options(parser) args = parser.parse_args(cmdl) if not (args.exactbw or args.smoothbw): parser.error('No output requested, use --exactbw and/or --smoothbw') return args
def _parse_cmdl(cmdl): parser = argparse.ArgumentParser( description="Automatic GEO SRA data downloader") parser.add_argument("-V", "--version", action="version", version="%(prog)s {v}".format(v=__version__)) # Required parser.add_argument( "-i", "--input", dest="input", required=True, help= "required: a GEO (GSE) accession, or a file with a list of GSE numbers" ) # Optional parser.add_argument("-n", "--name", help="Specify a project name. Defaults to GSE number") parser.add_argument("-m", "--metadata-folder", dest="metadata_folder", default="${SRAMETA}", help="Specify a location to store metadata " "[Default: $SRAMETA:" + safe_echo("SRAMETA") + "]") parser.add_argument( "-f", "--no-subfolder", action="store_true", help= "Don't automatically put metadata into a subfolder named with project name" ) parser.add_argument( "--just-metadata", action="store_true", help="If set, don't actually run downloads, just create metadata") parser.add_argument("-r", "--refresh-metadata", action="store_true", help="If set, re-download metadata even if it exists.") parser.add_argument( "--acc-anno", action="store_true", help="Also produce annotation sheets for each accession, not just" " for the whole project combined") parser.add_argument( "--use-key-subset", action="store_true", help= "Use just the keys defined in this module when writing out metadata.") parser.add_argument( "-x", "--split-experiments", action="store_true", help="""Split SRR runs into individual samples. By default, SRX experiments with multiple SRR Runs will have a single entry in the annotation table, with each run as a separate row in the subannotation table. This setting instead treats each run as a separate sample""") parser.add_argument("--config-template", default=None, help="Project config yaml file template.") parser.add_argument( "-p", "--processed", default=False, action="store_true", help="Download processed data [Default: download raw data].") parser.add_argument( "-g", "--geo-folder", default=safe_echo("GEODATA"), help="Optional: Specify a location to store processed GEO files " "[Default: $GEODATA:" + safe_echo("GEODATA") + "]") parser.add_argument( "-b", "--bam-folder", dest="bam_folder", default=safe_echo("SRABAM"), help="""Optional: Specify folder of bam files. Geofetch will not download sra files when corresponding bam files already exist. [Default: $SRABAM:""" + safe_echo("SRABAM") + "]") parser.add_argument( "-P", "--pipeline_interfaces", default=None, help= "Optional: Specify one or more filepaths to pipeline interface yaml files. " "These will be added to the project config file to make it immediately " "compatible with looper. [Default: null]") # Deprecated; these are for bam conversion which now happens in sra_convert # it still works here but I hide it so people don't use it, because it's confusing. parser.add_argument( "-s", "--sra-folder", dest="sra_folder", default=safe_echo("SRARAW"), help=argparse.SUPPRESS, # help="Optional: Specify a location to store sra files " # "[Default: $SRARAW:" + safe_echo("SRARAW") + "]" ) parser.add_argument( "--bam-conversion", action="store_true", # help="Turn on sequential bam conversion. Default: No conversion.", help=argparse.SUPPRESS) parser.add_argument( "--picard-path", dest="picard_path", default=safe_echo("PICARD"), # help="Specify a path to the picard jar, if you want to convert " # "fastq to bam [Default: $PICARD:" + safe_echo("PICARD") + "]", help=argparse.SUPPRESS) parser = add_logging_options(parser) return parser.parse_args(cmdl)
def main(): """ Primary workflow """ parser = logmuse.add_logging_options(build_argparser()) args, remaining_args = parser.parse_known_args() global _LOGGER _LOGGER = logmuse.logger_via_cli(args, make_root=True) _LOGGER.debug("refgenie {}".format(__version__)) _LOGGER.debug("Args: {}".format(args)) if not args.command: parser.print_help() _LOGGER.error("No command given") sys.exit(1) gencfg = refgenconf.select_genome_config( filename=args.genome_config, check_exist=not args.command == INIT_CMD, on_missing=lambda fp: fp, strict_env=True) if gencfg is None: raise MissingGenomeConfigError(args.genome_config) _LOGGER.debug("Determined genome config: {}".format(gencfg)) # From user input we want to construct a list of asset dicts, where each # asset has a genome name, asset name, and tag if "asset_registry_paths" in args and args.asset_registry_paths: _LOGGER.debug("Found registry_path: {}".format( args.asset_registry_paths)) asset_list = [ parse_registry_path(x) for x in args.asset_registry_paths ] for a in asset_list: # every asset must have a genome, either provided via registry path # or the args.genome arg. if not a["genome"]: if args.genome: a["genome"] = args.genome else: _LOGGER.error( "Provided asset registry path ({}/{}:{}) is invalid. See help for usage reference." .format(a["genome"], a["asset"], a["tag"])) sys.exit(1) else: if args.genome and args.genome != a["genome"]: _LOGGER.warn( "Two different genomes specified for asset '{}'.". format(a["asset"])) else: if args.command in GENOME_ONLY_REQUIRED and not args.genome: parser.error("You must provide either a genome or a registry path") sys.exit(1) if args.command in ASSET_REQUIRED: parser.error("You must provide an asset registry path") sys.exit(1) if args.command == INIT_CMD: _LOGGER.debug("Initializing refgenie genome configuration") rgc = RefGenConf(entries=OrderedDict( { CFG_VERSION_KEY: REQ_CFG_VERSION, CFG_FOLDER_KEY: os.path.dirname(os.path.abspath(gencfg)), CFG_SERVERS_KEY: args.genome_server or [DEFAULT_SERVER], CFG_GENOMES_KEY: None })) rgc.initialize_config_file(os.path.abspath(gencfg)) elif args.command == BUILD_CMD: if not all( [x["genome"] == asset_list[0]["genome"] for x in asset_list]): _LOGGER.error("Build can only build assets for one genome") sys.exit(1) recipe_name = None if args.recipe: if len(asset_list) > 1: _LOGGER.error( "Recipes cannot be specified for multi-asset builds") sys.exit(1) recipe_name = args.recipe if args.requirements: for a in asset_list: recipe = recipe_name or a["asset"] if recipe not in asset_build_packages.keys(): _raise_missing_recipe_error(recipe) _LOGGER.info("'{}' recipe requirements: ".format(recipe)) _make_asset_build_reqs(recipe) sys.exit(0) refgenie_build(gencfg, asset_list[0]["genome"], asset_list, recipe_name, args) elif args.command == GET_ASSET_CMD: rgc = RefGenConf(filepath=gencfg, writable=False) check = args.check_exists if args.check_exists else None for a in asset_list: _LOGGER.debug("getting asset: '{}/{}.{}:{}'".format( a["genome"], a["asset"], a["seek_key"], a["tag"])) print( rgc.seek(a["genome"], a["asset"], a["tag"], a["seek_key"], strict_exists=check)) return elif args.command == INSERT_CMD: rgc = RefGenConf(filepath=gencfg, writable=False) if len(asset_list) > 1: raise NotImplementedError("Can only add 1 asset at a time") else: refgenie_add(rgc, asset_list[0], args.path, args.force) elif args.command == PULL_CMD: rgc = RefGenConf(filepath=gencfg, writable=False) force = None if not args.force else True outdir = rgc[CFG_FOLDER_KEY] if not os.path.exists(outdir): raise MissingFolderError(outdir) target = _key_to_name(CFG_FOLDER_KEY) if not perm_check_x(outdir, target): return if not _single_folder_writeable(outdir): _LOGGER.error("Insufficient permissions to write to {}: {}".format( target, outdir)) return for a in asset_list: rgc.pull(a["genome"], a["asset"], a["tag"], unpack=not args.no_untar, force=force) elif args.command in [LIST_LOCAL_CMD, LIST_REMOTE_CMD]: rgc = RefGenConf(filepath=gencfg, writable=False) if args.command == LIST_REMOTE_CMD: num_servers = 0 # Keep all servers so that child updates maintain server list server_list = rgc[CFG_SERVERS_KEY] bad_servers = [] for server_url in rgc[CFG_SERVERS_KEY]: num_servers += 1 try: rgc[CFG_SERVERS_KEY] = server_url pfx, genomes, assets, recipes = _exec_list( rgc, args.command == LIST_REMOTE_CMD, args.genome) if assets is None and genomes is None: continue _LOGGER.info("{} genomes: {}".format(pfx, genomes)) if args.command != LIST_REMOTE_CMD: # Not implemented yet _LOGGER.info("{} recipes: {}".format(pfx, recipes)) _LOGGER.info("{} assets:\n{}\n".format(pfx, assets)) except (DownloadJsonError, ConnectionError): bad_servers.append(server_url) continue if num_servers >= len(server_list) and bad_servers: _LOGGER.error( "Could not list assets from the following server(s): {}". format(bad_servers)) # Restore original server list, even when we couldn't find assets on a server rgc[CFG_SERVERS_KEY] = server_list else: # Only check local assets once _LOGGER.info("Server subscriptions: {}".format(", ".join( rgc[CFG_SERVERS_KEY]))) pfx, genomes, assets, recipes = _exec_list( rgc, args.command == LIST_REMOTE_CMD, args.genome) _LOGGER.info("{} genomes: {}".format(pfx, genomes)) if args.command != LIST_REMOTE_CMD: # Not implemented yet _LOGGER.info("{} recipes: {}".format(pfx, recipes)) _LOGGER.info("{} assets:\n{}".format(pfx, assets)) elif args.command == GETSEQ_CMD: rgc = RefGenConf(filepath=gencfg, writable=False) rgc.getseq(rgc, args.genome, args.locus) elif args.command == REMOVE_CMD: force = args.force rgc = RefGenConf(filepath=gencfg) for a in asset_list: a["tag"] = a["tag"] or rgc.get_default_tag( a["genome"], a["asset"], use_existing=False) _LOGGER.debug("Determined tag for removal: {}".format(a["tag"])) if a["seek_key"] is not None: raise NotImplementedError( "You can't remove a specific seek_key.") bundle = [a["genome"], a["asset"], a["tag"]] try: if not rgc.is_asset_complete(*bundle): with rgc as r: r.cfg_remove_assets(*bundle) _LOGGER.info( "Removed an incomplete asset '{}/{}:{}'".format( *bundle)) return except (KeyError, MissingAssetError, MissingGenomeError): _LOGGER.info("Asset '{}/{}:{}' does not exist".format(*bundle)) return if len(asset_list) > 1: if not query_yes_no( "Are you sure you want to remove {} assets?".format( len(asset_list))): _LOGGER.info("Action aborted by the user") return force = True for a in asset_list: rgc.remove(genome=a["genome"], asset=a["asset"], tag=a["tag"], force=force) elif args.command == TAG_CMD: rgc = RefGenConf(filepath=gencfg) if len(asset_list) > 1: raise NotImplementedError("Can only tag 1 asset at a time") if args.default: # set the default tag and exit with rgc as r: r.set_default_pointer(a["genome"], a["asset"], a["tag"], True) sys.exit(0) rgc.tag(a["genome"], a["asset"], a["tag"], args.tag) elif args.command == ID_CMD: rgc = RefGenConf(filepath=gencfg, writable=False) if len(asset_list) == 1: g, a = asset_list[0]["genome"], asset_list[0]["asset"] t = asset_list[0]["tag"] or rgc.get_default_tag(g, a) print(rgc.id(g, a, t)) return for asset in asset_list: g, a = asset["genome"], asset["asset"] t = asset["tag"] or rgc.get_default_tag(g, a) print("{}/{}:{},".format(g, a, t) + rgc.id(g, a, t)) return elif args.command == SUBSCRIBE_CMD: rgc = RefGenConf(filepath=gencfg, writable=False) rgc.subscribe(urls=args.genome_server, reset=args.reset) return elif args.command == UNSUBSCRIBE_CMD: rgc = RefGenConf(filepath=gencfg, writable=False) rgc.unsubscribe(urls=args.genome_server) return
def test_all_options_are_added(parser, opt): """ If requested, all of the standard logging options are added. """ assert opt not in _get_optnames(parser) add_logging_options(parser) assert opt in _get_optnames(parser)
def main(): """ Primary workflow """ parser = logmuse.add_logging_options(build_argparser()) args, remaining_args = parser.parse_known_args() logger_kwargs = {"level": args.verbosity, "devmode": args.logdev} logmuse.init_logger(name="yacman", **logger_kwargs) global _LOGGER _LOGGER = logmuse.logger_via_cli(args) _LOGGER.debug("Command given: {}".format(args.command)) if not args.command: parser.print_help() _LOGGER.error("No command given") sys.exit(1) if args.command == "init": bulkercfg = args.config _LOGGER.debug("Initializing bulker configuration") _is_writable(os.path.dirname(bulkercfg), check_exist=False) bulker_init(bulkercfg, DEFAULT_CONFIG_FILEPATH, args.engine) sys.exit(0) bulkercfg = select_bulker_config(args.config) bulker_config = yacman.YacAttMap(filepath=bulkercfg, writable=False) if args.command == "list": # Output header via logger and content via print so the user can # redirect the list from stdout if desired without the header as clutter if args.simple: fmt = "{namespace}/{crate}:{tag}" else: _LOGGER.info("Available crates:") fmt = "{namespace}/{crate}:{tag} -- {path}" if bulker_config.bulker.crates: for namespace, crates in bulker_config.bulker.crates.items(): for crate, tags in crates.items(): for tag, path in tags.items(): print( fmt.format(namespace=namespace, crate=crate, tag=tag, path=path)) else: _LOGGER.info( "No crates available. Use 'bulker load' to load a crate.") sys.exit(1) # For all remaining commands we need a crate identifier _LOGGER.info("Bulker config: {}".format(bulkercfg)) if args.command == "activate": try: cratelist = parse_registry_paths( args.crate_registry_paths, bulker_config.bulker.default_namespace) _LOGGER.debug(cratelist) _LOGGER.info("Activating bulker crate: {}{}".format( args.crate_registry_paths, " (Strict)" if args.strict else "")) bulker_activate(bulker_config, cratelist, echo=args.echo, strict=args.strict, prompt=args.no_prompt) except KeyError as e: parser.print_help(sys.stderr) _LOGGER.error("{} is not an available crate".format(e)) sys.exit(1) except MissingCrateError as e: _LOGGER.error("Missing crate: {}".format(e)) sys.exit(1) except AttributeError as e: _LOGGER.error( "Your bulker config file is outdated, you need to re-initialize it: {}" .format(e)) sys.exit(1) if args.command == "run": try: cratelist = parse_registry_paths(args.crate_registry_paths) _LOGGER.info("Activating crate: {}\n".format( args.crate_registry_paths)) bulker_run(bulker_config, cratelist, args.cmd, strict=args.strict) except KeyError as e: parser.print_help(sys.stderr) _LOGGER.error("{} is not an available crate".format(e)) sys.exit(1) except MissingCrateError as e: _LOGGER.error("Missing crate: {}".format(e)) sys.exit(1) if args.command == "load": bulker_config.make_writable() manifest, cratevars = load_remote_registry_path( bulker_config, args.crate_registry_paths, args.manifest) exe_template_jinja = None build_template_jinja = None shell_template_jinja = None exe_template = mkabs(bulker_config.bulker.executable_template, os.path.dirname(bulker_config._file_path)) build_template = mkabs(bulker_config.bulker.build_template, os.path.dirname(bulker_config._file_path)) try: shell_template = mkabs(bulker_config.bulker.shell_template, os.path.dirname(bulker_config._file_path)) except AttributeError: _LOGGER.error( "You need to re-initialize your bulker config or add a 'shell_template' attribute." ) sys.exit(1) try: assert (os.path.exists(exe_template)) except AssertionError: _LOGGER.error( "Bulker config points to a missing executable template: {}". format(exe_template)) sys.exit(1) with open(exe_template, 'r') as f: # with open(DOCKER_TEMPLATE, 'r') as f: contents = f.read() exe_template_jinja = jinja2.Template(contents) try: assert (os.path.exists(shell_template)) except AssertionError: _LOGGER.error( "Bulker config points to a missing shell template: {}".format( shell_template)) sys.exit(1) with open(shell_template, 'r') as f: # with open(DOCKER_TEMPLATE, 'r') as f: contents = f.read() shell_template_jinja = jinja2.Template(contents) if args.build: try: assert (os.path.exists(build_template)) except AssertionError: _LOGGER.error( "Bulker config points to a missing build template: {}". format(build_template)) sys.exit(1) _LOGGER.info( "Building images with template: {}".format(build_template)) with open(build_template, 'r') as f: contents = f.read() build_template_jinja = jinja2.Template(contents) bulker_load(manifest, cratevars, bulker_config, exe_jinja2_template=exe_template_jinja, shell_jinja2_template=shell_template_jinja, crate_path=args.path, build=build_template_jinja, force=args.force) if args.command == "inspect": if args.crate_registry_paths == "": _LOGGER.error( "No active create. Inspect requires a provided crate, or a currently active create." ) sys.exit(1) manifest, cratevars = load_remote_registry_path( bulker_config, args.crate_registry_paths, None) manifest_name = cratevars['crate'] print("Bulker manifest: {}".format(args.crate_registry_paths)) crate_path = os.path.join(bulker_config.bulker.default_crate_folder, cratevars['namespace'], manifest_name, cratevars['tag']) if not os.path.isabs(crate_path): crate_path = os.path.join(os.path.dirname(bcfg._file_path), crate_path) print("Crate path: {}".format(crate_path)) import glob filenames = glob.glob(os.path.join(crate_path, "*")) available_commands = [ x for x in [os.path.basename(x) for x in filenames] if x[0] != "_" ] print("Available commands: {}".format(available_commands))
def main(): """ Primary workflow """ parser = logmuse.add_logging_options(build_argparser()) args, remaining_args = parser.parse_known_args() global _LOGGER _LOGGER = logmuse.logger_via_cli(args, make_root=True) _LOGGER.debug(f"versions: refgenie {__version__} | refgenconf {rgc_version}") _LOGGER.debug(f"Args: {args}") if not args.command: parser.print_help() _LOGGER.error("No command given") sys.exit(1) if args.command == ALIAS_CMD and not args.subcommand: parser.print_help() _LOGGER.error("No alias subcommand command given") sys.exit(1) gencfg = select_genome_config( filename=args.genome_config, check_exist=not args.command == INIT_CMD, on_missing=lambda fp: fp, strict_env=True, ) if gencfg is None: raise MissingGenomeConfigError(args.genome_config) _LOGGER.debug("Determined genome config: {}".format(gencfg)) skip_read_lock = _skip_lock(args.skip_read_lock, gencfg) # From user input we want to construct a list of asset dicts, where each # asset has a genome name, asset name, and tag if "asset_registry_paths" in args and args.asset_registry_paths: _LOGGER.debug("Found registry_path: {}".format(args.asset_registry_paths)) asset_list = [parse_registry_path(x) for x in args.asset_registry_paths] for a in asset_list: # every asset must have a genome, either provided via registry path # or the args.genome arg. if not a["genome"]: if args.genome: a["genome"] = args.genome else: _LOGGER.error( "Provided asset registry path ({}/{}:{}) is invalid. See help for usage reference.".format( a["genome"], a["asset"], a["tag"] ) ) sys.exit(1) else: if args.genome and args.genome != a["genome"]: _LOGGER.warn( "Two different genomes specified for asset '{}'.".format( a["asset"] ) ) else: if args.command in GENOME_ONLY_REQUIRED and not args.genome: parser.error("You must provide either a genome or a registry path") sys.exit(1) if args.command in ASSET_REQUIRED: parser.error("You must provide an asset registry path") sys.exit(1) if args.command == INIT_CMD: _LOGGER.debug("Initializing refgenie genome configuration") entries = OrderedDict( { CFG_VERSION_KEY: REQ_CFG_VERSION, CFG_FOLDER_KEY: os.path.dirname(os.path.abspath(gencfg)), CFG_SERVERS_KEY: args.genome_server or [DEFAULT_SERVER], CFG_GENOMES_KEY: None, } ) if args.settings_json: if os.path.isfile(args.settings_json): with open(args.settings_json, "r") as json_file: data = json.load(json_file) entries.update(data) else: raise FileNotFoundError( "JSON file with config init settings does not exist: {}".format( args.settings_json ) ) if args.genome_folder: entries.update({CFG_FOLDER_KEY: args.genome_folder}) if args.remote_url_base: entries.update({CFG_REMOTE_URL_BASE_KEY: args.remote_url_base}) if args.genome_archive_folder: entries.update({CFG_ARCHIVE_KEY: args.genome_archive_folder}) if args.genome_archive_config: entries.update({CFG_ARCHIVE_CONFIG_KEY: args.genome_archive_config}) _LOGGER.debug("initializing with entries: {}".format(entries)) rgc = RefGenConf(entries=entries, skip_read_lock=skip_read_lock) rgc.initialize_config_file(os.path.abspath(gencfg)) elif args.command == BUILD_CMD: if not all([x["genome"] == asset_list[0]["genome"] for x in asset_list]): _LOGGER.error("Build can only build assets for one genome") sys.exit(1) recipe_name = None if args.recipe: if len(asset_list) > 1: _LOGGER.error("Recipes cannot be specified for multi-asset builds") sys.exit(1) recipe_name = args.recipe if args.requirements: for a in asset_list: recipe = recipe_name or a["asset"] if recipe not in asset_build_packages.keys(): _raise_missing_recipe_error(recipe) _LOGGER.info("'{}' recipe requirements: ".format(recipe)) _make_asset_build_reqs(recipe) sys.exit(0) refgenie_build(gencfg, asset_list[0]["genome"], asset_list, recipe_name, args) elif args.command == GET_ASSET_CMD: rgc = RefGenConf(filepath=gencfg, writable=False, skip_read_lock=skip_read_lock) check = args.check_exists if args.check_exists else None for a in asset_list: _LOGGER.debug( "getting asset: '{}/{}.{}:{}'".format( a["genome"], a["asset"], a["seek_key"], a["tag"] ) ) print( rgc.seek( a["genome"], a["asset"], a["tag"], a["seek_key"], strict_exists=check, ) ) return elif args.command == INSERT_CMD: rgc = RefGenConf(filepath=gencfg, writable=False, skip_read_lock=skip_read_lock) if len(asset_list) > 1: raise NotImplementedError("Can only add 1 asset at a time") else: sk = args.seek_keys if sk: sk = json.loads(args.seek_keys) rgc.add( path=args.path, genome=asset_list[0]["genome"], asset=asset_list[0]["asset"], tag=asset_list[0]["tag"], seek_keys=sk, force=args.force, ) elif args.command == PULL_CMD: rgc = RefGenConf(filepath=gencfg, writable=False, skip_read_lock=skip_read_lock) # existing assets overwriting if args.no_overwrite: force = False elif args.force_overwrite: force = True else: force = None # large archive pulling if args.no_large: force_large = False elif args.pull_large: force_large = True else: force_large = None # batch mode takes precedence over other choices if args.batch: force_large = True force = False outdir = rgc.data_dir if not os.path.exists(outdir): raise MissingFolderError(outdir) if not perm_check_x(outdir): return if not _single_folder_writeable(outdir): _LOGGER.error("Insufficient permissions to write to: {}".format(outdir)) return for a in asset_list: rgc.pull( a["genome"], a["asset"], a["tag"], force=force, force_large=force_large, size_cutoff=args.size_cutoff, ) elif args.command in [LIST_LOCAL_CMD, LIST_REMOTE_CMD]: rgc = RefGenConf(filepath=gencfg, writable=False, skip_read_lock=skip_read_lock) console = Console() if args.command == LIST_REMOTE_CMD: num_servers = 0 bad_servers = [] for server_url in rgc[CFG_SERVERS_KEY]: num_servers += 1 try: table = rgc.get_asset_table( genomes=args.genome, server_url=server_url ) except (DownloadJsonError, ConnectionError, MissingSchema): bad_servers.append(server_url) continue else: console.print(table) if num_servers >= len(rgc[CFG_SERVERS_KEY]) and bad_servers: _LOGGER.error( "Could not list assets from the following servers: {}".format( bad_servers ) ) else: if args.recipes: print(", ".join(sorted(list(asset_build_packages.keys())))) else: console.print(rgc.get_asset_table(genomes=args.genome)) elif args.command == GETSEQ_CMD: rgc = RefGenConf(filepath=gencfg, writable=False, skip_read_lock=skip_read_lock) print(rgc.getseq(args.genome, args.locus)) elif args.command == REMOVE_CMD: force = args.force rgc = RefGenConf(filepath=gencfg, skip_read_lock=skip_read_lock) for a in asset_list: a["tag"] = a["tag"] or rgc.get_default_tag( a["genome"], a["asset"], use_existing=False ) _LOGGER.debug("Determined tag for removal: {}".format(a["tag"])) if a["seek_key"] is not None: raise NotImplementedError("You can't remove a specific seek_key.") gat = {"genome": a["genome"], "asset": a["asset"], "tag": a["tag"]} try: if not rgc.is_asset_complete(**gat): with rgc as r: r.cfg_remove_assets(**gat) _LOGGER.info( "Removed an incomplete asset " "'{genome}/{asset}:{tag}'".format(*gat) ) return except (KeyError, MissingAssetError, MissingGenomeError): _LOGGER.info( "Asset '{genome}/{asset}:{tag}' does not exist".format(**gat) ) return if len(asset_list) > 1: if not query_yes_no( "Are you sure you want to remove {} assets?".format(len(asset_list)) ): _LOGGER.info("Action aborted by the user") return force = True for a in asset_list: rgc.remove(genome=a["genome"], asset=a["asset"], tag=a["tag"], force=force) elif args.command == TAG_CMD: rgc = RefGenConf(filepath=gencfg, skip_read_lock=skip_read_lock) if len(asset_list) > 1: raise NotImplementedError("Can only tag 1 asset at a time") if args.default: # set the default tag and exit with rgc as r: r.set_default_pointer(a["genome"], a["asset"], a["tag"], True) sys.exit(0) rgc.tag(a["genome"], a["asset"], a["tag"], args.tag, force=args.force) elif args.command == ID_CMD: rgc = RefGenConf(filepath=gencfg, writable=False, skip_read_lock=skip_read_lock) if len(asset_list) == 1: g, a = asset_list[0]["genome"], asset_list[0]["asset"] t = asset_list[0]["tag"] or rgc.get_default_tag(g, a) print(rgc.id(g, a, t)) return for asset in asset_list: g, a = asset["genome"], asset["asset"] t = asset["tag"] or rgc.get_default_tag(g, a) print("{}/{}:{},".format(g, a, t) + rgc.id(g, a, t)) return elif args.command == SUBSCRIBE_CMD: rgc = RefGenConf(filepath=gencfg, writable=False, skip_read_lock=skip_read_lock) rgc.subscribe(urls=args.genome_server, reset=args.reset) return elif args.command == UNSUBSCRIBE_CMD: rgc = RefGenConf(filepath=gencfg, writable=False, skip_read_lock=skip_read_lock) rgc.unsubscribe(urls=args.genome_server) return elif args.command == ALIAS_CMD: rgc = RefGenConf(filepath=gencfg, skip_read_lock=skip_read_lock) if args.subcommand == ALIAS_GET_CMD: if args.aliases is not None: for a in args.aliases: print(rgc.get_genome_alias_digest(alias=a)) return console = Console() console.print(rgc.genome_aliases_table) if args.subcommand == ALIAS_SET_CMD: rgc.set_genome_alias( digest=args.digest, genome=args.aliases, reset_digest=args.reset, create_genome=args.force, ) return elif args.subcommand == ALIAS_REMOVE_CMD: rgc.remove_genome_aliases(digest=args.digest, aliases=args.aliases) return elif args.command == COMPARE_CMD: rgc = RefGenConf(filepath=gencfg, writable=False, skip_read_lock=skip_read_lock) res = rgc.compare( args.genome1[0], args.genome2[0], explain=not args.no_explanation ) if args.no_explanation: print(res) elif args.command == UPGRADE_CMD: upgrade_config( target_version=args.target_version, filepath=gencfg, force=args.force )
def parser(): """ Update empty argument parser with standard logging options. """ return add_logging_options(argparse.ArgumentParser())
def main(): """ Primary workflow """ parser = logmuse.add_logging_options(build_argparser()) args, remaining_args = parser.parse_known_args() global _LOGGER _LOGGER = logmuse.logger_via_cli(args) logmuse.logger_via_cli(args, name=refgenconf.__name__) _LOGGER.debug("Args: {}".format(args)) if not args.command: parser.print_help() _LOGGER.error("No command given") sys.exit(1) gencfg = yacman.select_config(args.genome_config, CFG_ENV_VARS, check_exist=not args.command == INIT_CMD, on_missing=lambda fp: fp) if gencfg is None: raise MissingGenomeConfigError(args.genome_config) _LOGGER.debug("Determined genome config: {}".format(gencfg)) if args.command == INIT_CMD: _LOGGER.info("Initializing refgenie genome configuration") _writeable(os.path.dirname(gencfg), strict_exists=True) refgenie_init(gencfg, args.genome_server) sys.exit(0) rgc = RefGenConf(gencfg) if args.command == BUILD_CMD: refgenie_build(rgc, args) elif args.command == GET_ASSET_CMD: _LOGGER.debug("getting asset: '{}/{}'".format(args.genome, args.asset)) print(" ".join( [rgc.get_asset(args.genome, asset) for asset in args.asset])) return elif args.command == INSERT_CMD: if len(args.asset) > 1: raise NotImplementedError("Can only add 1 asset at a time") else: # recast from list to str args.asset = args.asset[0] refgenie_add(rgc, args) elif args.command == PULL_CMD: outdir = rgc[CFG_FOLDER_KEY] if not os.path.exists(outdir): raise MissingFolderError(outdir) target = _key_to_name(CFG_FOLDER_KEY) if not perm_check_x(outdir, target): return if not _single_folder_writeable(outdir): _LOGGER.error("Insufficient permissions to write to {}: " "{}".format(target, outdir)) return rgc.pull_asset(args.genome, args.asset, gencfg, unpack=not args.no_untar) elif args.command in [LIST_LOCAL_CMD, LIST_REMOTE_CMD]: pfx, genomes, assets = _exec_list(rgc, args.command == LIST_REMOTE_CMD) _LOGGER.info("{} genomes: {}".format(pfx, genomes)) _LOGGER.info("{} assets:\n{}".format(pfx, assets))
def test_repeat_parser_configuration_is_exceptional(parser): """ add_logging_options must be called just once. """ with pytest.raises(argparse.ArgumentError): add_logging_options(parser) # Parser already has the logging options.
def main(): """Primary workflow""" parser = logmuse.add_logging_options(build_argparser()) # args, remaining_args = parser.parse_known_args() args = parser.parse_args() logger_kwargs = {"level": args.verbosity, "devmode": args.logdev} logmuse.init_logger("yacman", **logger_kwargs) global _LOGGER _LOGGER = logmuse.logger_via_cli(args) if not args.command: parser.print_help() _LOGGER.error("No command given") sys.exit(1) if args.command == "init": divcfg = args.config _LOGGER.debug("Initializing divvy configuration") is_writable(os.path.dirname(divcfg), check_exist=False) divvy_init(divcfg, DEFAULT_CONFIG_FILEPATH) sys.exit(0) _LOGGER.debug("Divvy config: {}".format(args.config)) divcfg = select_divvy_config(args.config) _LOGGER.info("Using divvy config: {}".format(divcfg)) dcc = ComputingConfiguration(filepath=divcfg) if args.command == "list": # Output header via logger and content via print so the user can # redirect the list from stdout if desired without the header as clutter _LOGGER.info("Available compute packages:\n") print("{}".format("\n".join(dcc.list_compute_packages()))) sys.exit(1) # Any non-divvy arguments will be passed along as key-value pairs # that can be used to populate the template. # keys = [str.replace(x, "--", "") for x in remaining_args[::2]] # cli_vars = dict(zip(keys, remaining_args[1::2])) if args.compute: cli_vars = {y[0]: y[1] for y in [x.split("=") for x in args.compute]} else: cli_vars = {} if args.command == "write" or args.command == "submit": try: dcc.activate_package(args.package) except AttributeError: parser.print_help(sys.stderr) sys.exit(1) if args.settings: _LOGGER.info("Loading settings file: %s", args.settings) with open(args.settings, "r") as f: vars_groups = [cli_vars, yaml.load(f, SafeLoader)] else: vars_groups = [cli_vars] _LOGGER.debug(vars_groups) if args.command == "write": dcc.write_script(args.outfile, vars_groups) elif args.command == "submit": dcc.submit(args.outfile, vars_groups)