def main(argv, from_file: BytesIO, to_file: BytesIO): parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, ) Storage.add_argparse_arg( parser, '--storage', required=True, help='JSON blob for creating a Storage instance.', ) parser.add_argument('--debug', action='store_true', help='Log more?') subparsers = parser.add_subparsers(help='Sub-commands have help.') parser_get = subparsers.add_parser('get', help='Download blob to stdout') parser_get.add_argument('storage_id', help='String of the form KEY:ID') parser_get.set_defaults(to_file=to_file) parser_get.set_defaults(func=get) parser_put = subparsers.add_parser( 'put', help='Write a blob from stdin, print its ID to stdout', ) parser_put.set_defaults(from_file=from_file) parser_put.set_defaults(to_file=to_file) # For the storage ID parser_put.set_defaults(func=put) args = Path.parse_args(parser, argv) init_logging(debug=args.debug) args.func(args)
def main(argv: List[str]): parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument('--rpm-installer', required=True, type=YumDnf) parser.add_argument( '--input-conf', required=True, type=Path.from_argparse, help='The `yum` or `dnf` config file to rewrite', ) parser.add_argument( '--output-dir', required=True, type=Path.from_argparse, help='Write new configs here, part of the snapshot build artifact', ) parser.add_argument( '--install-dir', required=True, type=Path.from_argparse, help='In the container, `--output-dir` will be available here', ) parser.add_argument( '--repo-server-ports', required=True, help='The rewritten config will direct the RPM installer to talk to ' '`repo-server` proxies listenting on `localhost:<PORT>`s') args = Path.parse_args(parser, argv) with populate_temp_dir_and_rename(args.output_dir) as td, \ open(args.input_conf, 'r') as infile: write_yum_dnf_conf( yum_dnf=args.rpm_installer, infile=infile, out_dir=td, install_dir=args.install_dir, ports=[int(p) for p in args.repo_server_ports.split()], )
def snapshot_repos_from_args(argv: List[str]): parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, ) add_standard_args(parser) parser.add_argument( '--dnf-conf', type=Path.from_argparse, help='Snapshot this `dnf.conf`, and all the repos that it lists. ' 'Can be set together with `--yum-conf`, in which case repos from ' 'both configs must be identical. At least one of these `--*-conf` ' 'options is required.', ) parser.add_argument( '--yum-conf', type=Path.from_argparse, help='Snapshot this `yum.conf`; see help for `--dnf-conf`', ) parser.add_argument( '--exclude', action='append', default=[], help='Repos to be excluded in the snapshot.', ) universe_warning = ( 'Described in the `repo_db.py` docblock. In production, it is ' 'important for the universe name to match existing conventions -- ' 'DO NOT JUST MAKE ONE UP.') universe_group = parser.add_mutually_exclusive_group(required=True) universe_group.add_argument( '--repo-to-universe-json', type=Path.from_argparse, help='JSON dict of repo name to universe name. ' + universe_warning, ) universe_group.add_argument( '--one-universe-for-all-repos', help='Snapshot all repos under this universe name. ' + universe_warning, ) args = Path.parse_args(parser, argv) init_logging(debug=args.debug) if args.one_universe_for_all_repos: def repo_to_universe(_repo): return args.one_universe_for_all_repos elif args.repo_to_universe_json: with open(args.repo_to_universe_json) as ru_json: repo_to_universe_json = json.load(ru_json) def repo_to_universe(repo): return repo_to_universe_json[repo.name] else: # pragma: no cover raise AssertionError(args) with populate_temp_dir_and_rename(args.snapshot_dir, overwrite=True) as td: snapshot_repos( dest=td, repo_to_universe=repo_to_universe, yum_conf_content=args.yum_conf.read_text() if args.yum_conf else None, dnf_conf_content=args.dnf_conf.read_text() if args.dnf_conf else None, db_cfg=args.db, storage_cfg=args.storage, rpm_shard=args.rpm_shard, gpg_key_whitelist_dir=args.gpg_key_whitelist_dir, exclude=frozenset(args.exclude), threads=args.threads, )
def _parse_cli_args(argv, *, allow_debug_only_opts) -> _NspawnOpts: 'Keep in sync with `_NspawnCLIArgs`' parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, ) # This is outside of `_NspawnOpts` because it actually changes the # API of the call substantially (there is one more process spawned, # with an extra FD, the console). parser.add_argument( '--boot', action='store_true', help='Boot the container with nspawn. This means invoke systemd ' 'as pid 1 and let it start up services', ) parser.add_argument( '--append-boot-console', default=None, help='Use with `--boot` to redirect output from the systemd console ' 'PTY into a file. By default it goes to stdout for easy debugging.', ) parser.add_argument( '--serve-rpm-snapshot', action='append', dest='serve_rpm_snapshots', default=[], type=Path.from_argparse, help='Container-relative path to an RPM repo snapshot directory, ' 'normally located under `RPM_SNAPSHOT_BASE_DIR`. Your container ' 'will be provided with `repo-server`s listening on the ports ' 'specified in the `etc/{yum,dnf}/{yum,dnf}.conf` of the snapshot, ' 'so you can simply run `{yum,dnf} -c PATH_TO_CONF` to use them. ' 'This option may be repeated to serve multiple snapshots.', ) parser.add_argument( '--snapshot-to-versionlock', action='append', nargs=2, metavar=('SNAPSHOT_PATH', 'VERSIONLOCK_PATH'), default=[], type=Path.from_argparse, help='Restrict available versions for some of the snapshots specified ' 'via `--serve-rpm-snapshot`. Each version-lock file lists allowed ' 'RPM versions, one per line, in the following TAB-separated ' 'format: N\\tE\\tV\\tR\\tA. Snapshot is a container path, while ' 'versionlock is a host path.', ) _parser_add_nspawn_opts(parser) if allow_debug_only_opts: _parser_add_debug_only_not_for_prod_opts(parser) args = Path.parse_args(parser, argv) assert args.boot or not args.append_boot_console, args args.snapshot_to_versionlock = _normalize_snapshot_to_versionlock( args.snapshot_to_versionlock, args.serve_rpm_snapshots, ) return _extract_opts_from_dict( _new_nspawn_cli_args, _NspawnCLIArgs._fields, args.__dict__, opts=_extract_opts_from_dict( new_nspawn_opts, _NspawnOpts._fields, args.__dict__, debug_only_opts=_extract_opts_from_dict( _new_nspawn_debug_only_not_for_prod_opts, _NspawnDebugOnlyNotForProdOpts._fields if allow_debug_only_opts else (), args.__dict__, ), ), )
def snapshot_repo(argv): parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, ) add_standard_args(parser) parser.add_argument( '--repo-universe', required=True, help='This is explained in the `repo_db.py` docblock. In production, ' 'it is important for the universe name to match existing ' 'conventions -- DO NOT JUST MAKE ONE UP.', ) parser.add_argument( '--repo-name', required=True, help="Used to distinguish this repo's metadata from others' in the DB.", ) parser.add_argument( '--repo-url', required=True, help='The base URL of the repo -- the part before repodata/repomd.xml. ' 'Supported protocols include file://, https://, and http://.', ) parser.add_argument( '--gpg-url', required=True, action='append', help='(May be repeated) Yum will need to import this key to gpgcheck ' 'the repo. To avoid placing blind trust in these keys (e.g. in ' 'case this is an HTTP URL), they are verified against ' '`--gpg-key-whitelist-dir`', ) args = Path.parse_args(parser, argv) init_logging(debug=args.debug) with populate_temp_dir_and_rename( args.snapshot_dir, overwrite=True, ) as td, RepoSnapshot.add_sqlite_to_storage( Storage.from_json(args.storage), td) as sqlite_db: sizer = RepoSizer() snapshot_gpg_keys( key_urls=args.gpg_url, whitelist_dir=args.gpg_key_whitelist_dir, snapshot_dir=td, ) repo = YumDnfConfRepo(name=args.repo_name, base_url=args.repo_url, gpg_key_urls=args.gpg_url) _, snapshot = next( download_repos( repos_and_universes=[(repo, args.repo_universe)], cfg=DownloadConfig( db_cfg=args.db, storage_cfg=args.storage, rpm_shard=args.rpm_shard, threads=args.threads, ), )) snapshot.visit(sizer).to_sqlite(args.repo_name, sqlite_db) log.info(sizer.get_report(f'This {args.rpm_shard} snapshot weighs'))
def parse_args(args) -> argparse.Namespace: parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawTextHelpFormatter, ) parser.add_argument( '--subvolumes-dir', required=True, help='A directory on a btrfs volume to store the compiled subvolume ' 'representing the new layer', ) # We separate this from `--subvolumes-dir` in order to help keep our # JSON output ignorant of the absolute path of the repo. parser.add_argument( '--subvolume-rel-path', required=True, help='Path underneath --subvolumes-dir where we should create ' 'the subvolume. Note that all path components but the basename ' 'should already exist.', ) parser.add_argument( '--build-appliance-json', help='Path to the JSON output of target referred by build_appliance', ) parser.add_argument( '--rpm-installer', type=YumDnf, help='Name of a supported RPM package manager (e.g. `yum` or `dnf`). ' 'Required if your image installs RPMs.', ) parser.add_argument( '--rpm-repo-snapshot', type=Path.from_argparse, help='Path to snapshot directory in the build appliance image. ' 'The default is the BA symlink `/__fs_image__/rpm/' 'default-snapshot-for-installer/<--rpm-installer>`.', ) parser.add_argument( '--preserve-yum-dnf-cache', action='store_true', help='RpmActionItem will write to `/var/cache/{dnf,yum}` on the ' 'subvolume when installing RPMs. The caches will contain repodata ' 'from the current repo snapshot, so this is most useful for ' 'constructing build-appliance images.', ) parser.add_argument( '--artifacts-may-require-repo', action='store_true', help='Buck @mode/dev produces "in-place" build artifacts that are ' 'not truly standalone. It is important to be able to execute ' 'code from images built in this mode to support rapid ' 'development and debugging, even though it is not a "true" ' 'self-contained image. To allow execution of in-place binaries, ' 'fs_image runtimes will automatically mount the repo into any ' '`--artifacts-may-require-repo` image at runtime (e.g. when ' 'running image unit-tests, when using `-container` or `-boot` ' 'targets, when using the image as a build appliance).', ) parser.add_argument( '--child-layer-target', required=True, help='The name of the Buck target describing the layer being built', ) parser.add_argument( '--child-feature-json', action='append', default=[], help='The path of the JSON output of any `image.feature`s that are' 'directly included by the layer being built', ) parser.add_argument( '--child-dependencies', nargs=argparse.REMAINDER, metavar=['TARGET', 'PATH'], default=(), help='Consumes the remaining arguments on the command-line, with ' 'arguments at positions 1, 3, 5, 7, ... used as Buck target names ' '(to be matched with the targets in per-feature JSON outputs). ' 'The argument immediately following each target name must be a ' 'path to the output of that target on disk.', ) parser.add_argument('--debug', action='store_true', help='Log more') parser.add_argument( '--allowed-host-mount-target', action='append', default=[], help='Target name that is allowed to contain host mounts used as ' 'build_sources. Can be specified more than once.') return Path.parse_args(parser, args)