def main(argv, from_file: BytesIO, to_file: BytesIO):
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
Storage.add_argparse_arg(
parser, '--storage', required=True,
help='JSON blob for creating a Storage instance.',
)
parser.add_argument('--debug', action='store_true', help='Log more?')
subparsers = parser.add_subparsers(help='Sub-commands have help.')
parser_get = subparsers.add_parser('get', help='Download blob to stdout')
parser_get.add_argument('storage_id', help='String of the form KEY:ID')
parser_get.set_defaults(to_file=to_file)
parser_get.set_defaults(func=get)
parser_put = subparsers.add_parser(
'put', help='Write a blob from stdin, print its ID to stdout',
)
parser_put.set_defaults(from_file=from_file)
parser_put.set_defaults(to_file=to_file) # For the storage ID
parser_put.set_defaults(func=put)
args = Path.parse_args(parser, argv)
init_logging(debug=args.debug)
args.func(args)
def main(argv: List[str]):
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument('--rpm-installer', required=True, type=YumDnf)
parser.add_argument(
'--input-conf',
required=True,
type=Path.from_argparse,
help='The `yum` or `dnf` config file to rewrite',
)
parser.add_argument(
'--output-dir',
required=True,
type=Path.from_argparse,
help='Write new configs here, part of the snapshot build artifact',
)
parser.add_argument(
'--install-dir',
required=True,
type=Path.from_argparse,
help='In the container, `--output-dir` will be available here',
)
parser.add_argument(
'--repo-server-ports',
required=True,
help='The rewritten config will direct the RPM installer to talk to '
'`repo-server` proxies listenting on `localhost:<PORT>`s')
args = Path.parse_args(parser, argv)
with populate_temp_dir_and_rename(args.output_dir) as td, \
open(args.input_conf, 'r') as infile:
write_yum_dnf_conf(
yum_dnf=args.rpm_installer,
infile=infile,
out_dir=td,
install_dir=args.install_dir,
ports=[int(p) for p in args.repo_server_ports.split()],
)
def snapshot_repos_from_args(argv: List[str]):
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
add_standard_args(parser)
parser.add_argument(
'--dnf-conf',
type=Path.from_argparse,
help='Snapshot this `dnf.conf`, and all the repos that it lists. '
'Can be set together with `--yum-conf`, in which case repos from '
'both configs must be identical. At least one of these `--*-conf` '
'options is required.',
)
parser.add_argument(
'--yum-conf',
type=Path.from_argparse,
help='Snapshot this `yum.conf`; see help for `--dnf-conf`',
)
parser.add_argument(
'--exclude',
action='append',
default=[],
help='Repos to be excluded in the snapshot.',
)
universe_warning = (
'Described in the `repo_db.py` docblock. In production, it is '
'important for the universe name to match existing conventions -- '
'DO NOT JUST MAKE ONE UP.')
universe_group = parser.add_mutually_exclusive_group(required=True)
universe_group.add_argument(
'--repo-to-universe-json',
type=Path.from_argparse,
help='JSON dict of repo name to universe name. ' + universe_warning,
)
universe_group.add_argument(
'--one-universe-for-all-repos',
help='Snapshot all repos under this universe name. ' +
universe_warning,
)
args = Path.parse_args(parser, argv)
init_logging(debug=args.debug)
if args.one_universe_for_all_repos:
def repo_to_universe(_repo):
return args.one_universe_for_all_repos
elif args.repo_to_universe_json:
with open(args.repo_to_universe_json) as ru_json:
repo_to_universe_json = json.load(ru_json)
def repo_to_universe(repo):
return repo_to_universe_json[repo.name]
else: # pragma: no cover
raise AssertionError(args)
with populate_temp_dir_and_rename(args.snapshot_dir, overwrite=True) as td:
snapshot_repos(
dest=td,
repo_to_universe=repo_to_universe,
yum_conf_content=args.yum_conf.read_text()
if args.yum_conf else None,
dnf_conf_content=args.dnf_conf.read_text()
if args.dnf_conf else None,
db_cfg=args.db,
storage_cfg=args.storage,
rpm_shard=args.rpm_shard,
gpg_key_whitelist_dir=args.gpg_key_whitelist_dir,
exclude=frozenset(args.exclude),
threads=args.threads,
)
def _parse_cli_args(argv, *, allow_debug_only_opts) -> _NspawnOpts:
'Keep in sync with `_NspawnCLIArgs`'
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
# This is outside of `_NspawnOpts` because it actually changes the
# API of the call substantially (there is one more process spawned,
# with an extra FD, the console).
parser.add_argument(
'--boot',
action='store_true',
help='Boot the container with nspawn. This means invoke systemd '
'as pid 1 and let it start up services',
)
parser.add_argument(
'--append-boot-console',
default=None,
help='Use with `--boot` to redirect output from the systemd console '
'PTY into a file. By default it goes to stdout for easy debugging.',
)
parser.add_argument(
'--serve-rpm-snapshot',
action='append',
dest='serve_rpm_snapshots',
default=[],
type=Path.from_argparse,
help='Container-relative path to an RPM repo snapshot directory, '
'normally located under `RPM_SNAPSHOT_BASE_DIR`. Your container '
'will be provided with `repo-server`s listening on the ports '
'specified in the `etc/{yum,dnf}/{yum,dnf}.conf` of the snapshot, '
'so you can simply run `{yum,dnf} -c PATH_TO_CONF` to use them. '
'This option may be repeated to serve multiple snapshots.',
)
parser.add_argument(
'--snapshot-to-versionlock',
action='append',
nargs=2,
metavar=('SNAPSHOT_PATH', 'VERSIONLOCK_PATH'),
default=[],
type=Path.from_argparse,
help='Restrict available versions for some of the snapshots specified '
'via `--serve-rpm-snapshot`. Each version-lock file lists allowed '
'RPM versions, one per line, in the following TAB-separated '
'format: N\\tE\\tV\\tR\\tA. Snapshot is a container path, while '
'versionlock is a host path.',
)
_parser_add_nspawn_opts(parser)
if allow_debug_only_opts:
_parser_add_debug_only_not_for_prod_opts(parser)
args = Path.parse_args(parser, argv)
assert args.boot or not args.append_boot_console, args
args.snapshot_to_versionlock = _normalize_snapshot_to_versionlock(
args.snapshot_to_versionlock,
args.serve_rpm_snapshots,
)
return _extract_opts_from_dict(
_new_nspawn_cli_args,
_NspawnCLIArgs._fields,
args.__dict__,
opts=_extract_opts_from_dict(
new_nspawn_opts,
_NspawnOpts._fields,
args.__dict__,
debug_only_opts=_extract_opts_from_dict(
_new_nspawn_debug_only_not_for_prod_opts,
_NspawnDebugOnlyNotForProdOpts._fields
if allow_debug_only_opts else (),
args.__dict__,
),
),
)
def snapshot_repo(argv):
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
add_standard_args(parser)
parser.add_argument(
'--repo-universe',
required=True,
help='This is explained in the `repo_db.py` docblock. In production, '
'it is important for the universe name to match existing '
'conventions -- DO NOT JUST MAKE ONE UP.',
)
parser.add_argument(
'--repo-name',
required=True,
help="Used to distinguish this repo's metadata from others' in the DB.",
)
parser.add_argument(
'--repo-url',
required=True,
help='The base URL of the repo -- the part before repodata/repomd.xml. '
'Supported protocols include file://, https://, and http://.',
)
parser.add_argument(
'--gpg-url',
required=True,
action='append',
help='(May be repeated) Yum will need to import this key to gpgcheck '
'the repo. To avoid placing blind trust in these keys (e.g. in '
'case this is an HTTP URL), they are verified against '
'`--gpg-key-whitelist-dir`',
)
args = Path.parse_args(parser, argv)
init_logging(debug=args.debug)
with populate_temp_dir_and_rename(
args.snapshot_dir,
overwrite=True,
) as td, RepoSnapshot.add_sqlite_to_storage(
Storage.from_json(args.storage), td) as sqlite_db:
sizer = RepoSizer()
snapshot_gpg_keys(
key_urls=args.gpg_url,
whitelist_dir=args.gpg_key_whitelist_dir,
snapshot_dir=td,
)
repo = YumDnfConfRepo(name=args.repo_name,
base_url=args.repo_url,
gpg_key_urls=args.gpg_url)
_, snapshot = next(
download_repos(
repos_and_universes=[(repo, args.repo_universe)],
cfg=DownloadConfig(
db_cfg=args.db,
storage_cfg=args.storage,
rpm_shard=args.rpm_shard,
threads=args.threads,
),
))
snapshot.visit(sizer).to_sqlite(args.repo_name, sqlite_db)
log.info(sizer.get_report(f'This {args.rpm_shard} snapshot weighs'))
def parse_args(args) -> argparse.Namespace:
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawTextHelpFormatter,
)
parser.add_argument(
'--subvolumes-dir', required=True,
help='A directory on a btrfs volume to store the compiled subvolume '
'representing the new layer',
)
# We separate this from `--subvolumes-dir` in order to help keep our
# JSON output ignorant of the absolute path of the repo.
parser.add_argument(
'--subvolume-rel-path', required=True,
help='Path underneath --subvolumes-dir where we should create '
'the subvolume. Note that all path components but the basename '
'should already exist.',
)
parser.add_argument(
'--build-appliance-json',
help='Path to the JSON output of target referred by build_appliance',
)
parser.add_argument(
'--rpm-installer', type=YumDnf,
help='Name of a supported RPM package manager (e.g. `yum` or `dnf`). '
'Required if your image installs RPMs.',
)
parser.add_argument(
'--rpm-repo-snapshot', type=Path.from_argparse,
help='Path to snapshot directory in the build appliance image. '
'The default is the BA symlink `/__fs_image__/rpm/'
'default-snapshot-for-installer/<--rpm-installer>`.',
)
parser.add_argument(
'--preserve-yum-dnf-cache', action='store_true',
help='RpmActionItem will write to `/var/cache/{dnf,yum}` on the '
'subvolume when installing RPMs. The caches will contain repodata '
'from the current repo snapshot, so this is most useful for '
'constructing build-appliance images.',
)
parser.add_argument(
'--artifacts-may-require-repo', action='store_true',
help='Buck @mode/dev produces "in-place" build artifacts that are '
'not truly standalone. It is important to be able to execute '
'code from images built in this mode to support rapid '
'development and debugging, even though it is not a "true" '
'self-contained image. To allow execution of in-place binaries, '
'fs_image runtimes will automatically mount the repo into any '
'`--artifacts-may-require-repo` image at runtime (e.g. when '
'running image unit-tests, when using `-container` or `-boot` '
'targets, when using the image as a build appliance).',
)
parser.add_argument(
'--child-layer-target', required=True,
help='The name of the Buck target describing the layer being built',
)
parser.add_argument(
'--child-feature-json', action='append', default=[],
help='The path of the JSON output of any `image.feature`s that are'
'directly included by the layer being built',
)
parser.add_argument(
'--child-dependencies',
nargs=argparse.REMAINDER, metavar=['TARGET', 'PATH'], default=(),
help='Consumes the remaining arguments on the command-line, with '
'arguments at positions 1, 3, 5, 7, ... used as Buck target names '
'(to be matched with the targets in per-feature JSON outputs). '
'The argument immediately following each target name must be a '
'path to the output of that target on disk.',
)
parser.add_argument('--debug', action='store_true', help='Log more')
parser.add_argument(
'--allowed-host-mount-target', action='append', default=[],
help='Target name that is allowed to contain host mounts used as '
'build_sources. Can be specified more than once.')
return Path.parse_args(parser, args)