def pack(
crates: List[Crate],
crates_root: Path,
bundle_path: Optional[Path],
archive_path: Path,
keep_going: bool,
) -> None:
num_good_paths = 0
num_bad_paths = 0
with common.tar_context(archive_path, "w") as tar_f:
if bundle_path is not None:
packed_name = INDEX_BUNDLE_PACKED_NAME
common.vprint("[pack] {}".format(packed_name))
tar_f.add(str(bundle_path), packed_name)
for rel_path in sorted(crate.rel_path() for crate in crates):
path = crates_root / rel_path
packed_name = "crates/" + rel_path.as_posix()
try:
common.vprint("[pack] {}".format(rel_path.name))
tar_f.add(str(path), packed_name)
num_good_paths += 1
except FileNotFoundError:
num_bad_paths += 1
common.eprint("Error: Missing {}".format(rel_path))
if not keep_going:
raise error.AbortError()
common.iprint("{} bad paths, {} good paths".format(num_bad_paths,
num_good_paths))
def cmd_list(self) -> None:
max_verbosity = common.get_max_verbosity()
show_details = max_verbosity >= common.VERBOSITY_INFO
for spec in self.adjust_wild_specs(self.specs):
common.vprint("List: {}".format(spec))
manifest = self.select_manifest(spec, download=False)
if show_details:
available_packages = manifest.available_packages()
available_targets = manifest.available_targets()
packages = self.downloaded_packages(manifest)
targets = self.downloaded_targets(manifest)
target_out = "targets[{}/{}]".format(
len(targets),
len(available_targets),
)
package_out = "packages[{}/{}]".format(
len(packages),
len(available_packages),
)
# Example output:
# stable-2020-01-30(1.41.0) \
# targets[84/84], packages[272/326]
common.iprint("{:28} {:16} {:18}".format(
manifest.ident, target_out, package_out))
for target in targets:
common.iprint(" {}".format(target))
else:
common.eprint(manifest.ident)
def download(self, url: str, dest_path: Path) -> None:
if dest_path.is_file():
common.eprint("[{}] file exists, unlinking".format(dest_path))
dest_path.unlink()
tmp_dest_path = common.tmp_path_for(dest_path)
self._download(url, tmp_dest_path)
tmp_dest_path.rename(dest_path)
def cmd_all_targets(self) -> None:
for spec in self.adjust_wild_specs(self.specs):
common.iprint("All targets: {}".format(spec))
manifest = self.select_manifest(spec, download=False)
common.iprint(" ident: {}".format(manifest.ident))
for target in manifest.all_targets():
common.eprint(target)
def main() -> None:
parser = make_parser()
args = parser.parse_args()
common.set_max_verbosity(common.VERBOSITY_INFO + args.verbose - args.quiet)
try:
cmd = args.subparser_name
if args.readme:
readme()
elif cmd is None:
raise error.UsageError("missing OPERATION (try --help)")
elif cmd == "crate":
romt.crate.Main(args).run()
elif cmd == "rustup":
romt.rustup.Main(args).run()
elif cmd == "toolchain":
romt.toolchain.Main(args).run()
elif cmd == "serve":
romt.serve.Main(args).run()
except error.Error as e:
common.eprint(e)
sys.exit(1)
except KeyboardInterrupt:
common.eprint("Keyboard interrupt")
def sig_verify(self, path: Path, sig_path: Path) -> None:
try:
self.sig_verifier.verify(path, sig_path)
except (error.MissingFileError, error.IntegrityError):
if self._warn_signature:
common.eprint("Warning: Signature failure for {}".format(path))
else:
raise
def download_fileobj(self, url: str, fileobj: BinaryIO) -> None:
try:
response = self._session.get(url, stream=True)
response.raise_for_status()
for chunk in response.iter_content(chunk_size=4096):
fileobj.write(chunk)
except requests.exceptions.RequestException as e:
common.eprint("[{}] download error: {}".format(url, e))
raise error.DownloadError(url, e)
def mark(repo: git.Repo, end: str) -> None:
for branch in ["mark", "master"]:
if branch == repo.head.reference.name:
common.eprint(
"Will not move branch {} (it is current HEAD)".format(
repr(branch)))
else:
common.iprint("Move branch {} to point to {}".format(
repr(branch), repr(end)))
repo.create_head("refs/heads/{}".format(branch), end, force=True)
def _download(self, url: str, dest_path: Path) -> None:
common.make_dirs_for(dest_path)
try:
with open(dest_path, "wb") as f:
self.download_fileobj(url, f)
except error.DownloadError as err:
common.eprint("[{}] download error: {}".format(dest_path, err))
if dest_path.is_file():
common.eprint("[{}] file exists, unlinking".format(dest_path))
dest_path.unlink()
raise
def verify_hash(self, path: Path, hash: str) -> None:
"""
Raises:
MissingFileError - path doesn't exist
IntegrityError - path exists with bad hash
"""
common.vprint("[verify] {}".format(path))
try:
integrity.verify_hash(path, hash)
except Exception as err:
common.eprint("[{}] verification failed: {}".format(path, err))
def _upgrade_to_working(repo: git.Repo) -> None:
working = git.Reference(repo, "refs/heads/working")
if repo.head.reference.name != "working" and not working.is_valid():
# Time to upgrade working tree to use "working" branch.
common.eprint("""Upgrade index to use "working" branch as HEAD""")
if repo.head.reference.is_valid():
# Create "working" branch based on current HEAD.
common.iprint(
"""Checkout new "working" branch from current HEAD""")
repo.create_head("refs/heads/working", "HEAD")
repo.head.set_reference(working)
def run(self) -> None:
common.iprint("server running")
setup_git_cgi()
# Avoid type hint warning about missing http.server.test by
# using getattr() to fetch the function.
serve_func = getattr(http.server, "test")
try:
serve_func(HandlerClass=Handler,
bind=self.args.bind,
port=self.args.port)
except Exception as err:
common.eprint("Error serving: {}".format(err))
raise
def _get_gpg(self) -> Optional[gnupg.GPG]:
if self._gpg_home is None:
self._gpg_home = tempfile.TemporaryDirectory()
try:
gpg = gnupg.GPG(gnupghome=self._gpg_home.name)
for key in RUST_KEYS:
result = gpg.import_keys(key)
if not result:
raise ValueError(
"Failed to import GPG key:\n{}".format(key))
self._gpg = gpg
except Exception:
common.eprint("** Warning: GPG setup failed; "
"signature verification disabled")
return self._gpg
def adjust_targets(self, version: str,
base_targets: List[str]) -> List[str]:
targets = set()
known_targets = set(ALL_KNOWN_TARGETS)
for target in base_targets:
if target == "all":
targets.update(known_targets)
elif target == "*":
targets.update(self.downloaded_targets(version))
else:
if target not in known_targets:
common.eprint("warning: unknown target {}".format(
repr(target)))
targets.add(target)
return sorted(targets)
def setup_git_cgi() -> None:
os.environ["GIT_HTTP_EXPORT_ALL"] = ""
os.environ["GIT_PROJECT_ROOT"] = os.path.abspath("git")
git_cgi_path = find_git_cgi_path()
if git_cgi_path:
if not common.is_windows and not common.is_executable(git_cgi_path):
common.eprint(
"Warning: setting executable flag on {}".format(git_cgi_path))
common.chmod_executable(git_cgi_path)
else:
git_http_backend_path = get_git_http_backend_path()
if git_http_backend_path:
make_git_cgi_script(git_http_backend_path)
else:
common.eprint("Warning: missing git-http-backend; no Git support")
def init_import(index_path: Path, crates_root_path: Path) -> None:
bundle_path = index_path / INDEX_BUNDLE_NAME
bundle_location = str(bundle_path.absolute())
repo = _init_common(index_path, bundle_location, crates_root_path)
try:
with repo.config_writer() as writer:
writer.set_value('remote "origin"', "fetch",
"+refs/heads/*:refs/remotes/origin/*")
writer.add_value(
'remote "origin"',
"fetch",
"+refs/heads/bundle/*:refs/remotes/origin/*",
)
except Exception as err:
common.eprint("Error while importing: {}".format(err))
raise
def _detect_version_targets(
self, rel_paths: Set[str]) -> Tuple[List[str], List[str]]:
# rel_paths should be: "archive/<version>/<target>/<file>".
versions = set()
targets = set()
for p in rel_paths:
parts = p.split("/")
if len(parts) < 4:
common.eprint("warning: unexpected path {}".format(p))
else:
version = parts[1]
target = parts[2]
if common.is_version(version):
versions.add(version)
targets.add(target)
return sorted(versions), sorted(targets)
def verify_hash(path: Path, expected_hash: str) -> None:
"""
Raises:
MissingFileError - path doesn't exist
IntegrityError - path exists with bad hash
"""
if not path.exists():
common.eprint("[{}] verify hash missing file".format(path))
raise MissingFileError(str(path))
h = hash_file(path)
if h != expected_hash:
common.eprint("[{}] verify hash bad hash".format(path))
raise IntegrityError(path.name,
actual_hash=h,
expected_hash=expected_hash)
def cmd_list(self) -> None:
max_verbosity = common.get_max_verbosity()
show_details = max_verbosity >= common.VERBOSITY_INFO
for spec in self.adjust_wild_specs(self.specs):
common.iprint("List: {}".format(spec))
version = self.version_from_spec(spec, download=False)
if show_details:
targets = self.downloaded_targets(version)
target_out = "targets[{}]".format(len(targets))
# Example output:
# 1.41.0 targets[84]
common.iprint("{:8} {}".format(version, target_out))
for target in targets:
common.iprint(" {}".format(target))
else:
common.eprint(version)
def _process_crates(downloader: romt.download.Downloader,
dl_template: Optional[str], crates: List[Crate],
crates_root: Path, good_paths_log_path: str,
bad_paths_log_path: str, *, keep_going: bool,
assume_ok: bool) -> None:
good_paths_file = common.open_optional(good_paths_log_path, "w")
bad_paths_file = common.open_optional(bad_paths_log_path, "w")
num_good_paths = 0
num_bad_paths = 0
for crate in crates:
rel_path = crate.rel_path()
path = crates_root / rel_path
is_good = False
try:
if dl_template is None:
downloader.verify_hash(path, crate.hash)
else:
url = dl_template.format(crate=crate.name,
version=crate.version)
downloader.download_verify_hash(url,
path,
crate.hash,
assume_ok=assume_ok)
is_good = True
except error.DownloadError as e:
common.eprint("Error: Download failure for {}: {}".format(
e.name, e.exception))
except error.MissingFileError as e:
common.eprint("Error: Missing {}".format(e.name))
except error.IntegrityError as e:
common.eprint(str(e))
except Exception as e:
common.eprint(
"Unknown error while processing crates: {}".format(e))
raise
if is_good:
num_good_paths += 1
common.log(good_paths_file, path)
else:
num_bad_paths += 1
common.log(bad_paths_file, path)
common.iprint("{} bad paths, {} good paths".format(num_bad_paths,
num_good_paths))
common.close_optional(good_paths_file)
common.close_optional(bad_paths_file)
if num_bad_paths > 0 and not keep_going:
raise error.AbortError()
def unpack(
repo: git.Repo,
crates_root: Path,
bundle_path: Path,
archive_path: Path,
keep_going: bool,
) -> None:
num_crates = 0
crates_prefix = "crates/"
found_bundle = False
with common.tar_context(archive_path, "r") as tar_f:
for tar_info in tar_f:
if tar_info.isdir():
continue
elif tar_info.name == INDEX_BUNDLE_PACKED_NAME:
found_bundle = True
tar_info.name = str(bundle_path)
common.vprint("[unpack] {}".format(tar_info.name))
tar_f.extract(tar_info)
elif tar_info.name.startswith(crates_prefix):
num_crates += 1
tar_info.name = tar_info.name[len(crates_prefix):]
common.vprint("[unpack] {}".format(
os.path.basename(tar_info.name)))
tar_f.extract(tar_info, str(crates_root))
else:
common.eprint("Unexpected archive member {}".format(
tar_info.name))
if not keep_going:
raise error.AbortError()
if not found_bundle:
common.eprint("Missing {} in archive".format(INDEX_BUNDLE_PACKED_NAME))
if not keep_going:
raise error.AbortError()
common.iprint("{} extracted crates".format(num_crates))
def cmd_all_targets(self) -> None:
common.iprint("All known targets:")
for target in ALL_KNOWN_TARGETS:
common.eprint(target)
