def pack( crates: List[Crate], crates_root: Path, bundle_path: Optional[Path], archive_path: Path, keep_going: bool, ) -> None: num_good_paths = 0 num_bad_paths = 0 with common.tar_context(archive_path, "w") as tar_f: if bundle_path is not None: packed_name = INDEX_BUNDLE_PACKED_NAME common.vprint("[pack] {}".format(packed_name)) tar_f.add(str(bundle_path), packed_name) for rel_path in sorted(crate.rel_path() for crate in crates): path = crates_root / rel_path packed_name = "crates/" + rel_path.as_posix() try: common.vprint("[pack] {}".format(rel_path.name)) tar_f.add(str(path), packed_name) num_good_paths += 1 except FileNotFoundError: num_bad_paths += 1 common.eprint("Error: Missing {}".format(rel_path)) if not keep_going: raise error.AbortError() common.iprint("{} bad paths, {} good paths".format(num_bad_paths, num_good_paths))
def cmd_list(self) -> None: max_verbosity = common.get_max_verbosity() show_details = max_verbosity >= common.VERBOSITY_INFO for spec in self.adjust_wild_specs(self.specs): common.vprint("List: {}".format(spec)) manifest = self.select_manifest(spec, download=False) if show_details: available_packages = manifest.available_packages() available_targets = manifest.available_targets() packages = self.downloaded_packages(manifest) targets = self.downloaded_targets(manifest) target_out = "targets[{}/{}]".format( len(targets), len(available_targets), ) package_out = "packages[{}/{}]".format( len(packages), len(available_packages), ) # Example output: # stable-2020-01-30(1.41.0) \ # targets[84/84], packages[272/326] common.iprint("{:28} {:16} {:18}".format( manifest.ident, target_out, package_out)) for target in targets: common.iprint(" {}".format(target)) else: common.eprint(manifest.ident)
def download(self, url: str, dest_path: Path) -> None: if dest_path.is_file(): common.eprint("[{}] file exists, unlinking".format(dest_path)) dest_path.unlink() tmp_dest_path = common.tmp_path_for(dest_path) self._download(url, tmp_dest_path) tmp_dest_path.rename(dest_path)
def cmd_all_targets(self) -> None: for spec in self.adjust_wild_specs(self.specs): common.iprint("All targets: {}".format(spec)) manifest = self.select_manifest(spec, download=False) common.iprint(" ident: {}".format(manifest.ident)) for target in manifest.all_targets(): common.eprint(target)
def main() -> None: parser = make_parser() args = parser.parse_args() common.set_max_verbosity(common.VERBOSITY_INFO + args.verbose - args.quiet) try: cmd = args.subparser_name if args.readme: readme() elif cmd is None: raise error.UsageError("missing OPERATION (try --help)") elif cmd == "crate": romt.crate.Main(args).run() elif cmd == "rustup": romt.rustup.Main(args).run() elif cmd == "toolchain": romt.toolchain.Main(args).run() elif cmd == "serve": romt.serve.Main(args).run() except error.Error as e: common.eprint(e) sys.exit(1) except KeyboardInterrupt: common.eprint("Keyboard interrupt")
def sig_verify(self, path: Path, sig_path: Path) -> None: try: self.sig_verifier.verify(path, sig_path) except (error.MissingFileError, error.IntegrityError): if self._warn_signature: common.eprint("Warning: Signature failure for {}".format(path)) else: raise
def download_fileobj(self, url: str, fileobj: BinaryIO) -> None: try: response = self._session.get(url, stream=True) response.raise_for_status() for chunk in response.iter_content(chunk_size=4096): fileobj.write(chunk) except requests.exceptions.RequestException as e: common.eprint("[{}] download error: {}".format(url, e)) raise error.DownloadError(url, e)
def mark(repo: git.Repo, end: str) -> None: for branch in ["mark", "master"]: if branch == repo.head.reference.name: common.eprint( "Will not move branch {} (it is current HEAD)".format( repr(branch))) else: common.iprint("Move branch {} to point to {}".format( repr(branch), repr(end))) repo.create_head("refs/heads/{}".format(branch), end, force=True)
def _download(self, url: str, dest_path: Path) -> None: common.make_dirs_for(dest_path) try: with open(dest_path, "wb") as f: self.download_fileobj(url, f) except error.DownloadError as err: common.eprint("[{}] download error: {}".format(dest_path, err)) if dest_path.is_file(): common.eprint("[{}] file exists, unlinking".format(dest_path)) dest_path.unlink() raise
def verify_hash(self, path: Path, hash: str) -> None: """ Raises: MissingFileError - path doesn't exist IntegrityError - path exists with bad hash """ common.vprint("[verify] {}".format(path)) try: integrity.verify_hash(path, hash) except Exception as err: common.eprint("[{}] verification failed: {}".format(path, err))
def _upgrade_to_working(repo: git.Repo) -> None: working = git.Reference(repo, "refs/heads/working") if repo.head.reference.name != "working" and not working.is_valid(): # Time to upgrade working tree to use "working" branch. common.eprint("""Upgrade index to use "working" branch as HEAD""") if repo.head.reference.is_valid(): # Create "working" branch based on current HEAD. common.iprint( """Checkout new "working" branch from current HEAD""") repo.create_head("refs/heads/working", "HEAD") repo.head.set_reference(working)
def run(self) -> None: common.iprint("server running") setup_git_cgi() # Avoid type hint warning about missing http.server.test by # using getattr() to fetch the function. serve_func = getattr(http.server, "test") try: serve_func(HandlerClass=Handler, bind=self.args.bind, port=self.args.port) except Exception as err: common.eprint("Error serving: {}".format(err)) raise
def _get_gpg(self) -> Optional[gnupg.GPG]: if self._gpg_home is None: self._gpg_home = tempfile.TemporaryDirectory() try: gpg = gnupg.GPG(gnupghome=self._gpg_home.name) for key in RUST_KEYS: result = gpg.import_keys(key) if not result: raise ValueError( "Failed to import GPG key:\n{}".format(key)) self._gpg = gpg except Exception: common.eprint("** Warning: GPG setup failed; " "signature verification disabled") return self._gpg
def adjust_targets(self, version: str, base_targets: List[str]) -> List[str]: targets = set() known_targets = set(ALL_KNOWN_TARGETS) for target in base_targets: if target == "all": targets.update(known_targets) elif target == "*": targets.update(self.downloaded_targets(version)) else: if target not in known_targets: common.eprint("warning: unknown target {}".format( repr(target))) targets.add(target) return sorted(targets)
def setup_git_cgi() -> None: os.environ["GIT_HTTP_EXPORT_ALL"] = "" os.environ["GIT_PROJECT_ROOT"] = os.path.abspath("git") git_cgi_path = find_git_cgi_path() if git_cgi_path: if not common.is_windows and not common.is_executable(git_cgi_path): common.eprint( "Warning: setting executable flag on {}".format(git_cgi_path)) common.chmod_executable(git_cgi_path) else: git_http_backend_path = get_git_http_backend_path() if git_http_backend_path: make_git_cgi_script(git_http_backend_path) else: common.eprint("Warning: missing git-http-backend; no Git support")
def init_import(index_path: Path, crates_root_path: Path) -> None: bundle_path = index_path / INDEX_BUNDLE_NAME bundle_location = str(bundle_path.absolute()) repo = _init_common(index_path, bundle_location, crates_root_path) try: with repo.config_writer() as writer: writer.set_value('remote "origin"', "fetch", "+refs/heads/*:refs/remotes/origin/*") writer.add_value( 'remote "origin"', "fetch", "+refs/heads/bundle/*:refs/remotes/origin/*", ) except Exception as err: common.eprint("Error while importing: {}".format(err)) raise
def _detect_version_targets( self, rel_paths: Set[str]) -> Tuple[List[str], List[str]]: # rel_paths should be: "archive/<version>/<target>/<file>". versions = set() targets = set() for p in rel_paths: parts = p.split("/") if len(parts) < 4: common.eprint("warning: unexpected path {}".format(p)) else: version = parts[1] target = parts[2] if common.is_version(version): versions.add(version) targets.add(target) return sorted(versions), sorted(targets)
def verify_hash(path: Path, expected_hash: str) -> None: """ Raises: MissingFileError - path doesn't exist IntegrityError - path exists with bad hash """ if not path.exists(): common.eprint("[{}] verify hash missing file".format(path)) raise MissingFileError(str(path)) h = hash_file(path) if h != expected_hash: common.eprint("[{}] verify hash bad hash".format(path)) raise IntegrityError(path.name, actual_hash=h, expected_hash=expected_hash)
def cmd_list(self) -> None: max_verbosity = common.get_max_verbosity() show_details = max_verbosity >= common.VERBOSITY_INFO for spec in self.adjust_wild_specs(self.specs): common.iprint("List: {}".format(spec)) version = self.version_from_spec(spec, download=False) if show_details: targets = self.downloaded_targets(version) target_out = "targets[{}]".format(len(targets)) # Example output: # 1.41.0 targets[84] common.iprint("{:8} {}".format(version, target_out)) for target in targets: common.iprint(" {}".format(target)) else: common.eprint(version)
def _process_crates(downloader: romt.download.Downloader, dl_template: Optional[str], crates: List[Crate], crates_root: Path, good_paths_log_path: str, bad_paths_log_path: str, *, keep_going: bool, assume_ok: bool) -> None: good_paths_file = common.open_optional(good_paths_log_path, "w") bad_paths_file = common.open_optional(bad_paths_log_path, "w") num_good_paths = 0 num_bad_paths = 0 for crate in crates: rel_path = crate.rel_path() path = crates_root / rel_path is_good = False try: if dl_template is None: downloader.verify_hash(path, crate.hash) else: url = dl_template.format(crate=crate.name, version=crate.version) downloader.download_verify_hash(url, path, crate.hash, assume_ok=assume_ok) is_good = True except error.DownloadError as e: common.eprint("Error: Download failure for {}: {}".format( e.name, e.exception)) except error.MissingFileError as e: common.eprint("Error: Missing {}".format(e.name)) except error.IntegrityError as e: common.eprint(str(e)) except Exception as e: common.eprint( "Unknown error while processing crates: {}".format(e)) raise if is_good: num_good_paths += 1 common.log(good_paths_file, path) else: num_bad_paths += 1 common.log(bad_paths_file, path) common.iprint("{} bad paths, {} good paths".format(num_bad_paths, num_good_paths)) common.close_optional(good_paths_file) common.close_optional(bad_paths_file) if num_bad_paths > 0 and not keep_going: raise error.AbortError()
def unpack( repo: git.Repo, crates_root: Path, bundle_path: Path, archive_path: Path, keep_going: bool, ) -> None: num_crates = 0 crates_prefix = "crates/" found_bundle = False with common.tar_context(archive_path, "r") as tar_f: for tar_info in tar_f: if tar_info.isdir(): continue elif tar_info.name == INDEX_BUNDLE_PACKED_NAME: found_bundle = True tar_info.name = str(bundle_path) common.vprint("[unpack] {}".format(tar_info.name)) tar_f.extract(tar_info) elif tar_info.name.startswith(crates_prefix): num_crates += 1 tar_info.name = tar_info.name[len(crates_prefix):] common.vprint("[unpack] {}".format( os.path.basename(tar_info.name))) tar_f.extract(tar_info, str(crates_root)) else: common.eprint("Unexpected archive member {}".format( tar_info.name)) if not keep_going: raise error.AbortError() if not found_bundle: common.eprint("Missing {} in archive".format(INDEX_BUNDLE_PACKED_NAME)) if not keep_going: raise error.AbortError() common.iprint("{} extracted crates".format(num_crates))
def cmd_all_targets(self) -> None: common.iprint("All known targets:") for target in ALL_KNOWN_TARGETS: common.eprint(target)