def pack( crates: List[Crate], crates_root: Path, bundle_path: Optional[Path], archive_path: Path, keep_going: bool, ) -> None: num_good_paths = 0 num_bad_paths = 0 with common.tar_context(archive_path, "w") as tar_f: if bundle_path is not None: packed_name = INDEX_BUNDLE_PACKED_NAME common.vprint("[pack] {}".format(packed_name)) tar_f.add(str(bundle_path), packed_name) for rel_path in sorted(crate.rel_path() for crate in crates): path = crates_root / rel_path packed_name = "crates/" + rel_path.as_posix() try: common.vprint("[pack] {}".format(rel_path.name)) tar_f.add(str(path), packed_name) num_good_paths += 1 except FileNotFoundError: num_bad_paths += 1 common.eprint("Error: Missing {}".format(rel_path)) if not keep_going: raise error.AbortError() common.iprint("{} bad paths, {} good paths".format(num_bad_paths, num_good_paths))
def cmd_unpack(self) -> None: archive_path = self.get_archive_path() common.iprint("Unpacking archive: {}".format(archive_path)) dist_prefix = "dist/" extracted = set() with common.tar_context(archive_path, "r") as tar_f: for tar_info in tar_f: if tar_info.isdir(): continue if not tar_info.name.startswith(dist_prefix): raise error.UnexpectedArchiveMemberError(tar_info.name) rel_path = tar_info.name[len(dist_prefix):] dest_path = self.dest_path_from_rel_path(rel_path) tar_info.name = str(dest_path) common.vprint("[unpack] {}".format(rel_path)) tar_f.extract(tar_info) extracted.add(rel_path) specs = self._detect_specs(extracted) targets = self._detect_targets(specs, extracted) common.iprint("Unpacked specs: {}".format(len(specs))) for spec in specs: common.iprint(" {}".format(spec)) common.iprint("Unpacked targets: {}".format(len(targets))) for target in targets: common.iprint(" {}".format(target)) self.specs = specs self.targets = targets
def cmd_list(self) -> None: max_verbosity = common.get_max_verbosity() show_details = max_verbosity >= common.VERBOSITY_INFO for spec in self.adjust_wild_specs(self.specs): common.vprint("List: {}".format(spec)) manifest = self.select_manifest(spec, download=False) if show_details: available_packages = manifest.available_packages() available_targets = manifest.available_targets() packages = self.downloaded_packages(manifest) targets = self.downloaded_targets(manifest) target_out = "targets[{}/{}]".format( len(targets), len(available_targets), ) package_out = "packages[{}/{}]".format( len(packages), len(available_packages), ) # Example output: # stable-2020-01-30(1.41.0) \ # targets[84/84], packages[272/326] common.iprint("{:28} {:16} {:18}".format( manifest.ident, target_out, package_out)) for target in targets: common.iprint(" {}".format(target)) else: common.eprint(manifest.ident)
def cmd_fixup(self) -> None: for spec in self.adjust_wild_specs(self.specs): common.iprint("Fixup: {}".format(spec)) manifest = self.select_manifest(spec, download=False, canonical=True) common.vprint(" ident: {}".format(manifest.ident)) self._write_manifest_variations(manifest)
def pack_path(rel_path: str) -> None: dest_path = self.dest_path_from_rel_path(rel_path) packed_name = "dist/" + rel_path common.vprint("[pack] {}".format(rel_path)) try: tar_f.add(str(dest_path), packed_name) except FileNotFoundError: raise error.MissingFileError(str(dest_path))
def get_crates(self) -> List[Crate]: if self._crates is None: common.vprint("[calculating crate list]") self._crates = list( crates_in_range(self.get_repo(), self.get_start(), self.args.end)) common.vprint("[{} crates in range]".format(len(self._crates))) return self._crates
def update_config_json(repo: git.Repo, config: bytes) -> None: old_config = read_config_json(repo) if old_config is None or config != old_config: config_path = _config_json_path(repo) common.vprint("update-config: {}".format(config_path)) config_path.write_bytes(config) repo.index.add(str(config_path)) repo.index.commit("Apply config.json adjustments")
def verify_hash(self, path: Path, hash: str) -> None: """ Raises: MissingFileError - path doesn't exist IntegrityError - path exists with bad hash """ common.vprint("[verify] {}".format(path)) integrity.verify_hash(path, hash)
def download_cached(self, dest_url: str, dest_path: Path, *, cached: bool = True) -> None: if cached and dest_path.is_file(): common.vprint("[cached file] {}".format(dest_path)) else: common.vprint("[downloading] {}".format(dest_path)) self.download(dest_url, dest_path)
def get_release_stable_version(self, *, download: bool) -> str: url, path = self.release_stable_url_path if download: # This file changes unexpectedly. Avoid caching to ensure the # correct version is used. self.downloader.download_cached(url, path, cached=False) elif path.is_file(): common.vprint("[read] {}".format(path)) else: raise error.MissingFileError(str(path)) return toml.load(path)["version"]
def verify_hash(self, path: Path, hash: str) -> None: """ Raises: MissingFileError - path doesn't exist IntegrityError - path exists with bad hash """ common.vprint("[verify] {}".format(path)) try: integrity.verify_hash(path, hash) except Exception as err: common.eprint("[{}] verification failed: {}".format(path, err))
def verify(self, path: Path, *, with_sig: bool = False) -> None: """ Raises: MissingFileError - path or associated sha256path doesn't exist IntegrityError - paths exists with bad hash """ hash_path = integrity.path_append_hash_suffix(path) common.vprint("[verify] {}".format(path)) integrity.verify(path, hash_path) if with_sig: sig_path = signature.path_append_sig_suffix(path) self.sig_verify(path, sig_path)
def download_verify(self, dest_url: str, dest_path: Path, *, cached: bool = True, assume_ok: bool = False, with_sig: bool = False) -> None: hash_path = integrity.path_append_hash_suffix(dest_path) sig_path = signature.path_append_sig_suffix(dest_path) if cached: if (assume_ok and dest_path.is_file() and hash_path.is_file() and (not with_sig or sig_path.is_file())): common.vvprint("[assuming OK] {}".format(dest_path)) return try: integrity.verify(dest_path, hash_path) if with_sig: self.sig_verify(dest_path, sig_path) common.vprint("[cached file] {}".format(dest_path)) return except (error.MissingFileError, error.IntegrityError): pass common.vprint("[downloading] {}".format(dest_path)) # Download the (small) hash and signature files first. hash_url = integrity.append_hash_suffix(dest_url) self.download(hash_url, hash_path) if with_sig: sig_url = signature.append_sig_suffix(dest_url) self.download(sig_url, sig_path) # If dest_path exists and has the correct hash, bypass the downloading # step to save download time. download_required = True if dest_path.is_file(): try: integrity.verify(dest_path, hash_path) download_required = False except (error.MissingFileError, error.IntegrityError): pass if download_required: self.download(dest_url, dest_path) integrity.verify(dest_path, hash_path) if with_sig: self.sig_verify(dest_path, sig_path)
def unpack( repo: git.Repo, crates_root: Path, bundle_path: Path, archive_path: Path, keep_going: bool, ) -> None: num_crates = 0 crates_prefix = "crates/" found_bundle = False try: with common.tar_context(archive_path, "r") as tar_f: for tar_info in tar_f: if tar_info.isdir(): continue elif tar_info.name == INDEX_BUNDLE_PACKED_NAME: found_bundle = True tar_info.name = str(bundle_path) common.vprint("[unpack] {}".format(tar_info.name)) tar_f.extract(tar_info) elif tar_info.name.startswith(crates_prefix): num_crates += 1 tar_info.name = tar_info.name[len(crates_prefix):] common.vprint("[unpack] {}".format( os.path.basename(tar_info.name))) tar_f.extract(tar_info, str(crates_root)) else: common.eprint("Unexpected archive member {}".format( tar_info.name)) if not keep_going: raise error.AbortError() except Exception as err: common.eprint("Exception unpacking: {}".format(err)) raise if not found_bundle: common.eprint("Missing {} in archive".format(INDEX_BUNDLE_PACKED_NAME)) if not keep_going: raise error.AbortError() common.iprint("{} extracted crates".format(num_crates))
def download_verify_hash(self, dest_url: str, dest_path: Path, hash: str, *, cached: bool = True, assume_ok: bool = False) -> None: if cached: if assume_ok and dest_path.is_file(): common.vvprint("[assuming OK] {}".format(dest_path)) return try: integrity.verify_hash(dest_path, hash) common.vprint("[cached file] {}".format(dest_path)) return except (error.MissingFileError, error.IntegrityError): pass common.vprint("[downloading] {}".format(dest_path)) self.download(dest_url, dest_path) integrity.verify_hash(dest_path, hash)
def merge_origin_master(repo: git.Repo) -> None: _upgrade_to_working(repo) initial_config = read_config_json(repo) try: common.vprint("merge-index: merge origin/master") repo.git.merge("remotes/origin/master", "-m", "Merge origin/master") except git.GitError: common.iprint("merge-index: merge failed; reconstructing") common.vprint("merge-index: reset to recover failed merge state") repo.head.reset(working_tree=True, index=True) common.vprint("merge-index: reset to remotes/origin/master") repo.head.reset("remotes/origin/master", working_tree=True, index=True) # Restore initial_config if necessary. if initial_config is not None: update_config_json(repo, initial_config)