def cmd_fixup(self) -> None:
for spec in self.adjust_wild_specs(self.specs):
common.iprint("Fixup: {}".format(spec))
version = self.version_from_spec(spec, download=False)
version_path = self.artifact_version_path(version)
# Artifacts are arranged as: <version_path>/<target>/<artifact>
# If no artifacts exist for any targets, claim the version
# is not present.
artifacts = list(version_path.glob("*/*"))
if not artifacts:
raise error.UsageError(
"version {} not present".format(version))
self._fixup_version(version)
# Copy rustup/archive/<stable_version>/ to rustup/dist/.
stable_version = self.get_release_stable_version(download=False)
archive_version_path = self.dest_path_from_rel_path(
"archive/{}".format(stable_version))
if not archive_version_path.is_dir():
raise error.MissingDirectoryError(str(archive_version_path))
dist_path = self.dest_path_from_rel_path("dist")
if dist_path.is_dir():
shutil.rmtree(str(dist_path))
common.iprint("[copytree] {} -> {}".format(archive_version_path,
dist_path))
shutil.copytree(str(archive_version_path), str(dist_path))
def readme() -> None:
desc = readme_from_pkg_resources()
if not desc:
desc = readme_from_file()
if not desc:
desc = "README.rst is not available."
common.iprint(desc)
def cmd_all_targets(self) -> None:
for spec in self.adjust_wild_specs(self.specs):
common.iprint("All targets: {}".format(spec))
manifest = self.select_manifest(spec, download=False)
common.iprint(" ident: {}".format(manifest.ident))
for target in manifest.all_targets():
common.eprint(target)
def cmd_list(self) -> None:
max_verbosity = common.get_max_verbosity()
show_details = max_verbosity >= common.VERBOSITY_INFO
for spec in self.adjust_wild_specs(self.specs):
common.vprint("List: {}".format(spec))
manifest = self.select_manifest(spec, download=False)
if show_details:
available_packages = manifest.available_packages()
available_targets = manifest.available_targets()
packages = self.downloaded_packages(manifest)
targets = self.downloaded_targets(manifest)
target_out = "targets[{}/{}]".format(
len(targets),
len(available_targets),
)
package_out = "packages[{}/{}]".format(
len(packages),
len(available_packages),
)
# Example output:
# stable-2020-01-30(1.41.0) \
# targets[84/84], packages[272/326]
common.iprint("{:28} {:16} {:18}".format(
manifest.ident, target_out, package_out))
for target in targets:
common.iprint(" {}".format(target))
else:
common.eprint(manifest.ident)
def cmd_unpack(self) -> None:
archive_path = self.get_archive_path()
common.iprint("Unpacking archive: {}".format(archive_path))
dist_prefix = "dist/"
extracted = set()
with common.tar_context(archive_path, "r") as tar_f:
for tar_info in tar_f:
if tar_info.isdir():
continue
if not tar_info.name.startswith(dist_prefix):
raise error.UnexpectedArchiveMemberError(tar_info.name)
rel_path = tar_info.name[len(dist_prefix):]
dest_path = self.dest_path_from_rel_path(rel_path)
tar_info.name = str(dest_path)
common.vprint("[unpack] {}".format(rel_path))
tar_f.extract(tar_info)
extracted.add(rel_path)
specs = self._detect_specs(extracted)
targets = self._detect_targets(specs, extracted)
common.iprint("Unpacked specs: {}".format(len(specs)))
for spec in specs:
common.iprint(" {}".format(spec))
common.iprint("Unpacked targets: {}".format(len(targets)))
for target in targets:
common.iprint(" {}".format(target))
self.specs = specs
self.targets = targets
def pack(
crates: List[Crate],
crates_root: Path,
bundle_path: Optional[Path],
archive_path: Path,
keep_going: bool,
) -> None:
num_good_paths = 0
num_bad_paths = 0
with common.tar_context(archive_path, "w") as tar_f:
if bundle_path is not None:
packed_name = INDEX_BUNDLE_PACKED_NAME
common.vprint("[pack] {}".format(packed_name))
tar_f.add(str(bundle_path), packed_name)
for rel_path in sorted(crate.rel_path() for crate in crates):
path = crates_root / rel_path
packed_name = "crates/" + rel_path.as_posix()
try:
common.vprint("[pack] {}".format(rel_path.name))
tar_f.add(str(path), packed_name)
num_good_paths += 1
except FileNotFoundError:
num_bad_paths += 1
common.eprint("Error: Missing {}".format(rel_path))
if not keep_going:
raise error.AbortError()
common.iprint("{} bad paths, {} good paths".format(num_bad_paths,
num_good_paths))
def cmd_fixup(self) -> None:
for spec in self.adjust_wild_specs(self.specs):
common.iprint("Fixup: {}".format(spec))
manifest = self.select_manifest(spec,
download=False,
canonical=True)
common.vprint(" ident: {}".format(manifest.ident))
self._write_manifest_variations(manifest)
def mark(repo: git.Repo, end: str) -> None:
for branch in ["mark", "master"]:
if branch == repo.head.reference.name:
common.eprint(
"Will not move branch {} (it is current HEAD)".format(
repr(branch)))
else:
common.iprint("Move branch {} to point to {}".format(
repr(branch), repr(end)))
repo.create_head("refs/heads/{}".format(branch), end, force=True)
def _upgrade_to_working(repo: git.Repo) -> None:
working = git.Reference(repo, "refs/heads/working")
if repo.head.reference.name != "working" and not working.is_valid():
# Time to upgrade working tree to use "working" branch.
common.eprint("""Upgrade index to use "working" branch as HEAD""")
if repo.head.reference.is_valid():
# Create "working" branch based on current HEAD.
common.iprint(
"""Checkout new "working" branch from current HEAD""")
repo.create_head("refs/heads/working", "HEAD")
repo.head.set_reference(working)
def run(self) -> None:
common.iprint("server running")
setup_git_cgi()
# Avoid type hint warning about missing http.server.test by
# using getattr() to fetch the function.
serve_func = getattr(http.server, "test")
serve_func(HandlerClass=Handler,
bind=self.args.bind,
port=self.args.port)
def _write_manifest(self, manifest: Manifest, date: str,
channel: str) -> None:
src_path = self.manifest_path(manifest.date, manifest.channel)
dst_path = self.manifest_path(date, channel)
common.iprint("[publish] {}".format(dst_path))
shutil.copyfile(str(src_path), str(dst_path))
src_hash_path = integrity.path_append_hash_suffix(src_path)
dst_hash_path = integrity.path_append_hash_suffix(dst_path)
shutil.copyfile(str(src_hash_path), str(dst_hash_path))
src_sig_path = signature.path_append_sig_suffix(src_path)
dst_sig_path = signature.path_append_sig_suffix(dst_path)
shutil.copyfile(str(src_sig_path), str(dst_sig_path))
def _process_crates(downloader: romt.download.Downloader,
dl_template: Optional[str], crates: List[Crate],
crates_root: Path, good_paths_log_path: str,
bad_paths_log_path: str, *, keep_going: bool,
assume_ok: bool) -> None:
good_paths_file = common.open_optional(good_paths_log_path, "w")
bad_paths_file = common.open_optional(bad_paths_log_path, "w")
num_good_paths = 0
num_bad_paths = 0
for crate in crates:
rel_path = crate.rel_path()
path = crates_root / rel_path
is_good = False
try:
if dl_template is None:
downloader.verify_hash(path, crate.hash)
else:
url = dl_template.format(crate=crate.name,
version=crate.version)
downloader.download_verify_hash(url,
path,
crate.hash,
assume_ok=assume_ok)
is_good = True
except error.DownloadError as e:
common.eprint("Error: Download failure for {}: {}".format(
e.name, e.exception))
except error.MissingFileError as e:
common.eprint("Error: Missing {}".format(e.name))
except error.IntegrityError as e:
common.eprint(str(e))
except Exception as e:
common.eprint(
"Unknown error while processing crates: {}".format(e))
raise
if is_good:
num_good_paths += 1
common.log(good_paths_file, path)
else:
num_bad_paths += 1
common.log(bad_paths_file, path)
common.iprint("{} bad paths, {} good paths".format(num_bad_paths,
num_good_paths))
common.close_optional(good_paths_file)
common.close_optional(bad_paths_file)
if num_bad_paths > 0 and not keep_going:
raise error.AbortError()
def _fixup_version(self, version: str) -> None:
# Write release_stable unless a newer one already exists.
path = self.release_stable_path
if path.is_file():
old_version = self.get_release_stable_version(download=False)
new_key = common.version_sort_key(version)
old_key = common.version_sort_key(old_version)
write = new_key >= old_key
else:
write = True
if write:
common.iprint("[write] {} (version={})".format(path, version))
self._write_release_stable(version)
def cmd_pack(self) -> None:
base_targets = dist.require_targets(self.targets, default="*")
archive_path = self.get_archive_path()
common.iprint("Packing archive: {}".format(archive_path))
with common.tar_context(archive_path, "w") as tar_f:
def pack_path(rel_path: str) -> None:
dest_path = self.dest_path_from_rel_path(rel_path)
packed_name = "rustup/" + rel_path
common.vprint("[pack] {}".format(rel_path))
try:
tar_f.add(str(dest_path), packed_name)
except FileNotFoundError:
raise error.MissingFileError(str(dest_path))
def pack_rel_path(rel_path: str) -> None:
pack_path(rel_path)
pack_path(integrity.append_hash_suffix(rel_path))
for spec in self.adjust_wild_specs(self.specs):
common.iprint("Pack: {}".format(spec))
version = self.version_from_spec(spec, download=False)
common.iprint(" version: {}".format(version))
targets = self.adjust_targets(version, base_targets)
common.iprint(" targets: {}".format(len(targets)))
for t in targets:
common.vvprint(" target: {}".format(t))
for target in targets:
rel_path = self.rustup_init_rel_path(version, target)
pack_rel_path(rel_path)
def run(self) -> None:
common.iprint("server running")
setup_git_cgi()
# Avoid type hint warning about missing http.server.test by
# using getattr() to fetch the function.
serve_func = getattr(http.server, "test")
try:
serve_func(HandlerClass=Handler,
bind=self.args.bind,
port=self.args.port)
except Exception as err:
common.eprint("Error serving: {}".format(err))
raise
def merge_origin_master(repo: git.Repo) -> None:
_upgrade_to_working(repo)
initial_config = read_config_json(repo)
try:
common.vprint("merge-index: merge origin/master")
repo.git.merge("remotes/origin/master", "-m", "Merge origin/master")
except git.GitError:
common.iprint("merge-index: merge failed; reconstructing")
common.vprint("merge-index: reset to recover failed merge state")
repo.head.reset(working_tree=True, index=True)
common.vprint("merge-index: reset to remotes/origin/master")
repo.head.reset("remotes/origin/master", working_tree=True, index=True)
# Restore initial_config if necessary.
if initial_config is not None:
update_config_json(repo, initial_config)
def make_git_cgi_script(git_http_backend_path: Path) -> None:
cgi_path = Path("cgi-bin")
if not cgi_path.is_dir():
common.iprint("mkdir {}".format(cgi_path))
cgi_path.mkdir()
if common.is_windows:
extension = ".bat"
template = '@echo off\n"{}"\n'
else:
extension = ".sh"
template = "#!/bin/sh\nexec '{}'\n"
script_path = cgi_path / ("git-http-backend" + extension)
script = template.format(git_http_backend_path)
common.iprint("Create script {} ({})".format(script_path, repr(script)))
script_path.write_text(script, encoding="utf-8")
if not common.is_windows:
common.chmod_executable(script_path)
def run(self) -> None:
valid_commands = [
"fetch-manifest",
"download",
"verify",
"list",
"all-targets",
"pack",
"unpack",
"fixup",
]
commands = self.args.commands
if commands:
base.verify_commands(commands, valid_commands)
else:
common.iprint("nothing to do; try a COMMAND")
return
for cmd in commands:
if cmd == "fetch-manifest":
self.cmd_fetch_manifest()
elif cmd == "download":
self.cmd_download()
self.cmd_fixup()
elif cmd == "verify":
self.cmd_verify()
elif cmd == "list":
self.cmd_list()
elif cmd == "all-targets":
self.cmd_all_targets()
elif cmd == "pack":
self.cmd_pack()
elif cmd == "unpack":
self.cmd_unpack()
self.cmd_verify()
self.cmd_fixup()
elif cmd == "fixup":
self.cmd_fixup()
def unpack(
repo: git.Repo,
crates_root: Path,
bundle_path: Path,
archive_path: Path,
keep_going: bool,
) -> None:
num_crates = 0
crates_prefix = "crates/"
found_bundle = False
try:
with common.tar_context(archive_path, "r") as tar_f:
for tar_info in tar_f:
if tar_info.isdir():
continue
elif tar_info.name == INDEX_BUNDLE_PACKED_NAME:
found_bundle = True
tar_info.name = str(bundle_path)
common.vprint("[unpack] {}".format(tar_info.name))
tar_f.extract(tar_info)
elif tar_info.name.startswith(crates_prefix):
num_crates += 1
tar_info.name = tar_info.name[len(crates_prefix):]
common.vprint("[unpack] {}".format(
os.path.basename(tar_info.name)))
tar_f.extract(tar_info, str(crates_root))
else:
common.eprint("Unexpected archive member {}".format(
tar_info.name))
if not keep_going:
raise error.AbortError()
except Exception as err:
common.eprint("Exception unpacking: {}".format(err))
raise
if not found_bundle:
common.eprint("Missing {} in archive".format(INDEX_BUNDLE_PACKED_NAME))
if not keep_going:
raise error.AbortError()
common.iprint("{} extracted crates".format(num_crates))
def cmd_pack(self) -> None:
if self.get_crates():
bundle_path = self.get_bundle_path()
git_bundle_create(
self.get_repo(),
str(bundle_path),
self.get_start(),
self.args.end,
)
pack(
self.get_crates(),
get_crates_root_path(self.args.crates),
bundle_path,
self.get_archive_path(),
self.args.keep_going,
)
else:
common.iprint("No crates to pack")
def _init_common(index_path: Path, origin_location: str,
crates_root_path: Path) -> git.Repo:
if index_path.is_dir():
raise error.UsageError("index directory {} already exists".format(
repr(str(index_path))))
common.iprint("create index repository at {}:".format(repr(
str(index_path))))
index_path.mkdir(parents=True)
repo = git.Repo.init(str(index_path))
common.iprint(" remote add origin {}".format(origin_location))
repo.create_remote("origin", origin_location)
# Setup "HEAD" to new "working" branch.
working = git.Reference(repo, "refs/heads/working")
repo.head.set_reference(working)
# Setup default remote and merge branch for "working".
with repo.config_writer() as writer:
writer.set_value('branch "working"', "remote", "origin")
writer.set_value('branch "working"', "merge", "refs/heads/master")
if not crates_root_path.is_dir():
common.iprint("create crates directory at {}:".format(
repr(str(crates_root_path))))
crates_root_path.mkdir(parents=True)
return repo
def _download_verify(self, download: bool, specs: List[str],
base_targets: List[str]) -> None:
for spec in specs:
common.iprint("{}: {}".format("Download" if download else "Verify",
spec))
manifest = self.select_manifest(spec,
download=download,
canonical=True)
common.iprint(" ident: {}".format(manifest.ident))
targets = self.adjust_targets(manifest, base_targets)
packages = list(manifest.gen_available_packages(targets=targets))
common.iprint(" packages: {}, targets: {}".format(
len(packages), len(targets)))
for t in targets:
common.vvprint(" target: {}".format(t))
for package in packages:
rel_path = package.rel_path
dest_path = self.dest_path_from_rel_path(rel_path)
dest_url = self.url_from_rel_path(rel_path)
if download:
self.downloader.download_verify(
dest_url,
dest_path,
assume_ok=self.args.assume_ok,
with_sig=self._with_sig,
)
else:
self.downloader.verify(dest_path, with_sig=self._with_sig)
def run(self) -> None:
if not self.args.start:
self.args.start = "mark"
self.args.allow_missing_start = True
commands = self.get_commands()
while commands:
command = commands.pop(0)
common.iprint("{}...".format(command))
if command in ("update", "export", "import"):
if command == "update":
cmd = "pull download mark"
elif command == "export":
cmd = "pull download pack mark"
else:
cmd = "unpack pull verify mark"
commands = cmd.split() + commands
else:
cmd_func_name = "cmd_{}".format(command.replace("-", "_"))
getattr(self, cmd_func_name)()
def get_commands(self) -> List[str]:
valid_commands = [
"pull",
"download",
"verify",
"pack",
"mark",
"unpack",
"list",
"update",
"export",
"import",
"init",
"init-import",
"config",
]
commands = self.args.commands[:]
if commands:
base.verify_commands(commands, valid_commands)
else:
common.iprint("Nothing to do (try --help)")
return commands
def _rewrite_path(self) -> None:
path = self.path
if path.startswith("/git/"):
git_cgi_path = find_git_cgi_path()
if git_cgi_path is not None:
path = path.replace("git", git_cgi_path.as_posix(), 1)
elif path.startswith("/crates/"):
# /crates/.../<name>/<name>-<version>.crate
# ->
# /crates/<prefix>/<name>/<name>-<version>.crate
parent = os.path.dirname(path)
name = os.path.basename(parent)
rel_path = "crates/{}/{}/{}".format(
crate_prefix_from_name(name),
name,
os.path.basename(path),
)
if os.path.isfile(rel_path):
path = "/" + rel_path
if self.path != path:
common.iprint("Rewrite URL: {} -> {}".format(self.path, path))
self.path = path
def cmd_pack(self) -> None:
base_targets = dist.require_targets(self.targets, default="*")
archive_path = self.get_archive_path()
common.iprint("Packing archive: {}".format(archive_path))
with common.tar_context(archive_path, "w") as tar_f:
def pack_path(rel_path: str) -> None:
dest_path = self.dest_path_from_rel_path(rel_path)
packed_name = "dist/" + rel_path
common.vprint("[pack] {}".format(rel_path))
try:
tar_f.add(str(dest_path), packed_name)
except FileNotFoundError:
raise error.MissingFileError(str(dest_path))
def pack_rel_path(rel_path: str) -> None:
pack_path(rel_path)
pack_path(integrity.append_hash_suffix(rel_path))
if self._with_sig:
pack_path(signature.append_sig_suffix(rel_path))
for spec in self.adjust_wild_specs(self.specs):
common.iprint("Pack: {}".format(spec))
manifest = self.select_manifest(spec,
download=False,
canonical=True)
common.iprint(" ident: {}".format(manifest.ident))
targets = self.adjust_targets(manifest, base_targets)
packages = list(
manifest.gen_available_packages(targets=targets))
common.iprint(" packages: {}, targets: {}".format(
len(packages), len(targets)))
for t in targets:
common.vvprint(" target: {}".format(t))
# Pack channel file.
pack_rel_path(channel_rel_path(manifest.date,
manifest.channel))
# Pack up package file parts.
for package in packages:
pack_rel_path(package.rel_path)
def cmd_list(self) -> None:
max_verbosity = common.get_max_verbosity()
show_details = max_verbosity >= common.VERBOSITY_INFO
for spec in self.adjust_wild_specs(self.specs):
common.iprint("List: {}".format(spec))
version = self.version_from_spec(spec, download=False)
if show_details:
targets = self.downloaded_targets(version)
target_out = "targets[{}]".format(len(targets))
# Example output:
# 1.41.0 targets[84]
common.iprint("{:8} {}".format(version, target_out))
for target in targets:
common.iprint(" {}".format(target))
else:
common.eprint(version)
def _download_verify(self, download: bool, specs: List[str],
base_targets: List[str]) -> None:
for spec in specs:
common.iprint("{}: {}".format("Download" if download else "Verify",
spec))
version = self.version_from_spec(spec, download=download)
common.iprint(" version: {}".format(version))
targets = self.adjust_targets(version, base_targets)
common.iprint(" targets: {}".format(len(targets)))
for t in targets:
common.vvprint(" target: {}".format(t))
for target in targets:
rel_path = self.rustup_init_rel_path(version, target)
dest_path = self.dest_path_from_rel_path(rel_path)
dest_url = self.url_from_rel_path(rel_path)
if download:
self.downloader.download_verify(
dest_url, dest_path, assume_ok=self.args.assume_ok)
else:
self.downloader.verify(dest_path)
def cmd_all_targets(self) -> None:
common.iprint("All known targets:")
for target in ALL_KNOWN_TARGETS:
common.eprint(target)
def cmd_fetch_manifest(self) -> None:
for spec in self.adjust_download_specs(self.specs):
common.iprint("Fetch manifest: {}".format(spec))
manifest = self.select_manifest(spec, download=True)
common.iprint(" ident: {}".format(manifest.ident))