def add_virtiowin_media(self, paths): """ Move iso/vfd media to the local tree. Set up symlinks and htaccess magic for the non-versioned links """ virtiodir = os.path.join( self.LOCAL_DIRECT_DIR, self.virtio_basedir) if os.path.exists(virtiodir): fail("dir=%s already exists? Make sure we aren't " "overwriting anything." % virtiodir) os.mkdir(virtiodir) htaccess = "" for versionfile, symlink in paths: shellcomm("cp %s %s" % (versionfile, virtiodir)) shellcomm("cp %s %s" % (symlink, virtiodir)) htaccess += _make_redirect( os.path.join(self.HTTP_DIRECT_DIR, self.virtio_basedir), os.path.basename(symlink), os.path.basename(versionfile)) # Write .htaccess, redirecting symlinks to versioned files, so # nobody ends up with unversioned files locally, since that # will make for crappy bug reports open(os.path.join(virtiodir, ".htaccess"), "w").write(htaccess)
def main(): options = parse_args() output_dir = options.output_dir if not os.path.exists(output_dir): os.mkdir(output_dir) if os.listdir(output_dir): fail("%s is not empty." % output_dir) shellcomm("git submodule init") shellcomm("git submodule update") driverdir = os.path.abspath(options.driverdir) vdagent_x64_msi = os.path.abspath(options.vdagent_x64_msi) vdagent_x86_msi = os.path.abspath(options.vdagent_x86_msi) qxlwddm_x64_msi = os.path.abspath(options.qxlwddm_x64_msi) qxlwddm_x86_msi = os.path.abspath(options.qxlwddm_x86_msi) ga_x64_msi = os.path.abspath(options.ga_x64_msi) ga_x86_msi = os.path.abspath(options.ga_x86_msi) win_fsp_msi = os.path.abspath(options.win_fsp_msi) os.chdir("virtio-win-guest-tools-installer") shellcomm("git clean -xdf") shellcomm( "./automation/build-artifacts.sh %s %s %s %s %s %s %s %s %s" % (driverdir, vdagent_x64_msi, vdagent_x86_msi, qxlwddm_x64_msi, qxlwddm_x86_msi, ga_x64_msi, ga_x86_msi, win_fsp_msi, options.nvr)) shellcomm("mv ./exported-artifacts/* %s" % output_dir) return 0
def main() -> None: parser = argparse.ArgumentParser( "Identifies matching functions by looking at function calls in matching functions" ) parser.add_argument("-f", "--fn", help="Functions to analyze", nargs="*") args = parser.parse_args() functions_to_analyze = set(args.fn) if args.fn else set() functions_by_addr: Dict[int, utils.FunctionInfo] = { fn.addr: fn for fn in utils.get_functions() } fn_checker = Checker() for fn in functions_by_addr.values(): if functions_to_analyze and fn.decomp_name not in functions_to_analyze: continue if fn.status != utils.FunctionStatus.Matching: continue base_fn = elf.get_fn_from_base_elf(fn.addr, fn.size) try: my_fn = elf.get_fn_from_my_elf(fn.decomp_name) except KeyError: utils.warn(f"could not find function {fn.decomp_name}") continue fn_checker.checking = fn.decomp_name fn_checker.check(base_fn, my_fn) if fn_checker.invalid_call_descriptions: for x in fn_checker.invalid_call_descriptions: utils.print_note(x) utils.fail("invalid calls detected") new_matches: Dict[int, str] = dict() calls = fn_checker.get_possible_calls().copy() for base_target, my_target in calls.items(): target_info = functions_by_addr.get(base_target) if target_info is None: continue if target_info.status != utils.FunctionStatus.NotDecompiled: continue base_fn = elf.get_fn_from_base_elf(target_info.addr, target_info.size) try: name = fn_checker.addr_to_symbol[my_target] my_fn = elf.get_fn_from_my_elf(name) except KeyError: continue if fn_checker.check(base_fn, my_fn): new_matches[base_target] = name utils.print_note( f"new match: {Fore.BLUE}{cxxfilt.demangle(name)}{Fore.RESET}") utils.add_decompiled_functions(new_matches)
def dump_fn(name: str) -> None: expected_dir = utils.get_repo_root() / "expected" try: fn = util.elf.get_fn_from_my_elf(name) path = expected_dir / f"{name}.bin" path.parent.mkdir(exist_ok=True) path.write_bytes(fn.data) except KeyError: utils.fail("could not find function")
def _get_local_dir(): """ Directory on the local machine we are using as the virtio-win mirror. We will update this locally and then rsync it to fedorapeople """ ret = os.path.expanduser("~/src/fedora/virt-group-repos/virtio-win") if not os.path.exists(ret): fail("Expected local virtio-win mirror does not exist: %s" % ret) return ret
def _get_fas_username(): """ Get fedora username. Uses FAS_USERNAME environment variable which is used by some other fedora tools """ ret = os.environ.get("FAS_USERNAME") if not ret: fail("You must set FAS_USERNAME environment variable to your " "fedorapeople account name") return ret
def set_internal_url(): config_path = os.path.expanduser( "~/.config/virtio-win-pkg-scripts/fetch-latest-builds.ini") if not os.path.exists(config_path): fail("Config file not found, see the docs: %s" % config_path) script_cfg = configparser.ConfigParser() script_cfg.read(config_path) global INTERNAL_URL INTERNAL_URL = script_cfg.get("config", "internal_url")
def copy_virtio_drivers(input_dir, output_dir): # Create a flat list of every leaf directory in the virtio-win directory alldirs = [] for dirpath, dirnames, files in os.walk(input_dir): dummy = files if dirnames: continue ostuple = dirpath[len(input_dir) + 1:] if ostuple not in alldirs: alldirs.append(ostuple) drivers = list(filemap.DRIVER_OS_MAP.keys())[:] copymap = {} missing_patterns = [] for drivername in drivers: for ostuple in sorted(filemap.DRIVER_OS_MAP[drivername]): # ./rhel is only used on RHEL builds for the qemupciserial # driver, so if it's not present on public builds, ignore it if (drivername == "qemupciserial" and ostuple == "./rhel"): continue if os.path.normpath(ostuple) not in alldirs and ostuple != "./": fail("driver=%s ostuple=%s not found in input=%s" % (drivername, ostuple, input_dir)) # We know that the ostuple dir contains bits for this driver, # figure out what files we want to copy. ret = _update_copymap_for_driver(input_dir, ostuple, drivername, copymap) missing_patterns.extend(ret) if missing_patterns: msg = ( "\nDid not find any files matching these patterns:\n %s\n\n" % "\n ".join(missing_patterns)) msg += textwrap.fill( "This means we expected to find that file in the " "virtio-win-prewhql archive, but it wasn't found. This means the " "build output changed. Assuming this file was intentionally " "removed, you'll need to update the file whitelists in " "filemap.py to accurately reflect the current new file layout.") msg += "\n\n" fail(msg) # Actually copy the files, and track the ones we've seen for srcfile, dests in list(copymap.items()): for d in dests: d = os.path.join(output_dir, d) if not os.path.exists(d): os.makedirs(d) shutil.copy2(srcfile, d) # The keys here are all a list of files we actually copied return list(copymap.keys())
def extract_files(filename): """ Passed in either a zip, tar.gz, or RPM, extract the contents, including the contents of any contained vfd or iso files. Move these to a temp directory for easy comparison. """ output_dir = tempfile.mkdtemp(prefix="virtio-win-archive-compare-") atexit.register(lambda: shutil.rmtree(output_dir)) # Extract the content if os.path.isdir(filename): shutil.copytree(filename, os.path.join(output_dir, "dircopy")) else: extract_dir = os.path.join(output_dir, "extracted-archive") os.mkdir(extract_dir) if os.path.isdir(filename): pass elif filename.endswith(".zip"): shellcomm("unzip %s -d %s > /dev/null" % (filename, extract_dir)) elif filename.endswith(".tar.gz"): shellcomm("tar -xvf %s --directory %s > /dev/null" % (filename, extract_dir)) elif filename.endswith(".rpm"): shellcomm("cd %s && rpm2cpio %s | cpio -idm --quiet" % (extract_dir, filename)) else: fail("Unexpected filename %s, only expecting .zip, *.tar.gz, .rpm, " "or a directory" % filename) # Find .vfd files mediafiles = [] for root, dirs, files in os.walk(output_dir): dummy = dirs mediafiles += [ os.path.join(root, name) for name in files if name.endswith(".vfd") or name.endswith(".iso") ] # Extract vfd file contents with guestfish for mediafile in mediafiles: if os.path.islink(mediafile): continue media_out_dir = os.path.join( output_dir, os.path.basename(mediafile) + "-extracted") os.mkdir(media_out_dir) shellcomm( "guestfish --ro --add %s --mount /dev/sda:/ glob copy-out '/*' %s" " > /dev/null" % (mediafile, media_out_dir)) shellcomm("chmod -R 777 %s" % media_out_dir) return output_dir
def add_link(src, link): fullsrc = os.path.join(self.LOCAL_DIRECT_DIR, src) linkpath = os.path.join(self.LOCAL_DIRECT_DIR, link) if not os.path.exists(fullsrc): fail("Nonexistent link src=%s for target=%s" % (fullsrc, linkpath)) if os.path.exists(linkpath): os.unlink(linkpath) shellcomm("ln -s %s %s" % (src, linkpath)) return _make_redirect(self.HTTP_DIRECT_DIR, link, src)
def main(): options = parse_args() if not options.regenerate_only and not options.resync: if not options.rpm_output or not options.rpm_buildroot: fail("--rpm-output and --rpm-buildroot must both " "be specified, or pass --regenerate-only to " "regen just the repo.") buildversions = BuildVersions() _populate_local_tree(buildversions, options.rpm_output, options.rpm_buildroot) if not options.resync: _generate_repos() _push_repos(reverse=options.resync) return 0
def dump_table(name: str) -> None: try: symbols = util.elf.build_addr_to_symbol_table(util.elf.my_symtab) decomp_symbols = { fn.decomp_name for fn in utils.get_functions() if fn.decomp_name } offset, size = util.elf.get_symbol_file_offset_and_size( util.elf.my_elf, util.elf.my_symtab, name) util.elf.my_elf.stream.seek(offset) vtable_bytes = util.elf.my_elf.stream.read(size) if not vtable_bytes: utils.fail( "empty vtable; has the key function been implemented? (https://lld.llvm.org/missingkeyfunction.html)" ) print( f"{Fore.WHITE}{Style.BRIGHT}{cxxfilt.demangle(name)}{Style.RESET_ALL}" ) print(f"{Fore.YELLOW}{Style.BRIGHT}vtable @ 0x0{Style.RESET_ALL}") assert size % 8 == 0 for i in range(size // 8): word: int = struct.unpack_from("<Q", vtable_bytes, 8 * i)[0] name = symbols.get(word, None) if word == 0: pass elif name is not None: demangled_name: str = cxxfilt.demangle(name) color = Fore.GREEN if name in decomp_symbols else Fore.BLUE print(f"{color}{bold(demangled_name)}{Style.RESET_ALL}") print(f" {name}") elif word & (1 << 63): offset = -struct.unpack_from("<q", vtable_bytes, 8 * i)[0] print() print( f"{Fore.YELLOW}{Style.BRIGHT}vtable @ {offset:#x}{Style.RESET_ALL}" ) else: print(f"{Fore.RED}unknown data: {word:016x}{Style.RESET_ALL}") except KeyError: utils.fail("could not find symbol")
def _add_relative_link(topdir, srcname, linkname): """ Create symlink for passed paths, but using relative path resolution """ srcpath = os.path.join(topdir, srcname) linkpath = os.path.join(topdir, linkname) if not os.path.exists(srcpath): fail("Nonexistent link src=%s for target=%s" % (srcpath, linkpath)) srcrelpath = os.path.relpath(srcname, os.path.dirname(linkname)) if os.path.exists(linkpath): if (os.path.islink(linkpath) and os.readlink(linkpath) == srcrelpath): print("link path=%s already points to src=%s, nothing to do" % (linkpath, srcrelpath)) return os.unlink(linkpath) shellcomm("ln -s %s %s" % (srcrelpath, linkpath))
def main(): options = parse_args() output_dir = options.output_dir if not os.path.exists(output_dir): os.mkdir(output_dir) if os.listdir(output_dir): fail("%s is not empty." % output_dir) options.input_dir = os.path.abspath(os.path.expanduser(options.input_dir)) # Actually move the files seenfiles = [] seenfiles += copy_virtio_drivers(options.input_dir, output_dir) seenfiles += copy_license(options.input_dir, output_dir) # Verify that there is nothing left over that we missed check_remaining_files(options.input_dir, seenfiles) print("Generated %s" % output_dir) return 0
def _generate_repos(): """ Create repo trees, run createrepo_c """ LOCAL_REPO_DIR = LocalRepo.LOCAL_REPO_DIR # Generate stable symlinks shellcomm("rm -rf %s/*" % os.path.join(LOCAL_REPO_DIR, "stable")) for stablever in STABLE_RPMS: filename = "virtio-win-%s.noarch.rpm" % stablever fullpath = os.path.join(LOCAL_REPO_DIR, "rpms", filename) if not os.path.exists(fullpath): fail("Didn't find stable RPM path %s" % fullpath) shellcomm("ln -s ../rpms/%s %s" % (filename, os.path.join(LOCAL_REPO_DIR, "stable", os.path.basename(fullpath)))) # Generate latest symlinks shellcomm("rm -rf %s/*" % os.path.join(LOCAL_REPO_DIR, "latest")) for fullpath in glob.glob(os.path.join(LOCAL_REPO_DIR, "rpms", "*.rpm")): filename = os.path.basename(fullpath) shellcomm("ln -s ../rpms/%s %s" % (filename, os.path.join(LOCAL_REPO_DIR, "latest", os.path.basename(fullpath)))) # Generate repodata for rpmdir in ["latest", "stable", "srpms"]: shellcomm("rm -rf %s" % os.path.join(LOCAL_REPO_DIR, rpmdir, "repodata")) shellcomm("createrepo_c %s > /dev/null" % os.path.join(LOCAL_REPO_DIR, rpmdir)) # Put the repo file in place shellcomm("cp -f data/virtio-win.repo %s" % LocalRepo.LOCAL_ROOT_DIR) # Use the RPM changelog as a changelog file for the whole tree shellcomm("cp -f data/rpm_changelog %s/CHANGELOG" % LocalRepo.LOCAL_ROOT_DIR)
def add_virtiowin_media(self, isopath, rpmpath, srpmpath): """ Move iso media to the local tree. Set up symlinks and htaccess magic for the non-versioned links """ virtiodir = os.path.join( self.LOCAL_DIRECT_DIR, self.virtio_basedir) if os.path.exists(virtiodir): fail("dir=%s already exists? Make sure we aren't " "overwriting anything." % virtiodir) os.mkdir(virtiodir) htaccess = "" def add_stable_path(path, stablename): versionname = os.path.basename(path) _add_relative_link(virtiodir, versionname, stablename) nonlocal htaccess htaccess += _make_redirect( os.path.join(self.HTTP_DIRECT_DIR, self.virtio_basedir), stablename, versionname) def add_rpm(path, stablename): # RPMs are already in the repo tree, so symlink the full path _add_relative_link(virtiodir, os.path.relpath(path, virtiodir), os.path.basename(path)) add_stable_path(path, stablename) add_rpm(rpmpath, "virtio-win.noarch.rpm") add_rpm(srpmpath, "virtio-win.src.rpm") shellcomm("cp %s %s" % (isopath, virtiodir)) add_stable_path(isopath, "virtio-win.iso") # Write .htaccess, redirecting symlinks to versioned files, so # nobody ends up with unversioned files locally, since that # will make for crappy bug reports open(os.path.join(virtiodir, ".htaccess"), "w").write(htaccess)
def _distill_links(url, extension, want, skip): """ :param want: Whitelist of files we want from find_links :param skip: Blacklist of expected files that we don't want Reason for the explicit list approach is so that any new brew output will cause the script to error, so we are forced to decide whether it's something to ship or not. """ origzipnames = find_links(url, extension) zipnames = origzipnames[:] for f in skip: if f not in zipnames: fail("Didn't find blacklisted '%s' at URL=%s\nOnly found: %s" % (f, url, origzipnames)) zipnames.remove(f) for f in want: if f not in zipnames: fail("Didn't find whitelisted '%s' at URL=%s\nOnly found: %s" % (f, url, origzipnames)) return [url + w for w in want]
def _glob(pattern, recursive=False): ret = list(glob.glob(pattern, recursive=recursive)) if not ret: fail("Didn't find any matching files: %s" % pattern) return ret
def check_remaining_files(input_dir, seenfiles): # Expected files that we want to skip. The reason we are so strict here # is to make sure that we don't forget to ship important files that appear # in new virtio-win builds. If a new file appears, we probably need to ask # the driver developers whether to ship it or not. whitelist = [ # vadim confirmed these files should _not_ be shipped # (private mail May 2015) r".*DVL\.XML", ".*vioser-test.*", ".*viorngtest.*", # Added in 171 build in May 2019, similar to above XML so I # presume it shouldn't be shipped r".*DVL-compat\.XML", # These are files that are needed for the vfd build process. They # were added to the prewhql sources in July 2015. # See: https://bugzilla.redhat.com/show_bug.cgi?id=1217799 # # We could possibly use them in this repo, but it's a bit # difficult because of the RHEL build process sharing. # Bug: https://bugzilla.redhat.com/show_bug.cgi?id=1251770 ".*/disk1", ".*/txtsetup-i386.oem", ".*/txtsetup-amd64.oem", # qxlwddm changelogs ".*/spice-qxl-wddm-dod/w10/Changelog", ".*/spice-qxl-wddm-dod-8.1-compatible/Changelog", ".*/spice-qxl-wddm-dod/w10/QxlWddmDod_0.20.0.0_x64.msi", ".*/spice-qxl-wddm-dod/w10/QxlWddmDod_0.20.0.0_x86.msi", # virtio-win build system unconditionally builds every driver # for every windows platform that supports it. However, depending # on the driver, functionally identical binaries might be # generated. In those cases, we ship only one build of the driver # for every windows version it will work on (see filemap.py # DRIVER_OS_MAP) # # This also simplifies the WHQL submission process, one submission # can cover multiple windows versions. # # In those cases, we end up with unused virtio-win build output. # That's what the below drivers cover. # # If you add to this list, be sure it's not a newly introduced # driver that you are ignoring! Everything listed here needs # be covered by a mapping in DRIVER_OS_MAP # Added in virtio-win build 137, for rhel only, and this # script is only used for non-rhel (Fedora) builds "/rhel/qemupciserial.cat", "/rhel/qemupciserial.inf", ] remaining = [] for dirpath, dirnames, files in os.walk(input_dir): dummy = dirnames for f in files: remaining.append(os.path.join(dirpath, f)) notseen = [f for f in remaining if f not in seenfiles] seenpatterns = [] for pattern in whitelist: for f in notseen[:]: if not re.match(pattern, f[len(input_dir):]): continue notseen.remove(f) if pattern not in seenpatterns: seenpatterns.append(pattern) if notseen: msg = ("\nUnhandled virtio-win files:\n %s\n\n" % "\n ".join([f[len(input_dir):] for f in sorted(notseen)])) msg += textwrap.fill("This means the above files were not tracked " "in filemap.py _and_ not tracked in the internal whitelist " "in this script. This probably means that there is new build " "output. You need to determine if it's something we should " "be shipping (add it to filemap.py) or something we should " "ignore (add it to the whitelist).") fail(msg) if len(seenpatterns) != len(whitelist): msg = ("\nDidn't match some whitelist entries:\n %s\n\n" % "\n ".join([p for p in whitelist if p not in seenpatterns])) msg += textwrap.fill("This means that the above pattern did not " "match anything in the build output. That pattern comes from " "the internal whitelist tracked as part of this script: they " "are files that we expect to see in the build output, but " "that we deliberately do _not_ ship as part of the RPM. If " "the whitelist entry didn't match, it likely means that the " "files are no longer output by the driver build, so you can " "just remove the explicit whitelist entry.") fail(msg)
for info in utils.get_functions(): if info.decomp_name == name or (find_wip and info.status == utils.FunctionStatus.Wip): return info for info in utils.get_functions(): if name in cxxfilt.demangle(info.decomp_name): return info return None info = find_function_info(args.function) if info is not None: if not info.decomp_name: utils.fail(f"{args.function} has not been decompiled") print( f"diffing: {Style.BRIGHT}{Fore.BLUE}{cxxfilt.demangle(info.decomp_name)}{Style.RESET_ALL} {Style.DIM}({info.decomp_name}){Style.RESET_ALL}" ) addr_end = info.addr + info.size subprocess.call([ "tools/asm-differ/diff.py", "-I", "-e", info.decomp_name, "0x%016x" % info.addr, "0x%016x" % addr_end ] + unknown) if info.status == utils.FunctionStatus.NonMatching: utils.warn( f"{info.decomp_name} is marked as non-matching and possibly NOT functionally equivalent" )
import diff_settings from util import utils _config: Dict[str, Any] = {} diff_settings.apply(_config, {}) _root = utils.get_repo_root() base_elf_data = io.BytesIO((_root / _config["baseimg"]).read_bytes()) my_elf_data = io.BytesIO((_root / _config["myimg"]).read_bytes()) base_elf = ELFFile(base_elf_data) my_elf = ELFFile(my_elf_data) my_symtab = my_elf.get_section_by_name(".symtab") if not my_symtab: utils.fail(f'{_config["myimg"]} has no symbol table') class Symbol(NamedTuple): addr: int name: str size: int class Function(NamedTuple): data: bytes addr: int _ElfSymFormat = struct.Struct("<IBBHQQ")