def add_virtiowin_media(self, paths):
"""
Move iso/vfd media to the local tree. Set up symlinks and
htaccess magic for the non-versioned links
"""
virtiodir = os.path.join(
self.LOCAL_DIRECT_DIR, self.virtio_basedir)
if os.path.exists(virtiodir):
fail("dir=%s already exists? Make sure we aren't "
"overwriting anything." % virtiodir)
os.mkdir(virtiodir)
htaccess = ""
for versionfile, symlink in paths:
shellcomm("cp %s %s" % (versionfile, virtiodir))
shellcomm("cp %s %s" % (symlink, virtiodir))
htaccess += _make_redirect(
os.path.join(self.HTTP_DIRECT_DIR, self.virtio_basedir),
os.path.basename(symlink), os.path.basename(versionfile))
# Write .htaccess, redirecting symlinks to versioned files, so
# nobody ends up with unversioned files locally, since that
# will make for crappy bug reports
open(os.path.join(virtiodir, ".htaccess"), "w").write(htaccess)
def main():
options = parse_args()
output_dir = options.output_dir
if not os.path.exists(output_dir):
os.mkdir(output_dir)
if os.listdir(output_dir):
fail("%s is not empty." % output_dir)
shellcomm("git submodule init")
shellcomm("git submodule update")
driverdir = os.path.abspath(options.driverdir)
vdagent_x64_msi = os.path.abspath(options.vdagent_x64_msi)
vdagent_x86_msi = os.path.abspath(options.vdagent_x86_msi)
qxlwddm_x64_msi = os.path.abspath(options.qxlwddm_x64_msi)
qxlwddm_x86_msi = os.path.abspath(options.qxlwddm_x86_msi)
ga_x64_msi = os.path.abspath(options.ga_x64_msi)
ga_x86_msi = os.path.abspath(options.ga_x86_msi)
win_fsp_msi = os.path.abspath(options.win_fsp_msi)
os.chdir("virtio-win-guest-tools-installer")
shellcomm("git clean -xdf")
shellcomm(
"./automation/build-artifacts.sh %s %s %s %s %s %s %s %s %s" %
(driverdir, vdagent_x64_msi, vdagent_x86_msi, qxlwddm_x64_msi,
qxlwddm_x86_msi, ga_x64_msi, ga_x86_msi, win_fsp_msi, options.nvr))
shellcomm("mv ./exported-artifacts/* %s" % output_dir)
return 0
def main() -> None:
parser = argparse.ArgumentParser(
"Identifies matching functions by looking at function calls in matching functions"
)
parser.add_argument("-f", "--fn", help="Functions to analyze", nargs="*")
args = parser.parse_args()
functions_to_analyze = set(args.fn) if args.fn else set()
functions_by_addr: Dict[int, utils.FunctionInfo] = {
fn.addr: fn
for fn in utils.get_functions()
}
fn_checker = Checker()
for fn in functions_by_addr.values():
if functions_to_analyze and fn.decomp_name not in functions_to_analyze:
continue
if fn.status != utils.FunctionStatus.Matching:
continue
base_fn = elf.get_fn_from_base_elf(fn.addr, fn.size)
try:
my_fn = elf.get_fn_from_my_elf(fn.decomp_name)
except KeyError:
utils.warn(f"could not find function {fn.decomp_name}")
continue
fn_checker.checking = fn.decomp_name
fn_checker.check(base_fn, my_fn)
if fn_checker.invalid_call_descriptions:
for x in fn_checker.invalid_call_descriptions:
utils.print_note(x)
utils.fail("invalid calls detected")
new_matches: Dict[int, str] = dict()
calls = fn_checker.get_possible_calls().copy()
for base_target, my_target in calls.items():
target_info = functions_by_addr.get(base_target)
if target_info is None:
continue
if target_info.status != utils.FunctionStatus.NotDecompiled:
continue
base_fn = elf.get_fn_from_base_elf(target_info.addr, target_info.size)
try:
name = fn_checker.addr_to_symbol[my_target]
my_fn = elf.get_fn_from_my_elf(name)
except KeyError:
continue
if fn_checker.check(base_fn, my_fn):
new_matches[base_target] = name
utils.print_note(
f"new match: {Fore.BLUE}{cxxfilt.demangle(name)}{Fore.RESET}")
utils.add_decompiled_functions(new_matches)
def dump_fn(name: str) -> None:
expected_dir = utils.get_repo_root() / "expected"
try:
fn = util.elf.get_fn_from_my_elf(name)
path = expected_dir / f"{name}.bin"
path.parent.mkdir(exist_ok=True)
path.write_bytes(fn.data)
except KeyError:
utils.fail("could not find function")
def _get_local_dir():
"""
Directory on the local machine we are using as the virtio-win mirror.
We will update this locally and then rsync it to fedorapeople
"""
ret = os.path.expanduser("~/src/fedora/virt-group-repos/virtio-win")
if not os.path.exists(ret):
fail("Expected local virtio-win mirror does not exist: %s" % ret)
return ret
def _get_fas_username():
"""
Get fedora username. Uses FAS_USERNAME environment variable which is
used by some other fedora tools
"""
ret = os.environ.get("FAS_USERNAME")
if not ret:
fail("You must set FAS_USERNAME environment variable to your "
"fedorapeople account name")
return ret
def set_internal_url():
config_path = os.path.expanduser(
"~/.config/virtio-win-pkg-scripts/fetch-latest-builds.ini")
if not os.path.exists(config_path):
fail("Config file not found, see the docs: %s" % config_path)
script_cfg = configparser.ConfigParser()
script_cfg.read(config_path)
global INTERNAL_URL
INTERNAL_URL = script_cfg.get("config", "internal_url")
def copy_virtio_drivers(input_dir, output_dir):
# Create a flat list of every leaf directory in the virtio-win directory
alldirs = []
for dirpath, dirnames, files in os.walk(input_dir):
dummy = files
if dirnames:
continue
ostuple = dirpath[len(input_dir) + 1:]
if ostuple not in alldirs:
alldirs.append(ostuple)
drivers = list(filemap.DRIVER_OS_MAP.keys())[:]
copymap = {}
missing_patterns = []
for drivername in drivers:
for ostuple in sorted(filemap.DRIVER_OS_MAP[drivername]):
# ./rhel is only used on RHEL builds for the qemupciserial
# driver, so if it's not present on public builds, ignore it
if (drivername == "qemupciserial" and ostuple == "./rhel"):
continue
if os.path.normpath(ostuple) not in alldirs and ostuple != "./":
fail("driver=%s ostuple=%s not found in input=%s" %
(drivername, ostuple, input_dir))
# We know that the ostuple dir contains bits for this driver,
# figure out what files we want to copy.
ret = _update_copymap_for_driver(input_dir, ostuple, drivername,
copymap)
missing_patterns.extend(ret)
if missing_patterns:
msg = (
"\nDid not find any files matching these patterns:\n %s\n\n" %
"\n ".join(missing_patterns))
msg += textwrap.fill(
"This means we expected to find that file in the "
"virtio-win-prewhql archive, but it wasn't found. This means the "
"build output changed. Assuming this file was intentionally "
"removed, you'll need to update the file whitelists in "
"filemap.py to accurately reflect the current new file layout.")
msg += "\n\n"
fail(msg)
# Actually copy the files, and track the ones we've seen
for srcfile, dests in list(copymap.items()):
for d in dests:
d = os.path.join(output_dir, d)
if not os.path.exists(d):
os.makedirs(d)
shutil.copy2(srcfile, d)
# The keys here are all a list of files we actually copied
return list(copymap.keys())
def extract_files(filename):
"""
Passed in either a zip, tar.gz, or RPM, extract the contents, including
the contents of any contained vfd or iso files. Move these to a temp
directory for easy comparison.
"""
output_dir = tempfile.mkdtemp(prefix="virtio-win-archive-compare-")
atexit.register(lambda: shutil.rmtree(output_dir))
# Extract the content
if os.path.isdir(filename):
shutil.copytree(filename, os.path.join(output_dir, "dircopy"))
else:
extract_dir = os.path.join(output_dir, "extracted-archive")
os.mkdir(extract_dir)
if os.path.isdir(filename):
pass
elif filename.endswith(".zip"):
shellcomm("unzip %s -d %s > /dev/null" % (filename, extract_dir))
elif filename.endswith(".tar.gz"):
shellcomm("tar -xvf %s --directory %s > /dev/null" %
(filename, extract_dir))
elif filename.endswith(".rpm"):
shellcomm("cd %s && rpm2cpio %s | cpio -idm --quiet" %
(extract_dir, filename))
else:
fail("Unexpected filename %s, only expecting .zip, *.tar.gz, .rpm, "
"or a directory" % filename)
# Find .vfd files
mediafiles = []
for root, dirs, files in os.walk(output_dir):
dummy = dirs
mediafiles += [
os.path.join(root, name) for name in files
if name.endswith(".vfd") or name.endswith(".iso")
]
# Extract vfd file contents with guestfish
for mediafile in mediafiles:
if os.path.islink(mediafile):
continue
media_out_dir = os.path.join(
output_dir,
os.path.basename(mediafile) + "-extracted")
os.mkdir(media_out_dir)
shellcomm(
"guestfish --ro --add %s --mount /dev/sda:/ glob copy-out '/*' %s"
" > /dev/null" % (mediafile, media_out_dir))
shellcomm("chmod -R 777 %s" % media_out_dir)
return output_dir
def add_link(src, link):
fullsrc = os.path.join(self.LOCAL_DIRECT_DIR, src)
linkpath = os.path.join(self.LOCAL_DIRECT_DIR, link)
if not os.path.exists(fullsrc):
fail("Nonexistent link src=%s for target=%s" %
(fullsrc, linkpath))
if os.path.exists(linkpath):
os.unlink(linkpath)
shellcomm("ln -s %s %s" % (src, linkpath))
return _make_redirect(self.HTTP_DIRECT_DIR, link, src)
def main():
options = parse_args()
if not options.regenerate_only and not options.resync:
if not options.rpm_output or not options.rpm_buildroot:
fail("--rpm-output and --rpm-buildroot must both "
"be specified, or pass --regenerate-only to "
"regen just the repo.")
buildversions = BuildVersions()
_populate_local_tree(buildversions,
options.rpm_output, options.rpm_buildroot)
if not options.resync:
_generate_repos()
_push_repos(reverse=options.resync)
return 0
def dump_table(name: str) -> None:
try:
symbols = util.elf.build_addr_to_symbol_table(util.elf.my_symtab)
decomp_symbols = {
fn.decomp_name
for fn in utils.get_functions() if fn.decomp_name
}
offset, size = util.elf.get_symbol_file_offset_and_size(
util.elf.my_elf, util.elf.my_symtab, name)
util.elf.my_elf.stream.seek(offset)
vtable_bytes = util.elf.my_elf.stream.read(size)
if not vtable_bytes:
utils.fail(
"empty vtable; has the key function been implemented? (https://lld.llvm.org/missingkeyfunction.html)"
)
print(
f"{Fore.WHITE}{Style.BRIGHT}{cxxfilt.demangle(name)}{Style.RESET_ALL}"
)
print(f"{Fore.YELLOW}{Style.BRIGHT}vtable @ 0x0{Style.RESET_ALL}")
assert size % 8 == 0
for i in range(size // 8):
word: int = struct.unpack_from("<Q", vtable_bytes, 8 * i)[0]
name = symbols.get(word, None)
if word == 0:
pass
elif name is not None:
demangled_name: str = cxxfilt.demangle(name)
color = Fore.GREEN if name in decomp_symbols else Fore.BLUE
print(f"{color}{bold(demangled_name)}{Style.RESET_ALL}")
print(f" {name}")
elif word & (1 << 63):
offset = -struct.unpack_from("<q", vtable_bytes, 8 * i)[0]
print()
print(
f"{Fore.YELLOW}{Style.BRIGHT}vtable @ {offset:#x}{Style.RESET_ALL}"
)
else:
print(f"{Fore.RED}unknown data: {word:016x}{Style.RESET_ALL}")
except KeyError:
utils.fail("could not find symbol")
def _add_relative_link(topdir, srcname, linkname):
"""
Create symlink for passed paths, but using relative path resolution
"""
srcpath = os.path.join(topdir, srcname)
linkpath = os.path.join(topdir, linkname)
if not os.path.exists(srcpath):
fail("Nonexistent link src=%s for target=%s" % (srcpath, linkpath))
srcrelpath = os.path.relpath(srcname, os.path.dirname(linkname))
if os.path.exists(linkpath):
if (os.path.islink(linkpath) and os.readlink(linkpath) == srcrelpath):
print("link path=%s already points to src=%s, nothing to do" %
(linkpath, srcrelpath))
return
os.unlink(linkpath)
shellcomm("ln -s %s %s" % (srcrelpath, linkpath))
def main():
options = parse_args()
output_dir = options.output_dir
if not os.path.exists(output_dir):
os.mkdir(output_dir)
if os.listdir(output_dir):
fail("%s is not empty." % output_dir)
options.input_dir = os.path.abspath(os.path.expanduser(options.input_dir))
# Actually move the files
seenfiles = []
seenfiles += copy_virtio_drivers(options.input_dir, output_dir)
seenfiles += copy_license(options.input_dir, output_dir)
# Verify that there is nothing left over that we missed
check_remaining_files(options.input_dir, seenfiles)
print("Generated %s" % output_dir)
return 0
def _generate_repos():
"""
Create repo trees, run createrepo_c
"""
LOCAL_REPO_DIR = LocalRepo.LOCAL_REPO_DIR
# Generate stable symlinks
shellcomm("rm -rf %s/*" % os.path.join(LOCAL_REPO_DIR, "stable"))
for stablever in STABLE_RPMS:
filename = "virtio-win-%s.noarch.rpm" % stablever
fullpath = os.path.join(LOCAL_REPO_DIR, "rpms", filename)
if not os.path.exists(fullpath):
fail("Didn't find stable RPM path %s" % fullpath)
shellcomm("ln -s ../rpms/%s %s" %
(filename,
os.path.join(LOCAL_REPO_DIR, "stable",
os.path.basename(fullpath))))
# Generate latest symlinks
shellcomm("rm -rf %s/*" % os.path.join(LOCAL_REPO_DIR, "latest"))
for fullpath in glob.glob(os.path.join(LOCAL_REPO_DIR, "rpms", "*.rpm")):
filename = os.path.basename(fullpath)
shellcomm("ln -s ../rpms/%s %s" %
(filename,
os.path.join(LOCAL_REPO_DIR, "latest",
os.path.basename(fullpath))))
# Generate repodata
for rpmdir in ["latest", "stable", "srpms"]:
shellcomm("rm -rf %s" %
os.path.join(LOCAL_REPO_DIR, rpmdir, "repodata"))
shellcomm("createrepo_c %s > /dev/null" %
os.path.join(LOCAL_REPO_DIR, rpmdir))
# Put the repo file in place
shellcomm("cp -f data/virtio-win.repo %s" % LocalRepo.LOCAL_ROOT_DIR)
# Use the RPM changelog as a changelog file for the whole tree
shellcomm("cp -f data/rpm_changelog %s/CHANGELOG" %
LocalRepo.LOCAL_ROOT_DIR)
def add_virtiowin_media(self, isopath, rpmpath, srpmpath):
"""
Move iso media to the local tree. Set up symlinks and
htaccess magic for the non-versioned links
"""
virtiodir = os.path.join(
self.LOCAL_DIRECT_DIR, self.virtio_basedir)
if os.path.exists(virtiodir):
fail("dir=%s already exists? Make sure we aren't "
"overwriting anything." % virtiodir)
os.mkdir(virtiodir)
htaccess = ""
def add_stable_path(path, stablename):
versionname = os.path.basename(path)
_add_relative_link(virtiodir, versionname, stablename)
nonlocal htaccess
htaccess += _make_redirect(
os.path.join(self.HTTP_DIRECT_DIR, self.virtio_basedir),
stablename, versionname)
def add_rpm(path, stablename):
# RPMs are already in the repo tree, so symlink the full path
_add_relative_link(virtiodir,
os.path.relpath(path, virtiodir),
os.path.basename(path))
add_stable_path(path, stablename)
add_rpm(rpmpath, "virtio-win.noarch.rpm")
add_rpm(srpmpath, "virtio-win.src.rpm")
shellcomm("cp %s %s" % (isopath, virtiodir))
add_stable_path(isopath, "virtio-win.iso")
# Write .htaccess, redirecting symlinks to versioned files, so
# nobody ends up with unversioned files locally, since that
# will make for crappy bug reports
open(os.path.join(virtiodir, ".htaccess"), "w").write(htaccess)
def _distill_links(url, extension, want, skip):
"""
:param want: Whitelist of files we want from find_links
:param skip: Blacklist of expected files that we don't want
Reason for the explicit list approach is so that any new brew output
will cause the script to error, so we are forced to decide whether it's
something to ship or not.
"""
origzipnames = find_links(url, extension)
zipnames = origzipnames[:]
for f in skip:
if f not in zipnames:
fail("Didn't find blacklisted '%s' at URL=%s\nOnly found: %s" %
(f, url, origzipnames))
zipnames.remove(f)
for f in want:
if f not in zipnames:
fail("Didn't find whitelisted '%s' at URL=%s\nOnly found: %s" %
(f, url, origzipnames))
return [url + w for w in want]
def _glob(pattern, recursive=False):
ret = list(glob.glob(pattern, recursive=recursive))
if not ret:
fail("Didn't find any matching files: %s" % pattern)
return ret
def check_remaining_files(input_dir, seenfiles):
# Expected files that we want to skip. The reason we are so strict here
# is to make sure that we don't forget to ship important files that appear
# in new virtio-win builds. If a new file appears, we probably need to ask
# the driver developers whether to ship it or not.
whitelist = [
# vadim confirmed these files should _not_ be shipped
# (private mail May 2015)
r".*DVL\.XML",
".*vioser-test.*",
".*viorngtest.*",
# Added in 171 build in May 2019, similar to above XML so I
# presume it shouldn't be shipped
r".*DVL-compat\.XML",
# These are files that are needed for the vfd build process. They
# were added to the prewhql sources in July 2015.
# See: https://bugzilla.redhat.com/show_bug.cgi?id=1217799
#
# We could possibly use them in this repo, but it's a bit
# difficult because of the RHEL build process sharing.
# Bug: https://bugzilla.redhat.com/show_bug.cgi?id=1251770
".*/disk1",
".*/txtsetup-i386.oem",
".*/txtsetup-amd64.oem",
# qxlwddm changelogs
".*/spice-qxl-wddm-dod/w10/Changelog",
".*/spice-qxl-wddm-dod-8.1-compatible/Changelog",
".*/spice-qxl-wddm-dod/w10/QxlWddmDod_0.20.0.0_x64.msi",
".*/spice-qxl-wddm-dod/w10/QxlWddmDod_0.20.0.0_x86.msi",
# virtio-win build system unconditionally builds every driver
# for every windows platform that supports it. However, depending
# on the driver, functionally identical binaries might be
# generated. In those cases, we ship only one build of the driver
# for every windows version it will work on (see filemap.py
# DRIVER_OS_MAP)
#
# This also simplifies the WHQL submission process, one submission
# can cover multiple windows versions.
#
# In those cases, we end up with unused virtio-win build output.
# That's what the below drivers cover.
#
# If you add to this list, be sure it's not a newly introduced
# driver that you are ignoring! Everything listed here needs
# be covered by a mapping in DRIVER_OS_MAP
# Added in virtio-win build 137, for rhel only, and this
# script is only used for non-rhel (Fedora) builds
"/rhel/qemupciserial.cat",
"/rhel/qemupciserial.inf",
]
remaining = []
for dirpath, dirnames, files in os.walk(input_dir):
dummy = dirnames
for f in files:
remaining.append(os.path.join(dirpath, f))
notseen = [f for f in remaining if f not in seenfiles]
seenpatterns = []
for pattern in whitelist:
for f in notseen[:]:
if not re.match(pattern, f[len(input_dir):]):
continue
notseen.remove(f)
if pattern not in seenpatterns:
seenpatterns.append(pattern)
if notseen:
msg = ("\nUnhandled virtio-win files:\n %s\n\n" %
"\n ".join([f[len(input_dir):] for f in sorted(notseen)]))
msg += textwrap.fill("This means the above files were not tracked "
"in filemap.py _and_ not tracked in the internal whitelist "
"in this script. This probably means that there is new build "
"output. You need to determine if it's something we should "
"be shipping (add it to filemap.py) or something we should "
"ignore (add it to the whitelist).")
fail(msg)
if len(seenpatterns) != len(whitelist):
msg = ("\nDidn't match some whitelist entries:\n %s\n\n" %
"\n ".join([p for p in whitelist if p not in seenpatterns]))
msg += textwrap.fill("This means that the above pattern did not "
"match anything in the build output. That pattern comes from "
"the internal whitelist tracked as part of this script: they "
"are files that we expect to see in the build output, but "
"that we deliberately do _not_ ship as part of the RPM. If "
"the whitelist entry didn't match, it likely means that the "
"files are no longer output by the driver build, so you can "
"just remove the explicit whitelist entry.")
fail(msg)
for info in utils.get_functions():
if info.decomp_name == name or (find_wip and info.status
== utils.FunctionStatus.Wip):
return info
for info in utils.get_functions():
if name in cxxfilt.demangle(info.decomp_name):
return info
return None
info = find_function_info(args.function)
if info is not None:
if not info.decomp_name:
utils.fail(f"{args.function} has not been decompiled")
print(
f"diffing: {Style.BRIGHT}{Fore.BLUE}{cxxfilt.demangle(info.decomp_name)}{Style.RESET_ALL} {Style.DIM}({info.decomp_name}){Style.RESET_ALL}"
)
addr_end = info.addr + info.size
subprocess.call([
"tools/asm-differ/diff.py", "-I", "-e", info.decomp_name,
"0x%016x" % info.addr,
"0x%016x" % addr_end
] + unknown)
if info.status == utils.FunctionStatus.NonMatching:
utils.warn(
f"{info.decomp_name} is marked as non-matching and possibly NOT functionally equivalent"
)
import diff_settings
from util import utils
_config: Dict[str, Any] = {}
diff_settings.apply(_config, {})
_root = utils.get_repo_root()
base_elf_data = io.BytesIO((_root / _config["baseimg"]).read_bytes())
my_elf_data = io.BytesIO((_root / _config["myimg"]).read_bytes())
base_elf = ELFFile(base_elf_data)
my_elf = ELFFile(my_elf_data)
my_symtab = my_elf.get_section_by_name(".symtab")
if not my_symtab:
utils.fail(f'{_config["myimg"]} has no symbol table')
class Symbol(NamedTuple):
addr: int
name: str
size: int
class Function(NamedTuple):
data: bytes
addr: int
_ElfSymFormat = struct.Struct("<IBBHQQ")
