def main():
if len(sys.argv) < 2:
print(f"USAGE: {sys.argv[0]} PATH [RPATH]")
sys.exit(2)
is_err = False
root = Path(sys.argv[1])
if root.is_dir():
libs = list(root.glob("*.so*"))
else:
libs, root = [root], root.parent
if len(sys.argv) > 2:
rpath = "$ORIGIN/{sys.argv[2]}"
else:
rpath = "$ORIGIN"
for lib in libs:
if lib.is_symlink():
continue
print(f"{lib}:")
libtree = lddtree(str(lib))
ext_refs = lddtree_external_references(libtree, root).get(POLICY, None)
if ext_refs:
for name, path in ext_refs["libs"].items():
print(f" -> {name} (@ {path})")
if path is None:
print(
f" ERROR: Couldn't find a real library path for: {name}"
)
is_err = True
continue
local_dep = root / name
if not local_dep.exists():
print(f" Copying {path} -> {local_dep}")
shutil.copy2(path, local_dep, follow_symlinks=True)
if not is_err:
print(f" re-setting RPATH for {lib} -> {rpath} ...")
check_call(["patchelf", "--remove-rpath", str(lib)])
check_call([
"patchelf", "--force-rpath", "--set-rpath", rpath,
str(lib)
])
if is_err:
print("Errors encountered")
sys.exit(1)
def relocate_elf_library(patchelf, output_dir, output_library, binary):
"""
Relocate an ELF shared library to be packaged on a wheel.
Given a shared library, find the transitive closure of its dependencies,
rename and copy them into the wheel while updating their respective rpaths.
"""
print("Relocating {0}".format(binary))
binary_path = osp.join(output_library, binary)
ld_tree = lddtree(binary_path)
tree_libs = ld_tree["libs"]
binary_queue = [(n, binary) for n in ld_tree["needed"]]
binary_paths = {binary: binary_path}
binary_dependencies = {}
while binary_queue != []:
library, parent = binary_queue.pop(0)
library_info = tree_libs[library]
print(library)
if library_info["path"] is None:
print("Omitting {0}".format(library))
continue
if library in ALLOWLIST:
# Omit glibc/gcc/system libraries
print("Omitting {0}".format(library))
continue
parent_dependencies = binary_dependencies.get(parent, [])
parent_dependencies.append(library)
binary_dependencies[parent] = parent_dependencies
if library in binary_paths:
continue
binary_paths[library] = library_info["path"]
binary_queue += [(n, library) for n in library_info["needed"]]
print("Copying dependencies to wheel directory")
new_libraries_path = osp.join(output_dir, "torchvision.libs")
os.makedirs(new_libraries_path)
new_names = {binary: binary_path}
for library in binary_paths:
if library != binary:
library_path = binary_paths[library]
new_library_path = patch_new_path(library_path, new_libraries_path)
print("{0} -> {1}".format(library, new_library_path))
shutil.copyfile(library_path, new_library_path)
new_names[library] = new_library_path
print("Updating dependency names by new files")
for library in binary_paths:
if library != binary:
if library not in binary_dependencies:
continue
library_dependencies = binary_dependencies[library]
new_library_name = new_names[library]
for dep in library_dependencies:
new_dep = osp.basename(new_names[dep])
print("{0}: {1} -> {2}".format(library, dep, new_dep))
subprocess.check_output([
patchelf, "--replace-needed", dep, new_dep,
new_library_name
],
cwd=new_libraries_path)
print("Updating library rpath")
subprocess.check_output(
[patchelf, "--set-rpath", "$ORIGIN", new_library_name],
cwd=new_libraries_path)
subprocess.check_output(
[patchelf, "--print-rpath", new_library_name],
cwd=new_libraries_path)
print("Update library dependencies")
library_dependencies = binary_dependencies[binary]
for dep in library_dependencies:
new_dep = osp.basename(new_names[dep])
print("{0}: {1} -> {2}".format(binary, dep, new_dep))
subprocess.check_output(
[patchelf, "--replace-needed", dep, new_dep, binary],
cwd=output_library)
print("Update library rpath")
subprocess.check_output([
patchelf, "--set-rpath", "$ORIGIN:$ORIGIN/../torchvision.libs",
binary_path
],
cwd=output_library)
def patch_linux():
# Get patchelf location
patchelf = find_program('patchelf')
if patchelf is None:
raise FileNotFoundError('Patchelf was not found in the system, please'
' make sure that is available on the PATH.')
# Produce wheel
print('Producing wheel...')
subprocess.check_output(
[
sys.executable,
'setup.py',
'bdist_wheel'
],
cwd=PACKAGE_ROOT
)
package_info = get_metadata()
version = package_info['version'].replace('-', '.')
wheel_name = 'pyduckling_native-{0}-cp{1}{2}-{3}-linux_{4}.whl'.format(
version, PYTHON_VERSION.major, PYTHON_VERSION.minor,
get_abi_tag(), PLATFORM_ARCH)
dist = osp.join(PACKAGE_ROOT, 'dist', wheel_name)
output_dir = osp.join(PACKAGE_ROOT, '.wheel-process')
print(glob.glob(osp.join(PACKAGE_ROOT, 'dist', '*.whl')))
if osp.exists(output_dir):
shutil.rmtree(output_dir)
os.makedirs(output_dir)
print('Unzipping wheel...')
unzip_file(dist, output_dir)
print('Finding ELF dependencies...')
main_binary = 'duckling.cpython-{0}-{1}-linux-gnu.so'.format(
get_abi_tag().replace('cp', ''), PLATFORM_ARCH)
output_library = osp.join(output_dir, 'duckling')
binary_path = osp.join(output_library, main_binary)
ld_tree = lddtree(binary_path)
tree_libs = ld_tree['libs']
binary_queue = [(n, main_binary) for n in ld_tree['needed']]
binary_paths = {main_binary: binary_path}
binary_dependencies = {}
while binary_queue != []:
library, parent = binary_queue.pop(0)
library_info = tree_libs[library]
print(library)
print(library_info)
if (library_info['path'].startswith('/lib') and
not library.startswith('libpcre')):
# Omit glibc/gcc/system libraries
continue
parent_dependencies = binary_dependencies.get(parent, [])
parent_dependencies.append(library)
binary_dependencies[parent] = parent_dependencies
if library in binary_paths:
continue
binary_paths[library] = library_info['path']
binary_queue += [(n, library) for n in library_info['needed']]
print('Copying dependencies to wheel directory')
new_libraries_path = osp.join(output_dir, 'duckling.libs')
os.makedirs(new_libraries_path)
new_names = {main_binary: binary_path}
for library in binary_paths:
if library != main_binary:
library_path = binary_paths[library]
new_library_path = patch_new_path(library_path, new_libraries_path)
print('{0} -> {1}'.format(library, new_library_path))
shutil.copyfile(library_path, new_library_path)
new_names[library] = new_library_path
print('Updating dependency names by new files')
for library in binary_paths:
if library != main_binary:
if library not in binary_dependencies:
continue
library_dependencies = binary_dependencies[library]
new_library_name = new_names[library]
for dep in library_dependencies:
new_dep = osp.basename(new_names[dep])
print('{0}: {1} -> {2}'.format(library, dep, new_dep))
subprocess.check_output(
[
patchelf,
'--replace-needed',
dep,
new_dep,
new_library_name
],
cwd=new_libraries_path)
print('Updating library rpath')
subprocess.check_output(
[
patchelf,
'--set-rpath',
"$ORIGIN",
new_library_name
],
cwd=new_libraries_path)
subprocess.check_output(
[
patchelf,
'--print-rpath',
new_library_name
],
cwd=new_libraries_path)
print("Update main library dependencies")
library_dependencies = binary_dependencies[main_binary]
for dep in library_dependencies:
new_dep = osp.basename(new_names[dep])
print('{0}: {1} -> {2}'.format(main_binary, dep, new_dep))
subprocess.check_output(
[
patchelf,
'--replace-needed',
dep,
new_dep,
main_binary
],
cwd=output_library)
print('Update main library rpath')
subprocess.check_output(
[
patchelf,
'--set-rpath',
"$ORIGIN:$ORIGIN/../duckling.libs",
binary_path
],
cwd=output_library
)
print('Update RECORD file in wheel')
dist_info = osp.join(
output_dir, 'pyduckling_native-{0}.dist-info'.format(version))
record_file = osp.join(dist_info, 'RECORD')
with open(record_file, 'w') as f:
for root, _, files in os.walk(output_dir):
for this_file in files:
full_file = osp.join(root, this_file)
rel_file = osp.relpath(full_file, output_dir)
if full_file == record_file:
f.write('{0},,\n'.format(rel_file))
else:
digest, size = rehash(full_file)
f.write('{0},{1},{2}\n'.format(rel_file, digest, size))
print('Compressing wheel')
shutil.make_archive(dist, 'zip', output_dir)
os.remove(dist)
shutil.move('{0}.zip'.format(dist), dist)
shutil.rmtree(output_dir)
def check_so_is_manylinux2014(path: str, allowed_imports: List = None):
# PEP 599
allowed_shared_objects = {
'libgcc_s.so.1',
'libstdc++.so.6',
'libm.so.6',
'libdl.so.2',
'librt.so.1',
'libc.so.6',
'libnsl.so.1',
'libutil.so.1',
'libpthread.so.0',
'libresolv.so.2',
'libX11.so.6',
'libXext.so.6',
'libXrender.so.1',
'libICE.so.6',
'libSM.so.6',
'libGL.so.1',
'libgobject-2.0.so.0',
'libgthread-2.0.so.0',
'libglib-2.0.so.0',
}
allowed_symbol_versions = {
'GLIBC': parse_version('2.17'),
'CXXABI': parse_version('1.3.7'),
'GLIBCXX': parse_version('3.4.19'),
'GCC': parse_version('4.8.0'),
}
allowed_imports_lower = {'ld-linux.so.2', 'ld-linux-x86-64.so.2'}
if allowed_imports:
for allowed_import in allowed_imports:
allowed_imports_lower.add(allowed_import)
print(f'Checking if file {path} is manylinux2014...')
try:
import auditwheel
except ImportError:
print('Install auditwheel: pip install auditwheel')
sys.exit(1)
from auditwheel.lddtree import lddtree
from auditwheel.elfutils import elf_find_versioned_symbols
from elftools.elf.elffile import ELFFile
if not os.path.exists(path):
print(f'File {path} is missing!')
sys.exit(1)
# Check if all libs are in the allowed_shared_objects or whitelisted
elftree = lddtree(path)
libs = elftree['libs'].keys()
for lib in libs:
if lib not in allowed_shared_objects and lib not in allowed_imports_lower:
print(
f'File {path} depends on {lib} which doesn\'t match the manylinux2014 requirements. '
'This file will need to be vendored in!')
sys.exit(1)
# Check if all versioned symbols are at the values in allowed_symbol_versions or lower
with open(path, 'rb') as file:
elffile = ELFFile(file)
for filename, symbol in elf_find_versioned_symbols(elffile):
symbol_name, _, version = symbol.partition('_')
if parse_version(version) > allowed_symbol_versions[symbol_name]:
print(
f'There is a call to {symbol_name} at version {version} which is not allowed for manylinux2014. '
'Rebuild the code using the manylinux2014 docker image!')
sys.exit(1)