def test_install(): def check(*path): return os.path.exists(os.path.join(*path)) def get_supported(): return list( wheel.pep425tags.get_supported()) + [('py3', 'none', 'win32')] tempdir = mkdtemp() whl = WheelFile(TESTWHEEL, context=get_supported) assert whl.supports_current_python(get_supported) try: locs = {} for key in ('purelib', 'platlib', 'scripts', 'headers', 'data'): locs[key] = os.path.join(tempdir, key) os.mkdir(locs[key]) whl.install(overrides=locs) assert len(os.listdir(locs['purelib'])) == 0 assert check(locs['platlib'], 'hello.pyd') assert check(locs['platlib'], 'hello', 'hello.py') assert check(locs['platlib'], 'hello', '__init__.py') assert check(locs['data'], 'hello.dat') assert check(locs['headers'], 'hello.dat') assert check(locs['scripts'], 'hello.sh') assert check(locs['platlib'], 'test-1.0.dist-info', 'RECORD') finally: shutil.rmtree(tempdir)
def test_can_extract_package(self): wheelhouse = os.path.join(self.tmpdir, 'extract_package') expected_install_at = os.path.join(wheelhouse, 'listeneragent-0.1') test_wheel_name = 'listeneragent-0.1-py2-none-any.whl' wheel_file = os.path.join(self.fixtureDir, test_wheel_name) installed_at = extract_package(wheel_file, wheelhouse) try: self.assertIsNotNone(installed_at) self.assertTrue(os.path.isdir(installed_at)) self.assertEqual(expected_install_at, installed_at) # use the wheel file to verify that everything was extracted # properly. wf = WheelFile(wheel_file) self.assertIsNone(wf.verify()) for o in wf.zipfile.infolist(): self.assertTrue( os.path.exists( os.path.join(expected_install_at, o.filename))) wf.zipfile.close() finally: shutil.rmtree(installed_at) shutil.rmtree(wheelhouse)
def _add_dist_zip(self, path, dist_name): # We need to distinguish between wheels and other zips. Most of the time, # when we have a zip, it contains its contents in an importable form. # But wheels don't have to be importable, so we need to force them # into an importable shape. We can do that by installing it into its own # wheel dir. if dist_name.endswith("whl"): from wheel.install import WheelFile tmp = safe_mkdtemp() whltmp = os.path.join(tmp, dist_name) os.mkdir(whltmp) wf = WheelFile(path) wf.install(overrides=self._get_installer_paths(whltmp), force=True) for (root, _, files) in os.walk(whltmp): pruned_dir = os.path.relpath(root, tmp) for f in files: fullpath = os.path.join(root, f) if os.path.isdir(fullpath): continue target = os.path.join(self._pex_info.internal_cache, pruned_dir, f) self._chroot.copy(fullpath, target) return CacheHelper.dir_hash(whltmp) with open_zip(path) as zf: for name in zf.namelist(): if name.endswith('/'): continue target = os.path.join(self._pex_info.internal_cache, dist_name, name) self._chroot.write(zf.read(name), target) return CacheHelper.zip_hash(zf)
def _ensure_setup_requires_site_dir(self, dist_targets, interpreter, site_dir): reqs_to_resolve = set() for tgt in dist_targets: for setup_req_lib_addr in tgt.setup_requires: for req_lib in self.context.build_graph.resolve(setup_req_lib_addr): for req in req_lib.requirements: reqs_to_resolve.add(req) if not reqs_to_resolve: return None self.context.log.debug('python_dist target(s) with setup_requires detected. ' 'Installing setup requirements: {}\n\n' .format([req.key for req in reqs_to_resolve])) setup_requires_dists = _resolve_multi(interpreter, reqs_to_resolve, ['current'], None) overrides = { 'purelib': site_dir, 'headers': os.path.join(site_dir, 'headers'), 'scripts': os.path.join(site_dir, 'bin'), 'platlib': site_dir, 'data': site_dir } # The `python_dist` target builds for the current platform only. for obj in setup_requires_dists['current']: wf = WheelFile(obj.location) wf.install(overrides=overrides, force=True) return site_dir
def ensure_setup_requires_site_dir(reqs_to_resolve, interpreter, site_dir, platforms=None): if not reqs_to_resolve: return None setup_requires_dists = resolve_multi(interpreter, reqs_to_resolve, platforms, None) # FIXME: there's no description of what this does or why it's necessary. overrides = { 'purelib': site_dir, 'headers': os.path.join(site_dir, 'headers'), 'scripts': os.path.join(site_dir, 'bin'), 'platlib': site_dir, 'data': site_dir } # The `python_dist` target builds for the current platform only. # FIXME: why does it build for the current platform only? for obj in setup_requires_dists['current']: wf = WheelFile(obj.location) wf.install(overrides=overrides, force=True) return SetupRequiresSiteDir(site_dir)
def test_can_extract_package(self): wheelhouse = os.path.join(self.tmpdir, 'extract_package') expected_install_at = os.path.join(wheelhouse, 'listeneragent-0.1') test_wheel_name = 'listeneragent-0.1-py2-none-any.whl' wheel_file = os.path.join(self.fixtureDir, test_wheel_name) installed_at = extract_package(wheel_file, wheelhouse) try: self.assertIsNotNone(installed_at) self.assertTrue(os.path.isdir(installed_at)) self.assertEqual(expected_install_at, installed_at) # use the wheel file to verify that everything was extracted # properly. wf = WheelFile(wheel_file) self.assertIsNone(wf.verify()) for o in wf.zipfile.infolist(): self.assertTrue( os.path.exists(os.path.join(expected_install_at, o.filename))) wf.zipfile.close() finally: shutil.rmtree(installed_at) shutil.rmtree(wheelhouse)
def test_can_create_package(self): ''' Tests that a proper wheel package is created from the create_package method of the AgentPackage class. ''' agent_name = AGENT_TESTCASE1_NAME package_tmp_dir = os.path.join(self.tmpdir, 'create_package') expected_package_name = 'listeneragent-0.1-py2-none-any.whl' returned_package = create_package( self.get_agent_fixture(agent_name), package_tmp_dir) self.assertIsNotNone( returned_package, "Invalid package name {}".format(returned_package)) self.assertTrue(os.path.exists(returned_package)) self.assertEqual( expected_package_name, os.path.basename(returned_package)) # Wheel is in the correct location. self.assertEqual( os.path.join(package_tmp_dir, expected_package_name), returned_package) self.assertTrue(os.path.exists(returned_package)) try: wf = WheelFile(returned_package) # sets up the expected hashes for all of the wheel directory. self.assertIsNone(wf.verify()) # Reading the files # if the hash doesn't match it will throw an exception. for o in wf.zipfile.infolist(): wf.zipfile.open(o).read() wf.zipfile.close() finally: shutil.rmtree(package_tmp_dir)
def package_exists(path): parsed_filename = WheelFile(path).parsed_filename package = parsed_filename.group('name') version = parsed_filename.group('ver') p = check_output('devpi list {}=={}'.format(package, version).split()) exists = True if p else False if exists: print('Package {}={} already exists.'.format(package, version)) return exists
def test_compatibility_tags(): """Test compatibilty tags are working.""" wf = WheelFile("package-1.0.0-cp32.cp33-noabi-noarch.whl") assert (list(wf.compatibility_tags) == [('cp32', 'noabi', 'noarch'), ('cp33', 'noabi', 'noarch')]) assert (wf.arity == 2) wf2 = WheelFile("package-1.0.0-1st-cp33-noabi-noarch.whl") wf2_info = wf2.parsed_filename.groupdict() assert wf2_info['build'] == '1st', wf2_info
def package_exists(path): parsed_filename = WheelFile(path).parsed_filename package, version = parsed_filename.group(2), parsed_filename.group(4) try: p = check_output('devpi list {}=={}'.format(package, version).split()) exists = True if p else False except CalledProcessError as e: if '404 Not Found: no project' in e.stdout.decode('utf-8'): exists = False else: raise if exists: print('Package {}={} already exists.'.format(package, version)) return exists
def _split_filename(filename): """ Split a .whl or .tar.gz distribution file name into a (package_name, version) tuple >>> _split_filename('abc-1.1.tar.gz') ('abc', <Version('1.1')>) >>> _split_filename('dir/abc-1.1.tar.gz') ('abc', <Version('1.1')>) >>> _split_filename('a_bc-1.1.tar.gz') ('a-bc', <Version('1.1')>) >>> _split_filename('a_b-c-1.1.tar.gz') ('a-b-c', <Version('1.1')>) >>> _split_filename('mis_builder-3.1.1.99.dev17-py2-none-any.whl') ('mis-builder', <Version('3.1.1.99.dev17')>) >>> _split_filename('a/b/mis_builder-3.1.1.99.dev17-py2-none-any.whl') ('mis-builder', <Version('3.1.1.99.dev17')>) """ basename = os.path.basename(filename) if basename.endswith('.whl'): wheelfile = WheelFile(basename) package_name = wheelfile.parsed_filename.group('name') package_ver = wheelfile.parsed_filename.group('ver') elif basename.endswith('.tar.gz'): package_ver = basename.split('-')[-1][:-7] package_name = basename[:-(len(package_ver) + 8)] else: raise RuntimeError("Unrecognized file type %s" % (filename, )) package_name = package_name.replace('_', '-') package_ver = parse_version(package_ver) return package_name, package_ver
def _split_wheelfilename(wheelfilename): wheelfile = WheelFile(wheelfilename) package_name = wheelfile.parsed_filename.group('name') package_name = package_name.replace('_', '-') package_ver = wheelfile.parsed_filename.group('ver') package_ver = parse_version(package_ver) return package_name, package_ver
def test_pick_best(): """Test the wheel ranking algorithm.""" def get_tags(res): info = res[-1].parsed_filename.groupdict() return info['pyver'], info['abi'], info['plat'] cand_tags = [('py27', 'noabi', 'noarch'), ('py26', 'noabi', 'noarch'), ('cp27', 'noabi', 'linux_i686'), ('cp26', 'noabi', 'linux_i686'), ('cp27', 'noabi', 'linux_x86_64'), ('cp26', 'noabi', 'linux_x86_64')] cand_wheels = [ WheelFile('testpkg-1.0-%s-%s-%s.whl' % t) for t in cand_tags ] supported = [('cp27', 'noabi', 'linux_i686'), ('py27', 'noabi', 'noarch')] supported2 = [('cp27', 'noabi', 'linux_i686'), ('py27', 'noabi', 'noarch'), ('cp26', 'noabi', 'linux_i686'), ('py26', 'noabi', 'noarch')] supported3 = [('cp26', 'noabi', 'linux_i686'), ('py26', 'noabi', 'noarch'), ('cp27', 'noabi', 'linux_i686'), ('py27', 'noabi', 'noarch')] for supp in (supported, supported2, supported3): context = lambda: list(supp) for wheel in cand_wheels: wheel.context = context best = max(cand_wheels) assert list(best.tags)[0] == supp[0]
def install_wheel(wheel, root_dir): importable_path = os.path.join(root_dir, 'install', os.path.basename(wheel)) overrides = { path: importable_path for path in ('purelib', 'platlib', 'headers', 'scripts', 'data') } WheelFile(wheel).install(force=True, overrides=overrides) return importable_path
def is_compatible(package): """ Check whether the given python package is a wheel compatible with the current platform and python interpreter. """ try: return WheelFile(package).compatible except BadWheelFile: return False
def make_wheel(name, ver, pyver, abi, arch): name = WHEELPAT % { 'name': name, 'ver': ver, 'pyver': pyver, 'abi': abi, 'arch': arch } return WheelFile(name)
def rebuild_html(): all_wheels = (WheelFile(name) for name in glob("simple/dist/*.whl")) packages = groupby(all_wheels, lambda w: w.parsed_filename.groupdict()['name']) package_names = [] for package_name, wheels in packages: package_names.append(package_name) write_package_html(package_name, wheels) write_index_html(package_names)
def _install_wheel(cls, wheel_path, install_dir): safe_mkdir(install_dir, clean=True) WheelFile(wheel_path).install(force=True, overrides={ 'purelib': install_dir, 'headers': os.path.join(install_dir, 'headers'), 'scripts': os.path.join(install_dir, 'bin'), 'platlib': install_dir, 'data': install_dir })
def get_winfo(info_fname, drop_version=True): """ Get wheel info from WHEEL file Drop "Wheel-Version" by default in case this changes in the future """ wf = WheelFile(info_fname) info = sorted(wf.parsed_wheel_info.items()) if drop_version: info = [(name, value) for (name, value) in info if name != "Wheel-Version"] return info
def is_pure(wheel): """ Check whether wheel given by the passed path is pure. Pure wheels operate independent of the specific Python version and platform. :param wheel: The path to the wheel to inspect :return: True if the wheel is pure """ return WheelFile( wheel).parsed_wheel_info['Root-Is-Purelib'] == 'true' # safe default
def parse_wheel_filename(filename): try: wf = WheelFile(filename) except BadWheelFile: raise ValueError("Wheel filename {} is not valid".format(filename)) return ( wf.parsed_filename.group('name'), wf.parsed_filename.group('ver'), wf.parsed_filename.group('pyver'), wf.parsed_filename.group('plat') )
def main(): build_tag = sys.argv[1] for wheel_fname in sys.argv[2:]: path, fname = psplit(wheel_fname) wf = WheelFile(fname) parsed = wf.parsed_filename.groupdict() parsed['build'] = build_tag out_fname = '{name}-{ver}-{build}-{pyver}-{abi}-{plat}.whl'.format( **parsed) out_path = pjoin(path, out_fname) print('Copying {} to {}'.format(wheel_fname, out_path)) copyfile(wheel_fname, out_path)
def _metadata(path): from wheel.install import WheelFile try: wf = WheelFile(path) except Exception as e: print("%s: %s" % (path, str(e))) metadata_file = "%s/metadata.json" % wf.distinfo_name with wf.zipfile.open(metadata_file) as mdf: import json md = json.loads(mdf.read().decode(encoding="UTF-8")) md["modified"] = _modtime(path) return md
def install_module(mod_name, mod_vals, debug=False): """Checks for dependencies, then installs module""" already_installed = check_install(mod_vals["ImportedName"], mod_vals["Version"]) if not already_installed: for dep in mod_vals["Dependencies"]: if not check_install(dep): if debug: print( "Did not install module; did not have all dependencies. Hopefully, this module will be placed on the waitlist" ) return False print("Installing module %s version %s" % (mod_name, mod_vals["Version"])) print("Downloading module") install_req = requests.get(mod_vals["DownloadLink"]) install_filename = os.path.join("/tmp", mod_vals["Filename"]) install_content = install_req.content with open(install_filename, "wb") as mod_file: mod_file.write(install_content) print("Installing module") wheelfile = WheelFile(install_filename) try: wheelfile.install(force=True) except ValueError as e: print(e) installed = check_install(mod_vals["ImportedName"], mod_vals["Version"]) os.remove(install_filename) if installed: print("Successfully installed") else: print("Failed to install") return installed else: if debug: print( "Not installing module. It is %s that this module was already installed" % already_installed) return already_installed
def test_install(): tempdir = mkdtemp() def get_supported(): return list(wheel.pep425tags.get_supported()) + [('py3', 'none', 'win32')] whl = WheelFile(TESTWHEEL, context=get_supported) assert whl.supports_current_python(get_supported) try: locs = {} for key in ('purelib', 'platlib', 'scripts', 'headers', 'data'): locs[key] = os.path.join(tempdir, key) os.mkdir(locs[key]) whl.install(overrides=locs) assert len(os.listdir(locs['purelib'])) == 0 assert check(locs['platlib'], 'hello.pyd') assert check(locs['platlib'], 'hello', 'hello.py') assert check(locs['platlib'], 'hello', '__init__.py') assert check(locs['data'], 'hello.dat') assert check(locs['headers'], 'hello.dat') assert check(locs['scripts'], 'hello.sh') assert check(locs['platlib'], 'test-1.0.dist-info', 'RECORD') finally: shutil.rmtree(tempdir)
def assert_winfo_similar(whl_fname, exp_items, drop_version=True): wf = WheelFile(whl_fname) wheel_parts = wf.parsed_filename.groupdict() # Info can contain duplicate keys (e.g. Tag) w_info = sorted(get_info(wf).items()) if drop_version: w_info = _filter_key(w_info, "Wheel-Version") exp_items = _filter_key(exp_items, "Wheel-Version") assert_equal(len(exp_items), len(w_info)) # Extract some information from actual values wheel_parts["pip_version"] = dict(w_info)["Generator"].split()[1] for (key1, value1), (key2, value2) in zip(exp_items, w_info): assert_equal(key1, key2) value1 = value1.format(**wheel_parts) assert_equal(value1, value2)
def extract_package(wheel_file, install_dir, include_uuid=False, specific_uuid=None): """ Extract a wheel file to the specified location. If include_uuid is True then a uuid will be generated under the passed location directory. The agent final directory will be based upon the wheel's data directory name in the following formats: .. code-block:: python if include_uuid == True install_dir/uuid/datadir_name else install_dir/datadir_name :param wheel_file: The wheel file to extract. :param install_dir: The root directory where to extract the wheel :param include_uuid: Auto-generates a uuuid under install_dir to place the wheel file data :param specific_uuid: A specific uuid to use for extracting the agent. :return: The folder where the wheel was extracted. """ real_dir = install_dir # Only include the uuid if the caller wants it. if include_uuid: if specific_uuid == None: real_dir = os.path.join(real_dir, str(uuid.uuid4())) else: real_dir = os.path.join(real_dir, specific_uuid) if not os.path.isdir(real_dir): os.makedirs(real_dir) wf = WheelFile(wheel_file) namever = wf.parsed_filename.group('namever') destination = os.path.join(real_dir, namever) sys.stderr.write("Unpacking to: %s\n" % (destination)) wf.zipfile.extractall(destination) wf.zipfile.close() return destination
def _find_wheel(self, name, version): """ Find a wheel with the given name and version """ candidates = [ WheelFile(filename) for filename in glob.iglob(path.join(self.wheelhouse, '*.whl')) ] matches = matches_requirement('{}=={}'.format(name, version), candidates) if len(matches) > 0: return str(matches[0]) else: raise BuildError( name, version, 'Failed to find the build wheel for {} {}'.format( name, version))
def _activate_wheel(wheel_path): install_dir = '{}-install'.format(wheel_path) safe_mkdir(install_dir, clean=True) WheelFile(wheel_path).install(force=True, overrides={ 'purelib': install_dir, 'headers': os.path.join(install_dir, 'headers'), 'scripts': os.path.join(install_dir, 'bin'), 'platlib': install_dir, 'data': install_dir }) # Activate any .pth files installed above. site.addsitedir(install_dir) return install_dir
def download_package(client, name, version): urls = client.release_urls(name, version) dists = defaultdict(list) for data in urls: dists[data.get('packagetype')].append(data) url = fname = None for sdist in dists['sdist']: url = sdist['url'] fname = sdist['filename'] break else: for bdist in dists['bdist_wheel']: if WheelFile(bdist['filename']).compatible: url = bdist['url'] fname = bdist['filename'] break if fname is not None: with _urlretrieve_lock: urlretrieve(url, fname) return fname
def verify_wheel(wheelfile): wf = WheelFile(wheelfile) sig_name = wf.distinfo_name + '/RECORD.jws' try: sig = json.loads(native(wf.zipfile.open(sig_name).read())) except KeyError: raise WheelValidationFailed("This wheel is not signed") verified = signatures.verify(sig) try: vk = verified[0][0]['jwk']['vk'] except (KeyError, IndexError, ValueError): raise WheelValidationFailed("Invalid signature") if vk != settings.WHEEL_USER: raise WheelValidationFailed("Wheel validation failed") kr = keyring.get_keyring() password = kr.get_password("wheel", settings.WHEEL_USER) if password != settings.WHEEL_PASSWORD: raise WheelValidationFailed("Wheel validation failed")
def __init__(self, req_string, parent=None, index_url=None, env=None): self.dist_path = None if req_string.endswith(".whl") and os.path.isfile(req_string): self.dist_path = req_string req_string = os.path.basename(req_string) whl = WheelFile(req_string) whl_name_info = whl.parsed_filename.groupdict() self.name = canonicalize_name(whl_name_info["name"]) self.specifier = "==" + canonicalize_version(whl_name_info["ver"]) self.req = pkg_resources.Requirement.parse(self.name + self.specifier) else: self.req = pkg_resources.Requirement.parse(req_string) self.name = canonicalize_name(self.req.name) self.specifier = str(self.req.specifier) self.extras_requested = sorted(self.req.extras) log = self.log = logger.bind(dist=str(self.req)) log.info("init johnnydist", parent=parent and str(parent.req)) if parent is not None: self.index_url = parent.index_url self.required_by = [str(parent.req)] self.env = parent.env self.env_data = parent.env_data else: self.index_url = index_url self.required_by = [] self.env = env if self.env is None: self.env_data = default_environment() else: self.env_data = dict(self.env) log.debug("target env", **self.env_data) if self.dist_path is None: log.debug("fetching best wheel") with wimpy.working_directory(self.tmp()): data = pipper.get(req_string, index_url=self.index_url, env=self.env) self.dist_path = data["path"] self.parent = parent self._recursed = False