Exemplo n.º 1
0
def test_pex_builder_from_requirements_pex():
  def build_from_req_pex(path, req_pex):
    pb = PEXBuilder(path=path)
    pb.add_from_requirements_pex(req_pex)
    with open(os.path.join(path, 'exe.py'), 'w') as fp:
      fp.write(exe_main)
    pb.set_executable(os.path.join(path, 'exe.py'))
    pb.freeze()
    return pb

  def verify(pb):
    success_txt = os.path.join(pb.path(), 'success.txt')
    PEX(pb.path(), interpreter=pb.interpreter).run(args=[success_txt])
    assert os.path.exists(success_txt)
    with open(success_txt) as fp:
      assert fp.read() == 'success'

  # Build from pex dir.
  with temporary_dir() as td2:
    with nested(temporary_dir(), make_bdist('p1')) as (td1, p1):
      pb1 = write_pex(td1, dists=[p1])
      pb2 = build_from_req_pex(td2, pb1.path())
    verify(pb2)

  # Build from .pex file.
  with temporary_dir() as td4:
    with nested(temporary_dir(), make_bdist('p1')) as (td3, p1):
      pb3 = write_pex(td3, dists=[p1])
      target = os.path.join(td3, 'foo.pex')
      pb3.build(target)
      pb4 = build_from_req_pex(td4, target)
    verify(pb4)
Exemplo n.º 2
0
def test_pex_builder_from_requirements_pex():
    # type: () -> None
    def build_from_req_pex(path, req_pex):
        # type: (str, PEXBuilder) -> PEXBuilder
        pb = PEXBuilder(path=path)
        pb.add_from_requirements_pex(req_pex)
        with open(os.path.join(path, "exe.py"), "w") as fp:
            fp.write(exe_main)
        pb.set_executable(os.path.join(path, "exe.py"))
        pb.freeze()
        return pb

    def verify(pb):
        # type: (PEXBuilder) -> None
        success_txt = os.path.join(pb.path(), "success.txt")
        PEX(pb.path(), interpreter=pb.interpreter).run(args=[success_txt])
        assert os.path.exists(success_txt)
        with open(success_txt) as fp:
            assert fp.read() == "success"

    # Build from pex dir.
    with temporary_dir() as td2:
        with nested(temporary_dir(), make_bdist("p1")) as (td1, p1):
            pb1 = write_pex(td1, dists=[p1])
            pb2 = build_from_req_pex(td2, pb1.path())
        verify(pb2)

    # Build from .pex file.
    with temporary_dir() as td4:
        with nested(temporary_dir(), make_bdist("p1")) as (td3, p1):
            pb3 = write_pex(td3, dists=[p1])
            target = os.path.join(td3, "foo.pex")
            pb3.build(target)
            pb4 = build_from_req_pex(td4, target)
        verify(pb4)
Exemplo n.º 3
0
def test_pex_builder():
  # test w/ and w/o zipfile dists
  with nested(temporary_dir(), make_bdist('p1', zipped=True)) as (td, p1):
    write_pex(td, exe_main, dists=[p1])

    success_txt = os.path.join(td, 'success.txt')
    PEX(td).run(args=[success_txt])
    assert os.path.exists(success_txt)
    with open(success_txt) as fp:
      assert fp.read() == 'success'

  # test w/ and w/o zipfile dists
  with nested(temporary_dir(), temporary_dir(), make_bdist('p1', zipped=True)) as (
      td1, td2, p1):
    target_egg_dir = os.path.join(td2, os.path.basename(p1.location))
    safe_mkdir(target_egg_dir)
    with closing(zipfile.ZipFile(p1.location, 'r')) as zf:
      zf.extractall(target_egg_dir)
    p1 = DistributionHelper.distribution_from_path(target_egg_dir)

    write_pex(td1, exe_main, dists=[p1])

    success_txt = os.path.join(td1, 'success.txt')
    PEX(td1).run(args=[success_txt])
    assert os.path.exists(success_txt)
    with open(success_txt) as fp:
      assert fp.read() == 'success'
Exemplo n.º 4
0
def test_pex_builder():
  # test w/ and w/o zipfile dists
  with nested(temporary_dir(), make_bdist('p1', zipped=True)) as (td, p1):
    write_pex(td, exe_main, dists=[p1])

    success_txt = os.path.join(td, 'success.txt')
    PEX(td).run(args=[success_txt])
    assert os.path.exists(success_txt)
    with open(success_txt) as fp:
      assert fp.read() == 'success'

  # test w/ and w/o zipfile dists
  with nested(temporary_dir(), temporary_dir(), make_bdist('p1', zipped=True)) as (
      td1, td2, p1):
    target_egg_dir = os.path.join(td2, os.path.basename(p1.location))
    safe_mkdir(target_egg_dir)
    with closing(zipfile.ZipFile(p1.location, 'r')) as zf:
      zf.extractall(target_egg_dir)
    p1 = DistributionHelper.distribution_from_path(target_egg_dir)

    write_pex(td1, exe_main, dists=[p1])

    success_txt = os.path.join(td1, 'success.txt')
    PEX(td1).run(args=[success_txt])
    assert os.path.exists(success_txt)
    with open(success_txt) as fp:
      assert fp.read() == 'success'
Exemplo n.º 5
0
def test_write_zipped_internal_cache():
    # zip_safe pex will not be written to install cache unless always_write_cache
    with nested(yield_pex_builder(zip_safe=True), temporary_dir(),
                temporary_file()) as (pb, pex_root, pex_file):

        pb.info.pex_root = pex_root
        pb.build(pex_file.name)

        existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(
            pex_file.name, pb.info)
        assert len(zip_safe) == 1
        assert normalize(zip_safe[0].location).startswith(
            normalize(os.path.join(pex_file.name, pb.info.internal_cache))), (
                'loc: %s, cache: %s' %
                (normalize(zip_safe[0].location),
                 normalize(os.path.join(pex_file.name,
                                        pb.info.internal_cache))))

        pb.info.always_write_cache = True
        existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(
            pex_file.name, pb.info)
        assert len(new) == 1
        assert normalize(new[0].location).startswith(
            normalize(pb.info.install_cache))

        # Check that we can read from the cache
        existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(
            pex_file.name, pb.info)
        assert len(existing) == 1
        assert normalize(existing[0].location).startswith(
            normalize(pb.info.install_cache))

    # non-zip_safe pex will be written to install cache
    with nested(yield_pex_builder(zip_safe=False), temporary_dir(),
                temporary_file()) as (pb, pex_root, pex_file):

        pb.info.pex_root = pex_root
        pb.build(pex_file.name)

        existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(
            pex_file.name, pb.info)
        assert len(new) == 1
        assert normalize(new[0].location).startswith(
            normalize(pb.info.install_cache))
        original_location = normalize(new[0].location)

        # do the second time to validate idempotence of caching
        existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(
            pex_file.name, pb.info)
        assert len(existing) == 1
        assert normalize(existing[0].location) == original_location
Exemplo n.º 6
0
def test_access_zipped_assets_integration():
  test_executable = dedent('''
      import os
      from _pex.util import DistributionHelper
      temp_dir = DistributionHelper.access_zipped_assets('my_package', 'submodule')
      with open(os.path.join(temp_dir, 'mod.py'), 'r') as fp:
        for line in fp:
          print(line)
  ''')
  with nested(temporary_dir(), temporary_dir()) as (td1, td2):
    pb = PEXBuilder(path=td1)
    with open(os.path.join(td1, 'exe.py'), 'w') as fp:
      fp.write(test_executable)
      pb.set_executable(fp.name)

    submodule = os.path.join(td1, 'my_package', 'submodule')
    safe_mkdir(submodule)
    mod_path = os.path.join(submodule, 'mod.py')
    with open(mod_path, 'w') as fp:
      fp.write('accessed')
      pb.add_source(fp.name, 'my_package/submodule/mod.py')

    pex = os.path.join(td2, 'app.pex')
    pb.build(pex)

    output, returncode = run_simple_pex(pex)
    try:
      output = output.decode('UTF-8')
    except ValueError:
      pass
    assert output == 'accessed\n'
    assert returncode == 0
Exemplo n.º 7
0
def assert_dist_cache(zip_safe):
    # type: (bool) -> None
    with nested(yield_pex_builder(zip_safe=zip_safe), temporary_dir(),
                temporary_filename()) as (
                    pb,
                    pex_root,
                    pex_file,
                ):

        pb.info.pex_root = pex_root
        pb.build(pex_file)

        with open_zip(pex_file) as zf:
            dists = PEXEnvironment._write_zipped_internal_cache(
                zf=zf, pex_info=pb.info)
            assert len(dists) == 1
            original_location = normalize(dists[0].location)
            assert original_location.startswith(
                normalize(pb.info.install_cache))

        # Call a second time to validate idempotence of caching.
        dists = PEXEnvironment._write_zipped_internal_cache(zf=None,
                                                            pex_info=pb.info)
        assert len(dists) == 1
        assert normalize(dists[0].location) == original_location
Exemplo n.º 8
0
def test_pex_builder_compilation():
    # type: () -> None
    with nested(temporary_dir(), temporary_dir(),
                temporary_dir()) as (td1, td2, td3):
        src = os.path.join(td1, "src.py")
        with open(src, "w") as fp:
            fp.write(exe_main)

        exe = os.path.join(td1, "exe.py")
        with open(exe, "w") as fp:
            fp.write(exe_main)

        def build_and_check(path, precompile):
            # type: (str, bool) -> None
            pb = PEXBuilder(path=path)
            pb.add_source(src, "lib/src.py")
            pb.set_executable(exe, "exe.py")
            pb.freeze(bytecode_compile=precompile)
            for pyc_file in ("exe.pyc", "lib/src.pyc", "__main__.pyc"):
                pyc_exists = os.path.exists(os.path.join(path, pyc_file))
                if precompile:
                    assert pyc_exists
                else:
                    assert not pyc_exists
            bootstrap_dir = os.path.join(path, BOOTSTRAP_DIR)
            bootstrap_pycs = []  # type: List[str]
            for _, _, files in os.walk(bootstrap_dir):
                bootstrap_pycs.extend(f for f in files if f.endswith(".pyc"))
            if precompile:
                assert len(bootstrap_pycs) > 0
            else:
                assert 0 == len(bootstrap_pycs)

        build_and_check(td2, False)
        build_and_check(td3, True)
Exemplo n.º 9
0
def yield_pex_builder(zip_safe=True, interpreter=None):
    with nested(temporary_dir(),
                make_bdist('p1', zip_safe=zip_safe,
                           interpreter=interpreter)) as (td, p1):
        pb = PEXBuilder(path=td, interpreter=interpreter)
        pb.add_dist_location(p1.location)
        yield pb
Exemplo n.º 10
0
def test_force_local():
    # type: () -> None
    with nested(yield_pex_builder(), temporary_dir(), temporary_filename()) as (
        pb,
        pex_root,
        pex_file,
    ):
        pb.info.pex_root = pex_root
        pb.build(pex_file)

        code_cache = PEXEnvironment._force_local(pex_file, pb.info)

        assert os.path.exists(pb.info.zip_unsafe_cache)
        listing = set(os.listdir(pb.info.zip_unsafe_cache))

        # The code_cache should be a write-locked directory.
        assert len(listing) == 2
        listing.remove(os.path.basename(code_cache))
        lockfile = listing.pop()
        assert os.path.isfile(os.path.join(pb.info.zip_unsafe_cache, lockfile))

        assert set(os.listdir(code_cache)) == {PexInfo.PATH, "__main__.py", "__main__.pyc"}

        # idempotence
        assert PEXEnvironment._force_local(pex_file, pb.info) == code_cache
Exemplo n.º 11
0
def test_access_zipped_assets():
  try:
    import __builtin__
    builtin_path = __builtin__.__name__
  except ImportError:
    # Python3
    import builtins
    builtin_path = builtins.__name__

  mock_open = mock.mock_open()
  with nested(mock.patch('%s.open' % builtin_path, mock_open, create=True),
      mock.patch('pex.util.resource_string', autospec=True, spec_set=True),
      mock.patch('pex.util.resource_isdir', autospec=True, spec_set=True),
      mock.patch('pex.util.resource_listdir', autospec=True, spec_set=True),
      mock.patch('pex.util.safe_mkdtemp', autospec=True, spec_set=True)) as (
          _, mock_resource_string, mock_resource_isdir, mock_resource_listdir, mock_safe_mkdtemp):

    mock_safe_mkdtemp.side_effect = iter(['tmpJIMMEH', 'faketmpDir'])
    mock_resource_listdir.side_effect = iter([['./__init__.py', './directory/'], ['file.py']])
    mock_resource_isdir.side_effect = iter([False, True, False])
    mock_resource_string.return_value = 'testing'

    temp_dir = DistributionHelper.access_zipped_assets('twitter.common', 'dirutil')

    assert mock_resource_listdir.call_count == 2
    assert mock_open.call_count == 2
    file_handle = mock_open.return_value.__enter__.return_value
    assert file_handle.write.call_count == 2
    assert mock_safe_mkdtemp.mock_calls == [mock.call()]
    assert temp_dir == 'tmpJIMMEH'
Exemplo n.º 12
0
def test_resolve_cache():
    # type: () -> None
    project_wheel = build_wheel(name="project")

    with nested(temporary_dir(), temporary_dir()) as (td, cache):
        safe_copy(project_wheel,
                  os.path.join(td, os.path.basename(project_wheel)))

        # Without a cache, each resolve should be isolated, but otherwise identical.
        resolved_dists1 = local_resolve_multi(["project"], find_links=[td])
        resolved_dists2 = local_resolve_multi(["project"], find_links=[td])
        assert resolved_dists1 != resolved_dists2
        assert len(resolved_dists1) == 1
        assert len(resolved_dists2) == 1
        assert resolved_dists1[0].requirement == resolved_dists2[0].requirement
        assert resolved_dists1[0].distribution.location != resolved_dists2[
            0].distribution.location

        # With a cache, each resolve should be identical.
        resolved_dists3 = local_resolve_multi(["project"],
                                              find_links=[td],
                                              cache=cache)
        resolved_dists4 = local_resolve_multi(["project"],
                                              find_links=[td],
                                              cache=cache)
        assert resolved_dists1 != resolved_dists3
        assert resolved_dists2 != resolved_dists3
        assert resolved_dists3 == resolved_dists4
Exemplo n.º 13
0
def test_pex_builder_copy_or_link():
    # type: () -> None
    with nested(temporary_dir(), temporary_dir(),
                temporary_dir()) as (td1, td2, td3):
        src = os.path.join(td1, "exe.py")
        with open(src, "w") as fp:
            fp.write(exe_main)

        def build_and_check(path, copy):
            # type: (str, bool) -> None
            pb = PEXBuilder(path=path, copy=copy)
            pb.add_source(src, "exe.py")

            path_clone = os.path.join(path, "__clone")
            pb.clone(into=path_clone)

            for root in path, path_clone:
                s1 = os.stat(src)
                s2 = os.stat(os.path.join(root, "exe.py"))
                is_link = (s1[stat.ST_INO],
                           s1[stat.ST_DEV]) == (s2[stat.ST_INO],
                                                s2[stat.ST_DEV])
                if copy:
                    assert not is_link
                else:
                    assert is_link

        build_and_check(td2, False)
        build_and_check(td3, True)
Exemplo n.º 14
0
def test_pex_builder_compilation():
  with nested(temporary_dir(), temporary_dir(), temporary_dir()) as (td1, td2, td3):
    src = os.path.join(td1, 'src.py')
    with open(src, 'w') as fp:
      fp.write(exe_main)

    exe = os.path.join(td1, 'exe.py')
    with open(exe, 'w') as fp:
      fp.write(exe_main)

    def build_and_check(path, precompile):
      pb = PEXBuilder(path)
      pb.add_source(src, 'lib/src.py')
      pb.set_executable(exe, 'exe.py')
      pb.freeze(bytecode_compile=precompile)
      for pyc_file in ('exe.pyc', 'lib/src.pyc', '__main__.pyc'):
        pyc_exists = os.path.exists(os.path.join(path, pyc_file))
        if precompile:
          assert pyc_exists
        else:
          assert not pyc_exists
      bootstrap_dir = os.path.join(path, PEXBuilder.BOOTSTRAP_DIR)
      bootstrap_pycs = []
      for _, _, files in os.walk(bootstrap_dir):
        bootstrap_pycs.extend(f for f in files if f.endswith('.pyc'))
      if precompile:
        assert len(bootstrap_pycs) > 0
      else:
        assert 0 == len(bootstrap_pycs)

    build_and_check(td2, False)
    build_and_check(td3, True)
Exemplo n.º 15
0
def assert_access_zipped_assets(distribution_helper_import):
    test_executable = dedent("""
      import os
      {distribution_helper_import}
      temp_dir = DistributionHelper.access_zipped_assets('my_package', 'submodule')
      with open(os.path.join(temp_dir, 'mod.py'), 'r') as fp:
        for line in fp:
          print(line)
  """.format(distribution_helper_import=distribution_helper_import))
    with nested(temporary_dir(), temporary_dir()) as (td1, td2):
        pb = PEXBuilder(path=td1)
        with open(os.path.join(td1, 'exe.py'), 'w') as fp:
            fp.write(test_executable)
            pb.set_executable(fp.name)

        submodule = os.path.join(td1, 'my_package', 'submodule')
        safe_mkdir(submodule)
        mod_path = os.path.join(submodule, 'mod.py')
        with open(mod_path, 'w') as fp:
            fp.write('accessed')
            pb.add_source(fp.name, 'my_package/submodule/mod.py')
        pb.add_source(None, 'my_package/__init__.py')
        pb.add_source(None, 'my_package/submodule/__init__.py')
        pex = os.path.join(td2, 'app.pex')
        pb.build(pex)

        process = PEX(pex,
                      interpreter=pb.interpreter).run(blocking=False,
                                                      stdout=subprocess.PIPE,
                                                      stderr=subprocess.PIPE)
        stdout, stderr = process.communicate()
        assert process.returncode == 0
        assert b'accessed\n' == stdout
        return stderr
Exemplo n.º 16
0
def test_access_zipped_assets_integration():
  test_executable = dedent('''
      import os
      from _pex.util import DistributionHelper
      temp_dir = DistributionHelper.access_zipped_assets('my_package', 'submodule')
      with open(os.path.join(temp_dir, 'mod.py'), 'r') as fp:
        for line in fp:
          print(line)
  ''')
  with nested(temporary_dir(), temporary_dir()) as (td1, td2):
    pb = PEXBuilder(path=td1)
    with open(os.path.join(td1, 'exe.py'), 'w') as fp:
      fp.write(test_executable)
      pb.set_executable(fp.name)

    submodule = os.path.join(td1, 'my_package', 'submodule')
    safe_mkdir(submodule)
    mod_path = os.path.join(submodule, 'mod.py')
    with open(mod_path, 'w') as fp:
      fp.write('accessed')
      pb.add_source(fp.name, 'my_package/submodule/mod.py')

    pex = os.path.join(td2, 'app.pex')
    pb.build(pex)

    output, returncode = run_simple_pex(pex)
    try:
      output = output.decode('UTF-8')
    except ValueError:
      pass
    assert output == 'accessed\n'
    assert returncode == 0
Exemplo n.º 17
0
def test_osx_platform_intel_issue_523():
  def bad_interpreter():
    return PythonInterpreter.from_binary(_KNOWN_BAD_APPLE_INTERPRETER)

  with temporary_dir() as cache:
    # We need to run the bad interpreter with a modern, non-Apple-Extras setuptools in order to
    # successfully install psutil; yield_pex_builder sets up the bad interpreter with our vendored
    # setuptools and wheel extras.
    with nested(yield_pex_builder(installer_impl=WheelInstaller, interpreter=bad_interpreter()),
                temporary_filename()) as (pb, pex_file):
      for resolved_dist in resolver.resolve(['psutil==5.4.3'],
                                            cache=cache,
                                            precedence=(SourcePackage, WheelPackage),
                                            interpreter=pb.interpreter):
        pb.add_dist_location(resolved_dist.distribution.location)
      pb.build(pex_file)

      # NB: We want PEX to find the bare bad interpreter at runtime.
      pex = PEX(pex_file, interpreter=bad_interpreter())

      def run(args, **env):
        pex_env = os.environ.copy()
        pex_env['PEX_VERBOSE'] = '1'
        pex_env.update(**env)
        process = pex.run(args=args,
                          env=pex_env,
                          blocking=False,
                          stdout=subprocess.PIPE,
                          stderr=subprocess.PIPE)
        stdout, stderr = process.communicate()
        return process.returncode, stdout, stderr

      returncode, _, stderr = run(['-c', 'import psutil'])
      assert 0 == returncode, (
        'Process failed with exit code {} and stderr:\n{}'.format(returncode, stderr)
      )

      returncode, stdout, stderr = run(['-c', 'import pkg_resources'])
      assert 0 != returncode, (
        'Isolated pex process succeeded but should not have found pkg-resources:\n'
        'STDOUT:\n'
        '{}\n'
        'STDERR:\n'
        '{}'
        .format(stdout, stdout, stderr)
      )

      returncode, stdout, stderr = run(
        ['-c', 'import pkg_resources; print(pkg_resources.get_supported_platform())'],
        # Let the bad interpreter site-packages setuptools leak in.
        PEX_INHERIT_PATH='1'
      )
      assert 0 == returncode, (
        'Process failed with exit code {} and stderr:\n{}'.format(returncode, stderr)
      )

      # Verify this worked along side the previously problematic pkg_resources-reported platform.
      release, _, _ = platform.mac_ver()
      major_minor = '.'.join(release.split('.')[:2])
      assert to_bytes('macosx-{}-intel'.format(major_minor)) == stdout.strip()
Exemplo n.º 18
0
def assert_access_zipped_assets(distribution_helper_import):
  test_executable = dedent("""
      import os
      {distribution_helper_import}
      temp_dir = DistributionHelper.access_zipped_assets('my_package', 'submodule')
      with open(os.path.join(temp_dir, 'mod.py'), 'r') as fp:
        for line in fp:
          print(line)
  """.format(distribution_helper_import=distribution_helper_import))
  with nested(temporary_dir(), temporary_dir()) as (td1, td2):
    pb = PEXBuilder(path=td1)
    with open(os.path.join(td1, 'exe.py'), 'w') as fp:
      fp.write(test_executable)
      pb.set_executable(fp.name)

    submodule = os.path.join(td1, 'my_package', 'submodule')
    safe_mkdir(submodule)
    mod_path = os.path.join(submodule, 'mod.py')
    with open(mod_path, 'w') as fp:
      fp.write('accessed')
      pb.add_source(fp.name, 'my_package/submodule/mod.py')
    pb.add_source(None, 'my_package/__init__.py')
    pb.add_source(None, 'my_package/submodule/__init__.py')
    pex = os.path.join(td2, 'app.pex')
    pb.build(pex)

    process = PEX(pex, interpreter=pb.interpreter).run(blocking=False,
                                                       stdout=subprocess.PIPE,
                                                       stderr=subprocess.PIPE)
    stdout, stderr = process.communicate()
    assert process.returncode == 0
    assert b'accessed\n' == stdout
    return stderr
Exemplo n.º 19
0
def test_site_libs():
  with nested(mock.patch.object(PEX, '_get_site_packages'), temporary_dir()) as (
          mock_site_packages, tempdir):
    site_packages = os.path.join(tempdir, 'site-packages')
    os.mkdir(site_packages)
    mock_site_packages.return_value = set([site_packages])
    site_libs = PEX.site_libs()
    assert site_packages in site_libs
Exemplo n.º 20
0
def test_site_libs():
    with nested(mock.patch.object(PEX, '_get_site_packages'),
                temporary_dir()) as (mock_site_packages, tempdir):
        site_packages = os.path.join(tempdir, 'site-packages')
        os.mkdir(site_packages)
        mock_site_packages.return_value = set([site_packages])
        site_libs = PEX.site_libs()
        assert site_packages in site_libs
Exemplo n.º 21
0
def test_clp_prereleases_resolver():
    # type: () -> None
    with nested(
            built_wheel(name="prerelease-dep", version="1.2.3b1"),
            built_wheel(name="transitive-dep",
                        install_reqs=["prerelease-dep"]),
            built_wheel(name="dep",
                        install_reqs=["prerelease-dep>=1.2",
                                      "transitive-dep"]),
            temporary_dir(),
            temporary_dir(),
    ) as (prerelease_dep, transitive_dep, dep, dist_dir, cache_dir):

        for dist in (prerelease_dep, transitive_dep, dep):
            safe_copy(dist, os.path.join(dist_dir, os.path.basename(dist)))

        parser = configure_clp()

        options, reqs = parser.parse_args(args=[
            "--no-index",
            "--find-links",
            dist_dir,
            "--cache-dir",
            cache_dir,  # Avoid dangling {pex_root}.
            "--no-pre",
            "dep",
        ])
        assert not options.allow_prereleases

        with pytest.raises(
                SystemExit,
                message="Should have failed to resolve prerelease dep"):
            build_pex(reqs, options)

        # When we specify `--pre`, allow_prereleases is True
        options, reqs = parser.parse_args(args=[
            "--no-index",
            "--find-links",
            dist_dir,
            "--cache-dir",
            cache_dir,  # Avoid dangling {pex_root}.
            "--pre",
            "dep",
        ])
        assert options.allow_prereleases

        # Without a corresponding fix in pex.py, this test failed for a dependency requirement of
        # dep==1.2.3b1 from one package and just dep (any version accepted) from another package.
        # The failure was an exit from build_pex() with the message:
        #
        # Could not satisfy all requirements for dep==1.2.3b1:
        #     dep==1.2.3b1, dep
        #
        # With a correct behavior the assert line is reached and pex_builder object created.
        pex_builder = build_pex(reqs, options)
        assert pex_builder is not None
        assert len(
            pex_builder.info.distributions) == 3, "Should have resolved deps"
Exemplo n.º 22
0
def test_pex_root():
  with nested(temporary_dir(), temporary_dir(), temporary_dir()) as (td, output_dir, tmp_home):
    output_path = os.path.join(output_dir, 'pex.pex')
    args = ['pex', '-o', output_path, '--not-zip-safe', '--pex-root={0}'.format(td)]
    results = run_pex_command(args=args, env=make_env(HOME=tmp_home, PEX_INTERPRETER='1'))
    results.assert_success()
    assert ['pex.pex'] == os.listdir(output_dir), 'Expected built pex file.'
    assert [] == os.listdir(tmp_home), 'Expected empty temp home dir.'
    assert 'build' in os.listdir(td), 'Expected build directory in tmp pex root.'
Exemplo n.º 23
0
def yield_pex_builder(zip_safe=True, interpreter=None):
    # type: (bool, Optional[PythonInterpreter]) -> Iterator[PEXBuilder]
    with nested(temporary_dir(), make_bdist("p1", zip_safe=zip_safe, interpreter=interpreter)) as (
        td,
        p1,
    ):
        pb = PEXBuilder(path=td, interpreter=interpreter)
        pb.add_dist_location(p1.location)
        yield pb
Exemplo n.º 24
0
def test_load_internal_cache_unzipped():
    # zip_safe pex will not be written to install cache unless always_write_cache
    with nested(yield_pex_builder(zip_safe=True), temporary_dir()) as (pb, pex_root):
        pb.info.pex_root = pex_root
        pb.freeze()

        dists = list(PEXEnvironment.load_internal_cache(pb.path(), pb.info))
        assert len(dists) == 1
        assert normalize(dists[0].location).startswith(normalize(os.path.join(pb.path(), pb.info.internal_cache)))
Exemplo n.º 25
0
def test_osx_platform_intel_issue_523():
    def bad_interpreter(include_site_extras=True):
        return PythonInterpreter.from_binary(
            _KNOWN_BAD_APPLE_INTERPRETER,
            include_site_extras=include_site_extras)

    interpreter = bad_interpreter(include_site_extras=False)
    with temporary_dir() as cache:
        # We need to run the bad interpreter with a modern, non-Apple-Extras setuptools in order to
        # successfully install psutil.
        for requirement in (SETUPTOOLS_REQUIREMENT, WHEEL_REQUIREMENT):
            for resolved_dist in resolver.resolve(
                [requirement],
                    cache=cache,
                    # We can't use wheels since we're bootstrapping them.
                    precedence=(SourcePackage, EggPackage),
                    interpreter=interpreter):
                dist = resolved_dist.distribution
                interpreter = interpreter.with_extra(dist.key, dist.version,
                                                     dist.location)

        with nested(
                yield_pex_builder(installer_impl=WheelInstaller,
                                  interpreter=interpreter),
                temporary_filename()) as (pb, pex_file):
            for resolved_dist in resolver.resolve(['psutil==5.4.3'],
                                                  cache=cache,
                                                  precedence=(SourcePackage,
                                                              WheelPackage),
                                                  interpreter=interpreter):
                pb.add_dist_location(resolved_dist.distribution.location)
            pb.build(pex_file)

            # NB: We want PEX to find the bare bad interpreter at runtime.
            pex = PEX(pex_file, interpreter=bad_interpreter())
            args = [
                '-c',
                'import pkg_resources; print(pkg_resources.get_supported_platform())'
            ]
            env = os.environ.copy()
            env['PEX_VERBOSE'] = '1'
            process = pex.run(args=args,
                              env=env,
                              blocking=False,
                              stdout=subprocess.PIPE,
                              stderr=subprocess.PIPE)
            stdout, stderr = process.communicate()
            assert 0 == process.returncode, (
                'Process failed with exit code {} and stderr:\n{}'.format(
                    process.returncode, stderr))

            # Verify this all worked under the previously problematic pkg_resources-reported platform.
            release, _, _ = platform.mac_ver()
            major_minor = '.'.join(release.split('.')[:2])
            assert to_bytes(
                'macosx-{}-intel'.format(major_minor)) == stdout.strip()
Exemplo n.º 26
0
def test_load_internal_cache_unzipped():
  # zip_safe pex will not be written to install cache unless always_write_cache
  with nested(yield_pex_builder(zip_safe=True), temporary_dir()) as (pb, pex_root):
    pb.info.pex_root = pex_root
    pb.freeze()

    dists = list(PEXEnvironment.load_internal_cache(pb.path(), pb.info))
    assert len(dists) == 1
    assert normalize(dists[0].location).startswith(
        normalize(os.path.join(pb.path(), pb.info.internal_cache)))
Exemplo n.º 27
0
def test_write_zipped_internal_cache():
  # zip_safe pex will not be written to install cache unless always_write_cache
  with nested(yield_pex_builder(zip_safe=True), temporary_dir(), temporary_file()) as (
      pb, pex_root, pex_file):

    pb.info.pex_root = pex_root
    pb.build(pex_file.name)

    existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(pex_file.name, pb.info)
    assert len(zip_safe) == 1
    assert normalize(zip_safe[0].location).startswith(
        normalize(os.path.join(pex_file.name, pb.info.internal_cache))), (
            'loc: %s, cache: %s' % (
                normalize(zip_safe[0].location),
                normalize(os.path.join(pex_file.name, pb.info.internal_cache))))

    pb.info.always_write_cache = True
    existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(pex_file.name, pb.info)
    assert len(new) == 1
    assert normalize(new[0].location).startswith(normalize(pb.info.install_cache))

    # Check that we can read from the cache
    existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(pex_file.name, pb.info)
    assert len(existing) == 1
    assert normalize(existing[0].location).startswith(normalize(pb.info.install_cache))

  # non-zip_safe pex will be written to install cache
  with nested(yield_pex_builder(zip_safe=False), temporary_dir(), temporary_file()) as (
      pb, pex_root, pex_file):

    pb.info.pex_root = pex_root
    pb.build(pex_file.name)

    existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(pex_file.name, pb.info)
    assert len(new) == 1
    assert normalize(new[0].location).startswith(normalize(pb.info.install_cache))
    original_location = normalize(new[0].location)

    # do the second time to validate idempotence of caching
    existing, new, zip_safe = PEXEnvironment.write_zipped_internal_cache(pex_file.name, pb.info)
    assert len(existing) == 1
    assert normalize(existing[0].location) == original_location
Exemplo n.º 28
0
def test_site_libs_excludes_prefix():
  """Windows returns sys.prefix as part of getsitepackages(). Make sure to exclude it."""

  with nested(mock.patch.object(PEX, '_get_site_packages'), temporary_dir()) as (
          mock_site_packages, tempdir):
    site_packages = os.path.join(tempdir, 'site-packages')
    os.mkdir(site_packages)
    mock_site_packages.return_value = set([site_packages, sys.prefix])
    site_libs = PEX.site_libs()
    assert site_packages in site_libs
    assert sys.prefix not in site_libs
Exemplo n.º 29
0
def test_pex_builder():
  # test w/ and w/o zipfile dists
  with nested(temporary_dir(), make_bdist('p1')) as (td, p1):
    pb = write_pex(td, exe_main, dists=[p1])

    success_txt = os.path.join(td, 'success.txt')
    PEX(td, interpreter=pb.interpreter).run(args=[success_txt])
    assert os.path.exists(success_txt)
    with open(success_txt) as fp:
      assert fp.read() == 'success'

  # test w/ and w/o zipfile dists
  with nested(temporary_dir(), temporary_dir(), make_bdist('p1')) as (td1, td2, p1):
    pb = write_pex(td1, exe_main, dists=[p1])

    success_txt = os.path.join(td1, 'success.txt')
    PEX(td1, interpreter=pb.interpreter).run(args=[success_txt])
    assert os.path.exists(success_txt)
    with open(success_txt) as fp:
      assert fp.read() == 'success'
Exemplo n.º 30
0
def test_site_libs():
    # type: () -> None
    with nested(mock.patch.object(PEX, "_get_site_packages"), temporary_dir()) as (
        mock_site_packages,
        tempdir,
    ):
        site_packages = os.path.join(tempdir, "site-packages")
        os.mkdir(site_packages)
        mock_site_packages.return_value = set([site_packages])
        site_libs = PEX.site_libs()
        assert site_packages in site_libs
Exemplo n.º 31
0
def test_pex_builder_wheeldep():
    """Repeat the pex_builder test, but this time include an import of something from a wheel that
    doesn't come in importable form."""
    with nested(temporary_dir(), make_bdist("p1")) as (td, p1):
        pyparsing_path = "./tests/example_packages/pyparsing-2.1.10-py2.py3-none-any.whl"
        pb = write_pex(td, wheeldeps_exe_main, dists=[p1, pyparsing_path])
        success_txt = os.path.join(td, "success.txt")
        PEX(td, interpreter=pb.interpreter).run(args=[success_txt])
        assert os.path.exists(success_txt)
        with open(success_txt) as fp:
            assert fp.read() == "success"
Exemplo n.º 32
0
def test_site_libs_excludes_prefix():
    """Windows returns sys.prefix as part of getsitepackages(). Make sure to exclude it."""

    with nested(mock.patch.object(PEX, '_get_site_packages'),
                temporary_dir()) as (mock_site_packages, tempdir):
        site_packages = os.path.join(tempdir, 'site-packages')
        os.mkdir(site_packages)
        mock_site_packages.return_value = set([site_packages, sys.prefix])
        site_libs = PEX.site_libs()
        assert site_packages in site_libs
        assert sys.prefix not in site_libs
Exemplo n.º 33
0
def test_load_internal_cache_unzipped():
    # Unzipped pexes should use distributions from the pex internal cache.
    with nested(yield_pex_builder(zip_safe=True),
                temporary_dir()) as (pb, pex_root):
        pb.info.pex_root = pex_root
        pb.freeze()

        dists = list(PEXEnvironment._load_internal_cache(pb.path(), pb.info))
        assert len(dists) == 1
        assert normalize(dists[0].location).startswith(
            normalize(os.path.join(pb.path(), pb.info.internal_cache)))
Exemplo n.º 34
0
def test_site_libs_symlink():
    with nested(mock.patch.object(PEX, '_get_site_packages'),
                temporary_dir()) as (mock_site_packages, tempdir):
        site_packages = os.path.join(tempdir, 'site-packages')
        os.mkdir(site_packages)
        site_packages_link = os.path.join(tempdir, 'site-packages-link')
        os.symlink(site_packages, site_packages_link)
        mock_site_packages.return_value = set([site_packages_link])

        site_libs = PEX.site_libs()
        assert os.path.realpath(site_packages) in site_libs
        assert site_packages_link in site_libs
Exemplo n.º 35
0
def test_site_libs_symlink():
  with nested(mock.patch.object(PEX, '_get_site_packages'), temporary_dir()) as (
          mock_site_packages, tempdir):
    site_packages = os.path.join(tempdir, 'site-packages')
    os.mkdir(site_packages)
    site_packages_link = os.path.join(tempdir, 'site-packages-link')
    os.symlink(site_packages, site_packages_link)
    mock_site_packages.return_value = set([site_packages_link])

    site_libs = PEX.site_libs()
    assert os.path.realpath(site_packages) in site_libs
    assert site_packages_link in site_libs
Exemplo n.º 36
0
def test_force_local():
  with nested(yield_pex_builder(), temporary_dir(), temporary_file()) as (pb, pex_root, pex_file):
    pb.info.pex_root = pex_root
    pb.build(pex_file.name)

    code_cache = PEXEnvironment.force_local(pex_file.name, pb.info)
    assert os.path.exists(pb.info.zip_unsafe_cache)
    assert len(os.listdir(pb.info.zip_unsafe_cache)) == 1
    assert [os.path.basename(code_cache)] == os.listdir(pb.info.zip_unsafe_cache)
    assert set(os.listdir(code_cache)) == set([PexInfo.PATH, '__main__.py'])

    # idempotence
    assert PEXEnvironment.force_local(pex_file.name, pb.info) == code_cache
Exemplo n.º 37
0
def test_pex_builder_wheeldep():
  """Repeat the pex_builder test, but this time include an import of
  something from a wheel that doesn't come in importable form.
  """
  with nested(temporary_dir(), make_bdist('p1', zipped=True)) as (td, p1):
    pyparsing_path = "./tests/example_packages/pyparsing-2.1.10-py2.py3-none-any.whl"
    dist = DistributionHelper.distribution_from_path(pyparsing_path)
    pb = write_pex(td, wheeldeps_exe_main, dists=[p1, dist])
    success_txt = os.path.join(td, 'success.txt')
    PEX(td, interpreter=pb.interpreter).run(args=[success_txt])
    assert os.path.exists(success_txt)
    with open(success_txt) as fp:
      assert fp.read() == 'success'
Exemplo n.º 38
0
def test_force_local():
    with nested(yield_pex_builder(), temporary_dir(), temporary_filename()) as (pb, pex_root, pex_file):
        pb.info.pex_root = pex_root
        pb.build(pex_file)

        code_cache = PEXEnvironment.force_local(pex_file, pb.info)
        assert os.path.exists(pb.info.zip_unsafe_cache)
        assert len(os.listdir(pb.info.zip_unsafe_cache)) == 1
        assert [os.path.basename(code_cache)] == os.listdir(pb.info.zip_unsafe_cache)
        assert set(os.listdir(code_cache)) == set([PexInfo.PATH, "__main__.py", "__main__.pyc"])

        # idempotence
        assert PEXEnvironment.force_local(pex_file, pb.info) == code_cache
Exemplo n.º 39
0
def test_pex_builder_wheeldep():
    """Repeat the pex_builder test, but this time include an import of
  something from a wheel that doesn't come in importable form.
  """
    with nested(temporary_dir(), make_bdist('p1', zipped=True)) as (td, p1):
        pyparsing_path = "./tests/example_packages/pyparsing-2.1.10-py2.py3-none-any.whl"
        dist = DistributionHelper.distribution_from_path(pyparsing_path)
        write_pex(td, wheeldeps_exe_main, dists=[p1, dist])
        success_txt = os.path.join(td, 'success.txt')
        PEX(td).run(args=[success_txt])
        assert os.path.exists(success_txt)
        with open(success_txt) as fp:
            assert fp.read() == 'success'
Exemplo n.º 40
0
def test_write_zipped_internal_cache():
  # zip_safe pex will not be written to install cache unless always_write_cache
  with nested(yield_pex_builder(zip_safe=True), temporary_dir(), temporary_file()) as (
      pb, pex_root, pex_file):

    pex = pb.path()
    pb.info.pex_root = pex_root
    pb.build(pex_file.name)

    dists = PEXEnvironment.write_zipped_internal_cache(pex_file.name, pb.info)
    assert len(dists) == 1
    assert normalize(dists[0].location).startswith(
        normalize(os.path.join(pex_file.name, pb.info.internal_cache))), (
        'loc: %s, cache: %s' % (
            normalize(dists[0].location),
            normalize(os.path.join(pex_file.name, pb.info.internal_cache))))

    pb.info.always_write_cache = True
    dists = PEXEnvironment.write_zipped_internal_cache(pex_file.name, pb.info)
    assert len(dists) == 1
    assert normalize(dists[0].location).startswith(normalize(pb.info.install_cache))

  # zip_safe pex will not be written to install cache unless always_write_cache
  with nested(yield_pex_builder(zip_safe=False), temporary_dir(), temporary_file()) as (
      pb, pex_root, pex_file):

    pex = pb.path()
    pb.info.pex_root = pex_root
    pb.build(pex_file.name)

    dists = PEXEnvironment.write_zipped_internal_cache(pex_file.name, pb.info)
    assert len(dists) == 1
    assert normalize(dists[0].location).startswith(normalize(pb.info.install_cache))
    original_location = normalize(dists[0].location)

    # do the second time to validate idempotence of caching
    dists = PEXEnvironment.write_zipped_internal_cache(pex_file.name, pb.info)
    assert len(dists) == 1
    assert normalize(dists[0].location) == original_location
Exemplo n.º 41
0
def test_write_zipped_internal_cache():
  # zip_safe pex will not be written to install cache unless always_write_cache
  with nested(yield_pex_builder(zip_safe=True), temporary_dir(), temporary_file()) as (
      pb, pex_root, pex_file):

    pex = pb.path()
    pb.info.pex_root = pex_root
    pb.build(pex_file.name)

    dists = PEXEnvironment.write_zipped_internal_cache(pex_file.name, pb.info)
    assert len(dists) == 1
    assert normalize(dists[0].location).startswith(
        normalize(os.path.join(pex_file.name, pb.info.internal_cache))), (
        'loc: %s, cache: %s' % (
            normalize(dists[0].location),
            normalize(os.path.join(pex_file.name, pb.info.internal_cache))))

    pb.info.always_write_cache = True
    dists = PEXEnvironment.write_zipped_internal_cache(pex_file.name, pb.info)
    assert len(dists) == 1
    assert normalize(dists[0].location).startswith(normalize(pb.info.install_cache))

  # zip_safe pex will not be written to install cache unless always_write_cache
  with nested(yield_pex_builder(zip_safe=False), temporary_dir(), temporary_file()) as (
      pb, pex_root, pex_file):

    pex = pb.path()
    pb.info.pex_root = pex_root
    pb.build(pex_file.name)

    dists = PEXEnvironment.write_zipped_internal_cache(pex_file.name, pb.info)
    assert len(dists) == 1
    assert normalize(dists[0].location).startswith(normalize(pb.info.install_cache))
    original_location = normalize(dists[0].location)

    # do the second time to validate idempotence of caching
    dists = PEXEnvironment.write_zipped_internal_cache(pex_file.name, pb.info)
    assert len(dists) == 1
    assert normalize(dists[0].location) == original_location
Exemplo n.º 42
0
def run_simple_pex_test(body,
                        args=(),
                        env=None,
                        dists=None,
                        coverage=False,
                        interpreter=None):
    with nested(temporary_dir(), temporary_dir()) as (td1, td2):
        pb = write_simple_pex(td1,
                              body,
                              dists=dists,
                              coverage=coverage,
                              interpreter=interpreter)
        pex = os.path.join(td2, 'app.pex')
        pb.build(pex)
        return run_simple_pex(pex, args=args, env=env, interpreter=interpreter)
Exemplo n.º 43
0
def test_pex_builder_copy_or_link():
  with nested(temporary_dir(), temporary_dir(), temporary_dir()) as (td1, td2, td3):
    src = os.path.join(td1, 'exe.py')
    with open(src, 'w') as fp:
      fp.write(exe_main)

    def build_and_check(path, copy):
      pb = PEXBuilder(path, copy=copy)
      pb.add_source(src, 'exe.py')

      s1 = os.stat(src)
      s2 = os.stat(os.path.join(path, 'exe.py'))
      is_link = (s1[stat.ST_INO], s1[stat.ST_DEV]) == (s2[stat.ST_INO], s2[stat.ST_DEV])
      if copy:
        assert not is_link
      else:
        assert is_link

    build_and_check(td2, False)
    build_and_check(td3, True)
Exemplo n.º 44
0
def yield_pex_builder(zip_safe=True):
    with nested(temporary_dir(), make_bdist("p1", zipped=True, zip_safe=zip_safe)) as (td, p1):
        pb = PEXBuilder(path=td)
        pb.add_egg(p1.location)
        yield pb
Exemplo n.º 45
0
def test_clp_prereleases_resolver():
  prerelease_dep = make_sdist(name='dep', version='1.2.3b1')
  with nested(temporary_dir(), temporary_dir()) as (dist_dir, cache_dir):
    safe_copy(prerelease_dep, os.path.join(dist_dir, os.path.basename(prerelease_dep)))
    fetcher = Fetcher([dist_dir])

    # When no specific options are specified, allow_prereleases is None
    parser, resolver_options_builder = configure_clp()
    assert resolver_options_builder._allow_prereleases is None

    # When we specify `--pre`, allow_prereleases is True
    options, reqs = parser.parse_args(args=['--cache-dir', cache_dir,  # Avoid dangling {pex_root}.
                                            '--pre', 'dep==1.2.3b1', 'dep'])
    assert resolver_options_builder._allow_prereleases
    # We need to use our own fetcher instead of PyPI
    resolver_options_builder._fetchers.insert(0, fetcher)

    #####
    # The resolver created during processing of command line options (configure_clp)
    # is not actually passed into the API call (resolve_multi) from build_pex().
    # Instead, resolve_multi() calls resolve() where a new ResolverOptionsBuilder instance
    # is created. The only way to supply our own fetcher to that new instance is to patch it
    # here in the test so that it can fetch our test package (dep-1.2.3b1). Hence, this class
    # below and the change in the `pex.resolver` module where the patched object resides.
    #
    import pex.resolver

    class BuilderWithFetcher(ResolverOptionsBuilder):
      def __init__(self,
                   fetchers=None,
                   allow_all_external=False,
                   allow_external=None,
                   allow_unverified=None,
                   allow_prereleases=None,
                   use_manylinux=None,
                   precedence=None,
                   context=None
                   ):
        super(BuilderWithFetcher, self).__init__(fetchers=fetchers,
                                                 allow_all_external=allow_all_external,
                                                 allow_external=allow_external,
                                                 allow_unverified=allow_unverified,
                                                 allow_prereleases=allow_prereleases,
                                                 use_manylinux=use_manylinux,
                                                 precedence=precedence,
                                                 context=context)
        self._fetchers.insert(0, fetcher)
    # end stub
    #####

    # Without a corresponding fix in pex.py, this test failed for a dependency requirement of
    # dep==1.2.3b1 from one package and just dep (any version accepted) from another package.
    # The failure was an exit from build_pex() with the message:
    #
    # Could not satisfy all requirements for dep==1.2.3b1:
    #     dep==1.2.3b1, dep
    #
    # With a correct behavior the assert line is reached and pex_builder object created.
    with mock.patch.object(pex.resolver, 'ResolverOptionsBuilder', BuilderWithFetcher):
      pex_builder = build_pex(reqs, options, resolver_options_builder)
      assert pex_builder is not None
Exemplo n.º 46
0
def run_simple_pex_test(body, args=(), env=None, dists=None, coverage=False, interpreter=None):
  with nested(temporary_dir(), temporary_dir()) as (td1, td2):
    pb = write_simple_pex(td1, body, dists=dists, coverage=coverage, interpreter=interpreter)
    pex = os.path.join(td2, 'app.pex')
    pb.build(pex)
    return run_simple_pex(pex, args=args, env=env, interpreter=interpreter)
Exemplo n.º 47
0
def run_test(body, env=None):
  with nested(temporary_dir(), temporary_dir()) as (td1, td2):
    pb = write_pex(td1, body)
    pex = os.path.join(td2, 'app.pex')
    pb.build(pex)
    return run_pex(pex, env=env)