Ejemplo n.º 1
0
def test_clp_requirements_txt():
    parser = configure_clp()
    options, _ = parser.parse_args(
        args='-r requirements1.txt -r requirements2.txt'.split())
    assert options.requirement_files == [
        'requirements1.txt', 'requirements2.txt'
    ]
Ejemplo n.º 2
0
    def run(self):
        parser, options_builder = configure_clp()
        options, reqs = parser.parse_args(self.pex_args)

        if options.entry_point or options.script or options.pex_name:
            die('Must not specify entry point, script or output file to --pex-args, given: {}'
                .format(' '.join(self.pex_args)))

        name = self.distribution.get_name()
        version = self.distribution.get_version()

        package_dir = os.path.dirname(
            os.path.realpath(os.path.expanduser(
                self.distribution.script_name)))
        if self.bdist_dir is None:
            self.bdist_dir = os.path.join(package_dir, 'dist')

        console_scripts = self.parse_entry_points()

        pex_specs = []
        if self.bdist_all:
            # Write all entry points into unversioned pex files.
            pex_specs.extend(
                (script_name, os.path.join(self.bdist_dir, script_name))
                for script_name in console_scripts)
        else:
            target = os.path.join(self.bdist_dir,
                                  name + '-' + version + '.pex')
            pex_specs.append(
                (name if name in console_scripts else None, target))

        # In order for code to run to here, pex is on the sys.path - make sure to propagate the
        # sys.path so the subprocess can find us.
        env = os.environ.copy()
        env['PYTHONPATH'] = os.pathsep.join(sys.path)

        args = [sys.executable, '-s', '-m', 'pex.bin.pex', package_dir
                ] + reqs + self.pex_args
        if self.get_log_level() < log.INFO and options.verbosity == 0:
            args.append('-v')

        for script_name, target in pex_specs:
            cmd = args + ['--output-file', target]
            if script_name:
                log.info('Writing %s to %s' % (script_name, target))
                cmd += ['--script', script_name]
            else:
                # The package has no namesake entry point, so build an environment pex.
                log.info('Writing environment pex into %s' % target)

            log.debug('Building pex via: {}'.format(' '.join(cmd)))
            process = Executor.open_process(cmd,
                                            stderr=subprocess.PIPE,
                                            env=env)
            _, stderr = process.communicate()
            result = process.returncode
            if result != 0:
                die(
                    'Failed to create pex via {}:\n{}'.format(
                        ' '.join(cmd), stderr.decode('utf-8')), result)
Ejemplo n.º 3
0
    def run(self):
        parser = configure_clp()
        options, reqs = parser.parse_args(self.pex_args)

        if options.entry_point or options.script or options.pex_name:
            die("Must not specify entry point, script or output file to --pex-args, given: {}"
                .format(" ".join(self.pex_args)))

        name = self.distribution.get_name()
        version = self.distribution.get_version()

        package_dir = os.path.dirname(
            os.path.realpath(os.path.expanduser(
                self.distribution.script_name)))
        if self.bdist_dir is None:
            self.bdist_dir = os.path.join(package_dir, "dist")

        console_scripts = self.parse_entry_points()

        pex_specs = []
        if self.bdist_all:
            # Write all entry points into unversioned pex files.
            pex_specs.extend(
                (script_name, os.path.join(self.bdist_dir, script_name))
                for script_name in console_scripts)
        else:
            target = os.path.join(self.bdist_dir,
                                  name + "-" + version + ".pex")
            pex_specs.append(
                (name if name in console_scripts else None, target))

        args = ["-m", "pex", package_dir] + reqs + self.pex_args
        if self.get_log_level() < log.INFO and options.verbosity == 0:
            args.append("-v")

        for script_name, target in pex_specs:
            pex_cmd = args + ["--output-file", target]
            if script_name:
                log.info("Writing %s to %s" % (script_name, target))
                pex_cmd += ["--script", script_name]
            else:
                # The package has no namesake entry point, so build an environment pex.
                log.info("Writing environment pex into %s" % target)

            cmd, process = PythonInterpreter.get().open_process(
                args=pex_cmd,
                stderr=subprocess.PIPE,
                # In order for code to run to here, pex is on the sys.path - make sure to propagate the
                # sys.path so the subprocess can find us.
                pythonpath=sys.path,
            )
            _, stderr = process.communicate()
            result = process.returncode
            if result != 0:
                die(
                    "Failed to create pex via {}:\n{}".format(
                        " ".join(cmd), stderr.decode("utf-8")),
                    result,
                )
Ejemplo n.º 4
0
def test_clp_requirements_txt():
    # type: () -> None
    parser = configure_clp()
    options, _ = parser.parse_args(
        args="-r requirements1.txt -r requirements2.txt".split())
    assert options.requirement_files == [
        "requirements1.txt", "requirements2.txt"
    ]
Ejemplo n.º 5
0
def test_clp_prereleases_resolver():
    # type: () -> None
    with nested(
            built_wheel(name="prerelease-dep", version="1.2.3b1"),
            built_wheel(name="transitive-dep",
                        install_reqs=["prerelease-dep"]),
            built_wheel(name="dep",
                        install_reqs=["prerelease-dep>=1.2",
                                      "transitive-dep"]),
            temporary_dir(),
            temporary_dir(),
    ) as (prerelease_dep, transitive_dep, dep, dist_dir, cache_dir):

        for dist in (prerelease_dep, transitive_dep, dep):
            safe_copy(dist, os.path.join(dist_dir, os.path.basename(dist)))

        parser = configure_clp()

        options, reqs = parser.parse_args(args=[
            "--no-index",
            "--find-links",
            dist_dir,
            "--cache-dir",
            cache_dir,  # Avoid dangling {pex_root}.
            "--no-pre",
            "dep",
        ])
        assert not options.allow_prereleases

        with pytest.raises(
                SystemExit,
                message="Should have failed to resolve prerelease dep"):
            build_pex(reqs, options)

        # When we specify `--pre`, allow_prereleases is True
        options, reqs = parser.parse_args(args=[
            "--no-index",
            "--find-links",
            dist_dir,
            "--cache-dir",
            cache_dir,  # Avoid dangling {pex_root}.
            "--pre",
            "dep",
        ])
        assert options.allow_prereleases

        # Without a corresponding fix in pex.py, this test failed for a dependency requirement of
        # dep==1.2.3b1 from one package and just dep (any version accepted) from another package.
        # The failure was an exit from build_pex() with the message:
        #
        # Could not satisfy all requirements for dep==1.2.3b1:
        #     dep==1.2.3b1, dep
        #
        # With a correct behavior the assert line is reached and pex_builder object created.
        pex_builder = build_pex(reqs, options)
        assert pex_builder is not None
        assert len(
            pex_builder.info.distributions) == 3, "Should have resolved deps"
Ejemplo n.º 6
0
def test_clp_prereleases_resolver():
  prerelease_dep = make_sdist(name='dep', version='1.2.3b1')
  with temporary_dir() as td:
    safe_copy(prerelease_dep, os.path.join(td, os.path.basename(prerelease_dep)))
    fetcher = Fetcher([td])

    # When no specific options are specified, allow_prereleases is None
    parser, resolver_options_builder = configure_clp()
    assert resolver_options_builder._allow_prereleases is None

    # When we specify `--pre`, allow_prereleases is True
    options, reqs = parser.parse_args(args=['--pre', 'dep==1.2.3b1', 'dep'])
    assert resolver_options_builder._allow_prereleases
    # We need to use our own fetcher instead of PyPI
    resolver_options_builder._fetchers.insert(0, fetcher)

    #####
    # The resolver created during processing of command line options (configure_clp)
    # is not actually passed into the API call (resolve_multi) from build_pex().
    # Instead, resolve_multi() calls resolve() where a new ResolverOptionsBuilder instance
    # is created. The only way to supply our own fetcher to that new instance is to patch it
    # here in the test so that it can fetch our test package (dep-1.2.3b1). Hence, this class
    # below and the change in the `pex.resolver` module where the patched object resides.
    #
    import pex.resolver

    class BuilderWithFetcher(ResolverOptionsBuilder):
      def __init__(self,
                   fetchers=None,
                   allow_all_external=False,
                   allow_external=None,
                   allow_unverified=None,
                   allow_prereleases=None,
                   precedence=None,
                   context=None
                   ):
        super(BuilderWithFetcher, self).__init__(fetchers=fetchers,
                                                 allow_all_external=allow_all_external,
                                                 allow_external=allow_external,
                                                 allow_unverified=allow_unverified,
                                                 allow_prereleases=allow_prereleases,
                                                 precedence=precedence,
                                                 context=context)
        self._fetchers.insert(0, fetcher)
    # end stub
    #####

    # Without a corresponding fix in pex.py, this test failed for a dependency requirement of
    # dep==1.2.3b1 from one package and just dep (any version accepted) from another package.
    # The failure was an exit from build_pex() with the message:
    #
    # Could not satisfy all requirements for dep==1.2.3b1:
    #     dep==1.2.3b1, dep
    #
    # With a correct behavior the assert line is reached and pex_builder object created.
    with mock.patch.object(pex.resolver, 'ResolverOptionsBuilder', BuilderWithFetcher):
      pex_builder = build_pex(reqs, options, resolver_options_builder)
      assert pex_builder is not None
Ejemplo n.º 7
0
def main():
    pparser = pexbin.configure_clp()
    poptions, args = pparser.parse_args(sys.argv)

    manifest_file = args[1]
    manifest_text = open(manifest_file, 'r').read()
    manifest = parse_manifest(manifest_text)

    reqs = manifest.get('requirements', [])

    with ENV.patch(PEX_VERBOSE=str(poptions.verbosity),
                   PEX_ROOT=poptions.pex_root or ENV.PEX_ROOT):
        with TRACER.timed('Building pex'):
            pex_builder = pexbin.build_pex(reqs, poptions)

        # Add source files from the manifest
        for modmap in manifest.get('modules', []):
            src = modmap.get('src')
            dst = modmap.get('dest')

            # NOTE(agallagher): calls the `add_source` and `add_resource` below
            # hard-link the given source into the PEX temp dir.  Since OS X and
            # Linux behave different when hard-linking a source that is a
            # symbolic link (Linux does *not* follow symlinks), resolve any
            # layers of symlinks here to get consistent behavior.
            try:
                pex_builder.add_source(dereference_symlinks(src), dst)
            except OSError as err:
                # Maybe we just can't use hardlinks? Try again.
                if not pex_builder._copy:
                    pex_builder._copy = True
                    pex_builder.add_source(dereference_symlinks(src), dst)
                else:
                    raise RuntimeError("Failed to add %s: %s" % (src, err))

        # Add resources from the manifest
        for reqmap in manifest.get('resources', []):
            src = reqmap.get('src')
            dst = reqmap.get('dest')
            pex_builder.add_resource(dereference_symlinks(src), dst)

        # Add eggs/wheels from the manifest
        for egg in manifest.get('prebuiltLibraries', []):
            try:
                pex_builder.add_dist_location(egg)
            except Exception as err:
                raise RuntimeError("Failed to add %s: %s" % (egg, err))

        # TODO(mikekap): Do something about manifest['nativeLibraries'].

        pexbin.log('Saving PEX file to %s' % poptions.pex_name,
                   V=poptions.verbosity)
        tmp_name = poptions.pex_name + '~'
        safe_delete(tmp_name)
        pex_builder.build(tmp_name)
        shutil.move(tmp_name, poptions.pex_name)
Ejemplo n.º 8
0
def test_clp_preamble_file():
    with NamedTemporaryFile() as tmpfile:
        tmpfile.write(to_bytes('print "foo!"'))
        tmpfile.flush()

        parser = configure_clp()
        options, reqs = parser.parse_args(args=["--preamble-file", tmpfile.name])
        assert options.preamble_file == tmpfile.name

        pex_builder = build_pex(reqs, options)
        assert pex_builder._preamble == 'print "foo!"'
Ejemplo n.º 9
0
def test_clp_arg_file():
    # type: () -> None
    with NamedTemporaryFile() as tmpfile:
        tmpfile.write(to_bytes("-r\nrequirements1.txt\r-r\nrequirements2.txt"))
        tmpfile.flush()

        parser = configure_clp()
        options = parser.parse_args(args=["@" + tmpfile.name])
        assert options.requirement_files == [
            "requirements1.txt", "requirements2.txt"
        ]
Ejemplo n.º 10
0
def test_clp_preamble_file():
  with NamedTemporaryFile() as tmpfile:
    tmpfile.write(to_bytes('print "foo!"'))
    tmpfile.flush()

    parser, resolver_options_builder = configure_clp()
    options, reqs = parser.parse_args(args=['--preamble-file', tmpfile.name])
    assert options.preamble_file == tmpfile.name

    pex_builder = build_pex(reqs, options, resolver_options_builder)
    assert pex_builder._preamble == 'print "foo!"'
Ejemplo n.º 11
0
 def _build(self):
     target_path = os.path.join(self.install_directory,
                                self.target.get("executable_name"))
     parser, resolver_options_builder = configure_clp()
     pex_builder = PEXBuilder(interpreter=self._get_interpreter(parser))
     pex_builder.set_entry_point(self.target.get("entry_point"))
     for dist in self._get_distributions(resolver_options_builder):
         pex_builder.add_distribution(dist)
         pex_builder.add_requirement(dist.as_requirement())
     pex_builder.build(target_path)
     self.directory.symlink_to_bin(self.target.get("executable_name"),
                                   target_path)
Ejemplo n.º 12
0
  def run(self):
    parser, options_builder = configure_clp()
    options, reqs = parser.parse_args(self.pex_args)

    if options.entry_point or options.script or options.pex_name:
      die('Must not specify entry point, script or output file to --pex-args, given: {}'
          .format(' '.join(self.pex_args)))

    name = self.distribution.get_name()
    version = self.distribution.get_version()

    package_dir = os.path.dirname(os.path.realpath(os.path.expanduser(
      self.distribution.script_name)))
    if self.bdist_dir is None:
      self.bdist_dir = os.path.join(package_dir, 'dist')

    console_scripts = self.parse_entry_points()

    pex_specs = []
    if self.bdist_all:
      # Write all entry points into unversioned pex files.
      pex_specs.extend((script_name, os.path.join(self.bdist_dir, script_name))
                       for script_name in console_scripts)
    else:
      target = os.path.join(self.bdist_dir, name + '-' + version + '.pex')
      pex_specs.append((name if name in console_scripts else None, target))

    # In order for code to run to here, pex is on the sys.path - make sure to propagate the
    # sys.path so the subprocess can find us.
    env = os.environ.copy()
    env['PYTHONPATH'] = os.pathsep.join(sys.path)

    args = [sys.executable, '-s', '-m', 'pex.bin.pex', package_dir] + reqs + self.pex_args
    if self.get_log_level() < log.INFO and options.verbosity == 0:
      args.append('-v')

    for script_name, target in pex_specs:
      cmd = args + ['--output-file', target]
      if script_name:
        log.info('Writing %s to %s' % (script_name, target))
        cmd += ['--script', script_name]
      else:
        # The package has no namesake entry point, so build an environment pex.
        log.info('Writing environment pex into %s' % target)

      log.debug('Building pex via: {}'.format(' '.join(cmd)))
      process = Executor.open_process(cmd, stderr=subprocess.PIPE, env=env)
      _, stderr = process.communicate()
      result = process.returncode
      if result != 0:
        die('Failed to create pex via {}:\n{}'.format(' '.join(cmd), stderr.decode('utf-8')),
            result)
Ejemplo n.º 13
0
 def _build(self):
     target_path = os.path.join(self.install_directory,
                                self.target.get("executable_name"))
     parser, resolver_options_builder = configure_clp()
     pex_builder = PEXBuilder(
         interpreter=self._get_interpreter(parser)
     )
     pex_builder.set_entry_point(self.target.get("entry_point"))
     for dist in self._get_distributions(resolver_options_builder):
         pex_builder.add_distribution(dist)
         pex_builder.add_requirement(dist.as_requirement())
     pex_builder.build(target_path)
     self.directory.symlink_to_bin(self.target.get("executable_name"),
                                   target_path)
Ejemplo n.º 14
0
    def run(self):
        name = self.distribution.get_name()
        version = self.distribution.get_version()
        parser, options_builder = configure_clp()
        package_dir = os.path.dirname(
            os.path.realpath(os.path.expanduser(
                self.distribution.script_name)))

        if self.bdist_dir is None:
            self.bdist_dir = os.path.join(package_dir, 'dist')

        options, reqs = parser.parse_args(self.pex_args)

        # Update cache_dir with pex_root in case this is being called directly.
        if options.cache_dir:
            options.cache_dir = make_relative_to_root(options.cache_dir)
        options.interpreter_cache_dir = make_relative_to_root(
            options.interpreter_cache_dir)

        if options.entry_point or options.script:
            die('Must not specify entry_point or script to --pex-args')

        reqs = [package_dir] + reqs

        with ENV.patch(PEX_VERBOSE=str(options.verbosity),
                       PEX_ROOT=options.pex_root):
            pex_builder = build_pex(reqs, options, options_builder)

        console_scripts = self.parse_entry_points()

        target = os.path.join(self.bdist_dir, name + '-' + version + '.pex')
        if self.bdist_all:
            # Write all entry points into unversioned pex files.
            for script_name in console_scripts:
                target = os.path.join(self.bdist_dir, script_name)
                log.info('Writing %s to %s' % (script_name, target))
                self._write(pex_builder, target, script=script_name)
        elif name in console_scripts:
            # The package has a namesake entry point, so use it.
            log.info('Writing %s to %s' % (name, target))
            self._write(pex_builder, target, script=name)
        else:
            # The package has no namesake entry point, so build an environment pex.
            log.info('Writing environment pex into %s' % target)
            self._write(pex_builder, target, script=None)
Ejemplo n.º 15
0
  def run(self):
    name = self.distribution.get_name()
    version = self.distribution.get_version()
    parser, options_builder = configure_clp()
    package_dir = os.path.dirname(os.path.realpath(os.path.expanduser(
        self.distribution.script_name)))

    if self.bdist_dir is None:
      self.bdist_dir = os.path.join(package_dir, 'dist')

    options, reqs = parser.parse_args(self.pex_args)

    if options.entry_point or options.script:
      die('Must not specify entry_point or script to --pex-args')

    reqs = [package_dir] + reqs

    with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
      pex_builder = build_pex(reqs, options, options_builder)

    def split_and_strip(entry_point):
      console_script, entry_point = entry_point.split('=', 2)
      return console_script.strip(), entry_point.strip()

    try:
      console_scripts = dict(split_and_strip(script)
          for script in self.distribution.entry_points.get('console_scripts', []))
    except ValueError:
      console_scripts = {}

    target = os.path.join(self.bdist_dir, name + '-' + version + '.pex')
    if self.bdist_all:
      # Write all entry points into unversioned pex files.
      for script_name in console_scripts:
        target = os.path.join(self.bdist_dir, script_name)
        log.info('Writing %s to %s' % (script_name, target))
        self._write(pex_builder, target, script=script_name)
    elif name in console_scripts:
      # The package has a namesake entry point, so use it.
      log.info('Writing %s to %s' % (name, target))
      self._write(pex_builder, target, script=name)
    else:
      # The package has no namesake entry point, so build an environment pex.
      log.info('Writing environment pex into %s' % target)
      self._write(pex_builder, target, script=None)
Ejemplo n.º 16
0
  def run(self):
    name = self.distribution.get_name()
    version = self.distribution.get_version()
    parser, options_builder = configure_clp()
    package_dir = os.path.dirname(os.path.realpath(os.path.expanduser(
        self.distribution.script_name)))

    if self.bdist_dir is None:
      self.bdist_dir = os.path.join(package_dir, 'dist')

    options, reqs = parser.parse_args(self.pex_args)

    # Update cache_dir with pex_root in case this is being called directly.
    if options.cache_dir:
      options.cache_dir = make_relative_to_root(options.cache_dir)
    options.interpreter_cache_dir = make_relative_to_root(options.interpreter_cache_dir)

    if options.entry_point or options.script:
      die('Must not specify entry_point or script to --pex-args')

    reqs = [package_dir] + reqs

    with ENV.patch(PEX_VERBOSE=str(options.verbosity), PEX_ROOT=options.pex_root):
      pex_builder = build_pex(reqs, options, options_builder)

    console_scripts = self.parse_entry_points()

    target = os.path.join(self.bdist_dir, name + '-' + version + '.pex')
    if self.bdist_all:
      # Write all entry points into unversioned pex files.
      for script_name in console_scripts:
        target = os.path.join(self.bdist_dir, script_name)
        log.info('Writing %s to %s' % (script_name, target))
        self._write(pex_builder, target, script=script_name)
    elif name in console_scripts:
      # The package has a namesake entry point, so use it.
      log.info('Writing %s to %s' % (name, target))
      self._write(pex_builder, target, script=name)
    else:
      # The package has no namesake entry point, so build an environment pex.
      log.info('Writing environment pex into %s' % target)
      self._write(pex_builder, target, script=None)
Ejemplo n.º 17
0
def main():
    # These are the options that this class will accept from the rule
    parser = optparse.OptionParser(usage="usage: %prog [options] output")
    parser.add_option('--entry-point', default='__main__')
    parser.add_option('--no-pypi', action='store_false', dest='pypi', default=True)
    parser.add_option('--not-zip-safe', action='store_false', dest='zip_safe', default=True)
    parser.add_option('--python', default="/usr/bin/python2.7")
    parser.add_option('--find-links', dest='find_links', default='')
    options, args = parser.parse_args()

    if len(args) == 2:
        output = args[0]
        manifest_text = open(args[1], 'r').read()
    elif len(args) == 1:
        output = args[0]
        manifest_text = sys.stdin.read()
    else:
        parser.error("'output' positional argument is required")
        return 1

    if manifest_text.startswith('"') and  manifest_text.endswith('"'):
        manifest_text = manifest_text[1:len(manifest_text) - 1]

    # The manifest is passed via stdin, as it can sometimes get too large
    # to be passed as a CLA.
    with open(os.path.join(tempfile.mkdtemp(dir="/tmp"), "stderr"), "w") as x:
        x.write(manifest_text)
    manifest = parse_manifest(manifest_text)

    # Setup a temp dir that the PEX builder will use as its scratch dir.
    tmp_dir = tempfile.mkdtemp()
    try:
        # These are the options that pex will use
        pparser, resolver_options_builder = configure_clp()

        # Disabling wheels since the PyYAML wheel is incompatible with the Travis CI linux host.
        # Enabling Trace logging in pex/tracer.py shows this upon failure:
        #  pex: Target package WheelPackage('file:///tmp/tmpR_gDlG/PyYAML-3.11-cp27-cp27mu-linux_x86_64.whl')
        #  is not compatible with CPython-2.7.3 / linux-x86_64
        poptions, preqs = pparser.parse_args(['--no-use-wheel'] + sys.argv)
        poptions.entry_point = options.entry_point
        poptions.find_links = options.find_links
        poptions.pypi = options.pypi
        poptions.python = options.python
        poptions.zip_safe = options.zip_safe

        print("pex options: %s" % poptions)
        os.environ["PATH"] = ".:%s:/bin:/usr/bin" % poptions.python

        # The version of pkg_resources.py (from setuptools) on some distros is too old for PEX. So
        # we keep a recent version in and force it into the process by constructing a custom
        # PythonInterpreter instance using it.
        interpreter = PythonInterpreter(
            poptions.python,
            PythonInterpreter.from_binary(options.python).identity,
            extras={
                # TODO: Fix this to resolve automatically
                ('setuptools', '18.0.1'): 'third_party/pex/setuptools-18.0.1-py2.py3-none-any.whl',
                ('wheel', '0.23.0'): 'third_party/pex/wheel-0.23.0-py2.7.egg'
            })

        # resolve setuptools
        interpreter = resolve_or_die(interpreter, SETUPTOOLS_REQUIREMENT, poptions)

        # possibly resolve wheel
        if interpreter and poptions.use_wheel:
          interpreter = resolve_or_die(interpreter, WHEEL_REQUIREMENT, poptions)

        # Add prebuilt libraries listed in the manifest.
        reqs = manifest.get('requirements', {}).keys()
        if len(reqs) > 0:
          print("pex requirements: %s" % reqs)
        pex_builder = build_pex(reqs, poptions,
                                resolver_options_builder, interpreter=interpreter)

        # Set whether this PEX as zip-safe, meaning everything will stayed zipped up
        # and we'll rely on python's zip-import mechanism to load modules from
        # the PEX.  This may not work in some situations (e.g. native
        # libraries, libraries that want to find resources via the FS).
        pex_builder.info.zip_safe = options.zip_safe

        # Set the starting point for this PEX.
        pex_builder.info.entry_point = options.entry_point

        pex_builder.add_source(
            dereference_symlinks(pkg_resources_py),
            os.path.join(pex_builder.BOOTSTRAP_DIR, 'pkg_resources.py'))

        # Add the sources listed in the manifest.
        for dst, src in manifest['modules'].iteritems():
            # NOTE(agallagher): calls the `add_source` and `add_resource` below
            # hard-link the given source into the PEX temp dir.  Since OS X and
            # Linux behave different when hard-linking a source that is a
            # symbolic link (Linux does *not* follow symlinks), resolve any
            # layers of symlinks here to get consistent behavior.
            try:
                pex_builder.add_source(dereference_symlinks(src), dst)
            except OSError as e:
                raise Exception("Failed to add {}: {}".format(src, e))

        # Add resources listed in the manifest.
        for dst, src in manifest['resources'].iteritems():
            # NOTE(agallagher): see rationale above.
            pex_builder.add_resource(dereference_symlinks(src), dst)

        # Add prebuilt libraries listed in the manifest.
        for req in manifest.get('prebuiltLibraries', []):
            try:
                pex_builder.add_dist_location(req)
            except Exception as e:
                raise Exception("Failed to add {}: {}".format(req, e))

        # TODO(mikekap): Do something about manifest['nativeLibraries'].

        # Generate the PEX file.
        pex_builder.build(output)

    # Always try cleaning up the scratch dir, ignoring failures.
    finally:
        shutil.rmtree(tmp_dir, True)
Ejemplo n.º 18
0
def main():
    """ Main """
    # Options that this wrapper will accept from the bazel rule
    parser = optparse.OptionParser(usage="usage: %prog [options] output")
    parser.add_option('--entry-point', default='__main__')
    parser.add_option('--no-pypi', action='store_false',
                      dest='pypi', default=True)
    parser.add_option('--not-zip-safe', action='store_false',
                      dest='zip_safe', default=True)
    parser.add_option('--python', default="python2.7")
    parser.add_option('--find-links', dest='find_links', default='')
    parser.add_option('--no-use-wheel', action='store_false',
                      dest='use_wheel', default=True)
    parser.add_option('--pex-root', dest='pex_root', default=".pex")
    options, args = parser.parse_args()

    # The manifest is passed via stdin or a file, as it can sometimes get too
    # large to be passed as a CLA.
    if len(args) == 2:
        output = args[0]
        manifest_text = open(args[1], 'r').read()
    elif len(args) == 1:
        output = args[0]
        manifest_text = sys.stdin.read()
    else:
        parser.error("'output' positional argument is required")
        return 1

    if manifest_text.startswith('"') and manifest_text.endswith('"'):
        manifest_text = manifest_text[1:len(manifest_text) - 1]

    manifest = parse_manifest(manifest_text)

    # These are the options that pex will use
    pparser, resolver_options_builder = configure_clp()

    poptions, preqs = pparser.parse_args(sys.argv)
    poptions.entry_point = options.entry_point
    poptions.find_links = options.find_links
    poptions.pypi = options.pypi
    poptions.python = options.python
    poptions.use_wheel = options.use_wheel
    poptions.zip_safe = options.zip_safe

    poptions.pex_root = options.pex_root
    poptions.cache_dir = options.pex_root + "/build"
    poptions.interpreter_cache_dir = options.pex_root + "/interpreters"

    # sys.stderr.write("pex options: %s\n" % poptions)
    os.environ["PATH"] = os.getenv("PATH",
                                   "%s:/bin:/usr/bin" % poptions.python)

    if os.path.exists(options.python):
        pybin = poptions.python
    else:
        pybin = distutils.spawn.find_executable(options.python)

    # The version of pkg_resources.py (from setuptools) on some distros is
    # too old for PEX. So we keep a recent version in and force it into the
    # process by constructing a custom PythonInterpreter instance using it.
    # interpreter = PythonInterpreter.from_binary(pybin,
    #                                             [SETUPTOOLS_PATH,
    #                                              WHEEL_PATH])
    interpreter = PythonInterpreter(
        pybin,
        PythonInterpreter.from_binary(pybin).identity,
        extras={
            # TODO: Fix this to resolve automatically
            ('setuptools', '18.0.1'): SETUPTOOLS_PATH,
            # FIXME: I don't think this accomplishes anything at all.
            ('wheel', '0.23.0'): WHEEL_PATH,
        })

    # resolve setuptools
    interpreter = resolve_or_die(interpreter,
                                 SETUPTOOLS_REQUIREMENT,
                                 poptions)

    # possibly resolve wheel
    if interpreter and poptions.use_wheel:
        interpreter = resolve_or_die(interpreter,
                                     WHEEL_REQUIREMENT,
                                     poptions)

    # Add prebuilt libraries listed in the manifest.
    reqs = manifest.get('requirements', {}).keys()
    # if len(reqs) > 0:
    #   sys.stderr.write("pex requirements: %s" % reqs)
    pex_builder = build_pex(reqs, poptions,
                            resolver_options_builder,
                            interpreter=interpreter)

    # Set whether this PEX is zip-safe, meaning everything will stay zipped
    # up and we'll rely on python's zip-import mechanism to load modules
    # from the PEX.  This may not work in some situations (e.g. native
    # libraries, libraries that want to find resources via the FS).
    pex_builder.info.zip_safe = options.zip_safe

    # Set the starting point for this PEX.
    pex_builder.info.entry_point = options.entry_point

    pex_builder.add_source(
        dereference_symlinks(PKG_RESOURCES_PATH),
        os.path.join(pex_builder.BOOTSTRAP_DIR, 'pkg_resources.py'))

    # Add the sources listed in the manifest.
    for dst, src in manifest['modules'].iteritems():
        # NOTE(agallagher): calls the `add_source` and `add_resource` below
        # hard-link the given source into the PEX temp dir.  Since OS X and
        # Linux behave different when hard-linking a source that is a
        # symbolic link (Linux does *not* follow symlinks), resolve any
        # layers of symlinks here to get consistent behavior.
        try:
            pex_builder.add_source(dereference_symlinks(src), dst)
        except OSError as err:
            raise Exception("Failed to add {}: {}".format(src, err))

    # Add resources listed in the manifest.
    for dst, src in manifest['resources'].iteritems():
        # NOTE(agallagher): see rationale above.
        pex_builder.add_resource(dereference_symlinks(src), dst)

    # Add prebuilt libraries listed in the manifest.
    for req in manifest.get('prebuiltLibraries', []):
        try:
            pex_builder.add_dist_location(req)
        except Exception as err:
            raise Exception("Failed to add {}: {}".format(req, err))

    # TODO(mikekap): Do something about manifest['nativeLibraries'].

    # Generate the PEX file.
    pex_builder.build(output)
Ejemplo n.º 19
0
def test_clp_requirements_txt():
  parser, builder = configure_clp()
  options, _ = parser.parse_args(args='-r requirements1.txt -r requirements2.txt'.split())
  assert options.requirement_files == ['requirements1.txt', 'requirements2.txt']
def convert_to_pex(unified_model, output_file_path: str) -> str:
    """
    Convert the given unified model into an executable PEX file.

    # Arguments
        unified_model (UnifiedModel or str): Unified model instance or path to model file
        output_file_path (string): Path to save the model.

    # Returns
    Full path to the converted model
    """
    # https://gist.github.com/simeonf/062af826e79259bc7686
    log.info("Start conversion to PEX")

    if isinstance(unified_model, UnifiedModel):
        pass
    elif os.path.exists(str(unified_model)):
        # load model instance
        unified_model = UnifiedModel.load(str(unified_model))
    else:
        raise Exception("Could not find model for: " + str(unified_model))

    from git import Repo
    from pex.bin import pex

    model_temp_folder = tempfile.mkdtemp()

    model_instance_name = "model_instance"
    model_instance_folder = os.path.join(model_temp_folder,
                                         model_instance_name)
    os.makedirs(model_instance_folder)

    with open(os.path.join(model_instance_folder, "setup.py"),
              "w") as text_file:
        text_file.write("from distutils.core import setup " +
                        "\nsetup(name='" + model_instance_name + "'," +
                        "\n\tpackages=['" + model_instance_name + "']," +
                        "\n\tversion='1.0'," + "\n\tpackage_data={'" +
                        model_instance_name + "': ['" + model_instance_name +
                        "']})")

    model_instance_package = os.path.join(model_instance_folder,
                                          model_instance_name)
    os.makedirs(model_instance_package)

    with open(os.path.join(model_instance_package, "__init__.py"),
              "w") as text_file:
        text_file.write(
            "import os, pkg_resources" +
            "\nfrom unified_model import cli_handler" +
            "\nos.environ[cli_handler.DEFAULT_MODEL_PATH_ENV] = pkg_resources.resource_filename(__name__, '"
            + model_instance_name + "')" + "\ncli_handler.cli()")

    with open(os.path.join(model_instance_package, "__main__.py"),
              "w") as text_file:
        text_file.write("")

    unified_model.save(
        os.path.join(model_instance_package, model_instance_name))

    lib_repo_folder = os.path.join(model_temp_folder, "unified-model")
    Repo.clone_from(UNIFIED_MODEL_REPO_URL, lib_repo_folder)

    parser, resolver_options_builder = pex.configure_clp()
    args = [lib_repo_folder, model_instance_folder, "--disable-cache"]
    for req in unified_model._requirements:
        if isinstance(req, six.string_types):
            args.append(req)
    options, reqs = parser.parse_args(args=args)
    pex_builder = pex.build_pex(reqs, options, resolver_options_builder)
    pex_builder.set_entry_point(model_instance_name)
    pex_builder.build(output_file_path)

    shutil.rmtree(model_temp_folder)

    log.info("Conversion to PEX successful: " + output_file_path)
    return output_file_path
Ejemplo n.º 21
0
def test_clp_prereleases_resolver():
    prerelease_dep = make_sdist(name='dep', version='1.2.3b1')
    with temporary_dir() as td:
        safe_copy(prerelease_dep,
                  os.path.join(td, os.path.basename(prerelease_dep)))
        fetcher = Fetcher([td])

        # When no specific options are specified, allow_prereleases is None
        parser, resolver_options_builder = configure_clp()
        assert resolver_options_builder._allow_prereleases is None

        # When we specify `--pre`, allow_prereleases is True
        options, reqs = parser.parse_args(
            args=['--pre', 'dep==1.2.3b1', 'dep'])
        assert resolver_options_builder._allow_prereleases
        # We need to use our own fetcher instead of PyPI
        resolver_options_builder._fetchers.insert(0, fetcher)

        #####
        # The resolver created during processing of command line options (configure_clp)
        # is not actually passed into the API call (resolve_multi) from build_pex().
        # Instead, resolve_multi() calls resolve() where a new ResolverOptionsBuilder instance
        # is created. The only way to supply our own fetcher to that new instance is to patch it
        # here in the test so that it can fetch our test package (dep-1.2.3b1). Hence, this class
        # below and the change in the `pex.resolver` module where the patched object resides.
        #
        import pex.resolver

        class BuilderWithFetcher(ResolverOptionsBuilder):
            def __init__(self,
                         fetchers=None,
                         allow_all_external=False,
                         allow_external=None,
                         allow_unverified=None,
                         allow_prereleases=None,
                         use_manylinux=None,
                         precedence=None,
                         context=None):
                super(BuilderWithFetcher,
                      self).__init__(fetchers=fetchers,
                                     allow_all_external=allow_all_external,
                                     allow_external=allow_external,
                                     allow_unverified=allow_unverified,
                                     allow_prereleases=allow_prereleases,
                                     use_manylinux=None,
                                     precedence=precedence,
                                     context=context)
                self._fetchers.insert(0, fetcher)

        # end stub
        #####

        # Without a corresponding fix in pex.py, this test failed for a dependency requirement of
        # dep==1.2.3b1 from one package and just dep (any version accepted) from another package.
        # The failure was an exit from build_pex() with the message:
        #
        # Could not satisfy all requirements for dep==1.2.3b1:
        #     dep==1.2.3b1, dep
        #
        # With a correct behavior the assert line is reached and pex_builder object created.
        with mock.patch.object(pex.resolver, 'ResolverOptionsBuilder',
                               BuilderWithFetcher):
            pex_builder = build_pex(reqs, options, resolver_options_builder)
            assert pex_builder is not None
Ejemplo n.º 22
0
def test_clp_constraints_txt():
  parser, builder = configure_clp()
  options, _ = parser.parse_args(args='--constraint requirements1.txt'.split())
  assert options.constraint_files == ['requirements1.txt']
Ejemplo n.º 23
0
def main(args=None):
    args = args[:] if args else sys.argv[1:]
    args = [pexbin.transform_legacy_arg(arg) for arg in args]
    pparser = pexbin.configure_clp()

    try:
        separator = args.index("--")
        args, cmdline = args[:separator], args[separator + 1:]
    except ValueError:
        args, cmdline = args, []

    pparser.add_argument(
            "--manifest-file",
            dest="manifest_file",
            default=None,
            metavar="FILE",
            type=str,
            help="pex_wrapper manifest file.",
        )
    poptions = pparser.parse_args(args=args)

    manifest_file = poptions.manifest_file
    manifest_text = open(manifest_file, 'r').read()
    manifest = parse_manifest(manifest_text)

    reqs = manifest.get('requirements', [])
    requirement_configuration = RequirementConfiguration(requirements=reqs)
    try:
        resolver_configuration = resolver_options.configure(poptions)
    except resolver_options.InvalidConfigurationError as e:
        die(str(e))

    try:
        target_configuration = target_options.configure(poptions)
    except target_options.InterpreterNotFound as e:
        die(str(e))
    except target_options.InterpreterConstraintsNotSatisfied as e:
        die(str(e), exit_code=pexbin.CANNOT_SETUP_INTERPRETER)

    with ENV.patch(PEX_VERBOSE=str(poptions.verbosity),
                   PEX_ROOT=poptions.pex_root or ENV.PEX_ROOT):
        with TRACER.timed('Building pex'):
            pex_builder = pexbin.build_pex(
                requirement_configuration=requirement_configuration,
                resolver_configuration=resolver_configuration,
                target_configuration=target_configuration,
                options=poptions)

        # Add source files from the manifest
        for modmap in manifest.get('modules', []):
            src = modmap.get('src')
            dst = modmap.get('dest')

            # NOTE(agallagher): calls the `add_source` and `add_resource` below
            # hard-link the given source into the PEX temp dir.  Since OS X and
            # Linux behave different when hard-linking a source that is a
            # symbolic link (Linux does *not* follow symlinks), resolve any
            # layers of symlinks here to get consistent behavior.
            try:
                pex_builder.add_source(dereference_symlinks(src), dst)
            except OSError as err:
                # Maybe we just can't use hardlinks? Try again.
                if not pex_builder._copy:
                    pex_builder._copy = True
                    pex_builder.add_source(dereference_symlinks(src), dst)
                else:
                    raise RuntimeError("Failed to add %s: %s" % (src, err))

        # Add resources from the manifest
        for reqmap in manifest.get('resources', []):
            src = reqmap.get('src')
            dst = reqmap.get('dest')
            pex_builder.add_source(dereference_symlinks(src), dst)

        # Add eggs/wheels from the manifest
        for egg in manifest.get('prebuiltLibraries', []):
            try:
                pex_builder.add_dist_location(egg)
            except Exception as err:
                raise RuntimeError("Failed to add %s: %s" % (egg, err))

        # TODO(mikekap): Do something about manifest['nativeLibraries'].

        pexbin.log('Saving PEX file to %s' % poptions.pex_name,
                   V=poptions.verbosity)
        tmp_name = poptions.pex_name + '~'
        safe_delete(tmp_name)
        pex_builder.build(tmp_name)
        shutil.move(tmp_name, poptions.pex_name)
Ejemplo n.º 24
0
def main():
    # These are the options that this class will accept from the rule
    parser = optparse.OptionParser(usage="usage: %prog [options] output")
    parser.add_option('--entry-point', default='__main__')
    parser.add_option('--no-pypi',
                      action='store_false',
                      dest='pypi',
                      default=True)
    parser.add_option('--disable-cache',
                      action='store_true',
                      dest='disable_cache',
                      default=False)
    parser.add_option('--not-zip-safe',
                      action='store_false',
                      dest='zip_safe',
                      default=True)
    parser.add_option('--python', default="/usr/bin/python2.7")
    parser.add_option('--find-links', dest='find_links', default='')
    options, args = parser.parse_args()

    if len(args) == 2:
        output = args[0]
        manifest_text = open(args[1], 'r').read()
    elif len(args) == 1:
        output = args[0]
        manifest_text = sys.stdin.read()
    else:
        parser.error("'output' positional argument is required")
        return 1

    if manifest_text.startswith('"') and manifest_text.endswith('"'):
        manifest_text = manifest_text[1:len(manifest_text) - 1]

    # The manifest is passed via stdin, as it can sometimes get too large
    # to be passed as a CLA.
    with open(os.path.join(tempfile.mkdtemp(dir="/tmp"), "stderr"), "w") as x:
        x.write(manifest_text)
    manifest = parse_manifest(manifest_text)

    # Setup a temp dir that the PEX builder will use as its scratch dir.
    tmp_dir = tempfile.mkdtemp()
    try:
        # These are the options that pex will use
        pparser, resolver_options_builder = configure_clp()

        # Disabling wheels since the PyYAML wheel is incompatible with the Travis CI linux host.
        # Enabling Trace logging in pex/tracer.py shows this upon failure:
        #  pex: Target package WheelPackage('file:///tmp/tmpR_gDlG/PyYAML-3.11-cp27-cp27mu-linux_x86_64.whl')
        #  is not compatible with CPython-2.7.3 / linux-x86_64
        poptions, preqs = pparser.parse_args(['--no-use-wheel'] + sys.argv)
        poptions.entry_point = options.entry_point
        poptions.find_links = options.find_links
        poptions.pypi = options.pypi
        poptions.python = options.python
        poptions.zip_safe = options.zip_safe
        poptions.disable_cache = options.disable_cache

        print("pex options: %s" % poptions)
        os.environ["PATH"] = ".:%s:/bin:/usr/bin" % poptions.python

        # The version of pkg_resources.py (from setuptools) on some distros is too old for PEX. So
        # we keep a recent version in and force it into the process by constructing a custom
        # PythonInterpreter instance using it.
        interpreter = PythonInterpreter(
            poptions.python,
            PythonInterpreter.from_binary(options.python).identity,
            extras={
                # TODO: Fix this to resolve automatically
                ('setuptools', '18.0.1'):
                'third_party/pex/setuptools-18.0.1-py2.py3-none-any.whl',
                ('wheel', '0.23.0'):
                'third_party/pex/wheel-0.23.0-py2.7.egg'
            })

        # resolve setuptools
        interpreter = resolve_or_die(interpreter, SETUPTOOLS_REQUIREMENT,
                                     poptions)

        # possibly resolve wheel
        if interpreter and poptions.use_wheel:
            interpreter = resolve_or_die(interpreter, WHEEL_REQUIREMENT,
                                         poptions)

        # Add prebuilt libraries listed in the manifest.
        reqs = manifest.get('requirements', {}).keys()
        if len(reqs) > 0:
            print("pex requirements: %s" % reqs)
        pex_builder = build_pex(reqs,
                                poptions,
                                resolver_options_builder,
                                interpreter=interpreter)

        # Set whether this PEX as zip-safe, meaning everything will stayed zipped up
        # and we'll rely on python's zip-import mechanism to load modules from
        # the PEX.  This may not work in some situations (e.g. native
        # libraries, libraries that want to find resources via the FS).
        pex_builder.info.zip_safe = options.zip_safe

        # Set the starting point for this PEX.
        pex_builder.info.entry_point = options.entry_point

        pex_builder.add_source(
            dereference_symlinks(pkg_resources_py),
            os.path.join(pex_builder.BOOTSTRAP_DIR, 'pkg_resources.py'))

        # Add the sources listed in the manifest.
        for dst, src in manifest['modules'].iteritems():
            # NOTE(agallagher): calls the `add_source` and `add_resource` below
            # hard-link the given source into the PEX temp dir.  Since OS X and
            # Linux behave different when hard-linking a source that is a
            # symbolic link (Linux does *not* follow symlinks), resolve any
            # layers of symlinks here to get consistent behavior.
            try:
                pex_builder.add_source(dereference_symlinks(src), dst)
            except OSError as e:
                raise Exception("Failed to add {}: {}".format(src, e))

        # Add resources listed in the manifest.
        for dst, src in manifest['resources'].iteritems():
            # NOTE(agallagher): see rationale above.
            pex_builder.add_resource(dereference_symlinks(src), dst)

        # Add prebuilt libraries listed in the manifest.
        for req in manifest.get('prebuiltLibraries', []):
            try:
                pex_builder.add_dist_location(req)
            except Exception as e:
                raise Exception("Failed to add {}: {}".format(req, e))

        # TODO(mikekap): Do something about manifest['nativeLibraries'].

        # Generate the PEX file.
        pex_builder.build(output)

    # Always try cleaning up the scratch dir, ignoring failures.
    finally:
        shutil.rmtree(tmp_dir, True)
Ejemplo n.º 25
0
def test_clp_constraints_txt():
    parser = configure_clp()
    options, _ = parser.parse_args(args="--constraint requirements1.txt".split())
    assert options.constraint_files == ["requirements1.txt"]
Ejemplo n.º 26
0
def main():
    pparser, resolver_options_builder = pexbin.configure_clp()
    poptions, args = pparser.parse_args(sys.argv)

    manifest_file = args[1]
    manifest_text = open(manifest_file, 'r').read()
    manifest = parse_manifest(manifest_text)

    if poptions.pex_root:
        ENV.set('PEX_ROOT', poptions.pex_root)
    else:
        poptions.pex_root = ENV.PEX_ROOT

    if poptions.cache_dir:
        poptions.cache_dir = pexbin.make_relative_to_root(poptions.cache_dir)
    poptions.interpreter_cache_dir = pexbin.make_relative_to_root(
        poptions.interpreter_cache_dir)

    reqs = manifest.get('requirements', [])

    with ENV.patch(PEX_VERBOSE=str(poptions.verbosity)):
        with TRACER.timed('Building pex'):
            pex_builder = pexbin.build_pex(reqs, poptions,
                                           resolver_options_builder)

        # Add source files from the manifest
        for modmap in manifest.get('modules', []):
            src = modmap.get('src')
            dst = modmap.get('dest')

            # NOTE(agallagher): calls the `add_source` and `add_resource` below
            # hard-link the given source into the PEX temp dir.  Since OS X and
            # Linux behave different when hard-linking a source that is a
            # symbolic link (Linux does *not* follow symlinks), resolve any
            # layers of symlinks here to get consistent behavior.
            try:
                pex_builder.add_source(dereference_symlinks(src), dst)
            except OSError as err:
                # Maybe we just can't use hardlinks? Try again.
                if not pex_builder._copy:
                    pex_builder._copy = True
                    pex_builder.add_source(dereference_symlinks(src), dst)
                else:
                    raise RuntimeError("Failed to add %s: %s" % (src, err))

        # Add resources from the manifest
        for reqmap in manifest.get('resources', []):
            src = reqmap.get('src')
            dst = reqmap.get('dest')
            pex_builder.add_resource(dereference_symlinks(src), dst)

        # Add eggs/wheels from the manifest
        for egg in manifest.get('prebuiltLibraries', []):
            try:
                pex_builder.add_dist_location(egg)
            except Exception as err:
                raise RuntimeError("Failed to add %s: %s" % (egg, err))

        # TODO(mikekap): Do something about manifest['nativeLibraries'].

        pexbin.log('Saving PEX file to %s' % poptions.pex_name,
                   v=poptions.verbosity)
        tmp_name = poptions.pex_name + '~'
        safe_delete(tmp_name)
        pex_builder.build(tmp_name)
        os.rename(tmp_name, poptions.pex_name)