def test_clp_requirements_txt(): parser = configure_clp() options, _ = parser.parse_args( args='-r requirements1.txt -r requirements2.txt'.split()) assert options.requirement_files == [ 'requirements1.txt', 'requirements2.txt' ]
def run(self): parser, options_builder = configure_clp() options, reqs = parser.parse_args(self.pex_args) if options.entry_point or options.script or options.pex_name: die('Must not specify entry point, script or output file to --pex-args, given: {}' .format(' '.join(self.pex_args))) name = self.distribution.get_name() version = self.distribution.get_version() package_dir = os.path.dirname( os.path.realpath(os.path.expanduser( self.distribution.script_name))) if self.bdist_dir is None: self.bdist_dir = os.path.join(package_dir, 'dist') console_scripts = self.parse_entry_points() pex_specs = [] if self.bdist_all: # Write all entry points into unversioned pex files. pex_specs.extend( (script_name, os.path.join(self.bdist_dir, script_name)) for script_name in console_scripts) else: target = os.path.join(self.bdist_dir, name + '-' + version + '.pex') pex_specs.append( (name if name in console_scripts else None, target)) # In order for code to run to here, pex is on the sys.path - make sure to propagate the # sys.path so the subprocess can find us. env = os.environ.copy() env['PYTHONPATH'] = os.pathsep.join(sys.path) args = [sys.executable, '-s', '-m', 'pex.bin.pex', package_dir ] + reqs + self.pex_args if self.get_log_level() < log.INFO and options.verbosity == 0: args.append('-v') for script_name, target in pex_specs: cmd = args + ['--output-file', target] if script_name: log.info('Writing %s to %s' % (script_name, target)) cmd += ['--script', script_name] else: # The package has no namesake entry point, so build an environment pex. log.info('Writing environment pex into %s' % target) log.debug('Building pex via: {}'.format(' '.join(cmd))) process = Executor.open_process(cmd, stderr=subprocess.PIPE, env=env) _, stderr = process.communicate() result = process.returncode if result != 0: die( 'Failed to create pex via {}:\n{}'.format( ' '.join(cmd), stderr.decode('utf-8')), result)
def run(self): parser = configure_clp() options, reqs = parser.parse_args(self.pex_args) if options.entry_point or options.script or options.pex_name: die("Must not specify entry point, script or output file to --pex-args, given: {}" .format(" ".join(self.pex_args))) name = self.distribution.get_name() version = self.distribution.get_version() package_dir = os.path.dirname( os.path.realpath(os.path.expanduser( self.distribution.script_name))) if self.bdist_dir is None: self.bdist_dir = os.path.join(package_dir, "dist") console_scripts = self.parse_entry_points() pex_specs = [] if self.bdist_all: # Write all entry points into unversioned pex files. pex_specs.extend( (script_name, os.path.join(self.bdist_dir, script_name)) for script_name in console_scripts) else: target = os.path.join(self.bdist_dir, name + "-" + version + ".pex") pex_specs.append( (name if name in console_scripts else None, target)) args = ["-m", "pex", package_dir] + reqs + self.pex_args if self.get_log_level() < log.INFO and options.verbosity == 0: args.append("-v") for script_name, target in pex_specs: pex_cmd = args + ["--output-file", target] if script_name: log.info("Writing %s to %s" % (script_name, target)) pex_cmd += ["--script", script_name] else: # The package has no namesake entry point, so build an environment pex. log.info("Writing environment pex into %s" % target) cmd, process = PythonInterpreter.get().open_process( args=pex_cmd, stderr=subprocess.PIPE, # In order for code to run to here, pex is on the sys.path - make sure to propagate the # sys.path so the subprocess can find us. pythonpath=sys.path, ) _, stderr = process.communicate() result = process.returncode if result != 0: die( "Failed to create pex via {}:\n{}".format( " ".join(cmd), stderr.decode("utf-8")), result, )
def test_clp_requirements_txt(): # type: () -> None parser = configure_clp() options, _ = parser.parse_args( args="-r requirements1.txt -r requirements2.txt".split()) assert options.requirement_files == [ "requirements1.txt", "requirements2.txt" ]
def test_clp_prereleases_resolver(): # type: () -> None with nested( built_wheel(name="prerelease-dep", version="1.2.3b1"), built_wheel(name="transitive-dep", install_reqs=["prerelease-dep"]), built_wheel(name="dep", install_reqs=["prerelease-dep>=1.2", "transitive-dep"]), temporary_dir(), temporary_dir(), ) as (prerelease_dep, transitive_dep, dep, dist_dir, cache_dir): for dist in (prerelease_dep, transitive_dep, dep): safe_copy(dist, os.path.join(dist_dir, os.path.basename(dist))) parser = configure_clp() options, reqs = parser.parse_args(args=[ "--no-index", "--find-links", dist_dir, "--cache-dir", cache_dir, # Avoid dangling {pex_root}. "--no-pre", "dep", ]) assert not options.allow_prereleases with pytest.raises( SystemExit, message="Should have failed to resolve prerelease dep"): build_pex(reqs, options) # When we specify `--pre`, allow_prereleases is True options, reqs = parser.parse_args(args=[ "--no-index", "--find-links", dist_dir, "--cache-dir", cache_dir, # Avoid dangling {pex_root}. "--pre", "dep", ]) assert options.allow_prereleases # Without a corresponding fix in pex.py, this test failed for a dependency requirement of # dep==1.2.3b1 from one package and just dep (any version accepted) from another package. # The failure was an exit from build_pex() with the message: # # Could not satisfy all requirements for dep==1.2.3b1: # dep==1.2.3b1, dep # # With a correct behavior the assert line is reached and pex_builder object created. pex_builder = build_pex(reqs, options) assert pex_builder is not None assert len( pex_builder.info.distributions) == 3, "Should have resolved deps"
def test_clp_prereleases_resolver(): prerelease_dep = make_sdist(name='dep', version='1.2.3b1') with temporary_dir() as td: safe_copy(prerelease_dep, os.path.join(td, os.path.basename(prerelease_dep))) fetcher = Fetcher([td]) # When no specific options are specified, allow_prereleases is None parser, resolver_options_builder = configure_clp() assert resolver_options_builder._allow_prereleases is None # When we specify `--pre`, allow_prereleases is True options, reqs = parser.parse_args(args=['--pre', 'dep==1.2.3b1', 'dep']) assert resolver_options_builder._allow_prereleases # We need to use our own fetcher instead of PyPI resolver_options_builder._fetchers.insert(0, fetcher) ##### # The resolver created during processing of command line options (configure_clp) # is not actually passed into the API call (resolve_multi) from build_pex(). # Instead, resolve_multi() calls resolve() where a new ResolverOptionsBuilder instance # is created. The only way to supply our own fetcher to that new instance is to patch it # here in the test so that it can fetch our test package (dep-1.2.3b1). Hence, this class # below and the change in the `pex.resolver` module where the patched object resides. # import pex.resolver class BuilderWithFetcher(ResolverOptionsBuilder): def __init__(self, fetchers=None, allow_all_external=False, allow_external=None, allow_unverified=None, allow_prereleases=None, precedence=None, context=None ): super(BuilderWithFetcher, self).__init__(fetchers=fetchers, allow_all_external=allow_all_external, allow_external=allow_external, allow_unverified=allow_unverified, allow_prereleases=allow_prereleases, precedence=precedence, context=context) self._fetchers.insert(0, fetcher) # end stub ##### # Without a corresponding fix in pex.py, this test failed for a dependency requirement of # dep==1.2.3b1 from one package and just dep (any version accepted) from another package. # The failure was an exit from build_pex() with the message: # # Could not satisfy all requirements for dep==1.2.3b1: # dep==1.2.3b1, dep # # With a correct behavior the assert line is reached and pex_builder object created. with mock.patch.object(pex.resolver, 'ResolverOptionsBuilder', BuilderWithFetcher): pex_builder = build_pex(reqs, options, resolver_options_builder) assert pex_builder is not None
def main(): pparser = pexbin.configure_clp() poptions, args = pparser.parse_args(sys.argv) manifest_file = args[1] manifest_text = open(manifest_file, 'r').read() manifest = parse_manifest(manifest_text) reqs = manifest.get('requirements', []) with ENV.patch(PEX_VERBOSE=str(poptions.verbosity), PEX_ROOT=poptions.pex_root or ENV.PEX_ROOT): with TRACER.timed('Building pex'): pex_builder = pexbin.build_pex(reqs, poptions) # Add source files from the manifest for modmap in manifest.get('modules', []): src = modmap.get('src') dst = modmap.get('dest') # NOTE(agallagher): calls the `add_source` and `add_resource` below # hard-link the given source into the PEX temp dir. Since OS X and # Linux behave different when hard-linking a source that is a # symbolic link (Linux does *not* follow symlinks), resolve any # layers of symlinks here to get consistent behavior. try: pex_builder.add_source(dereference_symlinks(src), dst) except OSError as err: # Maybe we just can't use hardlinks? Try again. if not pex_builder._copy: pex_builder._copy = True pex_builder.add_source(dereference_symlinks(src), dst) else: raise RuntimeError("Failed to add %s: %s" % (src, err)) # Add resources from the manifest for reqmap in manifest.get('resources', []): src = reqmap.get('src') dst = reqmap.get('dest') pex_builder.add_resource(dereference_symlinks(src), dst) # Add eggs/wheels from the manifest for egg in manifest.get('prebuiltLibraries', []): try: pex_builder.add_dist_location(egg) except Exception as err: raise RuntimeError("Failed to add %s: %s" % (egg, err)) # TODO(mikekap): Do something about manifest['nativeLibraries']. pexbin.log('Saving PEX file to %s' % poptions.pex_name, V=poptions.verbosity) tmp_name = poptions.pex_name + '~' safe_delete(tmp_name) pex_builder.build(tmp_name) shutil.move(tmp_name, poptions.pex_name)
def test_clp_preamble_file(): with NamedTemporaryFile() as tmpfile: tmpfile.write(to_bytes('print "foo!"')) tmpfile.flush() parser = configure_clp() options, reqs = parser.parse_args(args=["--preamble-file", tmpfile.name]) assert options.preamble_file == tmpfile.name pex_builder = build_pex(reqs, options) assert pex_builder._preamble == 'print "foo!"'
def test_clp_arg_file(): # type: () -> None with NamedTemporaryFile() as tmpfile: tmpfile.write(to_bytes("-r\nrequirements1.txt\r-r\nrequirements2.txt")) tmpfile.flush() parser = configure_clp() options = parser.parse_args(args=["@" + tmpfile.name]) assert options.requirement_files == [ "requirements1.txt", "requirements2.txt" ]
def test_clp_preamble_file(): with NamedTemporaryFile() as tmpfile: tmpfile.write(to_bytes('print "foo!"')) tmpfile.flush() parser, resolver_options_builder = configure_clp() options, reqs = parser.parse_args(args=['--preamble-file', tmpfile.name]) assert options.preamble_file == tmpfile.name pex_builder = build_pex(reqs, options, resolver_options_builder) assert pex_builder._preamble == 'print "foo!"'
def _build(self): target_path = os.path.join(self.install_directory, self.target.get("executable_name")) parser, resolver_options_builder = configure_clp() pex_builder = PEXBuilder(interpreter=self._get_interpreter(parser)) pex_builder.set_entry_point(self.target.get("entry_point")) for dist in self._get_distributions(resolver_options_builder): pex_builder.add_distribution(dist) pex_builder.add_requirement(dist.as_requirement()) pex_builder.build(target_path) self.directory.symlink_to_bin(self.target.get("executable_name"), target_path)
def run(self): parser, options_builder = configure_clp() options, reqs = parser.parse_args(self.pex_args) if options.entry_point or options.script or options.pex_name: die('Must not specify entry point, script or output file to --pex-args, given: {}' .format(' '.join(self.pex_args))) name = self.distribution.get_name() version = self.distribution.get_version() package_dir = os.path.dirname(os.path.realpath(os.path.expanduser( self.distribution.script_name))) if self.bdist_dir is None: self.bdist_dir = os.path.join(package_dir, 'dist') console_scripts = self.parse_entry_points() pex_specs = [] if self.bdist_all: # Write all entry points into unversioned pex files. pex_specs.extend((script_name, os.path.join(self.bdist_dir, script_name)) for script_name in console_scripts) else: target = os.path.join(self.bdist_dir, name + '-' + version + '.pex') pex_specs.append((name if name in console_scripts else None, target)) # In order for code to run to here, pex is on the sys.path - make sure to propagate the # sys.path so the subprocess can find us. env = os.environ.copy() env['PYTHONPATH'] = os.pathsep.join(sys.path) args = [sys.executable, '-s', '-m', 'pex.bin.pex', package_dir] + reqs + self.pex_args if self.get_log_level() < log.INFO and options.verbosity == 0: args.append('-v') for script_name, target in pex_specs: cmd = args + ['--output-file', target] if script_name: log.info('Writing %s to %s' % (script_name, target)) cmd += ['--script', script_name] else: # The package has no namesake entry point, so build an environment pex. log.info('Writing environment pex into %s' % target) log.debug('Building pex via: {}'.format(' '.join(cmd))) process = Executor.open_process(cmd, stderr=subprocess.PIPE, env=env) _, stderr = process.communicate() result = process.returncode if result != 0: die('Failed to create pex via {}:\n{}'.format(' '.join(cmd), stderr.decode('utf-8')), result)
def _build(self): target_path = os.path.join(self.install_directory, self.target.get("executable_name")) parser, resolver_options_builder = configure_clp() pex_builder = PEXBuilder( interpreter=self._get_interpreter(parser) ) pex_builder.set_entry_point(self.target.get("entry_point")) for dist in self._get_distributions(resolver_options_builder): pex_builder.add_distribution(dist) pex_builder.add_requirement(dist.as_requirement()) pex_builder.build(target_path) self.directory.symlink_to_bin(self.target.get("executable_name"), target_path)
def run(self): name = self.distribution.get_name() version = self.distribution.get_version() parser, options_builder = configure_clp() package_dir = os.path.dirname( os.path.realpath(os.path.expanduser( self.distribution.script_name))) if self.bdist_dir is None: self.bdist_dir = os.path.join(package_dir, 'dist') options, reqs = parser.parse_args(self.pex_args) # Update cache_dir with pex_root in case this is being called directly. if options.cache_dir: options.cache_dir = make_relative_to_root(options.cache_dir) options.interpreter_cache_dir = make_relative_to_root( options.interpreter_cache_dir) if options.entry_point or options.script: die('Must not specify entry_point or script to --pex-args') reqs = [package_dir] + reqs with ENV.patch(PEX_VERBOSE=str(options.verbosity), PEX_ROOT=options.pex_root): pex_builder = build_pex(reqs, options, options_builder) console_scripts = self.parse_entry_points() target = os.path.join(self.bdist_dir, name + '-' + version + '.pex') if self.bdist_all: # Write all entry points into unversioned pex files. for script_name in console_scripts: target = os.path.join(self.bdist_dir, script_name) log.info('Writing %s to %s' % (script_name, target)) self._write(pex_builder, target, script=script_name) elif name in console_scripts: # The package has a namesake entry point, so use it. log.info('Writing %s to %s' % (name, target)) self._write(pex_builder, target, script=name) else: # The package has no namesake entry point, so build an environment pex. log.info('Writing environment pex into %s' % target) self._write(pex_builder, target, script=None)
def run(self): name = self.distribution.get_name() version = self.distribution.get_version() parser, options_builder = configure_clp() package_dir = os.path.dirname(os.path.realpath(os.path.expanduser( self.distribution.script_name))) if self.bdist_dir is None: self.bdist_dir = os.path.join(package_dir, 'dist') options, reqs = parser.parse_args(self.pex_args) if options.entry_point or options.script: die('Must not specify entry_point or script to --pex-args') reqs = [package_dir] + reqs with ENV.patch(PEX_VERBOSE=str(options.verbosity)): pex_builder = build_pex(reqs, options, options_builder) def split_and_strip(entry_point): console_script, entry_point = entry_point.split('=', 2) return console_script.strip(), entry_point.strip() try: console_scripts = dict(split_and_strip(script) for script in self.distribution.entry_points.get('console_scripts', [])) except ValueError: console_scripts = {} target = os.path.join(self.bdist_dir, name + '-' + version + '.pex') if self.bdist_all: # Write all entry points into unversioned pex files. for script_name in console_scripts: target = os.path.join(self.bdist_dir, script_name) log.info('Writing %s to %s' % (script_name, target)) self._write(pex_builder, target, script=script_name) elif name in console_scripts: # The package has a namesake entry point, so use it. log.info('Writing %s to %s' % (name, target)) self._write(pex_builder, target, script=name) else: # The package has no namesake entry point, so build an environment pex. log.info('Writing environment pex into %s' % target) self._write(pex_builder, target, script=None)
def run(self): name = self.distribution.get_name() version = self.distribution.get_version() parser, options_builder = configure_clp() package_dir = os.path.dirname(os.path.realpath(os.path.expanduser( self.distribution.script_name))) if self.bdist_dir is None: self.bdist_dir = os.path.join(package_dir, 'dist') options, reqs = parser.parse_args(self.pex_args) # Update cache_dir with pex_root in case this is being called directly. if options.cache_dir: options.cache_dir = make_relative_to_root(options.cache_dir) options.interpreter_cache_dir = make_relative_to_root(options.interpreter_cache_dir) if options.entry_point or options.script: die('Must not specify entry_point or script to --pex-args') reqs = [package_dir] + reqs with ENV.patch(PEX_VERBOSE=str(options.verbosity), PEX_ROOT=options.pex_root): pex_builder = build_pex(reqs, options, options_builder) console_scripts = self.parse_entry_points() target = os.path.join(self.bdist_dir, name + '-' + version + '.pex') if self.bdist_all: # Write all entry points into unversioned pex files. for script_name in console_scripts: target = os.path.join(self.bdist_dir, script_name) log.info('Writing %s to %s' % (script_name, target)) self._write(pex_builder, target, script=script_name) elif name in console_scripts: # The package has a namesake entry point, so use it. log.info('Writing %s to %s' % (name, target)) self._write(pex_builder, target, script=name) else: # The package has no namesake entry point, so build an environment pex. log.info('Writing environment pex into %s' % target) self._write(pex_builder, target, script=None)
def main(): # These are the options that this class will accept from the rule parser = optparse.OptionParser(usage="usage: %prog [options] output") parser.add_option('--entry-point', default='__main__') parser.add_option('--no-pypi', action='store_false', dest='pypi', default=True) parser.add_option('--not-zip-safe', action='store_false', dest='zip_safe', default=True) parser.add_option('--python', default="/usr/bin/python2.7") parser.add_option('--find-links', dest='find_links', default='') options, args = parser.parse_args() if len(args) == 2: output = args[0] manifest_text = open(args[1], 'r').read() elif len(args) == 1: output = args[0] manifest_text = sys.stdin.read() else: parser.error("'output' positional argument is required") return 1 if manifest_text.startswith('"') and manifest_text.endswith('"'): manifest_text = manifest_text[1:len(manifest_text) - 1] # The manifest is passed via stdin, as it can sometimes get too large # to be passed as a CLA. with open(os.path.join(tempfile.mkdtemp(dir="/tmp"), "stderr"), "w") as x: x.write(manifest_text) manifest = parse_manifest(manifest_text) # Setup a temp dir that the PEX builder will use as its scratch dir. tmp_dir = tempfile.mkdtemp() try: # These are the options that pex will use pparser, resolver_options_builder = configure_clp() # Disabling wheels since the PyYAML wheel is incompatible with the Travis CI linux host. # Enabling Trace logging in pex/tracer.py shows this upon failure: # pex: Target package WheelPackage('file:///tmp/tmpR_gDlG/PyYAML-3.11-cp27-cp27mu-linux_x86_64.whl') # is not compatible with CPython-2.7.3 / linux-x86_64 poptions, preqs = pparser.parse_args(['--no-use-wheel'] + sys.argv) poptions.entry_point = options.entry_point poptions.find_links = options.find_links poptions.pypi = options.pypi poptions.python = options.python poptions.zip_safe = options.zip_safe print("pex options: %s" % poptions) os.environ["PATH"] = ".:%s:/bin:/usr/bin" % poptions.python # The version of pkg_resources.py (from setuptools) on some distros is too old for PEX. So # we keep a recent version in and force it into the process by constructing a custom # PythonInterpreter instance using it. interpreter = PythonInterpreter( poptions.python, PythonInterpreter.from_binary(options.python).identity, extras={ # TODO: Fix this to resolve automatically ('setuptools', '18.0.1'): 'third_party/pex/setuptools-18.0.1-py2.py3-none-any.whl', ('wheel', '0.23.0'): 'third_party/pex/wheel-0.23.0-py2.7.egg' }) # resolve setuptools interpreter = resolve_or_die(interpreter, SETUPTOOLS_REQUIREMENT, poptions) # possibly resolve wheel if interpreter and poptions.use_wheel: interpreter = resolve_or_die(interpreter, WHEEL_REQUIREMENT, poptions) # Add prebuilt libraries listed in the manifest. reqs = manifest.get('requirements', {}).keys() if len(reqs) > 0: print("pex requirements: %s" % reqs) pex_builder = build_pex(reqs, poptions, resolver_options_builder, interpreter=interpreter) # Set whether this PEX as zip-safe, meaning everything will stayed zipped up # and we'll rely on python's zip-import mechanism to load modules from # the PEX. This may not work in some situations (e.g. native # libraries, libraries that want to find resources via the FS). pex_builder.info.zip_safe = options.zip_safe # Set the starting point for this PEX. pex_builder.info.entry_point = options.entry_point pex_builder.add_source( dereference_symlinks(pkg_resources_py), os.path.join(pex_builder.BOOTSTRAP_DIR, 'pkg_resources.py')) # Add the sources listed in the manifest. for dst, src in manifest['modules'].iteritems(): # NOTE(agallagher): calls the `add_source` and `add_resource` below # hard-link the given source into the PEX temp dir. Since OS X and # Linux behave different when hard-linking a source that is a # symbolic link (Linux does *not* follow symlinks), resolve any # layers of symlinks here to get consistent behavior. try: pex_builder.add_source(dereference_symlinks(src), dst) except OSError as e: raise Exception("Failed to add {}: {}".format(src, e)) # Add resources listed in the manifest. for dst, src in manifest['resources'].iteritems(): # NOTE(agallagher): see rationale above. pex_builder.add_resource(dereference_symlinks(src), dst) # Add prebuilt libraries listed in the manifest. for req in manifest.get('prebuiltLibraries', []): try: pex_builder.add_dist_location(req) except Exception as e: raise Exception("Failed to add {}: {}".format(req, e)) # TODO(mikekap): Do something about manifest['nativeLibraries']. # Generate the PEX file. pex_builder.build(output) # Always try cleaning up the scratch dir, ignoring failures. finally: shutil.rmtree(tmp_dir, True)
def main(): """ Main """ # Options that this wrapper will accept from the bazel rule parser = optparse.OptionParser(usage="usage: %prog [options] output") parser.add_option('--entry-point', default='__main__') parser.add_option('--no-pypi', action='store_false', dest='pypi', default=True) parser.add_option('--not-zip-safe', action='store_false', dest='zip_safe', default=True) parser.add_option('--python', default="python2.7") parser.add_option('--find-links', dest='find_links', default='') parser.add_option('--no-use-wheel', action='store_false', dest='use_wheel', default=True) parser.add_option('--pex-root', dest='pex_root', default=".pex") options, args = parser.parse_args() # The manifest is passed via stdin or a file, as it can sometimes get too # large to be passed as a CLA. if len(args) == 2: output = args[0] manifest_text = open(args[1], 'r').read() elif len(args) == 1: output = args[0] manifest_text = sys.stdin.read() else: parser.error("'output' positional argument is required") return 1 if manifest_text.startswith('"') and manifest_text.endswith('"'): manifest_text = manifest_text[1:len(manifest_text) - 1] manifest = parse_manifest(manifest_text) # These are the options that pex will use pparser, resolver_options_builder = configure_clp() poptions, preqs = pparser.parse_args(sys.argv) poptions.entry_point = options.entry_point poptions.find_links = options.find_links poptions.pypi = options.pypi poptions.python = options.python poptions.use_wheel = options.use_wheel poptions.zip_safe = options.zip_safe poptions.pex_root = options.pex_root poptions.cache_dir = options.pex_root + "/build" poptions.interpreter_cache_dir = options.pex_root + "/interpreters" # sys.stderr.write("pex options: %s\n" % poptions) os.environ["PATH"] = os.getenv("PATH", "%s:/bin:/usr/bin" % poptions.python) if os.path.exists(options.python): pybin = poptions.python else: pybin = distutils.spawn.find_executable(options.python) # The version of pkg_resources.py (from setuptools) on some distros is # too old for PEX. So we keep a recent version in and force it into the # process by constructing a custom PythonInterpreter instance using it. # interpreter = PythonInterpreter.from_binary(pybin, # [SETUPTOOLS_PATH, # WHEEL_PATH]) interpreter = PythonInterpreter( pybin, PythonInterpreter.from_binary(pybin).identity, extras={ # TODO: Fix this to resolve automatically ('setuptools', '18.0.1'): SETUPTOOLS_PATH, # FIXME: I don't think this accomplishes anything at all. ('wheel', '0.23.0'): WHEEL_PATH, }) # resolve setuptools interpreter = resolve_or_die(interpreter, SETUPTOOLS_REQUIREMENT, poptions) # possibly resolve wheel if interpreter and poptions.use_wheel: interpreter = resolve_or_die(interpreter, WHEEL_REQUIREMENT, poptions) # Add prebuilt libraries listed in the manifest. reqs = manifest.get('requirements', {}).keys() # if len(reqs) > 0: # sys.stderr.write("pex requirements: %s" % reqs) pex_builder = build_pex(reqs, poptions, resolver_options_builder, interpreter=interpreter) # Set whether this PEX is zip-safe, meaning everything will stay zipped # up and we'll rely on python's zip-import mechanism to load modules # from the PEX. This may not work in some situations (e.g. native # libraries, libraries that want to find resources via the FS). pex_builder.info.zip_safe = options.zip_safe # Set the starting point for this PEX. pex_builder.info.entry_point = options.entry_point pex_builder.add_source( dereference_symlinks(PKG_RESOURCES_PATH), os.path.join(pex_builder.BOOTSTRAP_DIR, 'pkg_resources.py')) # Add the sources listed in the manifest. for dst, src in manifest['modules'].iteritems(): # NOTE(agallagher): calls the `add_source` and `add_resource` below # hard-link the given source into the PEX temp dir. Since OS X and # Linux behave different when hard-linking a source that is a # symbolic link (Linux does *not* follow symlinks), resolve any # layers of symlinks here to get consistent behavior. try: pex_builder.add_source(dereference_symlinks(src), dst) except OSError as err: raise Exception("Failed to add {}: {}".format(src, err)) # Add resources listed in the manifest. for dst, src in manifest['resources'].iteritems(): # NOTE(agallagher): see rationale above. pex_builder.add_resource(dereference_symlinks(src), dst) # Add prebuilt libraries listed in the manifest. for req in manifest.get('prebuiltLibraries', []): try: pex_builder.add_dist_location(req) except Exception as err: raise Exception("Failed to add {}: {}".format(req, err)) # TODO(mikekap): Do something about manifest['nativeLibraries']. # Generate the PEX file. pex_builder.build(output)
def test_clp_requirements_txt(): parser, builder = configure_clp() options, _ = parser.parse_args(args='-r requirements1.txt -r requirements2.txt'.split()) assert options.requirement_files == ['requirements1.txt', 'requirements2.txt']
def convert_to_pex(unified_model, output_file_path: str) -> str: """ Convert the given unified model into an executable PEX file. # Arguments unified_model (UnifiedModel or str): Unified model instance or path to model file output_file_path (string): Path to save the model. # Returns Full path to the converted model """ # https://gist.github.com/simeonf/062af826e79259bc7686 log.info("Start conversion to PEX") if isinstance(unified_model, UnifiedModel): pass elif os.path.exists(str(unified_model)): # load model instance unified_model = UnifiedModel.load(str(unified_model)) else: raise Exception("Could not find model for: " + str(unified_model)) from git import Repo from pex.bin import pex model_temp_folder = tempfile.mkdtemp() model_instance_name = "model_instance" model_instance_folder = os.path.join(model_temp_folder, model_instance_name) os.makedirs(model_instance_folder) with open(os.path.join(model_instance_folder, "setup.py"), "w") as text_file: text_file.write("from distutils.core import setup " + "\nsetup(name='" + model_instance_name + "'," + "\n\tpackages=['" + model_instance_name + "']," + "\n\tversion='1.0'," + "\n\tpackage_data={'" + model_instance_name + "': ['" + model_instance_name + "']})") model_instance_package = os.path.join(model_instance_folder, model_instance_name) os.makedirs(model_instance_package) with open(os.path.join(model_instance_package, "__init__.py"), "w") as text_file: text_file.write( "import os, pkg_resources" + "\nfrom unified_model import cli_handler" + "\nos.environ[cli_handler.DEFAULT_MODEL_PATH_ENV] = pkg_resources.resource_filename(__name__, '" + model_instance_name + "')" + "\ncli_handler.cli()") with open(os.path.join(model_instance_package, "__main__.py"), "w") as text_file: text_file.write("") unified_model.save( os.path.join(model_instance_package, model_instance_name)) lib_repo_folder = os.path.join(model_temp_folder, "unified-model") Repo.clone_from(UNIFIED_MODEL_REPO_URL, lib_repo_folder) parser, resolver_options_builder = pex.configure_clp() args = [lib_repo_folder, model_instance_folder, "--disable-cache"] for req in unified_model._requirements: if isinstance(req, six.string_types): args.append(req) options, reqs = parser.parse_args(args=args) pex_builder = pex.build_pex(reqs, options, resolver_options_builder) pex_builder.set_entry_point(model_instance_name) pex_builder.build(output_file_path) shutil.rmtree(model_temp_folder) log.info("Conversion to PEX successful: " + output_file_path) return output_file_path
def test_clp_prereleases_resolver(): prerelease_dep = make_sdist(name='dep', version='1.2.3b1') with temporary_dir() as td: safe_copy(prerelease_dep, os.path.join(td, os.path.basename(prerelease_dep))) fetcher = Fetcher([td]) # When no specific options are specified, allow_prereleases is None parser, resolver_options_builder = configure_clp() assert resolver_options_builder._allow_prereleases is None # When we specify `--pre`, allow_prereleases is True options, reqs = parser.parse_args( args=['--pre', 'dep==1.2.3b1', 'dep']) assert resolver_options_builder._allow_prereleases # We need to use our own fetcher instead of PyPI resolver_options_builder._fetchers.insert(0, fetcher) ##### # The resolver created during processing of command line options (configure_clp) # is not actually passed into the API call (resolve_multi) from build_pex(). # Instead, resolve_multi() calls resolve() where a new ResolverOptionsBuilder instance # is created. The only way to supply our own fetcher to that new instance is to patch it # here in the test so that it can fetch our test package (dep-1.2.3b1). Hence, this class # below and the change in the `pex.resolver` module where the patched object resides. # import pex.resolver class BuilderWithFetcher(ResolverOptionsBuilder): def __init__(self, fetchers=None, allow_all_external=False, allow_external=None, allow_unverified=None, allow_prereleases=None, use_manylinux=None, precedence=None, context=None): super(BuilderWithFetcher, self).__init__(fetchers=fetchers, allow_all_external=allow_all_external, allow_external=allow_external, allow_unverified=allow_unverified, allow_prereleases=allow_prereleases, use_manylinux=None, precedence=precedence, context=context) self._fetchers.insert(0, fetcher) # end stub ##### # Without a corresponding fix in pex.py, this test failed for a dependency requirement of # dep==1.2.3b1 from one package and just dep (any version accepted) from another package. # The failure was an exit from build_pex() with the message: # # Could not satisfy all requirements for dep==1.2.3b1: # dep==1.2.3b1, dep # # With a correct behavior the assert line is reached and pex_builder object created. with mock.patch.object(pex.resolver, 'ResolverOptionsBuilder', BuilderWithFetcher): pex_builder = build_pex(reqs, options, resolver_options_builder) assert pex_builder is not None
def test_clp_constraints_txt(): parser, builder = configure_clp() options, _ = parser.parse_args(args='--constraint requirements1.txt'.split()) assert options.constraint_files == ['requirements1.txt']
def main(args=None): args = args[:] if args else sys.argv[1:] args = [pexbin.transform_legacy_arg(arg) for arg in args] pparser = pexbin.configure_clp() try: separator = args.index("--") args, cmdline = args[:separator], args[separator + 1:] except ValueError: args, cmdline = args, [] pparser.add_argument( "--manifest-file", dest="manifest_file", default=None, metavar="FILE", type=str, help="pex_wrapper manifest file.", ) poptions = pparser.parse_args(args=args) manifest_file = poptions.manifest_file manifest_text = open(manifest_file, 'r').read() manifest = parse_manifest(manifest_text) reqs = manifest.get('requirements', []) requirement_configuration = RequirementConfiguration(requirements=reqs) try: resolver_configuration = resolver_options.configure(poptions) except resolver_options.InvalidConfigurationError as e: die(str(e)) try: target_configuration = target_options.configure(poptions) except target_options.InterpreterNotFound as e: die(str(e)) except target_options.InterpreterConstraintsNotSatisfied as e: die(str(e), exit_code=pexbin.CANNOT_SETUP_INTERPRETER) with ENV.patch(PEX_VERBOSE=str(poptions.verbosity), PEX_ROOT=poptions.pex_root or ENV.PEX_ROOT): with TRACER.timed('Building pex'): pex_builder = pexbin.build_pex( requirement_configuration=requirement_configuration, resolver_configuration=resolver_configuration, target_configuration=target_configuration, options=poptions) # Add source files from the manifest for modmap in manifest.get('modules', []): src = modmap.get('src') dst = modmap.get('dest') # NOTE(agallagher): calls the `add_source` and `add_resource` below # hard-link the given source into the PEX temp dir. Since OS X and # Linux behave different when hard-linking a source that is a # symbolic link (Linux does *not* follow symlinks), resolve any # layers of symlinks here to get consistent behavior. try: pex_builder.add_source(dereference_symlinks(src), dst) except OSError as err: # Maybe we just can't use hardlinks? Try again. if not pex_builder._copy: pex_builder._copy = True pex_builder.add_source(dereference_symlinks(src), dst) else: raise RuntimeError("Failed to add %s: %s" % (src, err)) # Add resources from the manifest for reqmap in manifest.get('resources', []): src = reqmap.get('src') dst = reqmap.get('dest') pex_builder.add_source(dereference_symlinks(src), dst) # Add eggs/wheels from the manifest for egg in manifest.get('prebuiltLibraries', []): try: pex_builder.add_dist_location(egg) except Exception as err: raise RuntimeError("Failed to add %s: %s" % (egg, err)) # TODO(mikekap): Do something about manifest['nativeLibraries']. pexbin.log('Saving PEX file to %s' % poptions.pex_name, V=poptions.verbosity) tmp_name = poptions.pex_name + '~' safe_delete(tmp_name) pex_builder.build(tmp_name) shutil.move(tmp_name, poptions.pex_name)
def main(): # These are the options that this class will accept from the rule parser = optparse.OptionParser(usage="usage: %prog [options] output") parser.add_option('--entry-point', default='__main__') parser.add_option('--no-pypi', action='store_false', dest='pypi', default=True) parser.add_option('--disable-cache', action='store_true', dest='disable_cache', default=False) parser.add_option('--not-zip-safe', action='store_false', dest='zip_safe', default=True) parser.add_option('--python', default="/usr/bin/python2.7") parser.add_option('--find-links', dest='find_links', default='') options, args = parser.parse_args() if len(args) == 2: output = args[0] manifest_text = open(args[1], 'r').read() elif len(args) == 1: output = args[0] manifest_text = sys.stdin.read() else: parser.error("'output' positional argument is required") return 1 if manifest_text.startswith('"') and manifest_text.endswith('"'): manifest_text = manifest_text[1:len(manifest_text) - 1] # The manifest is passed via stdin, as it can sometimes get too large # to be passed as a CLA. with open(os.path.join(tempfile.mkdtemp(dir="/tmp"), "stderr"), "w") as x: x.write(manifest_text) manifest = parse_manifest(manifest_text) # Setup a temp dir that the PEX builder will use as its scratch dir. tmp_dir = tempfile.mkdtemp() try: # These are the options that pex will use pparser, resolver_options_builder = configure_clp() # Disabling wheels since the PyYAML wheel is incompatible with the Travis CI linux host. # Enabling Trace logging in pex/tracer.py shows this upon failure: # pex: Target package WheelPackage('file:///tmp/tmpR_gDlG/PyYAML-3.11-cp27-cp27mu-linux_x86_64.whl') # is not compatible with CPython-2.7.3 / linux-x86_64 poptions, preqs = pparser.parse_args(['--no-use-wheel'] + sys.argv) poptions.entry_point = options.entry_point poptions.find_links = options.find_links poptions.pypi = options.pypi poptions.python = options.python poptions.zip_safe = options.zip_safe poptions.disable_cache = options.disable_cache print("pex options: %s" % poptions) os.environ["PATH"] = ".:%s:/bin:/usr/bin" % poptions.python # The version of pkg_resources.py (from setuptools) on some distros is too old for PEX. So # we keep a recent version in and force it into the process by constructing a custom # PythonInterpreter instance using it. interpreter = PythonInterpreter( poptions.python, PythonInterpreter.from_binary(options.python).identity, extras={ # TODO: Fix this to resolve automatically ('setuptools', '18.0.1'): 'third_party/pex/setuptools-18.0.1-py2.py3-none-any.whl', ('wheel', '0.23.0'): 'third_party/pex/wheel-0.23.0-py2.7.egg' }) # resolve setuptools interpreter = resolve_or_die(interpreter, SETUPTOOLS_REQUIREMENT, poptions) # possibly resolve wheel if interpreter and poptions.use_wheel: interpreter = resolve_or_die(interpreter, WHEEL_REQUIREMENT, poptions) # Add prebuilt libraries listed in the manifest. reqs = manifest.get('requirements', {}).keys() if len(reqs) > 0: print("pex requirements: %s" % reqs) pex_builder = build_pex(reqs, poptions, resolver_options_builder, interpreter=interpreter) # Set whether this PEX as zip-safe, meaning everything will stayed zipped up # and we'll rely on python's zip-import mechanism to load modules from # the PEX. This may not work in some situations (e.g. native # libraries, libraries that want to find resources via the FS). pex_builder.info.zip_safe = options.zip_safe # Set the starting point for this PEX. pex_builder.info.entry_point = options.entry_point pex_builder.add_source( dereference_symlinks(pkg_resources_py), os.path.join(pex_builder.BOOTSTRAP_DIR, 'pkg_resources.py')) # Add the sources listed in the manifest. for dst, src in manifest['modules'].iteritems(): # NOTE(agallagher): calls the `add_source` and `add_resource` below # hard-link the given source into the PEX temp dir. Since OS X and # Linux behave different when hard-linking a source that is a # symbolic link (Linux does *not* follow symlinks), resolve any # layers of symlinks here to get consistent behavior. try: pex_builder.add_source(dereference_symlinks(src), dst) except OSError as e: raise Exception("Failed to add {}: {}".format(src, e)) # Add resources listed in the manifest. for dst, src in manifest['resources'].iteritems(): # NOTE(agallagher): see rationale above. pex_builder.add_resource(dereference_symlinks(src), dst) # Add prebuilt libraries listed in the manifest. for req in manifest.get('prebuiltLibraries', []): try: pex_builder.add_dist_location(req) except Exception as e: raise Exception("Failed to add {}: {}".format(req, e)) # TODO(mikekap): Do something about manifest['nativeLibraries']. # Generate the PEX file. pex_builder.build(output) # Always try cleaning up the scratch dir, ignoring failures. finally: shutil.rmtree(tmp_dir, True)
def test_clp_constraints_txt(): parser = configure_clp() options, _ = parser.parse_args(args="--constraint requirements1.txt".split()) assert options.constraint_files == ["requirements1.txt"]
def main(): pparser, resolver_options_builder = pexbin.configure_clp() poptions, args = pparser.parse_args(sys.argv) manifest_file = args[1] manifest_text = open(manifest_file, 'r').read() manifest = parse_manifest(manifest_text) if poptions.pex_root: ENV.set('PEX_ROOT', poptions.pex_root) else: poptions.pex_root = ENV.PEX_ROOT if poptions.cache_dir: poptions.cache_dir = pexbin.make_relative_to_root(poptions.cache_dir) poptions.interpreter_cache_dir = pexbin.make_relative_to_root( poptions.interpreter_cache_dir) reqs = manifest.get('requirements', []) with ENV.patch(PEX_VERBOSE=str(poptions.verbosity)): with TRACER.timed('Building pex'): pex_builder = pexbin.build_pex(reqs, poptions, resolver_options_builder) # Add source files from the manifest for modmap in manifest.get('modules', []): src = modmap.get('src') dst = modmap.get('dest') # NOTE(agallagher): calls the `add_source` and `add_resource` below # hard-link the given source into the PEX temp dir. Since OS X and # Linux behave different when hard-linking a source that is a # symbolic link (Linux does *not* follow symlinks), resolve any # layers of symlinks here to get consistent behavior. try: pex_builder.add_source(dereference_symlinks(src), dst) except OSError as err: # Maybe we just can't use hardlinks? Try again. if not pex_builder._copy: pex_builder._copy = True pex_builder.add_source(dereference_symlinks(src), dst) else: raise RuntimeError("Failed to add %s: %s" % (src, err)) # Add resources from the manifest for reqmap in manifest.get('resources', []): src = reqmap.get('src') dst = reqmap.get('dest') pex_builder.add_resource(dereference_symlinks(src), dst) # Add eggs/wheels from the manifest for egg in manifest.get('prebuiltLibraries', []): try: pex_builder.add_dist_location(egg) except Exception as err: raise RuntimeError("Failed to add %s: %s" % (egg, err)) # TODO(mikekap): Do something about manifest['nativeLibraries']. pexbin.log('Saving PEX file to %s' % poptions.pex_name, v=poptions.verbosity) tmp_name = poptions.pex_name + '~' safe_delete(tmp_name) pex_builder.build(tmp_name) os.rename(tmp_name, poptions.pex_name)