def __init__(self,
address=None,
sources=None,
resources=None, # Old-style resources (file list, Fileset).
resource_targets=None, # New-style resources (Resources target specs).
provides=None,
compatibility=None,
**kwargs):
payload = PythonPayload(sources_rel_path=address.spec_path,
sources=sources or [],
resources=resources)
super(PythonTarget, self).__init__(address=address, payload=payload, **kwargs)
self._resource_target_specs = resource_targets
self.add_labels('python')
self._synthetic_resources_target = None
if provides and not isinstance(provides, PythonArtifact):
raise TargetDefinitionException(self,
"Target must provide a valid pants setup_py object. Received a '%s' object instead." %
provides.__class__.__name__)
self._provides = provides
self._compatibility = maybe_list(compatibility or ())
# Check that the compatibility requirements are well-formed.
for req in self._compatibility:
try:
PythonIdentity.parse_requirement(req)
except ValueError as e:
raise TargetDefinitionException(self, str(e))
def __init__(self,
address=None,
payload=None,
sources=None,
resources=None, # Old-style resources (file list, Fileset).
resource_targets=None, # New-style resources (Resources target specs).
provides=None,
compatibility=None,
**kwargs):
"""
:param dependencies: Other targets that this target depends on.
These dependencies may
be ``python_library``-like targets (``python_library``,
``python_thrift_library``, ``python_antlr_library`` and so forth) or
``python_requirement_library`` targets.
:type dependencies: List of target specs
:param sources: Files to "include". Paths are relative to the
BUILD file's directory.
:type sources: ``Fileset`` or list of strings
:param resources: non-Python resources, e.g. templates, keys, other data
(it is
recommended that your application uses the pkgutil package to access these
resources in a .zip-module friendly way.)
:param provides:
The `setup_py <#setup_py>`_ to publish that represents this
target outside the repo.
:param compatibility: either a string or list of strings that represents
interpreter compatibility for this target, using the Requirement-style
format, e.g. ``'CPython>=3', or just ['>=2.7','<3']`` for requirements
agnostic to interpreter class.
"""
self.address = address
payload = payload or Payload()
payload.add_fields({
'sources': self.create_sources_field(sources, address.spec_path, key_arg='sources'),
'resources': self.create_sources_field(resources, address.spec_path, key_arg='resources'),
'provides': provides,
'compatibility': PrimitiveField(maybe_list(compatibility or ())),
})
super(PythonTarget, self).__init__(address=address,
payload=payload,
**kwargs)
self._resource_target_specs = resource_targets
self.add_labels('python')
self._synthetic_resources_target = None
if provides and not isinstance(provides, PythonArtifact):
raise TargetDefinitionException(self,
"Target must provide a valid pants setup_py object. Received a '{}' object instead.".format(
provides.__class__.__name__))
self._provides = provides
# Check that the compatibility requirements are well-formed.
for req in self.payload.compatibility:
try:
PythonIdentity.parse_requirement(req)
except ValueError as e:
raise TargetDefinitionException(self, str(e))
def __init__(self,
address=None,
payload=None,
sources=None,
resources=None, # Old-style resources (file list, Fileset).
resource_targets=None, # New-style resources (Resources target specs).
provides=None,
compatibility=None,
**kwargs):
"""
:param dependencies: Other targets that this target depends on.
These dependencies may
be ``python_library``-like targets (``python_library``,
``python_thrift_library``, ``python_antlr_library`` and so forth) or
``python_requirement_library`` targets.
:type dependencies: List of target specs
:param sources: Files to "include". Paths are relative to the
BUILD file's directory.
:type sources: ``Fileset`` or list of strings
:param resources: non-Python resources, e.g. templates, keys, other data
(it is
recommended that your application uses the pkgutil package to access these
resources in a .zip-module friendly way.)
:param provides:
The `setup_py <#setup_py>`_ to publish that represents this
target outside the repo.
:param compatibility: either a string or list of strings that represents
interpreter compatibility for this target, using the Requirement-style
format, e.g. ``'CPython>=3', or just ['>=2.7','<3']`` for requirements
agnostic to interpreter class.
"""
self.address = address
payload = payload or Payload()
payload.add_fields({
'sources': self.create_sources_field(sources, address.spec_path, key_arg='sources'),
'resources': self.create_sources_field(resources, address.spec_path, key_arg='resources'),
'provides': provides,
'compatibility': PrimitiveField(maybe_list(compatibility or ())),
})
super(PythonTarget, self).__init__(address=address,
payload=payload,
**kwargs)
self._resource_target_specs = resource_targets
self.add_labels('python')
if provides and not isinstance(provides, PythonArtifact):
raise TargetDefinitionException(self,
"Target must provide a valid pants setup_py object. Received a '{}' object instead.".format(
provides.__class__.__name__))
self._provides = provides
# Check that the compatibility requirements are well-formed.
for req in self.payload.compatibility:
try:
PythonIdentity.parse_requirement(req)
except ValueError as e:
raise TargetDefinitionException(self, str(e))
def __init__(self,
address=None,
payload=None,
sources=None,
provides=None,
compatibility=None,
**kwargs):
"""
:param dependencies: The addresses of targets that this target depends on.
These dependencies may
be ``python_library``-like targets (``python_library``,
``python_thrift_library``, ``python_antlr_library`` and so forth) or
``python_requirement_library`` targets.
:type dependencies: list of strings
:param sources: Files to "include". Paths are relative to the
BUILD file's directory.
:type sources: ``Fileset`` or list of strings
:param provides:
The `setup_py <#setup_py>`_ to publish that represents this
target outside the repo.
:param compatibility: either a string that represents interpreter compatibility for this target
using the Requirement-style format, e.g. ``'CPython>=2.7,<3'`` (Select a CPython interpreter
with version ``>=2.7`` AND version ``<3``) or a list of Requirement-style strings which will
be OR'ed together. If the compatibility requirement is agnostic to interpreter class, using
the example above, a Requirement-style compatibility constraint like '>=2.7,<3' (N.B.: not
prefixed with CPython) can be used.
"""
self.address = address
payload = payload or Payload()
payload.add_fields({
'sources':
self.create_sources_field(sources,
address.spec_path,
key_arg='sources'),
'provides':
provides,
'compatibility':
PrimitiveField(maybe_list(compatibility or ())),
})
super(PythonTarget, self).__init__(address=address,
payload=payload,
**kwargs)
if provides and not isinstance(provides, PythonArtifact):
raise TargetDefinitionException(
self,
"Target must provide a valid pants setup_py object. Received a '{}' object instead."
.format(provides.__class__.__name__))
self._provides = provides
# Check that the compatibility requirements are well-formed.
for req in self.payload.compatibility:
try:
PythonIdentity.parse_requirement(req)
except ValueError as e:
raise TargetDefinitionException(self, str(e))
def validate_constraints(constraints):
# TODO: add check to see if constraints are mutually exclusive (bad) so no time is wasted:
# https://github.com/pantsbuild/pex/issues/432
for req in constraints:
# Check that the compatibility requirements are well-formed.
try:
PythonIdentity.parse_requirement(req)
except ValueError as e:
die("Compatibility requirements are not formatted properly: %s" % str(e))
def validate_constraints(constraints):
# TODO: add check to see if constraints are mutually exclusive (bad) so no time is wasted:
# https://github.com/pantsbuild/pex/issues/432
for req in constraints:
# Check that the compatibility requirements are well-formed.
try:
PythonIdentity.parse_requirement(req)
except ValueError as e:
die("Compatibility requirements are not formatted properly: %s" % str(e))
def __init__(
self,
address=None,
sources=None,
resources=None, # Old-style resources (file list, Fileset).
resource_targets=None, # New-style resources (Resources target specs).
provides=None,
compatibility=None,
**kwargs):
"""
:param dependencies: Other targets that this target depends on.
These dependencies may
be ``python_library``-like targets (``python_library``,
``python_thrift_library``, ``python_antlr_library`` and so forth) or
``python_requirement_library`` targets.
:type dependencies: List of target specs
:param resources: non-Python resources, e.g. templates, keys, other data
(it is
recommended that your application uses the pkgutil package to access these
resources in a .zip-module friendly way.)
:param provides:
The :ref:`setup_py <bdict_setup_py>` to publish that represents this
target outside the repo.
:param compatibility: either a string or list of strings that represents
interpreter compatibility for this target, using the Requirement-style
format, e.g. ``'CPython>=3', or just ['>=2.7','<3']`` for requirements
agnostic to interpreter class.
"""
payload = PythonPayload(sources_rel_path=address.spec_path,
sources=sources or [],
resources=resources)
super(PythonTarget, self).__init__(address=address,
payload=payload,
**kwargs)
self._resource_target_specs = resource_targets
self.add_labels('python')
self._synthetic_resources_target = None
if provides and not isinstance(provides, PythonArtifact):
raise TargetDefinitionException(
self,
"Target must provide a valid pants setup_py object. Received a '%s' object instead."
% provides.__class__.__name__)
self._provides = provides
self._compatibility = maybe_list(compatibility or ())
# Check that the compatibility requirements are well-formed.
for req in self._compatibility:
try:
PythonIdentity.parse_requirement(req)
except ValueError as e:
raise TargetDefinitionException(self, str(e))
def __init__(self,
address=None,
payload=None,
sources=None,
provides=None,
compatibility=None,
**kwargs):
"""
:param dependencies: The addresses of targets that this target depends on.
These dependencies may
be ``python_library``-like targets (``python_library``,
``python_thrift_library``, ``python_antlr_library`` and so forth) or
``python_requirement_library`` targets.
:type dependencies: list of strings
:param sources: Files to "include". Paths are relative to the
BUILD file's directory.
:type sources: ``EagerFilesetWithSpec``
:param provides:
The `setup_py <#setup_py>`_ to publish that represents this
target outside the repo.
:param compatibility: either a string that represents interpreter compatibility for this target
using the Requirement-style format, e.g. ``'CPython>=2.7,<3'`` (Select a CPython interpreter
with version ``>=2.7`` AND version ``<3``) or a list of Requirement-style strings which will
be OR'ed together. If the compatibility requirement is agnostic to interpreter class, using
the example above, a Requirement-style compatibility constraint like '>=2.7,<3' (N.B.: not
prefixed with CPython) can be used.
"""
self.address = address
payload = payload or Payload()
payload.add_fields({
'sources': self.create_sources_field(sources, address.spec_path, key_arg='sources'),
'provides': provides,
'compatibility': PrimitiveField(maybe_list(compatibility or ())),
})
super(PythonTarget, self).__init__(address=address, payload=payload, **kwargs)
if provides and not isinstance(provides, PythonArtifact):
raise TargetDefinitionException(self,
"Target must provide a valid pants setup_py object. Received a '{}' object instead.".format(
provides.__class__.__name__))
self._provides = provides
# Check that the compatibility requirements are well-formed.
for req in self.payload.compatibility:
try:
PythonIdentity.parse_requirement(req)
except ValueError as e:
raise TargetDefinitionException(self, str(e))
def __init__(self,
address=None,
payload=None,
sources=None,
compatibility=None,
setup_requires=None,
**kwargs):
"""
:param address: The Address that maps to this Target in the BuildGraph.
:type address: :class:`pants.build_graph.address.Address`
:param payload: The configuration encapsulated by this target. Also in charge of most
fingerprinting details.
:type payload: :class:`pants.base.payload.Payload`
:param sources: Files to "include". Paths are relative to the
BUILD file's directory.
:type sources: ``Fileset`` or list of strings. Must include setup.py.
:param compatibility: either a string or list of strings that represents
interpreter compatibility for this target, using the Requirement-style
format, e.g. ``'CPython>=3', or just ['>=2.7','<3']`` for requirements
agnostic to interpreter class.
"""
payload = payload or Payload()
payload.add_fields({
'sources':
self.create_sources_field(sources,
address.spec_path,
key_arg='sources'),
'compatibility':
PrimitiveField(maybe_list(compatibility or ())),
'setup_requires':
PrimitiveField(maybe_list(setup_requires or ()))
})
super(PythonDistribution, self).__init__(address=address,
payload=payload,
**kwargs)
if not 'setup.py' in sources:
raise TargetDefinitionException(
self,
'A setup.py in the top-level directory relative to the target definition is required.'
)
# Check that the compatibility requirements are well-formed.
for req in self.payload.compatibility:
try:
PythonIdentity.parse_requirement(req)
except ValueError as e:
raise TargetDefinitionException(self, str(e))
def __init__(self, root_dir, relpath=None, must_exist=True):
"""Creates a BuildFile object representing the BUILD file set at the specified path.
:param string root_dir: The base directory of the project
:param string relpath: The path relative to root_dir where the BUILD file is found - this can either point
directly at the BUILD file or else to a directory which contains BUILD files
:param bool must_exist: If True, the specified BUILD file must exist or else an IOError is thrown
:raises IOError: if the root_dir path is not absolute
:raises MissingBuildFileError: if the path does not house a BUILD file and must_exist is True
"""
if not os.path.isabs(root_dir):
raise self.InvalidRootDirError('BuildFile root_dir {root_dir} must be an absolute path.'
.format(root_dir=root_dir))
path = os.path.join(root_dir, relpath) if relpath else root_dir
self._build_basename = BuildFile._BUILD_FILE_PREFIX
buildfile = os.path.join(path, self._build_basename) if os.path.isdir(path) else path
if must_exist:
# If the build file must exist then we want to make sure it's not a dir.
# In other cases we are ok with it being a dir, for example someone might have
# repo/scripts/build/doit.sh.
if os.path.isdir(buildfile):
raise self.MissingBuildFileError(
'Path to buildfile ({buildfile}) is a directory, but it must be a file.'
.format(buildfile=buildfile))
if not os.path.exists(os.path.dirname(buildfile)):
raise self.MissingBuildFileError('Path to BUILD file does not exist at: {path}'
.format(path=os.path.dirname(buildfile)))
# There is no BUILD file without a prefix so select any viable sibling
if not os.path.exists(buildfile) or os.path.isdir(buildfile):
for build in BuildFile._get_all_build_files(os.path.dirname(buildfile)):
self._build_basename = build
buildfile = os.path.join(path, self._build_basename)
break
if must_exist:
if not os.path.exists(buildfile):
raise self.MissingBuildFileError('BUILD file does not exist at: {path}'
.format(path=buildfile))
if not BuildFile._is_buildfile_name(os.path.basename(buildfile)):
raise self.MissingBuildFileError('{path} is not a BUILD file'
.format(path=buildfile))
self.root_dir = os.path.realpath(root_dir)
self.full_path = os.path.realpath(buildfile)
self.name = os.path.basename(self.full_path)
self.parent_path = os.path.dirname(self.full_path)
self._bytecode_path = os.path.join(self.parent_path,
'.{name}.{ident}.pyc'.format(name=self.name,
ident=PythonIdentity.get()))
self.relpath = os.path.relpath(self.full_path, self.root_dir)
self.spec_path = os.path.dirname(self.relpath)
def execute(self):
interpreter = None
python_tgts = self.context.targets(
lambda tgt: isinstance(tgt, PythonTarget))
fs = PythonInterpreterFingerprintStrategy(task=self)
with self.invalidated(python_tgts,
fingerprint_strategy=fs) as invalidation_check:
# If there are no relevant targets, we still go through the motions of selecting
# an interpreter, to prevent downstream tasks from having to check for this special case.
if invalidation_check.all_vts:
target_set_id = VersionedTargetSet.from_versioned_targets(
invalidation_check.all_vts).cache_key.hash
else:
target_set_id = 'no_targets'
interpreter_path_file = os.path.join(self.workdir, target_set_id,
'interpreter.path')
if not os.path.exists(interpreter_path_file):
interpreter_cache = PythonInterpreterCache(
PythonSetup.global_instance(),
PythonRepos.global_instance(),
logger=self.context.log.debug)
# We filter the interpreter cache itself (and not just the interpreters we pull from it)
# because setting up some python versions (e.g., 3<=python<3.3) crashes, and this gives us
# an escape hatch.
filters = self.get_options().constraints or [b'']
# Cache setup's requirement fetching can hang if run concurrently by another pants proc.
self.context.acquire_lock()
try:
interpreter_cache.setup(filters=filters)
finally:
self.context.release_lock()
interpreter = interpreter_cache.select_interpreter_for_targets(
python_tgts)
safe_mkdir_for(interpreter_path_file)
with open(interpreter_path_file, 'w') as outfile:
outfile.write(b'{}\t{}\n'.format(
interpreter.binary, str(interpreter.identity)))
for dist, location in interpreter.extras.items():
dist_name, dist_version = dist
outfile.write(b'{}\t{}\t{}\n'.format(
dist_name, dist_version, location))
if not interpreter:
with open(interpreter_path_file, 'r') as infile:
lines = infile.readlines()
binary, identity = lines[0].strip().split('\t')
extras = {}
for line in lines[1:]:
dist_name, dist_version, location = line.strip().split(
'\t')
extras[(dist_name, dist_version)] = location
interpreter = PythonInterpreter(binary,
PythonIdentity.from_path(identity),
extras)
self.context.products.get_data(PythonInterpreter, lambda: interpreter)
def _get_interpreter(interpreter_path_file):
with open(interpreter_path_file, 'r') as infile:
lines = infile.readlines()
binary, identity = lines[0].strip().split('\t')
extras = {}
for line in lines[1:]:
dist_name, dist_version, location = line.strip().split('\t')
extras[(dist_name, dist_version)] = location
return PythonInterpreter(binary, PythonIdentity.from_path(identity), extras)
def _get_interpreter(interpreter_path_file):
with open(interpreter_path_file, 'r') as infile:
lines = infile.readlines()
binary, identity = lines[0].strip().split('\t')
extras = {}
for line in lines[1:]:
dist_name, dist_version, location = line.strip().split('\t')
extras[(dist_name, dist_version)] = location
return PythonInterpreter(binary, PythonIdentity.from_path(identity),
extras)
def _interpreter_from_path(self, path, filters):
interpreter_dir = os.path.basename(path)
identity = PythonIdentity.from_path(interpreter_dir)
try:
executable = os.readlink(os.path.join(path, 'python'))
except OSError:
return None
interpreter = PythonInterpreter(executable, identity)
if self._matches(interpreter, filters):
return self._resolve(interpreter)
return None
def test_iter_supported_tags():
identity = PythonIdentity('CPython', 2, 6, 5)
platform = 'linux-x86_64'
def iter_solutions():
for interp in ('cp', 'py'):
for interp_suffix in ('2', '20', '21', '22', '23', '24', '25', '26'):
for platform in ('linux_x86_64', 'any'):
yield (interp + interp_suffix, 'none', platform)
assert set(PEP425.iter_supported_tags(identity, platform)) == set(iter_solutions())
def _interpreter_from_path(self, path, filters):
interpreter_dir = os.path.basename(path)
identity = PythonIdentity.from_path(interpreter_dir)
try:
executable = os.readlink(os.path.join(path, 'python'))
except OSError:
return None
interpreter = PythonInterpreter(executable, identity)
if self._matches(interpreter, filters):
return self._resolve(interpreter)
return None
def __init__(self,
address=None,
payload=None,
sources=None,
compatibility=None,
setup_requires=None,
**kwargs):
"""
:param address: The Address that maps to this Target in the BuildGraph.
:type address: :class:`pants.build_graph.address.Address`
:param payload: The configuration encapsulated by this target. Also in charge of most
fingerprinting details.
:type payload: :class:`pants.base.payload.Payload`
:param sources: Files to "include". Paths are relative to the
BUILD file's directory.
:type sources: ``Fileset`` or list of strings. Must include setup.py.
:param compatibility: either a string or list of strings that represents
interpreter compatibility for this target, using the Requirement-style
format, e.g. ``'CPython>=3', or just ['>=2.7','<3']`` for requirements
agnostic to interpreter class.
"""
payload = payload or Payload()
payload.add_fields({
'sources': self.create_sources_field(sources, address.spec_path, key_arg='sources'),
'compatibility': PrimitiveField(maybe_list(compatibility or ())),
'setup_requires': PrimitiveField(maybe_list(setup_requires or ()))
})
super(PythonDistribution, self).__init__(address=address, payload=payload, **kwargs)
if not 'setup.py' in sources:
raise TargetDefinitionException(
self, 'A setup.py in the top-level directory relative to the target definition is required.'
)
# Check that the compatibility requirements are well-formed.
for req in self.payload.compatibility:
try:
PythonIdentity.parse_requirement(req)
except ValueError as e:
raise TargetDefinitionException(self, str(e))
def execute(self):
interpreter = None
python_tgts = self.context.targets(lambda tgt: isinstance(tgt, PythonTarget))
fs = PythonInterpreterFingerprintStrategy(task=self)
with self.invalidated(python_tgts, fingerprint_strategy=fs) as invalidation_check:
# If there are no relevant targets, we still go through the motions of selecting
# an interpreter, to prevent downstream tasks from having to check for this special case.
if invalidation_check.all_vts:
target_set_id = VersionedTargetSet.from_versioned_targets(
invalidation_check.all_vts).cache_key.hash
else:
target_set_id = 'no_targets'
interpreter_path_file = os.path.join(self.workdir, target_set_id, 'interpreter.path')
if not os.path.exists(interpreter_path_file):
interpreter_cache = PythonInterpreterCache(PythonSetup.global_instance(),
PythonRepos.global_instance(),
logger=self.context.log.debug)
# We filter the interpreter cache itself (and not just the interpreters we pull from it)
# because setting up some python versions (e.g., 3<=python<3.3) crashes, and this gives us
# an escape hatch.
filters = self.get_options().constraints or [b'']
# Cache setup's requirement fetching can hang if run concurrently by another pants proc.
self.context.acquire_lock()
try:
interpreter_cache.setup(filters=filters)
finally:
self.context.release_lock()
interpreter = interpreter_cache.select_interpreter_for_targets(python_tgts)
safe_mkdir_for(interpreter_path_file)
with open(interpreter_path_file, 'w') as outfile:
outfile.write(b'{}\t{}\n'.format(interpreter.binary, str(interpreter.identity)))
for dist, location in interpreter.extras.items():
dist_name, dist_version = dist
outfile.write(b'{}\t{}\t{}\n'.format(dist_name, dist_version, location))
if not interpreter:
with open(interpreter_path_file, 'r') as infile:
lines = infile.readlines()
binary, identity = lines[0].strip().split('\t')
extras = {}
for line in lines[1:]:
dist_name, dist_version, location = line.strip().split('\t')
extras[(dist_name, dist_version)] = location
interpreter = PythonInterpreter(binary, PythonIdentity.from_path(identity), extras)
self.context.products.get_data(PythonInterpreter, lambda: interpreter)
def test_iter_supported_tags():
identity = PythonIdentity('CPython', 2, 6, 5)
platform = 'linux-x86_64'
def iter_solutions():
for interp in ('cp', 'py'):
for interp_suffix in ('2', '20', '21', '22', '23', '24', '25', '26'):
for platform in ('linux_x86_64', 'any'):
abis = ['none']
if interp == 'cp' and interp_suffix == '26' and platform == 'linux_x86_64':
abis.extend([
'cp%s' % interp_suffix,
'cp%sdmu' % interp_suffix, 'cp%sdm' % interp_suffix,
'cp%sdu' % interp_suffix, 'cp%sd' % interp_suffix,
'cp%smu' % interp_suffix, 'cp%sm' % interp_suffix,
'cp%su' % interp_suffix
])
for abi in abis:
yield (interp + interp_suffix, abi, platform)
assert set(PEP425.iter_supported_tags(identity, platform)) == set(iter_solutions())
def main():
parser = optparse.OptionParser(usage="usage: %prog [options] output")
parser.add_option("--entry-point", default="__main__")
parser.add_option("--directory", action="store_true", default=False)
parser.add_option(
"--no-zip-safe", action="store_false", dest="zip_safe", default=True
)
parser.add_option("--python", default="")
parser.add_option("--python-version", default="")
parser.add_option("--python-shebang", default=None)
parser.add_option("--preload", action="append", default=[])
options, args = parser.parse_args()
if len(args) == 1:
output = args[0]
else:
parser.error("'output' positional argument is required")
return 1
# The manifest is passed via stdin, as it can sometimes get too large
# to be passed as a CLA.
manifest = json.load(sys.stdin)
# The version of pkg_resources.py (from setuptools) on some distros is
# too old for PEX. So we keep a recent version in the buck repo and
# force it into the process by constructing a custom PythonInterpreter
# instance using it.
if not options.python:
options.python = sys.executable
identity = PythonIdentity.get()
elif not options.python_version:
# Note: this is expensive (~500ms). prefer passing --python-version when possible.
identity = PythonInterpreter.from_binary(options.python).identity
else:
# Convert "CPython 2.7" to "CPython 2 7 0"
python_version = options.python_version.replace(".", " ").split()
if len(python_version) == 3:
python_version.append("0")
identity = PythonIdentity.from_id_string(" ".join(python_version))
interpreter = PythonInterpreter(options.python, identity, extras={})
pex_builder = PEXBuilder(
path=output if options.directory else None, interpreter=interpreter
)
if options.python_shebang is not None:
pex_builder.set_shebang(options.python_shebang)
# Set whether this PEX as zip-safe, meaning everything will stayed zipped up
# and we'll rely on python's zip-import mechanism to load modules from
# the PEX. This may not work in some situations (e.g. native
# libraries, libraries that want to find resources via the FS).
pex_builder.info.zip_safe = options.zip_safe
# Set the starting point for this PEX.
pex_builder.info.entry_point = options.entry_point
# Copy in our version of `pkg_resources` & `_markerlib`.
copy_package(pex_builder, "pkg_resources", prefix=pex_builder.BOOTSTRAP_DIR)
copy_package(pex_builder, "_markerlib", prefix=pex_builder.BOOTSTRAP_DIR)
# Add the sources listed in the manifest.
for dst, src in manifest["modules"].iteritems():
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as e:
raise Exception("Failed to add {}: {}".format(src, e))
# Add resources listed in the manifest.
for dst, src in manifest["resources"].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Add resources listed in the manifest.
for dst, src in manifest["nativeLibraries"].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
if options.directory:
pex_builder.freeze(code_hash=False, bytecode_compile=False)
else:
pex_builder.build(output)
def __init__(self, root_dir, relpath=None, must_exist=True):
"""Creates a BuildFile object representing the BUILD file set at the specified path.
:param string root_dir: The base directory of the project
:param string relpath: The path relative to root_dir where the BUILD file is found - this can either point
directly at the BUILD file or else to a directory which contains BUILD files
:param bool must_exist: If True, the specified BUILD file must exist or else an IOError is thrown
:raises IOError: if the root_dir path is not absolute
:raises MissingBuildFileError: if the path does not house a BUILD file and must_exist is True
"""
if not os.path.isabs(root_dir):
raise self.InvalidRootDirError(
'BuildFile root_dir {root_dir} must be an absolute path.'.
format(root_dir=root_dir))
path = os.path.join(root_dir, relpath) if relpath else root_dir
self._build_basename = BuildFile._BUILD_FILE_PREFIX
buildfile = os.path.join(
path, self._build_basename) if os.path.isdir(path) else path
if must_exist:
# If the build file must exist then we want to make sure it's not a dir.
# In other cases we are ok with it being a dir, for example someone might have
# repo/scripts/build/doit.sh.
if os.path.isdir(buildfile):
raise self.MissingBuildFileError(
'Path to buildfile ({buildfile}) is a directory, but it must be a file.'
.format(buildfile=buildfile))
if not os.path.exists(os.path.dirname(buildfile)):
raise self.MissingBuildFileError(
'Path to BUILD file does not exist at: {path}'.format(
path=os.path.dirname(buildfile)))
# There is no BUILD file without a prefix so select any viable sibling
if not os.path.exists(buildfile) or os.path.isdir(buildfile):
for build in BuildFile._get_all_build_files(
os.path.dirname(buildfile)):
self._build_basename = build
buildfile = os.path.join(path, self._build_basename)
break
if must_exist:
if not os.path.exists(buildfile):
raise self.MissingBuildFileError(
'BUILD file does not exist at: {path}'.format(
path=buildfile))
if not BuildFile._is_buildfile_name(os.path.basename(buildfile)):
raise self.MissingBuildFileError(
'{path} is not a BUILD file'.format(path=buildfile))
self.root_dir = os.path.realpath(root_dir)
self.full_path = os.path.realpath(buildfile)
self.name = os.path.basename(self.full_path)
self.parent_path = os.path.dirname(self.full_path)
self._bytecode_path = os.path.join(
self.parent_path,
'.{name}.{ident}.pyc'.format(name=self.name,
ident=PythonIdentity.get()))
self.relpath = os.path.relpath(self.full_path, self.root_dir)
self.spec_path = os.path.dirname(self.relpath)
def _extract_sdist(
pex, # type: PEX
dest_dir, # type: str
):
# type: (...) -> None
chroot = safe_mkdtemp()
src = os.path.join(chroot, "src")
safe_mkdir(src)
excludes = ["__main__.py", "PEX-INFO"]
if zipfile.is_zipfile(pex.path()):
PEXEnvironment(pex.path()).explode_code(src, exclude=excludes)
else:
shutil.copytree(pex.path(), src, ignore=lambda _dir, _names: excludes)
pex_info = pex.pex_info()
name, _ = os.path.splitext(os.path.basename(pex.path()))
version = "0.0.0+{}".format(pex_info.code_hash)
zip_safe = pex_info.zip_safe
py_modules = [os.path.splitext(f)[0] for f in os.listdir(src) if f.endswith(".py")]
packages = [
os.path.relpath(os.path.join(root, d), src).replace(os.sep, ".")
for root, dirs, _ in os.walk(src)
for d in dirs
]
install_requires = [str(req) for req in pex_info.requirements]
python_requires = None
if len(pex_info.interpreter_constraints) == 1:
python_requires = str(
PythonIdentity.parse_requirement(pex_info.interpreter_constraints[0]).specifier
)
elif pex_info.interpreter_constraints:
pex_warnings.warn(
"Omitting `python_requires` for {name} sdist since {pex} has multiple "
"interpreter constraints:\n{interpreter_constraints}".format(
name=name,
pex=os.path.normpath(pex.path()),
interpreter_constraints="\n".join(
"{index}.) {constraint}".format(index=index, constraint=constraint)
for index, constraint in enumerate(
pex_info.interpreter_constraints, start=1
)
),
)
)
entry_points = []
if pex_info.entry_point and ":" in pex_info.entry_point:
entry_points = [(name, pex_info.entry_point)]
with open(os.path.join(chroot, "setup.cfg"), "w") as fp:
fp.write(
dedent(
"""\
[metadata]
name = {name}
version = {version}
[options]
zip_safe = {zip_safe}
{py_modules}
{packages}
package_dir =
=src
include_package_data = True
{python_requires}
{install_requires}
[options.entry_points]
{entry_points}
"""
).format(
name=name,
version=version,
zip_safe=zip_safe,
py_modules=(
"py_modules =\n {}".format("\n ".join(py_modules)) if py_modules else ""
),
packages=(
"packages = \n {}".format("\n ".join(packages)) if packages else ""
),
install_requires=(
"install_requires =\n {}".format("\n ".join(install_requires))
if install_requires
else ""
),
python_requires=(
"python_requires = {}".format(python_requires) if python_requires else ""
),
entry_points=(
"console_scripts =\n {}".format(
"\n ".join(
"{} = {}".format(name, entry_point)
for name, entry_point in entry_points
)
)
if entry_points
else ""
),
)
)
with open(os.path.join(chroot, "MANIFEST.in"), "w") as fp:
fp.write("recursive-include src *")
with open(os.path.join(chroot, "setup.py"), "w") as fp:
fp.write("import setuptools; setuptools.setup()")
spawn_python_job(
args=["setup.py", "sdist", "--dist-dir", dest_dir],
interpreter=pex.interpreter,
expose=["setuptools"],
cwd=chroot,
).wait()
def main():
parser = optparse.OptionParser(usage="usage: %prog [options] output")
parser.add_option("--entry-point", default="__main__")
parser.add_option("--directory", action="store_true", default=False)
parser.add_option(
"--no-zip-safe", action="store_false", dest="zip_safe", default=True
)
parser.add_option("--python", default="")
parser.add_option("--python-version", default="")
parser.add_option("--python-shebang", default=None)
parser.add_option("--preload", action="append", default=[])
options, args = parser.parse_args()
if len(args) == 1:
output = args[0]
else:
parser.error("'output' positional argument is required")
return 1
# The manifest is passed via stdin, as it can sometimes get too large
# to be passed as a CLA.
manifest = json.load(sys.stdin)
# The version of pkg_resources.py (from setuptools) on some distros is
# too old for PEX. So we keep a recent version in the buck repo and
# force it into the process by constructing a custom PythonInterpreter
# instance using it.
if not options.python:
options.python = sys.executable
identity = PythonIdentity.get()
elif not options.python_version:
# Note: this is expensive (~500ms). prefer passing --python-version when possible.
identity = PythonInterpreter.from_binary(options.python).identity
else:
# Convert "CPython 2.7" to "CPython 2 7 0"
python_version = options.python_version.replace(".", " ").split()
if len(python_version) == 3:
python_version.append("0")
identity = PythonIdentity.from_id_string(" ".join(python_version))
interpreter = PythonInterpreter(options.python, identity, extras={})
pex_builder = PEXBuilder(
path=output if options.directory else None, interpreter=interpreter
)
if options.python_shebang is not None:
pex_builder.set_shebang(options.python_shebang)
# Set whether this PEX as zip-safe, meaning everything will stayed zipped up
# and we'll rely on python's zip-import mechanism to load modules from
# the PEX. This may not work in some situations (e.g. native
# libraries, libraries that want to find resources via the FS).
pex_builder.info.zip_safe = options.zip_safe
# Set the starting point for this PEX.
pex_builder.info.entry_point = options.entry_point
# Copy in our version of `pkg_resources` & `_markerlib`.
copy_package(pex_builder, "pkg_resources", prefix=pex_builder.BOOTSTRAP_DIR)
copy_package(pex_builder, "_markerlib", prefix=pex_builder.BOOTSTRAP_DIR)
# Add the sources listed in the manifest.
for dst, src in manifest["modules"].items():
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as e:
raise Exception("Failed to add {}: {}".format(src, e))
# Add resources listed in the manifest.
for dst, src in manifest["resources"].items():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Add resources listed in the manifest.
for dst, src in manifest["nativeLibraries"].items():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
if options.directory:
pex_builder.freeze(code_hash=False, bytecode_compile=False)
else:
pex_builder.build(output)
def fake_interpreter(id_str):
return PythonInterpreter('/fake/binary', PythonIdentity.from_id_string(id_str))
def fake_interpreter(id_str):
return PythonInterpreter('/fake/binary', PythonIdentity.from_id_string(id_str))
