def execute(self):
dist_targets = self.context.targets(is_local_python_dist)
if dist_targets:
interpreter = self.context.products.get_data(PythonInterpreter)
shared_libs_product = self.context.products.get(SharedLibrary)
with self.invalidated(
dist_targets,
invalidate_dependents=True) as invalidation_check:
for vt in invalidation_check.invalid_vts:
self._prepare_and_create_dist(interpreter,
shared_libs_product, vt)
local_wheel_products = self.context.products.get(
'local_wheels')
for vt in invalidation_check.all_vts:
dist = self._get_whl_from_dir(vt.results_dir)
req_lib_addr = Address.parse('{}__req_lib'.format(
vt.target.address.spec))
self._inject_synthetic_dist_requirements(
dist, req_lib_addr)
# Make any target that depends on the dist depend on the synthetic req_lib,
# for downstream consumption.
for dependent in self.context.build_graph.dependents_of(
vt.target.address):
self.context.build_graph.inject_dependency(
dependent, req_lib_addr)
dist_dir, dist_base = split_basename_and_dirname(dist)
local_wheel_products.add(vt.target,
dist_dir).append(dist_base)
def _inject_synthetic_dist_requirements(self, dist, req_lib_addr):
"""Inject a synthetic requirements library that references a local wheel.
:param dist: Path of the locally built wheel to reference.
:param req_lib_addr: :class: `Address` to give to the synthetic target.
:return: a :class: `PythonRequirementLibrary` referencing the locally-built wheel.
"""
whl_dir, base = split_basename_and_dirname(dist)
whl_metadata = base.split('-')
req_name = '=='.join([whl_metadata[0], whl_metadata[1]])
req = PythonRequirement(req_name, repository=whl_dir)
self.context.build_graph.inject_synthetic_target(
req_lib_addr, PythonRequirementLibrary, requirements=[req])
def _inject_synthetic_dist_requirements(self, dist, req_lib_addr):
"""Inject a synthetic requirements library that references a local wheel.
:param dist: Path of the locally built wheel to reference.
:param req_lib_addr: :class: `Address` to give to the synthetic target.
:return: a :class: `PythonRequirementLibrary` referencing the locally-built wheel.
"""
whl_dir, base = split_basename_and_dirname(dist)
whl_metadata = base.split('-')
req_name = '=='.join([whl_metadata[0], whl_metadata[1]])
req = PythonRequirement(req_name, repository=whl_dir)
self.context.build_graph.inject_synthetic_target(req_lib_addr, PythonRequirementLibrary,
requirements=[req])
def _invoke_xz(self, xz_input_file):
"""Run the xz command and yield a file object for its stdout.
This allows streaming the decompressed tar archive directly into a tar decompression stream,
which is significantly faster in practice than making a temporary file.
"""
(xz_bin_dir,
xz_filename) = split_basename_and_dirname(self._xz_binary_path)
# TODO(cosmicexplorer): --threads=0 is supposed to use "the number of processor cores on the
# machine", but I see no more than 100% cpu used at any point. This seems like it could be a
# bug? If performance is an issue, investigate further.
cmd = [
xz_filename, '--decompress', '--stdout', '--keep', '--threads=0',
xz_input_file
]
env = {
# Isolate the path so we know we're using our provided version of xz.
'PATH': xz_bin_dir,
# Only allow our xz's lib directory to resolve the liblzma.so dependency at runtime.
'LD_LIBRARY_PATH': self._xz_library_path,
}
try:
# Pipe stderr to our own stderr, but leave stdout open so we can yield it.
process = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=sys.stderr,
env=env)
except OSError as e:
raise self.XZArchiverError(
"Error invoking xz with command {} and environment {} for input file {}: {}"
.format(cmd, env, xz_input_file, e), e)
# This is a file object.
yield process.stdout
rc = process.wait()
if rc != 0:
raise self.XZArchiverError(
"Error decompressing xz input with command {} and environment {} for input file {}. "
"Exit code was: {}. ".format(cmd, env, xz_input_file, rc))
def _invoke_xz(self, xz_input_file):
"""Run the xz command and yield a file object for its stdout.
This allows streaming the decompressed tar archive directly into a tar decompression stream,
which is significantly faster in practice than making a temporary file.
"""
(xz_bin_dir, xz_filename) = split_basename_and_dirname(self._xz_binary_path)
# TODO(cosmicexplorer): --threads=0 is supposed to use "the number of processor cores on the
# machine", but I see no more than 100% cpu used at any point. This seems like it could be a
# bug? If performance is an issue, investigate further.
cmd = [xz_filename, '--decompress', '--stdout', '--keep', '--threads=0', xz_input_file]
env = {
# Isolate the path so we know we're using our provided version of xz.
'PATH': xz_bin_dir,
# Only allow our xz's lib directory to resolve the liblzma.so dependency at runtime.
'LD_LIBRARY_PATH': self._xz_library_path,
}
try:
# Pipe stderr to our own stderr, but leave stdout open so we can yield it.
process = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=sys.stderr,
env=env)
except OSError as e:
raise self.XZArchiverError(
"Error invoking xz with command {} and environment {} for input file {}: {}"
.format(cmd, env, xz_input_file, e),
e)
# This is a file object.
yield process.stdout
rc = process.wait()
if rc != 0:
raise self.XZArchiverError(
"Error decompressing xz input with command {} and environment {} for input file {}. "
"Exit code was: {}. "
.format(cmd, env, xz_input_file, rc))
def execute(self):
dist_targets = self.context.targets(is_local_python_dist)
if dist_targets:
interpreter = self.context.products.get_data(PythonInterpreter)
shared_libs_product = self.context.products.get(SharedLibrary)
with self.invalidated(dist_targets, invalidate_dependents=True) as invalidation_check:
for vt in invalidation_check.invalid_vts:
self._prepare_and_create_dist(interpreter, shared_libs_product, vt)
local_wheel_products = self.context.products.get('local_wheels')
for vt in invalidation_check.all_vts:
dist = self._get_whl_from_dir(vt.results_dir)
req_lib_addr = Address.parse('{}__req_lib'.format(vt.target.address.spec))
self._inject_synthetic_dist_requirements(dist, req_lib_addr)
# Make any target that depends on the dist depend on the synthetic req_lib,
# for downstream consumption.
for dependent in self.context.build_graph.dependents_of(vt.target.address):
self.context.build_graph.inject_dependency(dependent, req_lib_addr)
dist_dir, dist_base = split_basename_and_dirname(dist)
local_wheel_products.add(vt.target, dist_dir).append(dist_base)
