class SetupPythonEnvironment(Task):
"""
Establishes the python intepreter(s) for downstream Python tasks e.g. Resolve, Run, PytestRun.
Populates the product namespace (for typename = 'python'):
'intepreters': ordered list of PythonInterpreter objects
"""
@classmethod
def setup_parser(cls, option_group, args, mkflag):
option_group.add_option(mkflag("force"), dest="python_setup_force",
action="store_true", default=False,
help="Force clean and install.")
option_group.add_option(mkflag("path"), dest="python_setup_paths",
action="append", default=[],
help="Add a path to search for interpreters, by default PATH.")
option_group.add_option(mkflag("interpreter"), dest="python_interpreter",
default=[], action='append',
help="Constrain what Python interpreters to use. Uses Requirement "
"format from pkg_resources, e.g. 'CPython>=2.6,<3' or 'PyPy'. "
"By default, no constraints are used. Multiple constraints may "
"be added. They will be ORed together.")
option_group.add_option(mkflag("multi"), dest="python_multi",
default=False, action='store_true',
help="Allow multiple interpreters to be bound to an upstream chroot.")
def __init__(self, context):
context.products.require('python')
self._cache = PythonInterpreterCache(context.config, logger=context.log.debug)
super(SetupPythonEnvironment, self).__init__(context)
def execute(self, _):
ifilters = self.context.options.python_interpreter
self._cache.setup(force=self.context.options.python_setup_force,
paths=self.context.options.python_setup_paths,
filters=ifilters or [''])
all_interpreters = set(self._cache.interpreters)
for target in self.context.targets(is_python_root):
self.context.log.info('Setting up interpreters for %s' % target)
closure = target.closure()
self.context.log.debug(' - Target closure: %d targets' % len(closure))
target_compatibilities = [
set(self._cache.matches(getattr(closure_target, 'compatibility', [''])))
for closure_target in closure]
target_compatibilities = reduce(set.intersection, target_compatibilities, all_interpreters)
self.context.log.debug(' - Target minimum compatibility: %s' % (
' '.join(interp.version_string for interp in target_compatibilities)))
interpreters = self._cache.select_interpreter(target_compatibilities,
allow_multiple=self.context.options.python_multi)
self.context.log.debug(' - Selected: %s' % interpreters)
if not interpreters:
raise TaskError('No compatible interpreters for %s' % target)
target.interpreters = interpreters
class SetupPythonEnvironment(Task):
"""
Establishes the python intepreter(s) for downstream Python tasks e.g. Resolve, Run, PytestRun.
Populates the product namespace (for typename = 'python'):
'intepreters': ordered list of PythonInterpreter objects
"""
@classmethod
def setup_parser(cls, option_group, args, mkflag):
option_group.add_option(mkflag("force"), dest="python_setup_force",
action="store_true", default=False,
help="Force clean and install.")
option_group.add_option(mkflag("path"), dest="python_setup_paths",
action="append", default=[],
help="Add a path to search for interpreters, by default PATH.")
option_group.add_option(mkflag("interpreter"), dest="python_interpreter",
default=[], action='append',
help="Constrain what Python interpreters to use. Uses Requirement "
"format from pkg_resources, e.g. 'CPython>=2.6,<3' or 'PyPy'. "
"By default, no constraints are used. Multiple constraints may "
"be added. They will be ORed together.")
option_group.add_option(mkflag("multi"), dest="python_multi",
default=False, action='store_true',
help="Allow multiple interpreters to be bound to an upstream chroot.")
def __init__(self, context, workdir):
context.products.require('python')
self._cache = PythonInterpreterCache(context.config, logger=context.log.debug)
super(SetupPythonEnvironment, self).__init__(context, workdir)
def execute(self, _):
ifilters = self.context.options.python_interpreter
self._cache.setup(force=self.context.options.python_setup_force,
paths=self.context.options.python_setup_paths,
filters=ifilters or [b''])
all_interpreters = set(self._cache.interpreters)
for target in self.context.targets(is_python_root):
self.context.log.info('Setting up interpreters for %s' % target)
closure = target.closure()
self.context.log.debug(' - Target closure: %d targets' % len(closure))
target_compatibilities = [
set(self._cache.matches(getattr(closure_target, 'compatibility', [''])))
for closure_target in closure]
target_compatibilities = reduce(set.intersection, target_compatibilities, all_interpreters)
self.context.log.debug(' - Target minimum compatibility: %s' % (
' '.join(interp.version_string for interp in target_compatibilities)))
interpreters = self._cache.select_interpreter(target_compatibilities,
allow_multiple=self.context.options.python_multi)
self.context.log.debug(' - Selected: %s' % interpreters)
if not interpreters:
raise TaskError('No compatible interpreters for %s' % target)
target.interpreters = interpreters
class Py(Command):
"""Python chroot manipulation."""
__command__ = 'py'
def setup_parser(self, parser, args):
parser.set_usage('\n'
' %prog py (options) [spec] args\n')
parser.disable_interspersed_args()
parser.add_option('-t', '--timeout', dest='conn_timeout', type='int',
default=Config.load().getdefault('connection_timeout'),
help='Number of seconds to wait for http connections.')
parser.add_option('--pex', dest='pex', default=False, action='store_true',
help='Dump a .pex of this chroot instead of attempting to execute it.')
parser.add_option('--ipython', dest='ipython', default=False, action='store_true',
help='Run the target environment in an IPython interpreter.')
parser.add_option('-r', '--req', dest='extra_requirements', default=[], action='append',
help='Additional Python requirements to add to this chroot.')
parser.add_option('-i', '--interpreter', dest='interpreter', default=None,
help='The interpreter requirement for this chroot.')
parser.add_option('-e', '--entry_point', dest='entry_point', default=None,
help='The entry point for the generated PEX.')
parser.add_option('-v', '--verbose', dest='verbose', default=False, action='store_true',
help='Show verbose output.')
parser.epilog = """Interact with the chroot of the specified target."""
def __init__(self, run_tracker, root_dir, parser, argv):
Command.__init__(self, run_tracker, root_dir, parser, argv)
self.target = None
self.extra_targets = []
self.config = Config.load()
self.interpreter_cache = PythonInterpreterCache(self.config, logger=self.debug)
self.interpreter_cache.setup()
interpreters = self.interpreter_cache.select_interpreter(
list(self.interpreter_cache.matches([self.options.interpreter]
if self.options.interpreter else [b''])))
if len(interpreters) != 1:
self.error('Unable to detect suitable interpreter.')
self.interpreter = interpreters[0]
for req in self.options.extra_requirements:
with ParseContext.temp():
self.extra_targets.append(PythonRequirement(req, use_2to3=True))
# We parse each arg in the context of the cli usage:
# ./pants command (options) [spec] (build args)
# ./pants command (options) [spec]... -- (build args)
# Our command token and our options are parsed out so we see args of the form:
# [spec] (build args)
# [spec]... -- (build args)
binaries = []
for k in range(len(self.args)):
arg = self.args.pop(0)
if arg == '--':
break
def not_a_target(debug_msg):
self.debug('Not a target, assuming option: %s.' % e)
# We failed to parse the arg as a target or else it was in valid address format but did not
# correspond to a real target. Assume this is the 1st of the build args and terminate
# processing args for target addresses.
self.args.insert(0, arg)
target = None
try:
address = Address.parse(root_dir, arg)
target = Target.get(address)
if target is None:
not_a_target(debug_msg='Unrecognized target')
break
except Exception as e:
not_a_target(debug_msg=e)
break
for resolved in filter(lambda t: t.is_concrete, target.resolve()):
if isinstance(resolved, PythonBinary):
binaries.append(resolved)
else:
self.extra_targets.append(resolved)
if len(binaries) == 0:
# treat as a chroot
pass
elif len(binaries) == 1:
# We found a binary and are done, the rest of the args get passed to it
self.target = binaries[0]
else:
self.error('Can only process 1 binary target, %s contains %d:\n\t%s' % (
arg, len(binaries), '\n\t'.join(str(binary.address) for binary in binaries)
))
if self.target is None:
if not self.extra_targets:
self.error('No valid target specified!')
self.target = self.extra_targets.pop(0)
def debug(self, message):
if self.options.verbose:
print(message, file=sys.stderr)
def execute(self):
if self.options.pex and self.options.ipython:
self.error('Cannot specify both --pex and --ipython!')
if self.options.entry_point and self.options.ipython:
self.error('Cannot specify both --entry_point and --ipython!')
if self.options.verbose:
print('Build operating on target: %s %s' % (self.target,
'Extra targets: %s' % ' '.join(map(str, self.extra_targets)) if self.extra_targets else ''))
builder = PEXBuilder(tempfile.mkdtemp(), interpreter=self.interpreter,
pex_info=self.target.pexinfo if isinstance(self.target, PythonBinary) else None)
if self.options.entry_point:
builder.set_entry_point(self.options.entry_point)
if self.options.ipython:
if not self.config.has_section('python-ipython'):
self.error('No python-ipython sections defined in your pants.ini!')
builder.info.entry_point = self.config.get('python-ipython', 'entry_point')
if builder.info.entry_point is None:
self.error('Must specify entry_point for IPython in the python-ipython section '
'of your pants.ini!')
requirements = self.config.getlist('python-ipython', 'requirements', default=[])
with ParseContext.temp():
for requirement in requirements:
self.extra_targets.append(PythonRequirement(requirement))
executor = PythonChroot(
self.target,
self.root_dir,
builder=builder,
interpreter=self.interpreter,
extra_targets=self.extra_targets,
conn_timeout=self.options.conn_timeout)
executor.dump()
if self.options.pex:
pex_name = os.path.join(self.root_dir, 'dist', '%s.pex' % self.target.name)
builder.build(pex_name)
print('Wrote %s' % pex_name)
return 0
else:
builder.freeze()
pex = PEX(builder.path(), interpreter=self.interpreter)
po = pex.run(args=list(self.args), blocking=False)
try:
return po.wait()
except KeyboardInterrupt:
po.send_signal(signal.SIGINT)
raise
class Build(Command):
"""Builds a specified target."""
__command__ = 'build'
def setup_parser(self, parser, args):
parser.set_usage("\n"
" %prog build (options) [spec] (build args)\n"
" %prog build (options) [spec]... -- (build args)")
parser.add_option("-t", "--timeout", dest="conn_timeout", type="int",
default=Config.load().getdefault('connection_timeout'),
help="Number of seconds to wait for http connections.")
parser.add_option('-i', '--interpreter', dest='interpreters', default=[], action='append',
help="Constrain what Python interpreters to use. Uses Requirement "
"format from pkg_resources, e.g. 'CPython>=2.6,<3' or 'PyPy'. "
"By default, no constraints are used. Multiple constraints may "
"be added. They will be ORed together.")
parser.add_option('-v', '--verbose', dest='verbose', default=False, action='store_true',
help='Show verbose output.')
parser.disable_interspersed_args()
parser.epilog = ('Builds the specified Python target(s). Use ./pants goal for JVM and other '
'targets.')
def __init__(self, run_tracker, root_dir, parser, argv):
Command.__init__(self, run_tracker, root_dir, parser, argv)
if not self.args:
self.error("A spec argument is required")
self.config = Config.load()
interpreters = self.options.interpreters or [b'']
self.interpreter_cache = PythonInterpreterCache(self.config, logger=self.debug)
self.interpreter_cache.setup(filters=interpreters)
interpreters = self.interpreter_cache.select_interpreter(
list(self.interpreter_cache.matches(interpreters)))
if len(interpreters) != 1:
self.error('Unable to detect suitable interpreter.')
else:
self.debug('Selected %s' % interpreters[0])
self.interpreter = interpreters[0]
try:
specs_end = self.args.index('--')
if len(self.args) > specs_end:
self.build_args = self.args[specs_end+1:len(self.args)+1]
else:
self.build_args = []
except ValueError:
specs_end = 1
self.build_args = self.args[1:] if len(self.args) > 1 else []
self.targets = OrderedSet()
spec_parser = SpecParser(root_dir, self.build_file_parser)
self.top_level_addresses = set()
for spec in self.args[0:specs_end]:
try:
addresses = spec_parser.parse_addresses(spec)
except:
self.error("Problem parsing spec %s: %s" % (spec, traceback.format_exc()))
for address in addresses:
self.top_level_addresses.add(address)
try:
self.build_file_parser.inject_address_closure_into_build_graph(address, self.build_graph)
target = self.build_graph.get_target(address)
except:
self.error("Problem parsing BUILD target %s: %s" % (address, traceback.format_exc()))
if not target:
self.error("Target %s does not exist" % address)
transitive_targets = self.build_graph.transitive_subgraph_of_addresses([target.address])
for transitive_target in transitive_targets:
self.targets.add(transitive_target)
self.targets = [target for target in self.targets if target.is_python]
def debug(self, message):
if self.options.verbose:
print(message, file=sys.stderr)
def execute(self):
print("Build operating on top level addresses: %s" % self.top_level_addresses)
python_targets = OrderedSet()
for target in self.targets:
if target.is_python:
python_targets.add(target)
else:
self.error("Cannot build target %s" % target)
if python_targets:
status = self._python_build(python_targets)
else:
status = -1
return status
def _python_build(self, targets):
try:
executor = PythonBuilder(self.run_tracker, self.root_dir)
return executor.build(
targets,
self.build_args,
interpreter=self.interpreter,
conn_timeout=self.options.conn_timeout)
except:
self.error("Problem executing PythonBuilder for targets %s: %s" % (targets,
traceback.format_exc()))
class Build(Command):
"""Builds a specified target."""
__command__ = 'build'
def setup_parser(self, parser, args):
parser.set_usage("\n"
" %prog build (options) [spec] (build args)\n"
" %prog build (options) [spec]... -- (build args)")
parser.add_option(
"-t",
"--timeout",
dest="conn_timeout",
type="int",
default=Config.load().getdefault('connection_timeout'),
help="Number of seconds to wait for http connections.")
parser.add_option(
'-i',
'--interpreter',
dest='interpreters',
default=[],
action='append',
help="Constrain what Python interpreters to use. Uses Requirement "
"format from pkg_resources, e.g. 'CPython>=2.6,<3' or 'PyPy'. "
"By default, no constraints are used. Multiple constraints may "
"be added. They will be ORed together.")
parser.add_option('-v',
'--verbose',
dest='verbose',
default=False,
action='store_true',
help='Show verbose output.')
parser.disable_interspersed_args()
parser.epilog = (
'Builds the specified Python target(s). Use ./pants goal for JVM and other '
'targets.')
def __init__(self, run_tracker, root_dir, parser, argv):
Command.__init__(self, run_tracker, root_dir, parser, argv)
if not self.args:
self.error("A spec argument is required")
self.config = Config.load()
interpreters = self.options.interpreters or [b'']
self.interpreter_cache = PythonInterpreterCache(self.config,
logger=self.debug)
self.interpreter_cache.setup(filters=interpreters)
interpreters = self.interpreter_cache.select_interpreter(
list(self.interpreter_cache.matches(interpreters)))
if len(interpreters) != 1:
self.error('Unable to detect suitable interpreter.')
else:
self.debug('Selected %s' % interpreters[0])
self.interpreter = interpreters[0]
try:
specs_end = self.args.index('--')
if len(self.args) > specs_end:
self.build_args = self.args[specs_end + 1:len(self.args) + 1]
else:
self.build_args = []
except ValueError:
specs_end = 1
self.build_args = self.args[1:] if len(self.args) > 1 else []
self.targets = OrderedSet()
spec_parser = SpecParser(root_dir, self.build_file_parser)
self.top_level_addresses = set()
for spec in self.args[0:specs_end]:
try:
addresses = spec_parser.parse_addresses(spec)
except:
self.error("Problem parsing spec %s: %s" %
(spec, traceback.format_exc()))
for address in addresses:
self.top_level_addresses.add(address)
try:
self.build_file_parser.inject_address_closure_into_build_graph(
address, self.build_graph)
target = self.build_graph.get_target(address)
except:
self.error("Problem parsing BUILD target %s: %s" %
(address, traceback.format_exc()))
if not target:
self.error("Target %s does not exist" % address)
transitive_targets = self.build_graph.transitive_subgraph_of_addresses(
[target.address])
for transitive_target in transitive_targets:
self.targets.add(transitive_target)
self.targets = [target for target in self.targets if target.is_python]
def debug(self, message):
if self.options.verbose:
print(message, file=sys.stderr)
def execute(self):
print("Build operating on top level addresses: %s" %
self.top_level_addresses)
python_targets = OrderedSet()
for target in self.targets:
if target.is_python:
python_targets.add(target)
else:
self.error("Cannot build target %s" % target)
if python_targets:
status = self._python_build(python_targets)
else:
status = -1
return status
def _python_build(self, targets):
try:
executor = PythonBuilder(self.run_tracker, self.root_dir)
return executor.build(targets,
self.build_args,
interpreter=self.interpreter,
conn_timeout=self.options.conn_timeout)
except:
self.error("Problem executing PythonBuilder for targets %s: %s" %
(targets, traceback.format_exc()))
class Py(Command):
"""Python chroot manipulation."""
__command__ = 'py'
def setup_parser(self, parser, args):
parser.set_usage('\n'
' %prog py (options) [spec] args\n')
parser.disable_interspersed_args()
parser.add_option('-t', '--timeout', dest='conn_timeout', type='int',
default=Config.load().getdefault('connection_timeout'),
help='Number of seconds to wait for http connections.')
parser.add_option('--pex', dest='pex', default=False, action='store_true',
help='Dump a .pex of this chroot instead of attempting to execute it.')
parser.add_option('--ipython', dest='ipython', default=False, action='store_true',
help='Run the target environment in an IPython interpreter.')
parser.add_option('-r', '--req', dest='extra_requirements', default=[], action='append',
help='Additional Python requirements to add to this chroot.')
parser.add_option('-i', '--interpreter', dest='interpreters', default=[], action='append',
help="Constrain what Python interpreters to use. Uses Requirement "
"format from pkg_resources, e.g. 'CPython>=2.6,<3' or 'PyPy'. "
"By default, no constraints are used. Multiple constraints may "
"be added. They will be ORed together.")
parser.add_option('-e', '--entry_point', dest='entry_point', default=None,
help='The entry point for the generated PEX.')
parser.add_option('-v', '--verbose', dest='verbose', default=False, action='store_true',
help='Show verbose output.')
parser.epilog = """Interact with the chroot of the specified target."""
def __init__(self, run_tracker, root_dir, parser, argv):
Command.__init__(self, run_tracker, root_dir, parser, argv)
self.target = None
self.extra_targets = []
self.extra_requirements = []
self.config = Config.load()
interpreters = self.options.interpreters or [b'']
self.interpreter_cache = PythonInterpreterCache(self.config, logger=self.debug)
self.interpreter_cache.setup(filters=interpreters)
interpreters = self.interpreter_cache.select_interpreter(
list(self.interpreter_cache.matches(interpreters)))
if len(interpreters) != 1:
self.error('Unable to detect suitable interpreter.')
self.interpreter = interpreters[0]
for req in self.options.extra_requirements:
self.extra_requirements.append(PythonRequirement(req, use_2to3=True))
# We parse each arg in the context of the cli usage:
# ./pants command (options) [spec] (build args)
# ./pants command (options) [spec]... -- (build args)
# Our command token and our options are parsed out so we see args of the form:
# [spec] (build args)
# [spec]... -- (build args)
binaries = []
for k in range(len(self.args)):
arg = self.args.pop(0)
if arg == '--':
break
def not_a_target(debug_msg):
self.debug('Not a target, assuming option: %s.' % e)
# We failed to parse the arg as a target or else it was in valid address format but did not
# correspond to a real target. Assume this is the 1st of the build args and terminate
# processing args for target addresses.
self.args.insert(0, arg)
target = None
try:
print(root_dir, arg)
# import pdb; pdb.set_trace()
self.build_file_parser.inject_spec_closure_into_build_graph(arg, self.build_graph)
spec_path, target_name = parse_spec(arg)
build_file = BuildFile(root_dir, spec_path)
address = BuildFileAddress(build_file, target_name)
target = self.build_graph.get_target(address)
if target is None:
not_a_target(debug_msg='Unrecognized target')
break
except Exception as e:
not_a_target(debug_msg=e)
break
if isinstance(target, PythonBinary):
binaries.append(target)
else:
self.extra_targets.append(target)
if len(binaries) == 0:
# treat as a chroot
pass
elif len(binaries) == 1:
# We found a binary and are done, the rest of the args get passed to it
self.target = binaries[0]
else:
self.error('Can only process 1 binary target, %s contains %d:\n\t%s' % (
arg, len(binaries), '\n\t'.join(str(binary.address) for binary in binaries)
))
if self.target is None:
if not self.extra_targets:
self.error('No valid target specified!')
self.target = self.extra_targets.pop(0)
def debug(self, message):
if self.options.verbose:
print(message, file=sys.stderr)
def execute(self):
if self.options.pex and self.options.ipython:
self.error('Cannot specify both --pex and --ipython!')
if self.options.entry_point and self.options.ipython:
self.error('Cannot specify both --entry_point and --ipython!')
if self.options.verbose:
print('Build operating on target: %s %s' % (self.target,
'Extra targets: %s' % ' '.join(map(str, self.extra_targets)) if self.extra_targets else ''))
builder = PEXBuilder(tempfile.mkdtemp(), interpreter=self.interpreter,
pex_info=self.target.pexinfo if isinstance(self.target, PythonBinary) else None)
if self.options.entry_point:
builder.set_entry_point(self.options.entry_point)
if self.options.ipython:
if not self.config.has_section('python-ipython'):
self.error('No python-ipython sections defined in your pants.ini!')
builder.info.entry_point = self.config.get('python-ipython', 'entry_point')
if builder.info.entry_point is None:
self.error('Must specify entry_point for IPython in the python-ipython section '
'of your pants.ini!')
requirements = self.config.getlist('python-ipython', 'requirements', default=[])
for requirement in requirements:
self.extra_requirements.append(PythonRequirement(requirement))
executor = PythonChroot(
self.target,
self.root_dir,
builder=builder,
interpreter=self.interpreter,
extra_targets=self.extra_targets,
extra_requirements=self.extra_requirements,
conn_timeout=self.options.conn_timeout)
executor.dump()
if self.options.pex:
pex_name = os.path.join(self.root_dir, 'dist', '%s.pex' % self.target.name)
builder.build(pex_name)
print('Wrote %s' % pex_name)
return 0
else:
builder.freeze()
pex = PEX(builder.path(), interpreter=self.interpreter)
po = pex.run(args=list(self.args), blocking=False)
try:
return po.wait()
except KeyboardInterrupt:
po.send_signal(signal.SIGINT)
raise