def setup(options=None):
if not options:
options, _ = OptionsInitializer(OptionsBootstrapper()).setup()
build_root = get_buildroot()
cmd_line_spec_parser = CmdLineSpecParser(build_root)
spec_roots = [cmd_line_spec_parser.parse_spec(spec) for spec in options.target_specs]
storage = Storage.create(debug=False)
# Ignore any dotfile below build_root except . itself
project_tree = FileSystemProjectTree(build_root, ['.*', 'build-support/*.venv/'])
symbol_table_cls = LegacyTable
# Register "literal" subjects required for these tasks.
# TODO: Replace with `Subsystems`.
address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
parser_cls=LegacyPythonCallbacksParser)
# Create a Scheduler containing graph and filesystem tasks, with no installed goals. The ExpGraph
# will explicitly request the products it needs.
tasks = (
create_legacy_graph_tasks() +
create_fs_tasks() +
create_graph_tasks(address_mapper, symbol_table_cls)
)
return (
LocalScheduler(dict(), tasks, storage, project_tree),
storage,
options,
spec_roots,
symbol_table_cls
)
def setup(options=None):
if not options:
options, _ = OptionsInitializer(OptionsBootstrapper()).setup()
build_root = get_buildroot()
cmd_line_spec_parser = CmdLineSpecParser(build_root)
spec_roots = [cmd_line_spec_parser.parse_spec(spec) for spec in options.target_specs]
storage = Storage.create(debug=False)
project_tree = FileSystemProjectTree(build_root)
symbol_table_cls = LegacyTable
# Register "literal" subjects required for these tasks.
# TODO: Replace with `Subsystems`.
address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
parser_cls=LegacyPythonCallbacksParser)
# Create a Scheduler containing graph and filesystem tasks, with no installed goals. The ExpGraph
# will explicitly request the products it needs.
tasks = (
create_legacy_graph_tasks() +
create_fs_tasks() +
create_graph_tasks(address_mapper, symbol_table_cls)
)
return (
LocalScheduler(dict(), tasks, storage, project_tree),
storage,
options,
spec_roots,
symbol_table_cls
)
def _run_and_check(self, target_specs, incremental_import=None):
"""
Invoke idea-plugin goal and check for target specs and project in the
generated project and workspace file.
:param target_specs: list of target specs
:return: n/a
"""
self.assertTrue("targets are empty", target_specs)
spec_parser = CmdLineSpecParser(get_buildroot())
# project_path is always the directory of the first target,
# which is where intellij is going to zoom in under project view.
project_path = spec_parser.parse_spec(target_specs[0]).directory
with self.temporary_workdir() as workdir:
with temporary_file(root_dir=workdir, cleanup=True) as output_file:
args = [
'idea-plugin',
'--output-file={}'.format(output_file.name),
'--no-open',
]
if incremental_import is not None:
args.append(
'--incremental-import={}'.format(incremental_import))
pants_run = self.run_pants_with_workdir(
args + target_specs, workdir)
self.assert_success(pants_run)
project_dir = self._get_project_dir(output_file.name)
self.assertTrue(os.path.exists(project_dir),
"{} does not exist".format(project_dir))
self._do_check(project_dir,
project_path,
target_specs,
incremental_import=incremental_import)
def main_addresses():
build_root, goals, args = pop_build_root_and_goals(
'[build root path] [goal]+ [address spec]*', sys.argv[1:])
cmd_line_spec_parser = CmdLineSpecParser(build_root)
spec_roots = [cmd_line_spec_parser.parse_spec(spec) for spec in args]
visualize_build_request(build_root, goals, spec_roots)
def _run_and_check(self, target_specs, incremental_import=None):
"""
Invoke idea-plugin goal and check for target specs and project in the
generated project and workspace file.
:param target_specs: list of target specs
:return: n/a
"""
self.assertTrue("targets are empty", target_specs)
spec_parser = CmdLineSpecParser(get_buildroot())
# project_path is always the directory of the first target,
# which is where intellij is going to zoom in under project view.
project_path = spec_parser.parse_spec(target_specs[0]).directory
with self.temporary_workdir() as workdir:
with temporary_file(root_dir=workdir, cleanup=True) as output_file:
args = [
'idea-plugin',
'--output-file={}'.format(output_file.name),
'--no-open',
]
if incremental_import is not None:
args.append('--incremental-import={}'.format(incremental_import))
pants_run = self.run_pants_with_workdir(args + target_specs, workdir)
self.assert_success(pants_run)
project_dir = self._get_project_dir(output_file.name)
self.assertTrue(os.path.exists(project_dir), "{} does not exist".format(project_dir))
self._do_check(project_dir, project_path, target_specs, incremental_import=incremental_import)
def set_start_time(self, start_time: Optional[float]) -> None:
# Launch RunTracker as early as possible (before .run() is called).
self._run_tracker = RunTracker.global_instance()
# Propagates parent_build_id to pants runs that may be called from this pants run.
os.environ["PANTS_PARENT_BUILD_ID"] = self._run_tracker.run_id
self._reporting = Reporting.global_instance()
self._reporting.initialize(self._run_tracker,
self.options,
start_time=start_time)
spec_parser = CmdLineSpecParser(get_buildroot())
specs = [
spec_parser.parse_spec(spec).to_spec_string()
for spec in self.options.specs
]
# Note: This will not include values from `--changed-*` flags.
self._run_tracker.run_info.add_info("specs_from_command_line",
specs,
stringify=False)
# Capture a repro of the 'before' state for this build, if needed.
self._repro = Reproducer.global_instance().create_repro()
if self._repro:
self._repro.capture(self._run_tracker.run_info.get_as_dict())
def parse_specs(
cls,
raw_specs: Iterable[str],
build_root: Optional[str] = None,
exclude_patterns: Optional[Iterable[str]] = None,
tags: Optional[Iterable[str]] = None,
) -> Specs:
"""Parse raw string specs into a Specs object."""
build_root = build_root or get_buildroot()
spec_parser = CmdLineSpecParser(build_root)
address_specs: OrderedSet[AddressSpec] = OrderedSet()
filesystem_specs: OrderedSet[FilesystemSpec] = OrderedSet()
for spec_str in raw_specs:
parsed_spec = spec_parser.parse_spec(spec_str)
if isinstance(parsed_spec, AddressSpec):
address_specs.add(parsed_spec)
else:
filesystem_specs.add(parsed_spec)
address_specs_collection = AddressSpecs(
dependencies=address_specs,
exclude_patterns=exclude_patterns if exclude_patterns else tuple(),
tags=tags,
)
filesystem_specs_collection = FilesystemSpecs(filesystem_specs)
return Specs(
address_specs=address_specs_collection,
filesystem_specs=filesystem_specs_collection,
)
def main_addresses():
build_root, goals, args = pop_build_root_and_goals(
'[build root path] [goal]+ [address spec]*', sys.argv[1:])
cmd_line_spec_parser = CmdLineSpecParser(build_root)
spec_roots = [cmd_line_spec_parser.parse_spec(spec) for spec in args]
visualize_build_request(build_root, goals, spec_roots)
def setup():
build_root = get_buildroot()
cmd_line_spec_parser = CmdLineSpecParser(build_root)
spec_roots = [cmd_line_spec_parser.parse_spec(spec) for spec in sys.argv[1:]]
storage = Storage.create(debug=False)
project_tree = FileSystemProjectTree(build_root)
symbol_table_cls = LegacyTable
# Register "literal" subjects required for these tasks.
# TODO: Replace with `Subsystems`.
address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
parser_cls=LegacyPythonCallbacksParser)
# Create a Scheduler containing graph and filesystem tasks, with no installed goals. The ExpGraph
# will explicitly request the products it needs.
tasks = (
create_legacy_graph_tasks() +
create_fs_tasks() +
create_graph_tasks(address_mapper, symbol_table_cls)
)
return (
LocalScheduler(dict(), tasks, symbol_table_cls, project_tree),
storage,
spec_roots,
symbol_table_cls
)
def execute(self):
# NOTE(pl): We now rely on the fact that we've scheduled Buildgen (the dummy task in the
# buildgen goal) to run before the real buildgen tasks, e.g. buildgen-scala, buildgen-thrift,
# etc. Since we are being run before the real tasks but after everything else upstream,
# we can fix the target roots back up to be whatever the buildgen tasks are supposed to
# operate on (instead of the entire build graph, which the upstream operated on).
build_graph = self.context.build_graph
bg_target_roots = set()
# NOTE(mateo): Using all source roots adds a scan of 3rdparty - may have a minor perf penalty.
# I would like to switch to the configured source roots if I can find a way to surface it everywhere.
buildgen_dirs = self.buildgen_subsystem.source_dirs + self.buildgen_subsystem.test_dirs
all_buildgen_specs = ['{}::'.format(d) for d in buildgen_dirs]
spec_parser = CmdLineSpecParser(get_buildroot())
target_specs = self.context.options._target_specs or all_buildgen_specs
parsed_specs = [
spec_parser.parse_spec(target_spec)
for target_spec in target_specs
]
for address in self.context.address_mapper.scan_specs(parsed_specs):
build_graph.inject_address_closure(address)
bg_target_roots.add(build_graph.get_target(address))
# Disable building from SCM for now, and instead always build everything unless the user
# specifies less.
# bg_target_roots.update(
# target for target in
# build_graph.transitive_dependees_of_addresses(t.address for t in self.targets_from_scm())
# if target.is_original
# )
self.context._replace_targets(list(bg_target_roots))
def _run_and_check(self, address_specs, incremental_import=None):
"""Invoke idea-plugin goal and check for target specs and project in the generated project
and workspace file.
:param address_specs: list of address specs
:return: n/a
"""
self.assertTrue(address_specs, "targets are empty")
spec_parser = CmdLineSpecParser(get_buildroot())
# project_path is always the directory of the first target,
# which is where intellij is going to zoom in under project view.
spec = spec_parser.parse_spec(address_specs[0])
assert isinstance(spec, (SingleAddress, SiblingAddresses, DescendantAddresses))
project_path = spec.directory
with self.temporary_workdir() as workdir:
with temporary_file(root_dir=workdir, cleanup=True) as output_file:
args = [
"idea-plugin",
f"--output-file={output_file.name}",
"--no-open",
]
if incremental_import is not None:
args.append(f"--incremental-import={incremental_import}")
pants_run = self.run_pants_with_workdir(args + address_specs, workdir)
self.assert_success(pants_run)
project_dir = self._get_project_dir(output_file.name)
self.assertTrue(os.path.exists(project_dir), f"{project_dir} does not exist")
self._do_check(
project_dir, project_path, address_specs, incremental_import=incremental_import
)
def parse_specs(cls,
target_specs,
build_root=None,
exclude_patterns=None,
tags=None):
"""Parse string specs into unique `Spec` objects.
:param iterable target_specs: An iterable of string specs.
:param string build_root: The path to the build root.
:returns: An `OrderedSet` of `Spec` objects.
"""
build_root = build_root or get_buildroot()
spec_parser = CmdLineSpecParser(build_root)
dependencies = tuple(
OrderedSet(
spec_parser.parse_spec(spec_str) for spec_str in target_specs))
if not dependencies:
return None
return [
Specs(dependencies=dependencies,
exclude_patterns=exclude_patterns
if exclude_patterns else tuple(),
tags=tags)
]
def parse_specs(cls, target_specs, build_root=None):
"""Parse string specs into unique `Spec` objects.
:param iterable target_specs: An iterable of string specs.
:param string build_root: The path to the build root.
:returns: An `OrderedSet` of `Spec` objects.
"""
build_root = build_root or get_buildroot()
spec_parser = CmdLineSpecParser(build_root)
return OrderedSet(spec_parser.parse_spec(spec_str) for spec_str in target_specs)
def parse_specs(cls, target_specs, build_root=None):
"""Parse string specs into unique `Spec` objects.
:param iterable target_specs: An iterable of string specs.
:param string build_root: The path to the build root.
:returns: An `OrderedSet` of `Spec` objects.
"""
build_root = build_root or get_buildroot()
spec_parser = CmdLineSpecParser(build_root)
return OrderedSet(spec_parser.parse_spec(spec_str) for spec_str in target_specs)
def _determine_goals(self, address_mapper, requested_goals):
"""Check and populate the requested goals for a given run."""
spec_parser = CmdLineSpecParser(self._root_dir)
for goal in requested_goals:
if address_mapper.is_valid_single_address(spec_parser.parse_spec(goal)):
logger.warning("Command-line argument '{0}' is ambiguous and was assumed to be "
"a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal))
return [Goal.by_name(goal) for goal in requested_goals]
def _determine_goals(self, requested_goals):
"""Check and populate the requested goals for a given run."""
spec_parser = CmdLineSpecParser(self._root_dir)
for goal in requested_goals:
if self._address_mapper.is_valid_single_address(spec_parser.parse_spec(goal)):
logger.warning("Command-line argument '{0}' is ambiguous and was assumed to be "
"a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal))
goals = [Goal.by_name(goal) for goal in requested_goals]
return goals
def parse_commandline_to_spec_roots(options=None,
args=None,
build_root=None):
if not options:
options, _ = OptionsInitializer(OptionsBootstrapper(args=args),
init_logging=False).setup()
cmd_line_spec_parser = CmdLineSpecParser(build_root or get_buildroot())
spec_roots = [
cmd_line_spec_parser.parse_spec(spec)
for spec in options.target_specs
]
return spec_roots
def _determine_goals(self, requested_goals):
"""Check and populate the requested goals for a given run."""
def is_quiet(goals):
return any(goal.has_task_of_type(QuietTaskMixin) for goal in goals) or self._explain
spec_parser = CmdLineSpecParser(self._root_dir)
for goal in requested_goals:
if self._address_mapper.is_valid_single_address(spec_parser.parse_spec(goal)):
logger.warning("Command-line argument '{0}' is ambiguous and was assumed to be "
"a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal))
goals = [Goal.by_name(goal) for goal in requested_goals]
return goals, is_quiet(goals)
def _set_start_time(self, start_time: float) -> None:
# Propagates parent_build_id to pants runs that may be called from this pants run.
os.environ["PANTS_PARENT_BUILD_ID"] = self._run_tracker.run_id
self._run_tracker.start(self.options, run_start_time=start_time)
spec_parser = CmdLineSpecParser(get_buildroot())
specs = [
str(spec_parser.parse_spec(spec)) for spec in self.options.specs
]
# Note: This will not include values from `--changed-*` flags.
self._run_tracker.run_info.add_info("specs_from_command_line",
specs,
stringify=False)
def _get_targets(spec_str):
spec_parser = CmdLineSpecParser(get_buildroot())
try:
spec = spec_parser.parse_spec(spec_str)
addresses = self.context.address_mapper.scan_specs([spec])
except AddressLookupError as e:
raise TaskError('Failed to parse address selector: {spec_str}\n {message}'.format(spec_str=spec_str, message=e))
# filter specs may not have been parsed as part of the context: force parsing
matches = set()
for address in addresses:
self.context.build_graph.inject_address_closure(address)
matches.add(self.context.build_graph.get_target(address))
if not matches:
raise TaskError('No matches for address selector: {spec_str}'.format(spec_str=spec_str))
return matches
def _get_targets(spec_str):
spec_parser = CmdLineSpecParser(get_buildroot())
try:
spec = spec_parser.parse_spec(spec_str)
addresses = self.context.address_mapper.scan_specs([spec])
except AddressLookupError as e:
raise TaskError('Failed to parse address selector: {spec_str}\n {message}'.format(spec_str=spec_str, message=e))
# filter specs may not have been parsed as part of the context: force parsing
matches = set()
for address in addresses:
self.context.build_graph.inject_address_closure(address)
matches.add(self.context.build_graph.get_target(address))
if not matches:
raise TaskError('No matches for address selector: {spec_str}'.format(spec_str=spec_str))
return matches
def parse_specs(cls, target_specs, build_root=None, exclude_patterns=None, tags=None):
"""Parse string specs into unique `Spec` objects.
:param iterable target_specs: An iterable of string specs.
:param string build_root: The path to the build root.
:returns: A `Specs` object.
"""
build_root = build_root or get_buildroot()
spec_parser = CmdLineSpecParser(build_root)
dependencies = tuple(OrderedSet(spec_parser.parse_spec(spec_str) for spec_str in target_specs))
return Specs(
dependencies=dependencies,
exclude_patterns=exclude_patterns if exclude_patterns else tuple(),
tags=tags)
def _expand_specs(self, spec_strs, fail_fast):
"""Populate the BuildGraph and target list from a set of input specs."""
with self._run_tracker.new_workunit(name='parse', labels=[WorkUnitLabel.SETUP]):
def filter_for_tag(tag):
return lambda target: tag in map(str, target.tags)
tag_filter = wrap_filters(create_filters(self._tag, filter_for_tag))
# Parse all specs into unique Spec objects.
spec_parser = CmdLineSpecParser(self._root_dir)
specs = OrderedSet()
for spec_str in spec_strs:
specs.add(spec_parser.parse_spec(spec_str))
# Then scan them to generate unique Addresses.
for address in self._build_graph.inject_specs_closure(specs, fail_fast):
target = self._build_graph.get_target(address)
if tag_filter(target):
self._targets.append(target)
def _expand_specs(self, spec_strs, fail_fast):
"""Populate the BuildGraph and target list from a set of input specs."""
with self._run_tracker.new_workunit(name='parse', labels=[WorkUnitLabel.SETUP]):
def filter_for_tag(tag):
return lambda target: tag in map(str, target.tags)
tag_filter = wrap_filters(create_filters(self._tag, filter_for_tag))
# Parse all specs into unique Spec objects.
spec_parser = CmdLineSpecParser(self._root_dir)
specs = OrderedSet()
for spec_str in spec_strs:
specs.add(spec_parser.parse_spec(spec_str))
# Then scan them to generate unique Addresses.
for address in self._address_mapper.scan_specs(specs, fail_fast, self._spec_excludes):
self._build_graph.inject_address_closure(address)
target = self._build_graph.get_target(address)
if tag_filter(target):
self._targets.append(target)
def set_start_time(self, start_time):
# Launch RunTracker as early as possible (before .run() is called).
self._run_tracker = RunTracker.global_instance()
self._reporting = Reporting.global_instance()
self._run_start_time = start_time
self._reporting.initialize(self._run_tracker, self._options, start_time=self._run_start_time)
spec_parser = CmdLineSpecParser(get_buildroot())
target_specs = [
spec_parser.parse_spec(spec).to_spec_string()
for spec in self._options.positional_args
]
# Note: This will not include values from `--owner-of` or `--changed-*` flags.
self._run_tracker.run_info.add_info("specs_from_command_line", target_specs, stringify=False)
# Capture a repro of the 'before' state for this build, if needed.
self._repro = Reproducer.global_instance().create_repro()
if self._repro:
self._repro.capture(self._run_tracker.run_info.get_as_dict())
class EngineTest(unittest.TestCase):
def setUp(self):
build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs')
self.scheduler = setup_json_scheduler(build_root)
self.spec_parser = CmdLineSpecParser(build_root)
self.java = Address.parse('src/java/codegen/simple')
def request(self, goals, *addresses):
specs = [self.spec_parser.parse_spec(str(a)) for a in addresses]
return BuildRequest(goals=goals, subjects=specs)
def assert_engine(self, engine):
result = engine.execute(self.request(['compile'], self.java))
self.assertEqual({SelectNode(self.java, Classpath, None, None): Return(Classpath(creator='javac'))},
result.root_products)
self.assertIsNone(result.error)
@contextmanager
def multiprocessing_engine(self, pool_size=None):
with closing(LocalMultiprocessEngine(self.scheduler, pool_size=pool_size, debug=True)) as e:
yield e
def test_serial_engine_simple(self):
engine = LocalSerialEngine(self.scheduler)
self.assert_engine(engine)
def test_multiprocess_engine_multi(self):
with self.multiprocessing_engine() as engine:
self.assert_engine(engine)
def test_multiprocess_engine_single(self):
with self.multiprocessing_engine(pool_size=1) as engine:
self.assert_engine(engine)
def test_multiprocess_unpickleable(self):
build_request = self.request(['unpickleable'], self.java)
with self.multiprocessing_engine() as engine:
with self.assertRaises(SerializationError):
engine.execute(build_request)
def parse_specs(cls,
raw_specs: Iterable[str],
*,
build_root: Optional[str] = None) -> Specs:
"""Parse raw string specs into a Specs object."""
build_root = build_root or get_buildroot()
spec_parser = CmdLineSpecParser(build_root)
address_specs: OrderedSet[AddressSpec] = OrderedSet()
filesystem_specs: OrderedSet[FilesystemSpec] = OrderedSet()
for spec_str in raw_specs:
parsed_spec = spec_parser.parse_spec(spec_str)
if isinstance(parsed_spec, AddressSpec):
address_specs.add(parsed_spec)
else:
filesystem_specs.add(parsed_spec)
return Specs(
AddressSpecs(address_specs, filter_by_global_options=True),
FilesystemSpecs(filesystem_specs),
)
def create(cls, options=None, args=None, build_root=None, change_calculator=None):
"""
:param Options options: An `Options` instance to use, if available.
:param string args: Raw cli args to use for parsing if an `Options` instance isn't available.
:param string build_root: The build root.
:param ChangeCalculator change_calculator: A `ChangeCalculator` for calculating changes.
"""
if not options:
assert args is not None, 'must pass `args` if not passing `options`'
options, _ = OptionsInitializer(OptionsBootstrapper(args=args)).setup(init_logging=False)
# Determine the literal target roots.
cmd_line_spec_parser = CmdLineSpecParser(build_root or get_buildroot())
spec_roots = [cmd_line_spec_parser.parse_spec(spec) for spec in options.target_specs]
# Determine `Changed` arguments directly from options to support pre-`Subsystem` initialization paths.
changed_options = options.for_scope('changed')
changed_request = ChangedRequest.from_options(changed_options)
logger.debug('args are: %s', args)
logger.debug('spec_roots are: %s', spec_roots)
logger.debug('changed_request is: %s', changed_request)
if change_calculator and changed_request.is_actionable():
if spec_roots:
# We've been provided spec roots (e.g. `./pants list ::`) AND a changed request. Error out.
raise InvalidSpecConstraint('cannot provide changed parameters and target specs!')
# We've been provided no spec roots (e.g. `./pants list`) AND a changed request. Compute
# alternate target roots.
changed_addresses = change_calculator.changed_target_addresses(changed_request)
logger.debug('changed addresses: %s', changed_addresses)
return ChangedTargetRoots(changed_addresses)
# If no spec roots are passed, assume `::` as the intended target.
return LiteralTargetRoots(spec_roots if spec_roots else [DescendantAddresses('')])
def parse_commandline_to_spec_roots(options=None, args=None, build_root=None):
if not options:
options, _ = OptionsInitializer(OptionsBootstrapper(args=args)).setup(init_logging=False)
cmd_line_spec_parser = CmdLineSpecParser(build_root or get_buildroot())
spec_roots = [cmd_line_spec_parser.parse_spec(spec) for spec in options.target_specs]
return spec_roots
class CmdLineSpecParserTest(BaseTest):
def setUp(self):
super(CmdLineSpecParserTest, self).setUp()
self._spec_parser = CmdLineSpecParser(self.build_root)
def test_normal(self):
self.assert_parsed(':root', single('', 'root'))
self.assert_parsed('//:root', single('', 'root'))
self.assert_parsed('a', single('a'))
self.assert_parsed('a:a', single('a', 'a'))
self.assert_parsed('a/b', single('a/b'))
self.assert_parsed('a/b:b', single('a/b', 'b'))
self.assert_parsed('a/b:c', single('a/b', 'c'))
def test_sibling(self):
self.assert_parsed(':', sib(''))
self.assert_parsed('//:', sib(''))
self.assert_parsed('a:', sib('a'))
self.assert_parsed('//a:', sib('a'))
self.assert_parsed('a/b:', sib('a/b'))
self.assert_parsed('//a/b:', sib('a/b'))
def test_sibling_or_descendents(self):
self.assert_parsed('::', desc(''))
self.assert_parsed('//::', desc(''))
self.assert_parsed('a::', desc('a'))
self.assert_parsed('//a::', desc('a'))
self.assert_parsed('a/b::', desc('a/b'))
self.assert_parsed('//a/b::', desc('a/b'))
def test_absolute(self):
self.assert_parsed(os.path.join(self.build_root, 'a'), single('a'))
self.assert_parsed(os.path.join(self.build_root, 'a:a'), single('a', 'a'))
self.assert_parsed(os.path.join(self.build_root, 'a:'), sib('a'))
self.assert_parsed(os.path.join(self.build_root, 'a::'), desc('a'))
with self.assertRaises(CmdLineSpecParser.BadSpecError):
self.assert_parsed('/not/the/buildroot/a', sib('a'))
def test_absolute_double_slashed(self):
# By adding a double slash, we are insisting that this absolute path is actually
# relative to the buildroot. Thus, it should parse correctly.
double_absolute = '/' + os.path.join(self.build_root, 'a')
self.assertEquals('//', double_absolute[:2],
'A sanity check we have a leading-// absolute spec')
self.assert_parsed(double_absolute, single(double_absolute[2:]))
def test_cmd_line_affordances(self):
self.assert_parsed('./:root', single('', 'root'))
self.assert_parsed('//./:root', single('', 'root'))
self.assert_parsed('//./a/../:root', single('', 'root'))
self.assert_parsed(os.path.join(self.build_root, './a/../:root'), single('', 'root'))
self.assert_parsed('a/', single('a'))
self.assert_parsed('./a/', single('a'))
self.assert_parsed(os.path.join(self.build_root, './a/'), single('a'))
self.assert_parsed('a/b/:b', single('a/b', 'b'))
self.assert_parsed('./a/b/:b', single('a/b', 'b'))
self.assert_parsed(os.path.join(self.build_root, './a/b/:b'), single('a/b', 'b'))
def assert_parsed(self, spec_str, expected_spec):
self.assertEqual(self._spec_parser.parse_spec(spec_str), expected_spec)
class SchedulerTest(unittest.TestCase):
def setUp(self):
build_root = os.path.join(os.path.dirname(__file__), "examples", "scheduler_inputs")
self.spec_parser = CmdLineSpecParser(build_root)
self.scheduler, storage = setup_json_scheduler(build_root)
self.storage = storage
self.engine = LocalSerialEngine(self.scheduler, storage)
self.guava = Address.parse("3rdparty/jvm:guava")
self.thrift = Address.parse("src/thrift/codegen/simple")
self.java = Address.parse("src/java/codegen/simple")
self.java_simple = Address.parse("src/java/simple")
self.java_multi = Address.parse("src/java/multiple_classpath_entries")
self.no_variant_thrift = Address.parse("src/java/codegen/selector:conflict")
self.unconfigured_thrift = Address.parse("src/thrift/codegen/unconfigured")
self.resources = Address.parse("src/resources/simple")
self.consumes_resources = Address.parse("src/java/consumes_resources")
self.consumes_managed_thirdparty = Address.parse("src/java/managed_thirdparty")
self.managed_guava = Address.parse("3rdparty/jvm/managed:guava")
self.managed_hadoop = Address.parse("3rdparty/jvm/managed:hadoop-common")
self.managed_resolve_latest = Address.parse("3rdparty/jvm/managed:latest-hadoop")
self.inferred_deps = Address.parse("src/scala/inferred_deps")
def assert_select_for_subjects(self, walk, product, subjects, variants=None, variant_key=None):
node_type = SelectNode
variants = tuple(variants.items()) if variants else None
self.assertEqual(
{node_type(subject, product, variants, variant_key) for subject in subjects},
{
node
for (node, _), _ in walk
if node.product == product and isinstance(node, node_type) and node.variants == variants
},
)
def build_and_walk(self, build_request, failures=False):
"""Build and then walk the given build_request, returning the walked graph as a list."""
predicate = (lambda _: True) if failures else None
result = self.engine.execute(build_request)
self.assertIsNone(result.error)
return list(self.scheduler.product_graph.walk(build_request.roots, predicate=predicate))
def request(self, goals, *addresses):
return self.request_specs(goals, *[self.spec_parser.parse_spec(str(a)) for a in addresses])
def request_specs(self, goals, *specs):
return self.scheduler.build_request(goals=goals, subjects=specs)
def assert_resolve_only(self, goals, root_specs, jars):
build_request = self.request(goals, *root_specs)
walk = self.build_and_walk(build_request)
# Expect a SelectNode for each of the Jar/Classpath.
self.assert_select_for_subjects(walk, Jar, jars)
self.assert_select_for_subjects(walk, Classpath, jars)
def assert_root(self, walk, node, return_value):
"""Asserts that the first Node in a walk was a DependenciesNode with the single given result."""
((root, root_state), dependencies) = walk[0]
self.assertEquals(type(root), DependenciesNode)
self.assertEquals(Return([return_value]), root_state)
self.assertIn((node, Return(return_value)), dependencies)
def assert_root_failed(self, walk, node, thrown_type):
"""Asserts that the first Node in a walk was a DependenciesNode with a Throw result."""
((root, root_state), dependencies) = walk[0]
self.assertEquals(type(root), DependenciesNode)
self.assertEquals(Throw, type(root_state))
self.assertIn((node, thrown_type), [(k, type(v.exc)) for k, v in dependencies if type(v) is Throw])
def test_resolve(self):
self.assert_resolve_only(goals=["resolve"], root_specs=["3rdparty/jvm:guava"], jars=[self.guava])
def test_compile_only_3rdparty(self):
self.assert_resolve_only(goals=["compile"], root_specs=["3rdparty/jvm:guava"], jars=[self.guava])
def test_gen_noop(self):
# TODO(John Sirois): Ask around - is this OK?
# This is different than today. There is a gen'able target reachable from the java target, but
# the scheduler 'pull-seeding' has ApacheThriftPlanner stopping short since the subject it's
# handed is not thrift.
build_request = self.request(["gen"], self.java)
walk = self.build_and_walk(build_request)
self.assert_select_for_subjects(walk, JavaSources, [self.java])
def test_gen(self):
build_request = self.request(["gen"], self.thrift)
walk = self.build_and_walk(build_request)
# Root: expect the synthetic GenGoal product.
self.assert_root(
walk,
SelectNode(self.thrift, GenGoal, None, None),
GenGoal("non-empty input to satisfy the Goal constructor"),
)
variants = {"thrift": "apache_java"}
# Expect ThriftSources to have been selected.
self.assert_select_for_subjects(walk, ThriftSources, [self.thrift], variants=variants)
# Expect an ApacheThriftJavaConfiguration to have been used via the default Variants.
self.assert_select_for_subjects(
walk, ApacheThriftJavaConfiguration, [self.thrift], variants=variants, variant_key="thrift"
)
def test_codegen_simple(self):
build_request = self.request(["compile"], self.java)
walk = self.build_and_walk(build_request)
# The subgraph below 'src/thrift/codegen/simple' will be affected by its default variants.
subjects = [self.guava, self.java, self.thrift]
variant_subjects = [
Jar(org="org.apache.thrift", name="libthrift", rev="0.9.2", type_alias="jar"),
Jar(org="commons-lang", name="commons-lang", rev="2.5", type_alias="jar"),
Address.parse("src/thrift:slf4j-api"),
]
# Root: expect a DependenciesNode depending on a SelectNode with compilation via javac.
self.assert_root(walk, SelectNode(self.java, Classpath, None, None), Classpath(creator="javac"))
# Confirm that exactly the expected subjects got Classpaths.
self.assert_select_for_subjects(walk, Classpath, subjects)
self.assert_select_for_subjects(walk, Classpath, variant_subjects, variants={"thrift": "apache_java"})
def test_consumes_resources(self):
build_request = self.request(["compile"], self.consumes_resources)
walk = self.build_and_walk(build_request)
# Validate the root.
self.assert_root(walk, SelectNode(self.consumes_resources, Classpath, None, None), Classpath(creator="javac"))
# Confirm a classpath for the resources target and other subjects. We know that they are
# reachable from the root (since it was involved in this walk).
subjects = [self.resources, self.consumes_resources, self.guava]
self.assert_select_for_subjects(walk, Classpath, subjects)
def test_managed_resolve(self):
"""A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars."""
build_request = self.request(["compile"], self.consumes_managed_thirdparty)
walk = self.build_and_walk(build_request)
# Validate the root.
self.assert_root(
walk, SelectNode(self.consumes_managed_thirdparty, Classpath, None, None), Classpath(creator="javac")
)
# Confirm that we produced classpaths for the managed jars.
managed_jars = [self.managed_guava, self.managed_hadoop]
self.assert_select_for_subjects(walk, Classpath, [self.consumes_managed_thirdparty])
self.assert_select_for_subjects(walk, Classpath, managed_jars, variants={"resolve": "latest-hadoop"})
# Confirm that the produced jars had the appropriate versions.
self.assertEquals(
{Jar("org.apache.hadoop", "hadoop-common", "2.7.0"), Jar("com.google.guava", "guava", "18.0")},
{ret.value for (node, ret), _ in walk if node.product == Jar and isinstance(node, SelectNode)},
)
def test_dependency_inference(self):
"""Scala dependency inference introduces dependencies that do not exist in BUILD files."""
build_request = self.request(["compile"], self.inferred_deps)
walk = self.build_and_walk(build_request)
# Validate the root.
self.assert_root(walk, SelectNode(self.inferred_deps, Classpath, None, None), Classpath(creator="scalac"))
# Confirm that we requested a classpath for the root and inferred targets.
self.assert_select_for_subjects(walk, Classpath, [self.inferred_deps, self.java_simple])
def test_multiple_classpath_entries(self):
"""Multiple Classpath products for a single subject currently cause a failure."""
build_request = self.request(["compile"], self.java_multi)
walk = self.build_and_walk(build_request, failures=True)
# Validate that the root failed.
self.assert_root_failed(walk, SelectNode(self.java_multi, Classpath, None, None), ConflictingProducersError)
def test_descendant_specs(self):
"""Test that Addresses are produced via recursive globs of the 3rdparty/jvm directory."""
spec = self.spec_parser.parse_spec("3rdparty/jvm::")
build_request = self.request_specs(["list"], spec)
walk = self.build_and_walk(build_request)
# Validate the root.
root, root_state = walk[0][0]
root_value = root_state.value
self.assertEqual(DependenciesNode(spec, Address, None, Addresses, None), root)
self.assertEqual(list, type(root_value))
# Confirm that a few expected addresses are in the list.
self.assertIn(self.guava, root_value)
self.assertIn(self.managed_guava, root_value)
self.assertIn(self.managed_resolve_latest, root_value)
def test_sibling_specs(self):
"""Test that sibling Addresses are parsed in the 3rdparty/jvm directory."""
spec = self.spec_parser.parse_spec("3rdparty/jvm:")
build_request = self.request_specs(["list"], spec)
walk = self.build_and_walk(build_request)
# Validate the root.
root, root_state = walk[0][0]
root_value = root_state.value
self.assertEqual(DependenciesNode(spec, Address, None, Addresses, None), root)
self.assertEqual(list, type(root_value))
# Confirm that an expected address is in the list.
self.assertIn(self.guava, root_value)
# And that an subdirectory address is not.
self.assertNotIn(self.managed_guava, root_value)
def test_scheduler_visualize(self):
spec = self.spec_parser.parse_spec("3rdparty/jvm:")
build_request = self.request_specs(["list"], spec)
self.build_and_walk(build_request)
graphviz_output = "\n".join(self.scheduler.product_graph.visualize(build_request.roots))
with temporary_dir() as td:
output_path = os.path.join(td, "output.dot")
self.scheduler.visualize_graph_to_file(build_request.roots, output_path)
with open(output_path, "rb") as fh:
graphviz_disk_output = fh.read().strip()
self.assertEqual(graphviz_output, graphviz_disk_output)
self.assertIn("digraph", graphviz_output)
self.assertIn(" -> ", graphviz_output)
class SchedulerTest(unittest.TestCase):
def setUp(self):
build_root = os.path.join(os.path.dirname(__file__), 'examples',
'scheduler_inputs')
self.spec_parser = CmdLineSpecParser(build_root)
self.scheduler = setup_json_scheduler(build_root, inline_nodes=False)
self.pg = self.scheduler.product_graph
self.engine = LocalSerialEngine(self.scheduler)
self.guava = Address.parse('3rdparty/jvm:guava')
self.thrift = Address.parse('src/thrift/codegen/simple')
self.java = Address.parse('src/java/codegen/simple')
self.java_simple = Address.parse('src/java/simple')
self.java_multi = Address.parse('src/java/multiple_classpath_entries')
self.no_variant_thrift = Address.parse(
'src/java/codegen/selector:conflict')
self.unconfigured_thrift = Address.parse(
'src/thrift/codegen/unconfigured')
self.resources = Address.parse('src/resources/simple')
self.consumes_resources = Address.parse('src/java/consumes_resources')
self.consumes_managed_thirdparty = Address.parse(
'src/java/managed_thirdparty')
self.managed_guava = Address.parse('3rdparty/jvm/managed:guava')
self.managed_hadoop = Address.parse(
'3rdparty/jvm/managed:hadoop-common')
self.managed_resolve_latest = Address.parse(
'3rdparty/jvm/managed:latest-hadoop')
self.inferred_deps = Address.parse('src/scala/inferred_deps')
def assert_select_for_subjects(self,
walk,
selector,
subjects,
variants=None):
node_type = SelectNode
variants = tuple(variants.items()) if variants else None
self.assertEqual(
{node_type(subject, variants, selector)
for subject in subjects}, {
node
for node, _ in walk if node.product == selector.product
and isinstance(node, node_type) and node.variants == variants
})
def build_and_walk(self, build_request):
"""Build and then walk the given build_request, returning the walked graph as a list."""
result = self.engine.execute(build_request)
self.assertIsNone(result.error)
return list(self.scheduler.product_graph.walk(build_request.roots))
def request(self, goals, *addresses):
return self.request_specs(
goals, *[self.spec_parser.parse_spec(str(a)) for a in addresses])
def request_specs(self, goals, *specs):
return self.scheduler.build_request(goals=goals, subjects=specs)
def assert_resolve_only(self, goals, root_specs, jars):
build_request = self.request(goals, *root_specs)
walk = self.build_and_walk(build_request)
# Expect a SelectNode for each of the Jar/Classpath.
self.assert_select_for_subjects(walk, Select(Jar), jars)
self.assert_select_for_subjects(walk, Select(Classpath), jars)
def assert_root(self, walk, node, return_value):
"""Asserts that the first Node in a walk was a DependenciesNode with the single given result."""
root, root_state = walk[0]
self.assertEquals(type(root), DependenciesNode)
self.assertEquals(Return([return_value]), root_state)
self.assertIn(
(node, Return(return_value)),
[(d, self.pg.state(d)) for d in self.pg.dependencies_of(root)])
def assert_root_failed(self, walk, node, thrown_type):
"""Asserts that the first Node in a walk was a DependenciesNode with a Throw result."""
root, root_state = walk[0]
self.assertEquals(type(root), DependenciesNode)
self.assertEquals(Throw, type(root_state))
dependencies = [(d, self.pg.state(d))
for d in self.pg.dependencies_of(root)]
self.assertIn(
(node, thrown_type),
[(k, type(v.exc)) for k, v in dependencies if type(v) is Throw])
def test_type_error_on_unexpected_subject_type(self):
with self.assertRaises(TypeError) as cm:
self.scheduler.build_request(goals={}, subjects=['string'])
self.assertEquals(
"Unsupported root subject type: <type 'unicode'> for u'string'",
str(cm.exception))
def test_resolve(self):
self.assert_resolve_only(goals=['resolve'],
root_specs=['3rdparty/jvm:guava'],
jars=[self.guava])
def test_compile_only_3rdparty(self):
self.assert_resolve_only(goals=['compile'],
root_specs=['3rdparty/jvm:guava'],
jars=[self.guava])
def test_gen_noop(self):
# TODO(John Sirois): Ask around - is this OK?
# This is different than today. There is a gen'able target reachable from the java target, but
# the scheduler 'pull-seeding' has ApacheThriftPlanner stopping short since the subject it's
# handed is not thrift.
build_request = self.request(['gen'], self.java)
walk = self.build_and_walk(build_request)
self.assert_select_for_subjects(walk, Select(JavaSources,
optional=True),
[self.java])
def test_gen(self):
build_request = self.request(['gen'], self.thrift)
walk = self.build_and_walk(build_request)
# Root: expect the synthetic GenGoal product.
self.assert_root(
walk, SelectNode(self.thrift, None, Select(GenGoal)),
GenGoal("non-empty input to satisfy the Goal constructor"))
variants = {'thrift': 'apache_java'}
# Expect ThriftSources to have been selected.
self.assert_select_for_subjects(walk,
Select(ThriftSources), [self.thrift],
variants=variants)
# Expect an ApacheThriftJavaConfiguration to have been used via the default Variants.
self.assert_select_for_subjects(walk,
SelectVariant(
ApacheThriftJavaConfiguration,
variant_key='thrift'),
[self.thrift],
variants=variants)
def test_codegen_simple(self):
build_request = self.request(['compile'], self.java)
walk = self.build_and_walk(build_request)
# The subgraph below 'src/thrift/codegen/simple' will be affected by its default variants.
subjects = [self.guava, self.java, self.thrift]
variant_subjects = [
Jar(org='org.apache.thrift',
name='libthrift',
rev='0.9.2',
type_alias='jar'),
Jar(org='commons-lang',
name='commons-lang',
rev='2.5',
type_alias='jar'),
Address.parse('src/thrift:slf4j-api')
]
# Root: expect a DependenciesNode depending on a SelectNode with compilation via javac.
self.assert_root(walk, SelectNode(self.java, None, Select(Classpath)),
Classpath(creator='javac'))
# Confirm that exactly the expected subjects got Classpaths.
self.assert_select_for_subjects(walk, Select(Classpath), subjects)
self.assert_select_for_subjects(walk,
Select(Classpath),
variant_subjects,
variants={'thrift': 'apache_java'})
def test_consumes_resources(self):
build_request = self.request(['compile'], self.consumes_resources)
walk = self.build_and_walk(build_request)
# Validate the root.
self.assert_root(
walk, SelectNode(self.consumes_resources, None, Select(Classpath)),
Classpath(creator='javac'))
# Confirm a classpath for the resources target and other subjects. We know that they are
# reachable from the root (since it was involved in this walk).
subjects = [self.resources, self.consumes_resources, self.guava]
self.assert_select_for_subjects(walk, Select(Classpath), subjects)
def test_managed_resolve(self):
"""A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars."""
build_request = self.request(['compile'],
self.consumes_managed_thirdparty)
walk = self.build_and_walk(build_request)
# Validate the root.
self.assert_root(
walk,
SelectNode(self.consumes_managed_thirdparty, None,
Select(Classpath)), Classpath(creator='javac'))
# Confirm that we produced classpaths for the managed jars.
managed_jars = [self.managed_guava, self.managed_hadoop]
self.assert_select_for_subjects(walk, Select(Classpath),
[self.consumes_managed_thirdparty])
self.assert_select_for_subjects(walk,
Select(Classpath),
managed_jars,
variants={'resolve': 'latest-hadoop'})
# Confirm that the produced jars had the appropriate versions.
self.assertEquals(
{
Jar('org.apache.hadoop', 'hadoop-common', '2.7.0'),
Jar('com.google.guava', 'guava', '18.0')
}, {
ret.value
for node, ret in walk
if node.product == Jar and isinstance(node, SelectNode)
})
def test_dependency_inference(self):
"""Scala dependency inference introduces dependencies that do not exist in BUILD files."""
build_request = self.request(['compile'], self.inferred_deps)
walk = self.build_and_walk(build_request)
# Validate the root.
self.assert_root(
walk, SelectNode(self.inferred_deps, None, Select(Classpath)),
Classpath(creator='scalac'))
# Confirm that we requested a classpath for the root and inferred targets.
self.assert_select_for_subjects(walk, Select(Classpath),
[self.inferred_deps, self.java_simple])
def test_multiple_classpath_entries(self):
"""Multiple Classpath products for a single subject currently cause a failure."""
build_request = self.request(['compile'], self.java_multi)
walk = self.build_and_walk(build_request)
# Validate that the root failed.
self.assert_root_failed(
walk, SelectNode(self.java_multi, None, Select(Classpath)),
ConflictingProducersError)
def test_descendant_specs(self):
"""Test that Addresses are produced via recursive globs of the 3rdparty/jvm directory."""
spec = self.spec_parser.parse_spec('3rdparty/jvm::')
build_request = self.request_specs(['list'], spec)
walk = self.build_and_walk(build_request)
# Validate the root.
root, root_state = walk[0]
root_value = root_state.value
self.assertEqual(
DependenciesNode(
spec, None,
SelectDependencies(Address, Addresses,
field_types=(Address, ))), root)
self.assertEqual(list, type(root_value))
# Confirm that a few expected addresses are in the list.
self.assertIn(self.guava, root_value)
self.assertIn(self.managed_guava, root_value)
self.assertIn(self.managed_resolve_latest, root_value)
def test_sibling_specs(self):
"""Test that sibling Addresses are parsed in the 3rdparty/jvm directory."""
spec = self.spec_parser.parse_spec('3rdparty/jvm:')
build_request = self.request_specs(['list'], spec)
walk = self.build_and_walk(build_request)
# Validate the root.
root, root_state = walk[0]
root_value = root_state.value
self.assertEqual(
DependenciesNode(
spec, None,
SelectDependencies(Address, Addresses,
field_types=(Address, ))), root)
self.assertEqual(list, type(root_value))
# Confirm that an expected address is in the list.
self.assertIn(self.guava, root_value)
# And that an subdirectory address is not.
self.assertNotIn(self.managed_guava, root_value)
def test_scheduler_visualize(self):
spec = self.spec_parser.parse_spec('3rdparty/jvm:')
build_request = self.request_specs(['list'], spec)
self.build_and_walk(build_request)
graphviz_output = '\n'.join(
self.scheduler.product_graph.visualize(build_request.roots))
with temporary_dir() as td:
output_path = os.path.join(td, 'output.dot')
self.scheduler.visualize_graph_to_file(build_request.roots,
output_path)
with open(output_path, 'rb') as fh:
graphviz_disk_output = fh.read().strip()
self.assertEqual(graphviz_output, graphviz_disk_output)
self.assertIn('digraph', graphviz_output)
self.assertIn(' -> ', graphviz_output)
class CmdLineSpecParserTest(TestBase):
def setUp(self) -> None:
super().setUp()
self._spec_parser = CmdLineSpecParser(self.build_root)
def test_normal_address_specs(self) -> None:
self.assert_address_spec_parsed(":root", single("", "root"))
self.assert_address_spec_parsed("//:root", single("", "root"))
self.assert_address_spec_parsed("a", single("a"))
self.assert_address_spec_parsed("a:a", single("a", "a"))
self.assert_address_spec_parsed("a/b", single("a/b"))
self.assert_address_spec_parsed("a/b:b", single("a/b", "b"))
self.assert_address_spec_parsed("a/b:c", single("a/b", "c"))
def test_sibling(self) -> None:
self.assert_address_spec_parsed(":", sib(""))
self.assert_address_spec_parsed("//:", sib(""))
self.assert_address_spec_parsed("a:", sib("a"))
self.assert_address_spec_parsed("//a:", sib("a"))
self.assert_address_spec_parsed("a/b:", sib("a/b"))
self.assert_address_spec_parsed("//a/b:", sib("a/b"))
def test_descendant(self) -> None:
self.assert_address_spec_parsed("::", desc(""))
self.assert_address_spec_parsed("//::", desc(""))
self.assert_address_spec_parsed("a::", desc("a"))
self.assert_address_spec_parsed("//a::", desc("a"))
self.assert_address_spec_parsed("a/b::", desc("a/b"))
self.assert_address_spec_parsed("//a/b::", desc("a/b"))
def test_files(self) -> None:
# We assume that specs with an extension are meant to be interpreted as filesystem specs.
for f in ["a.txt", "a.tmp.cache.txt.bak", "a/b/c.txt", ".a.txt"]:
self.assert_filesystem_spec_parsed(f, literal(f))
self.assert_filesystem_spec_parsed("./a.txt", literal("a.txt"))
self.assert_filesystem_spec_parsed("//./a.txt", literal("a.txt"))
def test_globs(self) -> None:
for glob_str in [
"*", "**/*", "a/b/*", "a/b/test_*.py", "a/b/**/test_*"
]:
self.assert_filesystem_spec_parsed(glob_str, glob(glob_str))
def test_excludes(self) -> None:
for glob_str in ["!", "!a/b/", "!/a/b/*"]:
self.assert_filesystem_spec_parsed(glob_str, ignore(glob_str[1:]))
def test_ambiguous_files(self) -> None:
# These could either be files or the shorthand for single addresses. We check if they exist
# on the file system to disambiguate.
for spec in ["a", "b/c"]:
self.assert_address_spec_parsed(spec, single(spec))
self.create_file(spec)
self.assert_filesystem_spec_parsed(spec, literal(spec))
def test_absolute(self) -> None:
self.assert_address_spec_parsed(os.path.join(self.build_root, "a"),
single("a"))
self.assert_address_spec_parsed(os.path.join(self.build_root, "a:a"),
single("a", "a"))
self.assert_address_spec_parsed(os.path.join(self.build_root, "a:"),
sib("a"))
self.assert_address_spec_parsed(os.path.join(self.build_root, "a::"),
desc("a"))
self.assert_filesystem_spec_parsed(
os.path.join(self.build_root, "a.txt"), literal("a.txt"))
with self.assertRaises(CmdLineSpecParser.BadSpecError):
self.assert_address_spec_parsed("/not/the/buildroot/a", sib("a"))
self.assert_filesystem_spec_parsed("/not/the/buildroot/a.txt",
literal("a.txt"))
def test_absolute_double_slashed(self) -> None:
# By adding a double slash, we are insisting that this absolute path is actually
# relative to the buildroot. Thus, it should parse correctly.
double_absolute_address = "/" + os.path.join(self.build_root, "a")
double_absolute_file = "/" + os.path.join(self.build_root, "a.txt")
for spec in [double_absolute_address, double_absolute_file]:
assert "//" == spec[:2]
self.assert_address_spec_parsed(double_absolute_address,
single(double_absolute_address[2:]))
self.assert_filesystem_spec_parsed(double_absolute_file,
literal(double_absolute_file[2:]))
def test_cmd_line_affordances(self) -> None:
self.assert_address_spec_parsed("./:root", single("", "root"))
self.assert_address_spec_parsed("//./:root", single("", "root"))
self.assert_address_spec_parsed("//./a/../:root", single("", "root"))
self.assert_address_spec_parsed(
os.path.join(self.build_root, "./a/../:root"), single("", "root"))
self.assert_address_spec_parsed("a/", single("a"))
self.assert_address_spec_parsed("./a/", single("a"))
self.assert_address_spec_parsed(os.path.join(self.build_root, "./a/"),
single("a"))
self.assert_address_spec_parsed("a/b/:b", single("a/b", "b"))
self.assert_address_spec_parsed("./a/b/:b", single("a/b", "b"))
self.assert_address_spec_parsed(
os.path.join(self.build_root, "./a/b/:b"), single("a/b", "b"))
def assert_address_spec_parsed(self, spec_str: str,
expected_spec: AddressSpec) -> None:
spec = self._spec_parser.parse_spec(spec_str)
assert isinstance(spec, AddressSpec)
assert spec == expected_spec
def assert_filesystem_spec_parsed(self, spec_str: str,
expected_spec: FilesystemSpec) -> None:
spec = self._spec_parser.parse_spec(spec_str)
assert isinstance(spec, FilesystemSpec)
assert spec == expected_spec
class SchedulerTest(unittest.TestCase):
_native = init_native()
def setUp(self):
build_root = os.path.join(os.path.dirname(__file__), 'examples',
'scheduler_inputs')
self.spec_parser = CmdLineSpecParser(build_root)
self.scheduler = setup_json_scheduler(build_root, self._native)
self.guava = Address.parse('3rdparty/jvm:guava')
self.thrift = Address.parse('src/thrift/codegen/simple')
self.java = Address.parse('src/java/codegen/simple')
self.java_simple = Address.parse('src/java/simple')
self.java_multi = Address.parse('src/java/multiple_classpath_entries')
self.no_variant_thrift = Address.parse(
'src/java/codegen/selector:conflict')
self.unconfigured_thrift = Address.parse(
'src/thrift/codegen/unconfigured')
self.resources = Address.parse('src/resources/simple')
self.consumes_resources = Address.parse('src/java/consumes_resources')
self.consumes_managed_thirdparty = Address.parse(
'src/java/managed_thirdparty')
self.managed_guava = Address.parse('3rdparty/jvm/managed:guava')
self.managed_hadoop = Address.parse(
'3rdparty/jvm/managed:hadoop-common')
self.managed_resolve_latest = Address.parse(
'3rdparty/jvm/managed:latest-hadoop')
self.inferred_deps = Address.parse('src/scala/inferred_deps')
def tearDown(self):
super(SchedulerTest, self).tearDown()
# Without eagerly dropping this reference, each instance created for a test method
# will live until all tests in this class have completed: can confirm by editing
# the `scheduler_destroy` call in `src/python/pants/engine/native.py`.
self.scheduler = None
def parse_specs(self, *specs):
return Specs(tuple(
self.spec_parser.parse_spec(spec) for spec in specs))
def assert_select_for_subjects(self,
walk,
selector,
subjects,
variants=None):
raise ValueError(walk)
def build(self, execution_request):
"""Execute the given request and return roots as a list of ((subject, product), value) tuples."""
result = self.scheduler.execute(execution_request)
self.assertIsNone(result.error)
return result.root_products
def request(self, products, *subjects):
return self.scheduler.execution_request(products, subjects)
def assert_root(self, root, subject, return_value):
"""Asserts that the given root has the given result."""
self.assertEquals(subject, root[0][0])
self.assertEquals(Return(return_value), root[1])
def assert_root_failed(self, root, subject, msg_str):
"""Asserts that the root was a Throw result containing the given msg string."""
self.assertEquals(subject, root[0][0])
self.assertEquals(Throw, type(root[1]))
self.assertIn(msg_str, str(root[1].exc))
def test_compile_only_3rdparty(self):
build_request = self.request([Classpath], self.guava)
root, = self.build(build_request)
self.assert_root(root, self.guava, Classpath(creator='ivy_resolve'))
@unittest.skip('Skipped to expedite landing #3821; see: #4027.')
def test_compile_only_3rdparty_internal(self):
build_request = self.request([Classpath], '3rdparty/jvm:guava')
root, = self.build(build_request)
# Expect a SelectNode for each of the Jar/Classpath.
self.assert_select_for_subjects(walk, Select(Jar), [self.guava])
self.assert_select_for_subjects(walk, Select(Classpath), [self.guava])
@unittest.skip('Skipped to expedite landing #3821; see: #4020.')
def test_gen(self):
build_request = self.request([GenGoal], self.thrift)
root, = self.build(build_request)
# Root: expect the synthetic GenGoal product.
self.assert_root(
root, self.thrift,
GenGoal("non-empty input to satisfy the Goal constructor"))
variants = {'thrift': 'apache_java'}
# Expect ThriftSources to have been selected.
self.assert_select_for_subjects(walk,
Select(ThriftSources), [self.thrift],
variants=variants)
# Expect an ApacheThriftJavaConfiguration to have been used via the default Variants.
self.assert_select_for_subjects(walk,
SelectVariant(
ApacheThriftJavaConfiguration,
variant_key='thrift'),
[self.thrift],
variants=variants)
@unittest.skip('Skipped to expedite landing #3821; see: #4020.')
def test_codegen_simple(self):
build_request = self.request([Classpath], self.java)
root, = self.build(build_request)
# The subgraph below 'src/thrift/codegen/simple' will be affected by its default variants.
subjects = [self.guava, self.java, self.thrift]
variant_subjects = [
Jar(org='org.apache.thrift',
name='libthrift',
rev='0.9.2',
type_alias='jar'),
Jar(org='commons-lang',
name='commons-lang',
rev='2.5',
type_alias='jar'),
Address.parse('src/thrift:slf4j-api')
]
# Root: expect a DependenciesNode depending on a SelectNode with compilation via javac.
self.assert_root(root, self.java, Classpath(creator='javac'))
# Confirm that exactly the expected subjects got Classpaths.
self.assert_select_for_subjects(walk, Select(Classpath), subjects)
self.assert_select_for_subjects(walk,
Select(Classpath),
variant_subjects,
variants={'thrift': 'apache_java'})
def test_consumes_resources(self):
build_request = self.request([Classpath], self.consumes_resources)
root, = self.build(build_request)
self.assert_root(root, self.consumes_resources,
Classpath(creator='javac'))
@unittest.skip('Skipped to expedite landing #3821; see: #4027.')
def test_consumes_resources_internal(self):
build_request = self.request([Classpath], self.consumes_resources)
root, = self.build(build_request)
# Confirm a classpath for the resources target and other subjects. We know that they are
# reachable from the root (since it was involved in this walk).
subjects = [self.resources, self.consumes_resources, self.guava]
self.assert_select_for_subjects(walk, Select(Classpath), subjects)
@unittest.skip('Skipped to expedite landing #3821; see: #4020.')
def test_managed_resolve(self):
"""A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars."""
build_request = self.request([Classpath],
self.consumes_managed_thirdparty)
root, = self.build(build_request)
# Validate the root.
self.assert_root(root, self.consumes_managed_thirdparty,
Classpath(creator='javac'))
# Confirm that we produced classpaths for the managed jars.
managed_jars = [self.managed_guava, self.managed_hadoop]
self.assert_select_for_subjects(walk, Select(Classpath),
[self.consumes_managed_thirdparty])
self.assert_select_for_subjects(walk,
Select(Classpath),
managed_jars,
variants={'resolve': 'latest-hadoop'})
# Confirm that the produced jars had the appropriate versions.
self.assertEquals(
{
Jar('org.apache.hadoop', 'hadoop-common', '2.7.0'),
Jar('com.google.guava', 'guava', '18.0')
}, {ret.value
for node, ret in walk if node.product == Jar})
def test_dependency_inference(self):
"""Scala dependency inference introduces dependencies that do not exist in BUILD files."""
build_request = self.request([Classpath], self.inferred_deps)
root, = self.build(build_request)
self.assert_root(root, self.inferred_deps, Classpath(creator='scalac'))
@unittest.skip('Skipped to expedite landing #3821; see: #4027.')
def test_dependency_inference_internal(self):
"""Scala dependency inference introduces dependencies that do not exist in BUILD files."""
build_request = self.request([Classpath], self.inferred_deps)
root, = self.build(build_request)
# Confirm that we requested a classpath for the root and inferred targets.
self.assert_select_for_subjects(walk, Select(Classpath),
[self.inferred_deps, self.java_simple])
def test_multiple_classpath_entries(self):
"""Multiple Classpath products for a single subject currently cause a failure."""
build_request = self.request([Classpath], self.java_multi)
root, = self.build(build_request)
# Validate that the root failed.
self.assert_root_failed(root, self.java_multi,
"Conflicting values produced for")
def test_descendant_specs(self):
"""Test that Addresses are produced via recursive globs of the 3rdparty/jvm directory."""
specs = self.parse_specs('3rdparty/jvm::')
build_request = self.scheduler.execution_request([BuildFileAddresses],
[specs])
((subject, _), root), = self.build(build_request)
# Validate the root.
self.assertEqual(specs, subject)
self.assertEqual(BuildFileAddresses, type(root.value))
# Confirm that a few expected addresses are in the list.
self.assertIn(self.guava, root.value.dependencies)
self.assertIn(self.managed_guava, root.value.dependencies)
self.assertIn(self.managed_resolve_latest, root.value.dependencies)
def test_sibling_specs(self):
"""Test that sibling Addresses are parsed in the 3rdparty/jvm directory."""
specs = self.parse_specs('3rdparty/jvm:')
build_request = self.scheduler.execution_request([BuildFileAddresses],
[specs])
((subject, _), root), = self.build(build_request)
# Validate the root.
self.assertEqual(specs, subject)
self.assertEqual(BuildFileAddresses, type(root.value))
# Confirm that an expected address is in the list.
self.assertIn(self.guava, root.value.dependencies)
# And that a subdirectory address is not.
self.assertNotIn(self.managed_guava, root.value.dependencies)
def test_scheduler_visualize(self):
specs = self.parse_specs('3rdparty/jvm::')
build_request = self.request([BuildFileAddresses], specs)
self.build(build_request)
with temporary_dir() as td:
output_path = os.path.join(td, 'output.dot')
self.scheduler.visualize_graph_to_file(build_request, output_path)
with open(output_path, 'rb') as fh:
graphviz_output = fh.read().strip()
self.assertIn('digraph', graphviz_output)
self.assertIn(' -> ', graphviz_output)
class SchedulerTest(unittest.TestCase):
def setUp(self):
build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs')
self.spec_parser = CmdLineSpecParser(build_root)
self.scheduler = setup_json_scheduler(build_root)
self.subjects = self.scheduler._subjects
self.engine = LocalSerialEngine(self.scheduler)
self.guava = Address.parse('3rdparty/jvm:guava')
self.thrift = Address.parse('src/thrift/codegen/simple')
self.java = Address.parse('src/java/codegen/simple')
self.java_simple = Address.parse('src/java/simple')
self.java_multi = Address.parse('src/java/multiple_classpath_entries')
self.no_variant_thrift = Address.parse('src/java/codegen/selector:conflict')
self.unconfigured_thrift = Address.parse('src/thrift/codegen/unconfigured')
self.resources = Address.parse('src/resources/simple')
self.consumes_resources = Address.parse('src/java/consumes_resources')
self.consumes_managed_thirdparty = Address.parse('src/java/managed_thirdparty')
self.managed_guava = Address.parse('3rdparty/jvm/managed:guava')
self.managed_hadoop = Address.parse('3rdparty/jvm/managed:hadoop-common')
self.managed_resolve_latest = Address.parse('3rdparty/jvm/managed:latest-hadoop')
self.inferred_deps = Address.parse('src/scala/inferred_deps')
def key(self, subject):
return self.subjects.put(subject)
def assert_select_for_subjects(self, walk, product, subjects, variants=None, variant_key=None):
node_type = SelectNode
variants = tuple(variants.items()) if variants else None
self.assertEqual({node_type(self.key(subject), product, variants, variant_key) for subject in subjects},
{node for (node, _), _ in walk
if node.product == product and isinstance(node, node_type) and node.variants == variants})
def build_and_walk(self, build_request, failures=False):
"""Build and then walk the given build_request, returning the walked graph as a list."""
predicate = (lambda _: True) if failures else None
result = self.engine.execute(build_request)
self.assertIsNone(result.error)
return list(self.scheduler.product_graph.walk(build_request.roots, predicate=predicate))
def request(self, goals, *addresses):
return self.request_specs(goals, *[self.spec_parser.parse_spec(str(a)) for a in addresses])
def request_specs(self, goals, *specs):
return self.scheduler.build_request(goals=goals, subjects=specs)
def assert_resolve_only(self, goals, root_specs, jars):
build_request = self.request(goals, *root_specs)
walk = self.build_and_walk(build_request)
# Expect a SelectNode for each of the Jar/Classpath.
self.assert_select_for_subjects(walk, Jar, jars)
self.assert_select_for_subjects(walk, Classpath, jars)
def assert_root(self, walk, node, return_value):
"""Asserts that the first Node in a walk was a DependenciesNode with the single given result."""
((root, root_state), dependencies) = walk[0]
self.assertEquals(type(root), DependenciesNode)
self.assertEquals(Return([return_value]), root_state)
self.assertIn((node, Return(return_value)), dependencies)
def assert_root_failed(self, walk, node, thrown_type):
"""Asserts that the first Node in a walk was a DependenciesNode with a Throw result."""
((root, root_state), dependencies) = walk[0]
self.assertEquals(type(root), DependenciesNode)
self.assertEquals(Throw, type(root_state))
self.assertIn((node, thrown_type), [(k, type(v.exc)) for k, v in dependencies
if type(v) is Throw])
def test_resolve(self):
self.assert_resolve_only(goals=['resolve'],
root_specs=['3rdparty/jvm:guava'],
jars=[self.guava])
def test_compile_only_3rdparty(self):
self.assert_resolve_only(goals=['compile'],
root_specs=['3rdparty/jvm:guava'],
jars=[self.guava])
def test_gen_noop(self):
# TODO(John Sirois): Ask around - is this OK?
# This is different than today. There is a gen'able target reachable from the java target, but
# the scheduler 'pull-seeding' has ApacheThriftPlanner stopping short since the subject it's
# handed is not thrift.
build_request = self.request(['gen'], self.java)
walk = self.build_and_walk(build_request)
self.assert_select_for_subjects(walk, JavaSources, [self.java])
def test_gen(self):
build_request = self.request(['gen'], self.thrift)
walk = self.build_and_walk(build_request)
# Root: expect the synthetic GenGoal product.
self.assert_root(walk,
SelectNode(self.key(self.thrift), GenGoal, None, None),
GenGoal("non-empty input to satisfy the Goal constructor"))
variants = {'thrift': 'apache_java'}
# Expect ThriftSources to have been selected.
self.assert_select_for_subjects(walk, ThriftSources, [self.thrift], variants=variants)
# Expect an ApacheThriftJavaConfiguration to have been used via the default Variants.
self.assert_select_for_subjects(walk, ApacheThriftJavaConfiguration, [self.thrift],
variants=variants, variant_key='thrift')
def test_codegen_simple(self):
build_request = self.request(['compile'], self.java)
walk = self.build_and_walk(build_request)
# TODO: Utter insanity. Pickle only encodes a unique object (ie, `if A is B`) a single
# time on the wire. Because the copy of this object that we're comparing to is coming
# from a file, the string will be encoded twice. Thus, to match (with pickle) we need
# to ensure that `(cl1 is not cl2)` here. See:
# https://github.com/pantsbuild/pants/issues/2969
cl1 = 'commons-lang'
cl2 = 'commons' + '-lang'
# The subgraph below 'src/thrift/codegen/simple' will be affected by its default variants.
subjects = [
self.guava,
self.java,
self.thrift]
variant_subjects = [
Jar(org='org.apache.thrift', name='libthrift', rev='0.9.2', type_alias='jar'),
Jar(org=cl1, name=cl2, rev='2.5', type_alias='jar'),
Address.parse('src/thrift:slf4j-api')]
# Root: expect a DependenciesNode depending on a SelectNode with compilation via javac.
self.assert_root(walk,
SelectNode(self.key(self.java), Classpath, None, None),
Classpath(creator='javac'))
# Confirm that exactly the expected subjects got Classpaths.
self.assert_select_for_subjects(walk, Classpath, subjects)
self.assert_select_for_subjects(walk, Classpath, variant_subjects,
variants={'thrift': 'apache_java'})
def test_consumes_resources(self):
build_request = self.request(['compile'], self.consumes_resources)
walk = self.build_and_walk(build_request)
# Validate the root.
self.assert_root(walk,
SelectNode(self.key(self.consumes_resources), Classpath, None, None),
Classpath(creator='javac'))
# Confirm a classpath for the resources target and other subjects. We know that they are
# reachable from the root (since it was involved in this walk).
subjects = [self.resources,
self.consumes_resources,
self.guava]
self.assert_select_for_subjects(walk, Classpath, subjects)
def test_managed_resolve(self):
"""A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars."""
build_request = self.request(['compile'], self.consumes_managed_thirdparty)
walk = self.build_and_walk(build_request)
# Validate the root.
self.assert_root(walk,
SelectNode(self.key(self.consumes_managed_thirdparty), Classpath, None, None),
Classpath(creator='javac'))
# Confirm that we produced classpaths for the managed jars.
managed_jars = [self.managed_guava,
self.managed_hadoop]
self.assert_select_for_subjects(walk, Classpath, [self.consumes_managed_thirdparty])
self.assert_select_for_subjects(walk, Classpath, managed_jars, variants={'resolve': 'latest-hadoop'})
# Confirm that the produced jars had the appropriate versions.
self.assertEquals({Jar('org.apache.hadoop', 'hadoop-common', '2.7.0'),
Jar('com.google.guava', 'guava', '18.0')},
{ret.value for (node, ret), _ in walk
if node.product == Jar and isinstance(node, SelectNode)})
def test_dependency_inference(self):
"""Scala dependency inference introduces dependencies that do not exist in BUILD files."""
build_request = self.request(['compile'], self.inferred_deps)
walk = self.build_and_walk(build_request)
# Validate the root.
self.assert_root(walk,
SelectNode(self.key(self.inferred_deps), Classpath, None, None),
Classpath(creator='scalac'))
# Confirm that we requested a classpath for the root and inferred targets.
self.assert_select_for_subjects(walk, Classpath, [self.inferred_deps, self.java_simple])
def test_multiple_classpath_entries(self):
"""Multiple Classpath products for a single subject currently cause a failure."""
build_request = self.request(['compile'], self.java_multi)
walk = self.build_and_walk(build_request, failures=True)
# Validate that the root failed.
self.assert_root_failed(walk,
SelectNode(self.key(self.java_multi), Classpath, None, None),
ConflictingProducersError)
def test_no_variant_thrift(self):
"""No `thrift` variant is configured, and so no configuration is selected."""
build_request = self.request(['compile'], self.no_variant_thrift)
with self.assertRaises(PartiallyConsumedInputsError):
self.build_and_walk(build_request)
def test_unconfigured_thrift(self):
"""The BuildPropertiesPlanner is able to produce a Classpath, but we should still fail.
A target with ThriftSources doesn't have a thrift config: that input is partially consumed.
"""
build_request = self.request(['compile'], self.unconfigured_thrift)
with self.assertRaises(PartiallyConsumedInputsError):
self.build_and_walk(build_request)
def test_descendant_specs(self):
"""Test that Addresses are produced via recursive globs of the 3rdparty/jvm directory."""
spec = self.spec_parser.parse_spec('3rdparty/jvm::')
build_request = self.request_specs(['list'], spec)
walk = self.build_and_walk(build_request)
# Validate the root.
root, root_state = walk[0][0]
root_value = root_state.value
self.assertEqual(DependenciesNode(self.key(spec), Address, None, Addresses, None), root)
self.assertEqual(list, type(root_value))
# Confirm that a few expected addresses are in the list.
self.assertIn(self.guava, root_value)
self.assertIn(self.managed_guava, root_value)
self.assertIn(self.managed_resolve_latest, root_value)
def test_sibling_specs(self):
"""Test that sibling Addresses are parsed in the 3rdparty/jvm directory."""
spec = self.spec_parser.parse_spec('3rdparty/jvm:')
build_request = self.request_specs(['list'], spec)
walk = self.build_and_walk(build_request)
# Validate the root.
root, root_state = walk[0][0]
root_value = root_state.value
self.assertEqual(DependenciesNode(self.key(spec), Address, None, Addresses, None), root)
self.assertEqual(list, type(root_value))
# Confirm that an expected address is in the list.
self.assertIn(self.guava, root_value)
# And that an subdirectory address is not.
self.assertNotIn(self.managed_guava, root_value)
class SchedulerTest(unittest.TestCase):
_native = init_native()
def setUp(self):
build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs')
self.spec_parser = CmdLineSpecParser(build_root)
self.scheduler = setup_json_scheduler(build_root, self._native)
self.engine = LocalSerialEngine(self.scheduler)
self.guava = Address.parse('3rdparty/jvm:guava')
self.thrift = Address.parse('src/thrift/codegen/simple')
self.java = Address.parse('src/java/codegen/simple')
self.java_simple = Address.parse('src/java/simple')
self.java_multi = Address.parse('src/java/multiple_classpath_entries')
self.no_variant_thrift = Address.parse('src/java/codegen/selector:conflict')
self.unconfigured_thrift = Address.parse('src/thrift/codegen/unconfigured')
self.resources = Address.parse('src/resources/simple')
self.consumes_resources = Address.parse('src/java/consumes_resources')
self.consumes_managed_thirdparty = Address.parse('src/java/managed_thirdparty')
self.managed_guava = Address.parse('3rdparty/jvm/managed:guava')
self.managed_hadoop = Address.parse('3rdparty/jvm/managed:hadoop-common')
self.managed_resolve_latest = Address.parse('3rdparty/jvm/managed:latest-hadoop')
self.inferred_deps = Address.parse('src/scala/inferred_deps')
def assert_select_for_subjects(self, walk, selector, subjects, variants=None):
raise ValueError(walk)
def build(self, build_request):
"""Execute the given request and return roots as a list of ((subject, product), value) tuples."""
result = self.engine.execute(build_request)
self.assertIsNone(result.error)
return self.scheduler.root_entries(build_request).items()
def request(self, goals, *subjects):
return self.scheduler.build_request(goals=goals, subjects=subjects)
def assert_root(self, root, subject, return_value):
"""Asserts that the given root has the given result."""
self.assertEquals(subject, root[0][0])
self.assertEquals(Return(return_value), root[1])
def assert_root_failed(self, root, subject, msg_str):
"""Asserts that the root was a Throw result containing the given msg string."""
self.assertEquals(subject, root[0][0])
self.assertEquals(Throw, type(root[1]))
self.assertIn(msg_str, str(root[1].exc))
def test_compile_only_3rdparty(self):
build_request = self.request(['compile'], self.guava)
root, = self.build(build_request)
self.assert_root(root, self.guava, Classpath(creator='ivy_resolve'))
@unittest.skip('Skipped to expedite landing #3821; see: #4027.')
def test_compile_only_3rdparty_internal(self):
build_request = self.request(['compile'], '3rdparty/jvm:guava')
root, = self.build(build_request)
# Expect a SelectNode for each of the Jar/Classpath.
self.assert_select_for_subjects(walk, Select(Jar), [self.guava])
self.assert_select_for_subjects(walk, Select(Classpath), [self.guava])
@unittest.skip('Skipped to expedite landing #3821; see: #4020.')
def test_gen(self):
build_request = self.request(['gen'], self.thrift)
root, = self.build(build_request)
# Root: expect the synthetic GenGoal product.
self.assert_root(root, self.thrift, GenGoal("non-empty input to satisfy the Goal constructor"))
variants = {'thrift': 'apache_java'}
# Expect ThriftSources to have been selected.
self.assert_select_for_subjects(walk, Select(ThriftSources), [self.thrift], variants=variants)
# Expect an ApacheThriftJavaConfiguration to have been used via the default Variants.
self.assert_select_for_subjects(walk, SelectVariant(ApacheThriftJavaConfiguration,
variant_key='thrift'),
[self.thrift],
variants=variants)
@unittest.skip('Skipped to expedite landing #3821; see: #4020.')
def test_codegen_simple(self):
build_request = self.request(['compile'], self.java)
root, = self.build(build_request)
# The subgraph below 'src/thrift/codegen/simple' will be affected by its default variants.
subjects = [self.guava, self.java, self.thrift]
variant_subjects = [
Jar(org='org.apache.thrift', name='libthrift', rev='0.9.2', type_alias='jar'),
Jar(org='commons-lang', name='commons-lang', rev='2.5', type_alias='jar'),
Address.parse('src/thrift:slf4j-api')]
# Root: expect a DependenciesNode depending on a SelectNode with compilation via javac.
self.assert_root(root, self.java, Classpath(creator='javac'))
# Confirm that exactly the expected subjects got Classpaths.
self.assert_select_for_subjects(walk, Select(Classpath), subjects)
self.assert_select_for_subjects(walk, Select(Classpath), variant_subjects,
variants={'thrift': 'apache_java'})
def test_consumes_resources(self):
build_request = self.request(['compile'], self.consumes_resources)
root, = self.build(build_request)
self.assert_root(root, self.consumes_resources, Classpath(creator='javac'))
@unittest.skip('Skipped to expedite landing #3821; see: #4027.')
def test_consumes_resources_internal(self):
build_request = self.request(['compile'], self.consumes_resources)
root, = self.build(build_request)
# Confirm a classpath for the resources target and other subjects. We know that they are
# reachable from the root (since it was involved in this walk).
subjects = [self.resources,
self.consumes_resources,
self.guava]
self.assert_select_for_subjects(walk, Select(Classpath), subjects)
@unittest.skip('Skipped to expedite landing #3821; see: #4020.')
def test_managed_resolve(self):
"""A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars."""
build_request = self.request(['compile'], self.consumes_managed_thirdparty)
root, = self.build(build_request)
# Validate the root.
self.assert_root(root, self.consumes_managed_thirdparty, Classpath(creator='javac'))
# Confirm that we produced classpaths for the managed jars.
managed_jars = [self.managed_guava, self.managed_hadoop]
self.assert_select_for_subjects(walk, Select(Classpath), [self.consumes_managed_thirdparty])
self.assert_select_for_subjects(walk, Select(Classpath), managed_jars,
variants={'resolve': 'latest-hadoop'})
# Confirm that the produced jars had the appropriate versions.
self.assertEquals({Jar('org.apache.hadoop', 'hadoop-common', '2.7.0'),
Jar('com.google.guava', 'guava', '18.0')},
{ret.value for node, ret in walk
if node.product == Jar})
def test_dependency_inference(self):
"""Scala dependency inference introduces dependencies that do not exist in BUILD files."""
build_request = self.request(['compile'], self.inferred_deps)
root, = self.build(build_request)
self.assert_root(root, self.inferred_deps, Classpath(creator='scalac'))
@unittest.skip('Skipped to expedite landing #3821; see: #4027.')
def test_dependency_inference_internal(self):
"""Scala dependency inference introduces dependencies that do not exist in BUILD files."""
build_request = self.request(['compile'], self.inferred_deps)
root, = self.build(build_request)
# Confirm that we requested a classpath for the root and inferred targets.
self.assert_select_for_subjects(walk, Select(Classpath), [self.inferred_deps, self.java_simple])
def test_multiple_classpath_entries(self):
"""Multiple Classpath products for a single subject currently cause a failure."""
build_request = self.request(['compile'], self.java_multi)
root, = self.build(build_request)
# Validate that the root failed.
self.assert_root_failed(root, self.java_multi, "Conflicting values produced for")
def test_descendant_specs(self):
"""Test that Addresses are produced via recursive globs of the 3rdparty/jvm directory."""
spec = self.spec_parser.parse_spec('3rdparty/jvm::')
selector = Select(BuildFileAddresses)
build_request = self.scheduler.selection_request([(selector, spec)])
((subject, _), root), = self.build(build_request)
# Validate the root.
self.assertEqual(spec, subject)
self.assertEqual(BuildFileAddresses, type(root.value))
# Confirm that a few expected addresses are in the list.
self.assertIn(self.guava, root.value.dependencies)
self.assertIn(self.managed_guava, root.value.dependencies)
self.assertIn(self.managed_resolve_latest, root.value.dependencies)
def test_sibling_specs(self):
"""Test that sibling Addresses are parsed in the 3rdparty/jvm directory."""
spec = self.spec_parser.parse_spec('3rdparty/jvm:')
selector = Select(BuildFileAddresses)
build_request = self.scheduler.selection_request([(selector,spec)])
((subject, _), root), = self.build(build_request)
# Validate the root.
self.assertEqual(spec, subject)
self.assertEqual(BuildFileAddresses, type(root.value))
# Confirm that an expected address is in the list.
self.assertIn(self.guava, root.value.dependencies)
# And that a subdirectory address is not.
self.assertNotIn(self.managed_guava, root.value.dependencies)
def test_scheduler_visualize(self):
spec = self.spec_parser.parse_spec('3rdparty/jvm:')
build_request = self.request(['list'], spec)
self.build(build_request)
with temporary_dir() as td:
output_path = os.path.join(td, 'output.dot')
self.scheduler.visualize_graph_to_file(output_path)
with open(output_path, 'rb') as fh:
graphviz_output = fh.read().strip()
self.assertIn('digraph', graphviz_output)
self.assertIn(' -> ', graphviz_output)
class BuildFileAddressMapperScanTest(BaseTest):
NO_FAIL_FAST_RE = re.compile(r"""^--------------------
.*
Exception message: name 'a_is_bad' is not defined
while executing BUILD file BuildFile\(bad/a/BUILD, FileSystemProjectTree\(.*\)\)
Loading addresses from 'bad/a' failed\.
.*
Exception message: name 'b_is_bad' is not defined
while executing BUILD file BuildFile\(bad/b/BUILD, FileSystemProjectTree\(.*\)\)
Loading addresses from 'bad/b' failed\.
Invalid BUILD files for \[::\]$""", re.DOTALL)
FAIL_FAST_RE = """^name 'a_is_bad' is not defined
while executing BUILD file BuildFile\(bad/a/BUILD\, FileSystemProjectTree\(.*\)\)
Loading addresses from 'bad/a' failed.$"""
def setUp(self):
super(BuildFileAddressMapperScanTest, self).setUp()
def add_target(path, name):
self.add_to_build_file(path, 'target(name="{name}")\n'.format(name=name))
add_target('BUILD', 'root')
add_target('a', 'a')
add_target('a', 'b')
add_target('a/b', 'b')
add_target('a/b', 'c')
self._spec_parser = CmdLineSpecParser(self.build_root)
def test_bad_build_files(self):
self.add_to_build_file('bad/a', 'a_is_bad')
self.add_to_build_file('bad/b', 'b_is_bad')
with self.assertRaisesRegexp(AddressLookupError, self.NO_FAIL_FAST_RE):
list(self.address_mapper.scan_specs([DescendantAddresses('')], fail_fast=False))
def test_bad_build_files_fail_fast(self):
self.add_to_build_file('bad/a', 'a_is_bad')
self.add_to_build_file('bad/b', 'b_is_bad')
with self.assertRaisesRegexp(AddressLookupError, self.FAIL_FAST_RE):
list(self.address_mapper.scan_specs([DescendantAddresses('')], fail_fast=True))
def test_normal(self):
self.assert_scanned([':root'], expected=[':root'])
self.assert_scanned(['//:root'], expected=[':root'])
self.assert_scanned(['a'], expected=['a'])
self.assert_scanned(['a:a'], expected=['a'])
self.assert_scanned(['a/b'], expected=['a/b'])
self.assert_scanned(['a/b:b'], expected=['a/b'])
self.assert_scanned(['a/b:c'], expected=['a/b:c'])
def test_sibling(self):
self.assert_scanned([':'], expected=[':root'])
self.assert_scanned(['//:'], expected=[':root'])
self.assert_scanned(['a:'], expected=['a', 'a:b'])
self.assert_scanned(['//a:'], expected=['a', 'a:b'])
self.assert_scanned(['a/b:'], expected=['a/b', 'a/b:c'])
self.assert_scanned(['//a/b:'], expected=['a/b', 'a/b:c'])
def test_sibling_or_descendents(self):
self.assert_scanned(['::'], expected=[':root', 'a', 'a:b', 'a/b', 'a/b:c'])
self.assert_scanned(['//::'], expected=[':root', 'a', 'a:b', 'a/b', 'a/b:c'])
self.assert_scanned(['a::'], expected=['a', 'a:b', 'a/b', 'a/b:c'])
self.assert_scanned(['//a::'], expected=['a', 'a:b', 'a/b', 'a/b:c'])
self.assert_scanned(['a/b::'], expected=['a/b', 'a/b:c'])
self.assert_scanned(['//a/b::'], expected=['a/b', 'a/b:c'])
def test_cmd_line_affordances(self):
self.assert_scanned(['./:root'], expected=[':root'])
self.assert_scanned(['//./:root'], expected=[':root'])
self.assert_scanned(['//./a/../:root'], expected=[':root'])
self.assert_scanned([os.path.join(self.build_root, './a/../:root')],
expected=[':root'])
self.assert_scanned(['a/'], expected=['a'])
self.assert_scanned(['./a/'], expected=['a'])
self.assert_scanned([os.path.join(self.build_root, './a/')], expected=['a'])
self.assert_scanned(['a/b/:b'], expected=['a/b'])
self.assert_scanned(['./a/b/:b'], expected=['a/b'])
self.assert_scanned([os.path.join(self.build_root, './a/b/:b')], expected=['a/b'])
def test_cmd_line_spec_list(self):
self.assert_scanned(['a', 'a/b'], expected=['a', 'a/b'])
self.assert_scanned(['::'], expected=[':root', 'a', 'a:b', 'a/b', 'a/b:c'])
def test_does_not_exist(self):
with self.assertRaises(AddressLookupError):
self.assert_scanned(['c'], expected=[])
with self.assertRaises(AddressLookupError):
self.assert_scanned(['c:'], expected=[])
with self.assertRaises(AddressLookupError):
self.assert_scanned(['c::'], expected=[])
def test_build_ignore_patterns(self):
expected_specs = [':root', 'a', 'a:b', 'a/b', 'a/b:c']
# This bogus BUILD file gets in the way of parsing.
self.add_to_build_file('some/dir', 'COMPLETELY BOGUS BUILDFILE)\n')
with self.assertRaises(AddressLookupError):
self.assert_scanned(['::'], expected=expected_specs)
address_mapper_with_ignore = BuildFileAddressMapper(self.build_file_parser,
self.project_tree,
build_ignore_patterns=['some'])
self.assert_scanned(['::'], expected=expected_specs, address_mapper=address_mapper_with_ignore)
def test_exclude_target_regexps(self):
address_mapper_with_exclude = BuildFileAddressMapper(self.build_file_parser,
self.project_tree,
exclude_target_regexps=[r'.*:b.*'])
self.assert_scanned(['::'], expected=[':root', 'a', 'a/b:c'],
address_mapper=address_mapper_with_exclude)
def assert_scanned(self, specs_strings, expected, address_mapper=None):
"""Parse and scan the given specs."""
address_mapper = address_mapper or self.address_mapper
def sort(addresses):
return sorted(addresses, key=lambda address: address.spec)
specs = [self._spec_parser.parse_spec(s) for s in specs_strings]
self.assertEqual(sort(Address.parse(addr) for addr in expected),
sort(address_mapper.scan_specs(specs)))
class CmdLineSpecParserTest(BaseTest):
def setUp(self):
super(CmdLineSpecParserTest, self).setUp()
self._spec_parser = CmdLineSpecParser(self.build_root)
def test_normal(self):
self.assert_parsed(':root', single('', 'root'))
self.assert_parsed('//:root', single('', 'root'))
self.assert_parsed('a', single('a'))
self.assert_parsed('a:a', single('a', 'a'))
self.assert_parsed('a/b', single('a/b'))
self.assert_parsed('a/b:b', single('a/b', 'b'))
self.assert_parsed('a/b:c', single('a/b', 'c'))
def test_sibling(self):
self.assert_parsed(':', sib(''))
self.assert_parsed('//:', sib(''))
self.assert_parsed('a:', sib('a'))
self.assert_parsed('//a:', sib('a'))
self.assert_parsed('a/b:', sib('a/b'))
self.assert_parsed('//a/b:', sib('a/b'))
def test_sibling_or_descendents(self):
self.assert_parsed('::', desc(''))
self.assert_parsed('//::', desc(''))
self.assert_parsed('a::', desc('a'))
self.assert_parsed('//a::', desc('a'))
self.assert_parsed('a/b::', desc('a/b'))
self.assert_parsed('//a/b::', desc('a/b'))
def test_absolute(self):
self.assert_parsed(os.path.join(self.build_root, 'a'), single('a'))
self.assert_parsed(os.path.join(self.build_root, 'a:a'),
single('a', 'a'))
self.assert_parsed(os.path.join(self.build_root, 'a:'), sib('a'))
self.assert_parsed(os.path.join(self.build_root, 'a::'), desc('a'))
with self.assertRaises(CmdLineSpecParser.BadSpecError):
self.assert_parsed('/not/the/buildroot/a', sib('a'))
def test_absolute_double_slashed(self):
# By adding a double slash, we are insisting that this absolute path is actually
# relative to the buildroot. Thus, it should parse correctly.
double_absolute = '/' + os.path.join(self.build_root, 'a')
self.assertEquals('//', double_absolute[:2],
'A sanity check we have a leading-// absolute spec')
self.assert_parsed(double_absolute, single(double_absolute[2:]))
def test_cmd_line_affordances(self):
self.assert_parsed('./:root', single('', 'root'))
self.assert_parsed('//./:root', single('', 'root'))
self.assert_parsed('//./a/../:root', single('', 'root'))
self.assert_parsed(os.path.join(self.build_root, './a/../:root'),
single('', 'root'))
self.assert_parsed('a/', single('a'))
self.assert_parsed('./a/', single('a'))
self.assert_parsed(os.path.join(self.build_root, './a/'), single('a'))
self.assert_parsed('a/b/:b', single('a/b', 'b'))
self.assert_parsed('./a/b/:b', single('a/b', 'b'))
self.assert_parsed(os.path.join(self.build_root, './a/b/:b'),
single('a/b', 'b'))
def assert_parsed(self, spec_str, expected_spec):
self.assertEqual(self._spec_parser.parse_spec(spec_str), expected_spec)
class SchedulerTest(unittest.TestCase):
def setUp(self):
build_root = os.path.join(os.path.dirname(__file__), 'examples',
'scheduler_inputs')
self.spec_parser = CmdLineSpecParser(build_root)
self.scheduler = setup_json_scheduler(build_root)
self.engine = LocalSerialEngine(self.scheduler)
self.guava = Address.parse('3rdparty/jvm:guava')
self.thrift = Address.parse('src/thrift/codegen/simple')
self.java = Address.parse('src/java/codegen/simple')
self.java_simple = Address.parse('src/java/simple')
self.java_multi = Address.parse('src/java/multiple_classpath_entries')
self.no_variant_thrift = Address.parse(
'src/java/codegen/selector:conflict')
self.unconfigured_thrift = Address.parse(
'src/thrift/codegen/unconfigured')
self.resources = Address.parse('src/resources/simple')
self.consumes_resources = Address.parse('src/java/consumes_resources')
self.consumes_managed_thirdparty = Address.parse(
'src/java/managed_thirdparty')
self.managed_guava = Address.parse('3rdparty/jvm/managed:guava')
self.managed_hadoop = Address.parse(
'3rdparty/jvm/managed:hadoop-common')
self.managed_resolve_latest = Address.parse(
'3rdparty/jvm/managed:latest-hadoop')
self.inferred_deps = Address.parse('src/scala/inferred_deps')
def assert_select_for_subjects(self,
walk,
product,
subjects,
variants=None,
variant_key=None):
node_type = SelectNode
variants = tuple(variants.items()) if variants else None
self.assertEqual(
{
node_type(subject, product, variants, variant_key)
for subject in subjects
}, {
node
for (node, _), _ in walk if node.product == product
and isinstance(node, node_type) and node.variants == variants
})
def build_and_walk(self, build_request, failures=False):
"""Build and then walk the given build_request, returning the walked graph as a list."""
predicate = (lambda _: True) if failures else None
result = self.engine.execute(build_request)
self.assertIsNone(result.error)
return list(
self.scheduler.walk_product_graph(build_request,
predicate=predicate))
def request(self, goals, *addresses):
return self.request_specs(
goals, *[self.spec_parser.parse_spec(str(a)) for a in addresses])
def request_specs(self, goals, *specs):
return BuildRequest(goals=goals, subjects=specs)
def assert_resolve_only(self, goals, root_specs, jars):
build_request = self.request(goals, *root_specs)
walk = self.build_and_walk(build_request)
# Expect a SelectNode for each of the Jar/Classpath.
self.assert_select_for_subjects(walk, Jar, jars)
self.assert_select_for_subjects(walk, Classpath, jars)
def test_resolve(self):
self.assert_resolve_only(goals=['resolve'],
root_specs=['3rdparty/jvm:guava'],
jars=[self.guava])
def test_compile_only_3rdparty(self):
self.assert_resolve_only(goals=['compile'],
root_specs=['3rdparty/jvm:guava'],
jars=[self.guava])
def test_gen_noop(self):
# TODO(John Sirois): Ask around - is this OK?
# This is different than today. There is a gen'able target reachable from the java target, but
# the scheduler 'pull-seeding' has ApacheThriftPlanner stopping short since the subject it's
# handed is not thrift.
build_request = self.request(['gen'], self.java)
walk = self.build_and_walk(build_request)
self.assert_select_for_subjects(walk, JavaSources, [self.java])
def test_gen(self):
build_request = self.request(['gen'], self.thrift)
walk = self.build_and_walk(build_request)
# Root: expect the synthetic GenGoal product.
root_entry = walk[0][0]
self.assertEqual(SelectNode(self.thrift, GenGoal, None, None),
root_entry[0])
self.assertIsInstance(root_entry[1], Return)
variants = {'thrift': 'apache_java'}
# Expect ThriftSources to have been selected.
self.assert_select_for_subjects(walk,
ThriftSources, [self.thrift],
variants=variants)
# Expect an ApacheThriftJavaConfiguration to have been used via the default Variants.
self.assert_select_for_subjects(walk,
ApacheThriftJavaConfiguration,
[self.thrift],
variants=variants,
variant_key='thrift')
def test_codegen_simple(self):
build_request = self.request(['compile'], self.java)
walk = self.build_and_walk(build_request)
# The subgraph below 'src/thrift/codegen/simple' will be affected by its default variants.
subjects = [self.guava, self.java, self.thrift]
variant_subjects = [
Jar(org='org.apache.thrift', name='libthrift', rev='0.9.2'),
Jar(org='commons-lang', name='commons-lang', rev='2.5'),
Address.parse('src/thrift:slf4j-api')
]
# Root: expect compilation via javac.
self.assertEqual(
(SelectNode(self.java, Classpath, None,
None), Return(Classpath(creator='javac'))), walk[0][0])
# Confirm that exactly the expected subjects got Classpaths.
self.assert_select_for_subjects(walk, Classpath, subjects)
self.assert_select_for_subjects(walk,
Classpath,
variant_subjects,
variants={'thrift': 'apache_java'})
def test_consumes_resources(self):
build_request = self.request(['compile'], self.consumes_resources)
walk = self.build_and_walk(build_request)
# Validate the root.
self.assertEqual(
(SelectNode(self.consumes_resources, Classpath, None,
None), Return(Classpath(creator='javac'))), walk[0][0])
# Confirm a classpath for the resources target and other subjects. We know that they are
# reachable from the root (since it was involved in this walk).
subjects = [self.resources, self.consumes_resources, self.guava]
self.assert_select_for_subjects(walk, Classpath, subjects)
def test_managed_resolve(self):
"""A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars."""
build_request = self.request(['compile'],
self.consumes_managed_thirdparty)
walk = self.build_and_walk(build_request)
# Validate the root.
self.assertEqual(
(SelectNode(self.consumes_managed_thirdparty, Classpath, None,
None), Return(Classpath(creator='javac'))), walk[0][0])
# Confirm that we produced classpaths for the managed jars.
managed_jars = [self.managed_guava, self.managed_hadoop]
self.assert_select_for_subjects(walk, Classpath,
[self.consumes_managed_thirdparty])
self.assert_select_for_subjects(walk,
Classpath,
managed_jars,
variants={'resolve': 'latest-hadoop'})
# Confirm that the produced jars had the appropriate versions.
self.assertEquals(
{
Jar('org.apache.hadoop', 'hadoop-common', '2.7.0'),
Jar('com.google.guava', 'guava', '18.0')
}, {
ret.value
for (node, ret), _ in walk
if node.product == Jar and isinstance(node, SelectNode)
})
def test_dependency_inference(self):
"""Scala dependency inference introduces dependencies that do not exist in BUILD files."""
build_request = self.request(['compile'], self.inferred_deps)
walk = self.build_and_walk(build_request)
# Validate the root.
self.assertEqual(
(SelectNode(self.inferred_deps, Classpath, None, None),
Return(Classpath(creator='scalac'))), walk[0][0])
# Confirm that we requested a classpath for the root and inferred targets.
self.assert_select_for_subjects(walk, Classpath,
[self.inferred_deps, self.java_simple])
def test_multiple_classpath_entries(self):
"""Multiple Classpath products for a single subject currently cause a failure."""
build_request = self.request(['compile'], self.java_multi)
walk = self.build_and_walk(build_request, failures=True)
# Validate that the root failed.
root_node, root_state = walk[0][0]
self.assertEqual(SelectNode(self.java_multi, Classpath, None, None),
root_node)
self.assertEqual(Throw, type(root_state))
def test_no_variant_thrift(self):
"""No `thrift` variant is configured, and so no configuration is selected."""
build_request = self.request(['compile'], self.no_variant_thrift)
with self.assertRaises(PartiallyConsumedInputsError):
self.build_and_walk(build_request)
def test_unconfigured_thrift(self):
"""The BuildPropertiesPlanner is able to produce a Classpath, but we should still fail.
A target with ThriftSources doesn't have a thrift config: that input is partially consumed.
"""
build_request = self.request(['compile'], self.unconfigured_thrift)
with self.assertRaises(PartiallyConsumedInputsError):
self.build_and_walk(build_request)
def test_descendant_specs(self):
"""Test that Addresses are produced via recursive globs of the 3rdparty/jvm directory."""
spec = self.spec_parser.parse_spec('3rdparty/jvm::')
build_request = self.request_specs(['list'], spec)
walk = self.build_and_walk(build_request)
# Validate the root.
root, root_state = walk[0][0]
root_value = root_state.value
self.assertEqual(
DependenciesNode(spec, Address, None, Addresses, None), root)
self.assertEqual(list, type(root_value))
# Confirm that a few expected addresses are in the list.
self.assertIn(self.guava, root_value)
self.assertIn(self.managed_guava, root_value)
self.assertIn(self.managed_resolve_latest, root_value)
def test_sibling_specs(self):
"""Test that sibling Addresses are parsed in the 3rdparty/jvm directory."""
spec = self.spec_parser.parse_spec('3rdparty/jvm:')
build_request = self.request_specs(['list'], spec)
walk = self.build_and_walk(build_request)
# Validate the root.
root, root_state = walk[0][0]
root_value = root_state.value
self.assertEqual(
DependenciesNode(spec, Address, None, Addresses, None), root)
self.assertEqual(list, type(root_value))
# Confirm that an expected address is in the list.
self.assertIn(self.guava, root_value)
# And that an subdirectory address is not.
self.assertNotIn(self.managed_guava, root_value)
class BuildFileAddressMapperScanTest(BaseTest):
NO_FAIL_FAST_RE = re.compile(r"""^--------------------
.*
Exception message: name 'a_is_bad' is not defined
while executing BUILD file BuildFile\(bad/a/BUILD, FileSystemProjectTree\(.*\)\)
Loading addresses from 'bad/a' failed\.
.*
Exception message: name 'b_is_bad' is not defined
while executing BUILD file BuildFile\(bad/b/BUILD, FileSystemProjectTree\(.*\)\)
Loading addresses from 'bad/b' failed\.
Invalid BUILD files for \[::\]$""", re.DOTALL)
FAIL_FAST_RE = """^name 'a_is_bad' is not defined
while executing BUILD file BuildFile\(bad/a/BUILD\, FileSystemProjectTree\(.*\)\)
Loading addresses from 'bad/a' failed.$"""
def setUp(self):
super(BuildFileAddressMapperScanTest, self).setUp()
def add_target(path, name):
self.add_to_build_file(path, 'target(name="{name}")\n'.format(name=name))
add_target('BUILD', 'root')
add_target('a', 'a')
add_target('a', 'b')
add_target('a/b', 'b')
add_target('a/b', 'c')
self._spec_parser = CmdLineSpecParser(self.build_root)
def test_bad_build_files(self):
self.add_to_build_file('bad/a', 'a_is_bad')
self.add_to_build_file('bad/b', 'b_is_bad')
with self.assertRaisesRegexp(AddressLookupError, self.NO_FAIL_FAST_RE):
list(self.address_mapper.scan_specs([DescendantAddresses('')], fail_fast=False))
def test_bad_build_files_fail_fast(self):
self.add_to_build_file('bad/a', 'a_is_bad')
self.add_to_build_file('bad/b', 'b_is_bad')
with self.assertRaisesRegexp(AddressLookupError, self.FAIL_FAST_RE):
list(self.address_mapper.scan_specs([DescendantAddresses('')], fail_fast=True))
def test_normal(self):
self.assert_scanned([':root'], expected=[':root'])
self.assert_scanned(['//:root'], expected=[':root'])
self.assert_scanned(['a'], expected=['a'])
self.assert_scanned(['a:a'], expected=['a'])
self.assert_scanned(['a/b'], expected=['a/b'])
self.assert_scanned(['a/b:b'], expected=['a/b'])
self.assert_scanned(['a/b:c'], expected=['a/b:c'])
def test_sibling(self):
self.assert_scanned([':'], expected=[':root'])
self.assert_scanned(['//:'], expected=[':root'])
self.assert_scanned(['a:'], expected=['a', 'a:b'])
self.assert_scanned(['//a:'], expected=['a', 'a:b'])
self.assert_scanned(['a/b:'], expected=['a/b', 'a/b:c'])
self.assert_scanned(['//a/b:'], expected=['a/b', 'a/b:c'])
def test_sibling_or_descendents(self):
self.assert_scanned(['::'], expected=[':root', 'a', 'a:b', 'a/b', 'a/b:c'])
self.assert_scanned(['//::'], expected=[':root', 'a', 'a:b', 'a/b', 'a/b:c'])
self.assert_scanned(['a::'], expected=['a', 'a:b', 'a/b', 'a/b:c'])
self.assert_scanned(['//a::'], expected=['a', 'a:b', 'a/b', 'a/b:c'])
self.assert_scanned(['a/b::'], expected=['a/b', 'a/b:c'])
self.assert_scanned(['//a/b::'], expected=['a/b', 'a/b:c'])
def test_cmd_line_affordances(self):
self.assert_scanned(['./:root'], expected=[':root'])
self.assert_scanned(['//./:root'], expected=[':root'])
self.assert_scanned(['//./a/../:root'], expected=[':root'])
self.assert_scanned([os.path.join(self.build_root, './a/../:root')],
expected=[':root'])
self.assert_scanned(['a/'], expected=['a'])
self.assert_scanned(['./a/'], expected=['a'])
self.assert_scanned([os.path.join(self.build_root, './a/')], expected=['a'])
self.assert_scanned(['a/b/:b'], expected=['a/b'])
self.assert_scanned(['./a/b/:b'], expected=['a/b'])
self.assert_scanned([os.path.join(self.build_root, './a/b/:b')], expected=['a/b'])
def test_cmd_line_spec_list(self):
self.assert_scanned(['a', 'a/b'], expected=['a', 'a/b'])
self.assert_scanned(['::'], expected=[':root', 'a', 'a:b', 'a/b', 'a/b:c'])
def test_does_not_exist(self):
with self.assertRaises(AddressLookupError):
self.assert_scanned(['c'], expected=[])
with self.assertRaises(AddressLookupError):
self.assert_scanned(['c:'], expected=[])
with self.assertRaises(AddressLookupError):
self.assert_scanned(['c::'], expected=[])
def test_build_ignore_patterns(self):
expected_specs = [':root', 'a', 'a:b', 'a/b', 'a/b:c']
# This bogus BUILD file gets in the way of parsing.
self.add_to_build_file('some/dir', 'COMPLETELY BOGUS BUILDFILE)\n')
with self.assertRaises(AddressLookupError):
self.assert_scanned(['::'], expected=expected_specs)
address_mapper_with_ignore = BuildFileAddressMapper(self.build_file_parser,
self.project_tree,
build_ignore_patterns=['some'])
self.assert_scanned(['::'], expected=expected_specs, address_mapper=address_mapper_with_ignore)
def test_exclude_target_regexps(self):
address_mapper_with_exclude = BuildFileAddressMapper(self.build_file_parser,
self.project_tree,
exclude_target_regexps=[r'.*:b.*'])
self.assert_scanned(['::'], expected=[':root', 'a', 'a/b:c'],
address_mapper=address_mapper_with_exclude)
def assert_scanned(self, specs_strings, expected, address_mapper=None):
"""Parse and scan the given specs."""
address_mapper = address_mapper or self.address_mapper
def sort(addresses):
return sorted(addresses, key=lambda address: address.spec)
specs = [self._spec_parser.parse_spec(s) for s in specs_strings]
self.assertEqual(sort(Address.parse(addr) for addr in expected),
sort(address_mapper.scan_specs(specs)))
class SchedulerTest(unittest.TestCase):
def setUp(self):
build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs')
self.spec_parser = CmdLineSpecParser(build_root)
self.scheduler = setup_json_scheduler(build_root, inline_nodes=False)
self.pg = self.scheduler.product_graph
self.engine = LocalSerialEngine(self.scheduler)
self.guava = Address.parse('3rdparty/jvm:guava')
self.thrift = Address.parse('src/thrift/codegen/simple')
self.java = Address.parse('src/java/codegen/simple')
self.java_simple = Address.parse('src/java/simple')
self.java_multi = Address.parse('src/java/multiple_classpath_entries')
self.no_variant_thrift = Address.parse('src/java/codegen/selector:conflict')
self.unconfigured_thrift = Address.parse('src/thrift/codegen/unconfigured')
self.resources = Address.parse('src/resources/simple')
self.consumes_resources = Address.parse('src/java/consumes_resources')
self.consumes_managed_thirdparty = Address.parse('src/java/managed_thirdparty')
self.managed_guava = Address.parse('3rdparty/jvm/managed:guava')
self.managed_hadoop = Address.parse('3rdparty/jvm/managed:hadoop-common')
self.managed_resolve_latest = Address.parse('3rdparty/jvm/managed:latest-hadoop')
self.inferred_deps = Address.parse('src/scala/inferred_deps')
def assert_select_for_subjects(self, walk, selector, subjects, variants=None):
node_type = SelectNode
variants = tuple(variants.items()) if variants else None
self.assertEqual({node_type(subject, variants, selector) for subject in subjects},
{node for node, _ in walk
if node.product == selector.product and isinstance(node, node_type) and node.variants == variants})
def build_and_walk(self, build_request):
"""Build and then walk the given build_request, returning the walked graph as a list."""
result = self.engine.execute(build_request)
self.assertIsNone(result.error)
return list(self.scheduler.product_graph.walk(build_request.roots))
def request(self, goals, *addresses):
return self.request_specs(goals, *[self.spec_parser.parse_spec(str(a)) for a in addresses])
def request_specs(self, goals, *specs):
return self.scheduler.build_request(goals=goals, subjects=specs)
def assert_resolve_only(self, goals, root_specs, jars):
build_request = self.request(goals, *root_specs)
walk = self.build_and_walk(build_request)
# Expect a SelectNode for each of the Jar/Classpath.
self.assert_select_for_subjects(walk, Select(Jar), jars)
self.assert_select_for_subjects(walk, Select(Classpath), jars)
def assert_root(self, walk, node, return_value):
"""Asserts that the first Node in a walk was a DependenciesNode with the single given result."""
root, root_state = walk[0]
self.assertEquals(type(root), DependenciesNode)
self.assertEquals(Return([return_value]), root_state)
self.assertIn((node, Return(return_value)),
[(d, self.pg.state(d)) for d in self.pg.dependencies_of(root)])
def assert_root_failed(self, walk, node, thrown_type):
"""Asserts that the first Node in a walk was a DependenciesNode with a Throw result."""
root, root_state = walk[0]
self.assertEquals(type(root), DependenciesNode)
self.assertEquals(Throw, type(root_state))
dependencies = [(d, self.pg.state(d)) for d in self.pg.dependencies_of(root)]
self.assertIn((node, thrown_type), [(k, type(v.exc))
for k, v in dependencies if type(v) is Throw])
def test_resolve(self):
self.assert_resolve_only(goals=['resolve'],
root_specs=['3rdparty/jvm:guava'],
jars=[self.guava])
def test_compile_only_3rdparty(self):
self.assert_resolve_only(goals=['compile'],
root_specs=['3rdparty/jvm:guava'],
jars=[self.guava])
def test_gen_noop(self):
# TODO(John Sirois): Ask around - is this OK?
# This is different than today. There is a gen'able target reachable from the java target, but
# the scheduler 'pull-seeding' has ApacheThriftPlanner stopping short since the subject it's
# handed is not thrift.
build_request = self.request(['gen'], self.java)
walk = self.build_and_walk(build_request)
self.assert_select_for_subjects(walk, Select(JavaSources, optional=True), [self.java])
def test_gen(self):
build_request = self.request(['gen'], self.thrift)
walk = self.build_and_walk(build_request)
# Root: expect the synthetic GenGoal product.
self.assert_root(walk,
SelectNode(self.thrift, None, Select(GenGoal)),
GenGoal("non-empty input to satisfy the Goal constructor"))
variants = {'thrift': 'apache_java'}
# Expect ThriftSources to have been selected.
self.assert_select_for_subjects(walk, Select(ThriftSources), [self.thrift], variants=variants)
# Expect an ApacheThriftJavaConfiguration to have been used via the default Variants.
self.assert_select_for_subjects(walk, SelectVariant(ApacheThriftJavaConfiguration,
variant_key='thrift'),
[self.thrift],
variants=variants)
def test_codegen_simple(self):
build_request = self.request(['compile'], self.java)
walk = self.build_and_walk(build_request)
# The subgraph below 'src/thrift/codegen/simple' will be affected by its default variants.
subjects = [
self.guava,
self.java,
self.thrift]
variant_subjects = [
Jar(org='org.apache.thrift', name='libthrift', rev='0.9.2', type_alias='jar'),
Jar(org='commons-lang', name='commons-lang', rev='2.5', type_alias='jar'),
Address.parse('src/thrift:slf4j-api')]
# Root: expect a DependenciesNode depending on a SelectNode with compilation via javac.
self.assert_root(walk,
SelectNode(self.java, None, Select(Classpath)),
Classpath(creator='javac'))
# Confirm that exactly the expected subjects got Classpaths.
self.assert_select_for_subjects(walk, Select(Classpath), subjects)
self.assert_select_for_subjects(walk, Select(Classpath), variant_subjects,
variants={'thrift': 'apache_java'})
def test_consumes_resources(self):
build_request = self.request(['compile'], self.consumes_resources)
walk = self.build_and_walk(build_request)
# Validate the root.
self.assert_root(walk,
SelectNode(self.consumes_resources, None, Select(Classpath)),
Classpath(creator='javac'))
# Confirm a classpath for the resources target and other subjects. We know that they are
# reachable from the root (since it was involved in this walk).
subjects = [self.resources,
self.consumes_resources,
self.guava]
self.assert_select_for_subjects(walk, Select(Classpath), subjects)
def test_managed_resolve(self):
"""A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars."""
build_request = self.request(['compile'], self.consumes_managed_thirdparty)
walk = self.build_and_walk(build_request)
# Validate the root.
self.assert_root(walk,
SelectNode(self.consumes_managed_thirdparty, None, Select(Classpath)),
Classpath(creator='javac'))
# Confirm that we produced classpaths for the managed jars.
managed_jars = [self.managed_guava,
self.managed_hadoop]
self.assert_select_for_subjects(walk, Select(Classpath), [self.consumes_managed_thirdparty])
self.assert_select_for_subjects(walk, Select(Classpath), managed_jars,
variants={'resolve': 'latest-hadoop'})
# Confirm that the produced jars had the appropriate versions.
self.assertEquals({Jar('org.apache.hadoop', 'hadoop-common', '2.7.0'),
Jar('com.google.guava', 'guava', '18.0')},
{ret.value for node, ret in walk
if node.product == Jar and isinstance(node, SelectNode)})
def test_dependency_inference(self):
"""Scala dependency inference introduces dependencies that do not exist in BUILD files."""
build_request = self.request(['compile'], self.inferred_deps)
walk = self.build_and_walk(build_request)
# Validate the root.
self.assert_root(walk,
SelectNode(self.inferred_deps, None, Select(Classpath)),
Classpath(creator='scalac'))
# Confirm that we requested a classpath for the root and inferred targets.
self.assert_select_for_subjects(walk, Select(Classpath), [self.inferred_deps, self.java_simple])
def test_multiple_classpath_entries(self):
"""Multiple Classpath products for a single subject currently cause a failure."""
build_request = self.request(['compile'], self.java_multi)
walk = self.build_and_walk(build_request)
# Validate that the root failed.
self.assert_root_failed(walk,
SelectNode(self.java_multi, None, Select(Classpath)),
ConflictingProducersError)
def test_descendant_specs(self):
"""Test that Addresses are produced via recursive globs of the 3rdparty/jvm directory."""
spec = self.spec_parser.parse_spec('3rdparty/jvm::')
build_request = self.request_specs(['list'], spec)
walk = self.build_and_walk(build_request)
# Validate the root.
root, root_state = walk[0]
root_value = root_state.value
self.assertEqual(DependenciesNode(spec, None, SelectDependencies(Address, Addresses)), root)
self.assertEqual(list, type(root_value))
# Confirm that a few expected addresses are in the list.
self.assertIn(self.guava, root_value)
self.assertIn(self.managed_guava, root_value)
self.assertIn(self.managed_resolve_latest, root_value)
def test_sibling_specs(self):
"""Test that sibling Addresses are parsed in the 3rdparty/jvm directory."""
spec = self.spec_parser.parse_spec('3rdparty/jvm:')
build_request = self.request_specs(['list'], spec)
walk = self.build_and_walk(build_request)
# Validate the root.
root, root_state = walk[0]
root_value = root_state.value
self.assertEqual(DependenciesNode(spec, None, SelectDependencies(Address, Addresses)), root)
self.assertEqual(list, type(root_value))
# Confirm that an expected address is in the list.
self.assertIn(self.guava, root_value)
# And that an subdirectory address is not.
self.assertNotIn(self.managed_guava, root_value)
def test_scheduler_visualize(self):
spec = self.spec_parser.parse_spec('3rdparty/jvm:')
build_request = self.request_specs(['list'], spec)
self.build_and_walk(build_request)
graphviz_output = '\n'.join(self.scheduler.product_graph.visualize(build_request.roots))
with temporary_dir() as td:
output_path = os.path.join(td, 'output.dot')
self.scheduler.visualize_graph_to_file(build_request.roots, output_path)
with open(output_path, 'rb') as fh:
graphviz_disk_output = fh.read().strip()
self.assertEqual(graphviz_output, graphviz_disk_output)
self.assertIn('digraph', graphviz_output)
self.assertIn(' -> ', graphviz_output)
