def execute_request(self, scheduler, storage, product, *subjects):
"""Creates, runs, and returns an ExecutionRequest for the given product and subjects."""
request = scheduler.execution_request([product], subjects)
res = LocalSerialEngine(scheduler, storage).execute(request)
if res.error:
raise res.error
return request
def _populate(self, scheduler, address): """Perform an ExecutionRequest to parse the given Address into a Struct.""" request = scheduler.execution_request([self._product], [address]) LocalSerialEngine(scheduler).reduce(request) root_entries = scheduler.root_entries(request).items() self.assertEquals(1, len(root_entries)) return root_entries[0]
def setUp(self):
build_root = os.path.join(os.path.dirname(__file__), 'examples',
'scheduler_inputs')
self.spec_parser = CmdLineSpecParser(build_root)
self.scheduler = setup_json_scheduler(build_root, inline_nodes=False)
self.pg = self.scheduler.product_graph
self.engine = LocalSerialEngine(self.scheduler)
self.guava = Address.parse('3rdparty/jvm:guava')
self.thrift = Address.parse('src/thrift/codegen/simple')
self.java = Address.parse('src/java/codegen/simple')
self.java_simple = Address.parse('src/java/simple')
self.java_multi = Address.parse('src/java/multiple_classpath_entries')
self.no_variant_thrift = Address.parse(
'src/java/codegen/selector:conflict')
self.unconfigured_thrift = Address.parse(
'src/thrift/codegen/unconfigured')
self.resources = Address.parse('src/resources/simple')
self.consumes_resources = Address.parse('src/java/consumes_resources')
self.consumes_managed_thirdparty = Address.parse(
'src/java/managed_thirdparty')
self.managed_guava = Address.parse('3rdparty/jvm/managed:guava')
self.managed_hadoop = Address.parse(
'3rdparty/jvm/managed:hadoop-common')
self.managed_resolve_latest = Address.parse(
'3rdparty/jvm/managed:latest-hadoop')
self.inferred_deps = Address.parse('src/scala/inferred_deps')
def setup_legacy_graph(path_ignore_patterns):
"""Construct and return the components necessary for LegacyBuildGraph construction.
:param list path_ignore_patterns: A list of path ignore patterns for FileSystemProjectTree,
usually taken from the `--pants-ignore` global option.
:returns: A tuple of (scheduler, engine, symbol_table_cls, build_graph_cls).
"""
build_root = get_buildroot()
project_tree = FileSystemProjectTree(build_root, path_ignore_patterns)
symbol_table_cls = LegacySymbolTable
# Register "literal" subjects required for these tasks.
# TODO: Replace with `Subsystems`.
address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
parser_cls=LegacyPythonCallbacksParser)
# Create a Scheduler containing graph and filesystem tasks, with no installed goals. The
# LegacyBuildGraph will explicitly request the products it needs.
tasks = (
create_legacy_graph_tasks() +
create_fs_tasks() +
create_graph_tasks(address_mapper, symbol_table_cls)
)
scheduler = LocalScheduler(dict(), tasks, project_tree)
engine = LocalSerialEngine(scheduler, Storage.create(debug=False))
return LegacyGraphHelper(scheduler, engine, symbol_table_cls, LegacyBuildGraph)
def execute_request(self, scheduler, product, *subjects):
"""Creates, runs, and returns an ExecutionRequest for the given product and subjects."""
request = scheduler.execution_request([product], subjects)
with closing(LocalSerialEngine(scheduler)) as engine:
res = engine.execute(request)
if res.error:
raise res.error
return request
def setup_legacy_graph(pants_ignore_patterns,
workdir,
build_root=None,
native=None,
symbol_table_cls=None,
build_ignore_patterns=None,
exclude_target_regexps=None,
subproject_roots=None):
"""Construct and return the components necessary for LegacyBuildGraph construction.
:param list pants_ignore_patterns: A list of path ignore patterns for FileSystemProjectTree,
usually taken from the '--pants-ignore' global option.
:param str workdir: The pants workdir.
:param str build_root: A path to be used as the build root. If None, then default is used.
:param Native native: An instance of the native-engine subsystem.
:param SymbolTable symbol_table_cls: A SymbolTable class to use for build file parsing, or
None to use the default.
:param list build_ignore_patterns: A list of paths ignore patterns used when searching for BUILD
files, usually taken from the '--build-ignore' global option.
:param list exclude_target_regexps: A list of regular expressions for excluding targets.
:param list subproject_roots: Paths that correspond with embedded build roots
under the current build root.
:returns: A tuple of (scheduler, engine, symbol_table_cls, build_graph_cls).
"""
build_root = build_root or get_buildroot()
scm = get_scm()
symbol_table_cls = symbol_table_cls or LegacySymbolTable
project_tree = FileSystemProjectTree(build_root, pants_ignore_patterns)
# Register "literal" subjects required for these tasks.
# TODO: Replace with `Subsystems`.
address_mapper = AddressMapper(
symbol_table_cls=symbol_table_cls,
parser_cls=LegacyPythonCallbacksParser,
build_ignore_patterns=build_ignore_patterns,
exclude_target_regexps=exclude_target_regexps,
subproject_roots=subproject_roots)
# Load the native backend.
native = native or Native.Factory.global_instance().create()
# Create a Scheduler containing graph and filesystem tasks, with no installed goals. The
# LegacyBuildGraph will explicitly request the products it needs.
tasks = (create_legacy_graph_tasks(symbol_table_cls) +
create_fs_rules() +
create_graph_rules(address_mapper, symbol_table_cls))
# TODO: Do not use the cache yet, as it incurs a high overhead.
scheduler = LocalScheduler(workdir, dict(), tasks, project_tree,
native)
engine = LocalSerialEngine(scheduler, use_cache=False)
change_calculator = EngineChangeCalculator(
scheduler, engine, symbol_table_cls, scm) if scm else None
return LegacyGraphHelper(scheduler, engine, symbol_table_cls,
change_calculator)
def visualize_build_request(build_root, goals, subjects):
with subsystem_instance(Native.Factory) as native_factory:
scheduler = setup_json_scheduler(build_root, native_factory.create())
execution_request = scheduler.build_request(goals, subjects)
# NB: Calls `reduce` independently of `execute`, in order to render a graph before validating it.
engine = LocalSerialEngine(scheduler, Storage.create())
engine.reduce(execution_request)
visualize_execution_graph(scheduler)
def _populate(self, scheduler, address):
"""Perform an ExecutionRequest to parse the given Address into a Struct."""
# NB: requesting any of the possible types that come from parsing a BUILD file
request = scheduler.execution_request([TestTable.constraint()],
[address])
LocalSerialEngine(scheduler).reduce(request)
root_entries = scheduler.root_entries(request).items()
self.assertEquals(1, len(root_entries))
return root_entries[0]
def visualize_build_request(build_root, goals, subjects):
scheduler, storage = setup_json_scheduler(build_root)
execution_request = scheduler.build_request(goals, subjects)
# NB: Calls `reduce` independently of `execute`, in order to render a graph before validating it.
engine = LocalSerialEngine(scheduler, storage)
engine.start()
try:
engine.reduce(execution_request)
visualize_execution_graph(scheduler, storage, execution_request)
finally:
engine.close()
def resolve(self, spec):
request = self.scheduler.execution_request([UnhydratedStruct], [spec])
result = LocalSerialEngine(self.scheduler, self.storage).execute(request)
if result.error:
raise result.error
# Expect a single root.
state, = result.root_products.values()
if type(state) is Throw:
raise state.exc
return state.value
def resolve(self, spec):
select = SelectDependencies(UnhydratedStruct, Addresses, field_types=(Address,))
request = self.scheduler.selection_request([(select, spec)])
result = LocalSerialEngine(self.scheduler).execute(request)
if result.error:
raise result.error
# Expect a single root.
state, = result.root_products.values()
if type(state) is Throw:
raise Exception(state.exc)
return state.value
def test_gather_snapshot_of_pathglobs(self):
project_tree = self.mk_example_fs_tree()
scheduler = self.mk_scheduler(project_tree=project_tree, tasks=create_snapshot_tasks(project_tree))
snapshot_archive_root = os.path.join(project_tree.build_root, '.snapshots')
request = scheduler.execution_request([Snapshot],
[PathGlobs.create('', globs=['fs_test/a/b/*'])])
LocalSerialEngine(scheduler).reduce(request)
root_entries = scheduler.root_entries(request).items()
self.assertEquals(1, len(root_entries))
state = self.assertFirstEntryIsReturn(root_entries, scheduler)
snapshot = state.value
self.assert_archive_files(['fs_test/a/b/1.txt', 'fs_test/a/b/2'], snapshot,
snapshot_archive_root)
def test_gather_snapshot_of_pathglobs(self):
project_tree = self.mk_example_fs_tree()
scheduler = self.mk_scheduler(project_tree=project_tree)
empty_step_context = StepContext(node_builder=None, project_tree=project_tree, node_states=[], inline_nodes=False)
request = scheduler.execution_request([Snapshot],
[PathGlobs.create('', globs=['fs_test/a/b/*'])])
LocalSerialEngine(scheduler).reduce(request)
root_entries = scheduler.root_entries(request).items()
self.assertEquals(1, len(root_entries))
state = self.assertFirstEntryIsReturn(root_entries, scheduler)
snapshot = state.value
self.assert_archive_files(['fs_test/a/b/1.txt', 'fs_test/a/b/2'], snapshot,
empty_step_context)
def setup_legacy_graph(pants_ignore_patterns,
symbol_table_cls=None,
build_ignore_patterns=None,
exclude_target_regexps=None):
"""Construct and return the components necessary for LegacyBuildGraph construction.
:param list pants_ignore_patterns: A list of path ignore patterns for FileSystemProjectTree,
usually taken from the '--pants-ignore' global option.
:param SymbolTable symbol_table_cls: A SymbolTable class to use for build file parsing, or
None to use the default.
:param list build_ignore_patterns: A list of paths ignore patterns used when searching for BUILD
files, usually taken from the '--build-ignore' global option.
:param list exclude_target_regexps: A list of regular expressions for excluding targets.
:returns: A tuple of (scheduler, engine, symbol_table_cls, build_graph_cls).
"""
build_root = get_buildroot()
scm = get_scm()
project_tree = FileSystemProjectTree(build_root, pants_ignore_patterns)
symbol_table_cls = symbol_table_cls or LegacySymbolTable
# Register "literal" subjects required for these tasks.
# TODO: Replace with `Subsystems`.
address_mapper = AddressMapper(
symbol_table_cls=symbol_table_cls,
parser_cls=LegacyPythonCallbacksParser,
build_ignore_patterns=build_ignore_patterns,
exclude_target_regexps=exclude_target_regexps)
# Create a Scheduler containing graph and filesystem tasks, with no installed goals. The
# LegacyBuildGraph will explicitly request the products it needs.
tasks = (create_legacy_graph_tasks(symbol_table_cls) +
create_fs_tasks() +
create_graph_tasks(address_mapper, symbol_table_cls))
scheduler = LocalScheduler(dict(), tasks, project_tree)
# TODO: Do not use the cache yet, as it incurs a high overhead.
engine = LocalSerialEngine(scheduler,
Storage.create(),
use_cache=False)
change_calculator = EngineChangeCalculator(engine,
scm) if scm else None
return LegacyGraphHelper(scheduler, engine, symbol_table_cls,
change_calculator)
def test_failed_output_conversion_propagates_throw(self):
scheduler = self.mk_scheduler_in_example_fs([
# subject to files / product of subject to files for snapshot.
SnapshottedProcess.create(product_type=Concatted,
binary_type=ShellCatToOutFile,
input_selectors=(Select(Snapshot),),
input_conversion=file_list_to_args_for_cat_with_snapshot_subjects_and_output_file,
output_conversion=fail_process_result),
[ShellCatToOutFile, [], ShellCatToOutFile]
])
request = scheduler.execution_request([Concatted],
[PathGlobs.create('', include=['fs_test/a/b/*'])])
LocalSerialEngine(scheduler).reduce(request)
root_entries = scheduler.root_entries(request).items()
self.assertEquals(1, len(root_entries))
self.assertFirstEntryIsThrow(root_entries,
in_msg='Failed in output conversion!')
def test_failed_command_propagates_throw(self):
scheduler = self.mk_scheduler_in_example_fs([
# subject to files / product of subject to files for snapshot.
SnapshottedProcess.create(product_type=Concatted,
binary_type=ShellFailCommand,
input_selectors=tuple(),
input_conversion=empty_process_request,
output_conversion=fail_process_result),
[ShellFailCommand, [], ShellFailCommand]
])
request = scheduler.execution_request([Concatted],
[PathGlobs.create('', include=['fs_test/a/b/*'])])
LocalSerialEngine(scheduler).reduce(request)
root_entries = scheduler.root_entries(request).items()
self.assertEquals(1, len(root_entries))
self.assertFirstEntryIsThrow(root_entries,
in_msg='Running ShellFailCommand failed with non-zero exit code: 1')
def test_integration_concat_with_snapshot_subjects_test(self):
scheduler = self.mk_scheduler_in_example_fs([
# subject to files / product of subject to files for snapshot.
SnapshottedProcess.create(product_type=Concatted,
binary_type=ShellCatToOutFile,
input_selectors=(Select(Snapshot),),
input_conversion=file_list_to_args_for_cat_with_snapshot_subjects_and_output_file,
output_conversion=process_result_to_concatted_from_outfile),
[ShellCatToOutFile, [], ShellCatToOutFile],
])
request = scheduler.execution_request([Concatted],
[PathGlobs.create('', include=['fs_test/a/b/*'])])
LocalSerialEngine(scheduler).reduce(request)
root_entries = scheduler.root_entries(request).items()
self.assertEquals(1, len(root_entries))
state = self.assertFirstEntryIsReturn(root_entries, scheduler)
concatted = state.value
self.assertEqual(Concatted('one\ntwo\n'), concatted)
def test_javac_compilation_example(self):
sources = PathGlobs.create('', include=['scheduler_inputs/src/java/simple/Simple.java'])
scheduler = self.mk_scheduler_in_example_fs([
SnapshottedProcess.create(ClasspathEntry,
Javac,
(Select(Snapshot), SelectLiteral(JavaOutputDir('build'), JavaOutputDir)),
java_sources_to_javac_args,
process_result_to_classpath_entry),
[Javac, [], Javac]
])
request = scheduler.execution_request(
[ClasspathEntry],
[sources])
LocalSerialEngine(scheduler).reduce(request)
root_entries = scheduler.root_entries(request).items()
self.assertEquals(1, len(root_entries))
state = self.assertFirstEntryIsReturn(root_entries, scheduler)
classpath_entry = state.value
self.assertIsInstance(classpath_entry, ClasspathEntry)
self.assertTrue(os.path.exists(os.path.join(classpath_entry.path, 'simple', 'Simple.class')))
def create_engine(self, root_subject_types, rules, include_trace_on_error):
engine = LocalSerialEngine(
self.scheduler(root_subject_types, rules),
include_trace_on_error=include_trace_on_error)
return engine
def test_serial_engine_simple(self):
with closing(LocalSerialEngine(self.scheduler)) as engine:
self.assert_engine(engine)
def serial_engine(self):
with closing(LocalSerialEngine(self.scheduler)) as e:
yield e
def serial_engine(self):
yield LocalSerialEngine(self.scheduler)
def test_serial_engine_simple(self):
engine = LocalSerialEngine(self.scheduler, self.storage, self.cache)
self.assert_engine(engine)
