def __init__(self): # Objects indexed by their keys, i.e, content digests self._objects = Storage.create() # Memoized object Ids. self._id_to_key = dict() self._key_to_id = dict() self._next_id = 0
def __init__(self, scheduler, storage=None, cache=None, pool_size=None, debug=True):
"""
:param scheduler: The local scheduler for creating execution graphs.
:type scheduler: :class:`pants.engine.scheduler.LocalScheduler`
:param storage: The storage instance for serializables keyed by their hashes.
:type storage: :class:`pants.engine.storage.Storage`
:param cache: The cache instance for storing execution results, by default it uses the same
Storage instance if not specified.
:type cache: :class:`pants.engine.storage.Cache`
:param int pool_size: The number of worker processes to use; by default 2 processes per core will
be used.
:param bool debug: `True` to turn on pickling error debug mode (slower); True by default.
"""
# This is the only place where non in-memory storage is needed, create one if not specified.
storage = storage or Storage.create(in_memory=False)
super(LocalMultiprocessEngine, self).__init__(scheduler, storage, cache)
self._pool_size = pool_size if pool_size and pool_size > 0 else 2 * multiprocessing.cpu_count()
execute_step = functools.partial(_execute_step, debug)
self._processed_queue = Queue()
self.node_builder = scheduler.node_builder
process_initializer = functools.partial(_process_initializer, self._storage)
self._pool = StatefulPool(self._pool_size, process_initializer, execute_step)
self._debug = debug
self._pool.start()
def multiprocessing_engine(self, pool_size=None):
storage = Storage.create(debug=True, in_memory=False)
cache = Cache.create(storage=storage)
with closing(LocalMultiprocessEngine(self.scheduler, storage, cache,
pool_size=pool_size, debug=True)) as e:
e.start()
yield e
def setup_legacy_graph(path_ignore_patterns):
"""Construct and return the components necessary for LegacyBuildGraph construction.
:param list path_ignore_patterns: A list of path ignore patterns for FileSystemProjectTree,
usually taken from the `--pants-ignore` global option.
:returns: A tuple of (scheduler, engine, symbol_table_cls, build_graph_cls).
"""
build_root = get_buildroot()
project_tree = FileSystemProjectTree(build_root, path_ignore_patterns)
symbol_table_cls = LegacySymbolTable
# Register "literal" subjects required for these tasks.
# TODO: Replace with `Subsystems`.
address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
parser_cls=LegacyPythonCallbacksParser)
# Create a Scheduler containing graph and filesystem tasks, with no installed goals. The
# LegacyBuildGraph will explicitly request the products it needs.
tasks = (
create_legacy_graph_tasks() +
create_fs_tasks() +
create_graph_tasks(address_mapper, symbol_table_cls)
)
scheduler = LocalScheduler(dict(), tasks, project_tree)
engine = LocalSerialEngine(scheduler, Storage.create(debug=False))
return LegacyGraphHelper(scheduler, engine, symbol_table_cls, LegacyBuildGraph)
def _process_initializer(node_builder, storage): """Another picklable top-level function that provides multi-processes' initial states. States are returned as a tuple. States are `Closable` so they can be cleaned up once processes are done. """ return (node_builder, Storage.clone(storage))
def hybrid_engine(self, pool_size=None):
async_nodes = (FilesystemNode,)
storage = Storage.create(in_memory=True)
cache = Cache.create(storage=storage)
with closing(ThreadHybridEngine(self.scheduler, storage,
threaded_node_types=async_nodes, cache=cache,
pool_size=pool_size, debug=True)) as e:
yield e
def _process_initializer(storage): """Another picklable top-level function that provides multi-processes' initial states. States are returned as a tuple. States are `Closable` so they can be cleaned up once processes are done. """ storage = Storage.clone(storage) return (storage, Cache.create(storage=storage))
def setUp(self):
self.storage = Storage.create(in_memory=True)
self.result = StepResult(state="something")
self.request = StepRequest(
step_id=123,
node="some node",
dependencies={"some dep": "some state", "another dep": "another state"},
project_tree="some project tree",
)
def visualize_build_request(build_root, goals, subjects):
with subsystem_instance(Native.Factory) as native_factory:
scheduler = setup_json_scheduler(build_root, native_factory.create())
execution_request = scheduler.build_request(goals, subjects)
# NB: Calls `reduce` independently of `execute`, in order to render a graph before validating it.
engine = LocalSerialEngine(scheduler, Storage.create())
engine.reduce(execution_request)
visualize_execution_graph(scheduler)
def setUp(self):
self.storage = Storage.create()
self.result = StepResult(state='something')
self.request = StepRequest(step_id=123,
node='some node',
dependencies={'some dep': 'some state',
'another dep': 'another state'},
inline_nodes=False,
project_tree='some project tree')
def visualize_build_request(build_root, goals, subjects):
scheduler = setup_json_scheduler(build_root)
execution_request = scheduler.build_request(goals, subjects)
# NB: Calls `reduce` independently of `execute`, in order to render a graph before validating it.
engine = LocalSerialEngine(scheduler, Storage.create())
try:
engine.reduce(execution_request)
visualize_execution_graph(scheduler, execution_request)
finally:
engine.close()
def setUp(self):
"""Setup cache as well as request and result."""
self.storage = Storage.create()
self.cache = Cache.create(storage=self.storage)
request = StepRequest(step_id=123,
node='some node',
dependencies={'some dep': 'some state',
'another dep': 'another state'},
inline_nodes=False,
project_tree='some project tree')
self.result = StepResult(state='something')
self.keyed_request = self.storage.key_for_request(request)
def setUp(self):
"""Setup cache as well as request and result."""
self.storage = Storage.create(in_memory=True)
self.cache = Cache.create(storage=self.storage)
request = StepRequest(
step_id=123,
node="some node",
dependencies={"some dep": "some state", "another dep": "another state"},
project_tree="some project tree",
)
self.result = StepResult(state="something")
self.keyed_request = self.storage.key_for_request(request)
def __init__(self, scheduler, storage=None, cache=None):
"""
:param scheduler: The local scheduler for creating execution graphs.
:type scheduler: :class:`pants.engine.scheduler.LocalScheduler`
:param storage: The storage instance for serializables keyed by their hashes.
:type storage: :class:`pants.engine.storage.Storage`
:param cache: The cache instance for storing execution results, by default it uses the same
Storage instance if not specified.
:type cache: :class:`pants.engine.storage.Cache`
"""
self._scheduler = scheduler
self._storage = storage or Storage.create()
self._cache = cache or Cache.create(storage)
def mk_scheduler(self,
tasks=None,
goals=None,
storage=None,
project_tree=None,
symbol_table_cls=EmptyTable):
"""Creates a Scheduler with "native" tasks already included, and the given additional tasks."""
goals = goals or dict()
tasks = tasks or []
storage = storage or Storage.create(in_memory=True)
project_tree = project_tree or self.mk_fs_tree()
tasks = list(tasks) + create_fs_tasks()
scheduler = LocalScheduler(goals, tasks, storage, project_tree)
return scheduler, storage
def __init__(self, scheduler, storage=None, cache=None, use_cache=True):
"""
:param scheduler: The local scheduler for creating execution graphs.
:type scheduler: :class:`pants.engine.scheduler.LocalScheduler`
:param storage: The storage instance for serializables keyed by their hashes.
:type storage: :class:`pants.engine.storage.Storage`
:param cache: The cache instance for storing execution results, by default it uses the same
Storage instance if not specified.
:type cache: :class:`pants.engine.storage.Cache`
:param use_cache: True to enable usage of the cache. The cache incurs a large amount of
overhead for small tasks, and needs TODO: further improvement.
:type use_cache: bool
"""
self._scheduler = scheduler
self._storage = storage or Storage.create()
self._cache = cache or Cache.create(storage)
self._use_cache = use_cache
def setup_legacy_graph(pants_ignore_patterns,
build_root=None,
symbol_table_cls=None,
build_ignore_patterns=None,
exclude_target_regexps=None):
"""Construct and return the components necessary for LegacyBuildGraph construction.
:param list pants_ignore_patterns: A list of path ignore patterns for FileSystemProjectTree,
usually taken from the '--pants-ignore' global option.
:param str build_root: A path to be used as the build root. If None, then default is used.
:param SymbolTable symbol_table_cls: A SymbolTable class to use for build file parsing, or
None to use the default.
:param list build_ignore_patterns: A list of paths ignore patterns used when searching for BUILD
files, usually taken from the '--build-ignore' global option.
:param list exclude_target_regexps: A list of regular expressions for excluding targets.
:returns: A tuple of (scheduler, engine, symbol_table_cls, build_graph_cls).
"""
build_root = build_root or get_buildroot()
scm = get_scm()
symbol_table_cls = symbol_table_cls or LegacySymbolTable
project_tree = FileSystemProjectTree(build_root, pants_ignore_patterns)
# Register "literal" subjects required for these tasks.
# TODO: Replace with `Subsystems`.
address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
parser_cls=LegacyPythonCallbacksParser,
build_ignore_patterns=build_ignore_patterns,
exclude_target_regexps=exclude_target_regexps)
# Create a Scheduler containing graph and filesystem tasks, with no installed goals. The
# LegacyBuildGraph will explicitly request the products it needs.
tasks = (
create_legacy_graph_tasks(symbol_table_cls) +
create_fs_tasks() +
create_graph_tasks(address_mapper, symbol_table_cls)
)
scheduler = LocalScheduler(dict(), tasks, project_tree)
# TODO: Do not use the cache yet, as it incurs a high overhead.
engine = LocalSerialEngine(scheduler, Storage.create(), use_cache=False)
change_calculator = EngineChangeCalculator(engine, scm) if scm else None
return LegacyGraphHelper(scheduler, engine, symbol_table_cls, change_calculator)
def setUp(self):
# Set up a scheduler that supports address mapping.
symbol_table_cls = TargetTable
self.storage = Storage.create(in_memory=True)
address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
parser_cls=JsonParser,
build_pattern=r'.+\.BUILD.json$')
tasks = create_graph_tasks(address_mapper, symbol_table_cls)
project_tree = self.mk_fs_tree(os.path.join(os.path.dirname(__file__), 'examples/mapper_test'))
self.build_root = project_tree.build_root
self.scheduler, _ = self.mk_scheduler(tasks=tasks,
project_tree=project_tree,
storage=self.storage,
symbol_table_cls=symbol_table_cls)
self.a_b = Address.parse('a/b')
self.a_b_target = Target(name='b',
dependencies=['//d:e'],
configurations=['//a', Struct(embedded='yes')],
type_alias='target')
def setUp(self):
"""Setup cache as well as request and result."""
self.storage = Storage.create()
self.cache = Cache.create(storage=self.storage)
self.request = Runnable(func=_runnable, args=("this is an arg",), cacheable=True)
self.result = "something"
def setUp(self):
self.storage = Storage.create()
self.result = "something"
self.request = Runnable(func=_runnable, args=("this is an arg",), cacheable=True)
def setUp(self):
super(GraphTestBase, self).setUp()
self.storage = Storage.create(in_memory=True)
def setUp(self):
self.storage = Storage.create()
self.result = 'something'
self.request = Runnable(func=_runnable, args=('this is an arg',))
def setUp(self):
"""Setup cache as well as request and result."""
self.storage = Storage.create()
self.cache = Cache.create(storage=self.storage)
self.request = Runnable(func=_runnable, args=('this is an arg',))
self.result = 'something'
def setUp(self):
build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs')
self.scheduler, self.storage = setup_json_scheduler(build_root, debug=True)
self.cache = Cache.create(Storage.create())
self.java = Address.parse('src/java/codegen/simple')
def setup_json_scheduler(build_root, debug=True):
"""Return a build graph and scheduler configured for BLD.json files under the given build root.
:rtype A tuple of :class:`pants.engine.scheduler.LocalScheduler`,
:class:`pants.engine.storage.Storage`.
"""
storage = Storage.create(debug=debug)
symbol_table_cls = ExampleTable
# Register "literal" subjects required for these tasks.
# TODO: Replace with `Subsystems`.
address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
build_pattern=r'^BLD.json$',
parser_cls=JsonParser)
source_roots = SourceRoots(('src/java','src/scala'))
scrooge_tool_address = Address.parse('src/scala/scrooge')
goals = {
'compile': Classpath,
# TODO: to allow for running resolve alone, should split out a distinct 'IvyReport' product.
'resolve': Classpath,
'list': Address,
GenGoal.name(): GenGoal,
'unpickleable': UnpickleableResult,
'ls': File,
'cat': FileContent,
}
tasks = [
# Codegen
GenGoal.signature(),
(JavaSources,
[Select(ThriftSources),
SelectVariant(ApacheThriftJavaConfiguration, 'thrift')],
gen_apache_thrift),
(PythonSources,
[Select(ThriftSources),
SelectVariant(ApacheThriftPythonConfiguration, 'thrift')],
gen_apache_thrift),
(ScalaSources,
[Select(ThriftSources),
SelectVariant(ScroogeScalaConfiguration, 'thrift'),
SelectLiteral(scrooge_tool_address, Classpath)],
gen_scrooge_thrift),
(JavaSources,
[Select(ThriftSources),
SelectVariant(ScroogeJavaConfiguration, 'thrift'),
SelectLiteral(scrooge_tool_address, Classpath)],
gen_scrooge_thrift),
] + [
# scala dependency inference
(ScalaSources,
[Select(ScalaInferredDepsSources),
SelectDependencies(Address, ImportedJVMPackages)],
reify_scala_sources),
(ImportedJVMPackages,
[SelectProjection(FilesContent, PathGlobs, ('path_globs',), ScalaInferredDepsSources)],
extract_scala_imports),
(Address,
[Select(JVMPackageName),
SelectDependencies(AddressFamily, Dirs)],
select_package_address),
(PathGlobs,
[Select(JVMPackageName),
SelectLiteral(source_roots, SourceRoots)],
calculate_package_search_path),
] + [
# Remote dependency resolution
(Classpath,
[Select(Jar)],
ivy_resolve),
(Jar,
[Select(ManagedJar),
SelectVariant(ManagedResolve, 'resolve')],
select_rev),
] + [
# Compilers
(Classpath,
[Select(ResourceSources)],
isolate_resources),
(Classpath,
[Select(BuildPropertiesConfiguration)],
write_name_file),
(Classpath,
[Select(JavaSources),
SelectDependencies(Classpath, JavaSources)],
javac),
(Classpath,
[Select(ScalaSources),
SelectDependencies(Classpath, ScalaSources)],
scalac),
] + [
# TODO
(UnpickleableOutput,
[],
unpickleable_output),
(UnpickleableResult,
[Select(UnpickleableOutput)],
unpickleable_input),
] + (
create_graph_tasks(address_mapper, symbol_table_cls)
) + (
create_fs_tasks()
)
project_tree = FileSystemProjectTree(build_root)
return LocalScheduler(goals, tasks, storage, project_tree, None, GraphValidator(symbol_table_cls)), storage
def setUp(self):
self.storage = Storage.create()
self.result = 'something'
self.request = Runnable(func=_runnable, args=('this is an arg',))
def setup_json_scheduler(build_root, debug=True):
"""Return a build graph and scheduler configured for BLD.json files under the given build root.
:rtype A tuple of :class:`pants.engine.scheduler.LocalScheduler`,
:class:`pants.engine.storage.Storage`.
"""
storage = Storage.create(debug=debug)
symbol_table_cls = ExampleTable
# Register "literal" subjects required for these tasks.
# TODO: Replace with `Subsystems`.
address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
build_pattern=r'^BLD.json$',
parser_cls=JsonParser)
source_roots = SourceRoots(('src/java', 'src/scala'))
scrooge_tool_address = Address.parse('src/scala/scrooge')
goals = {
'compile': Classpath,
# TODO: to allow for running resolve alone, should split out a distinct 'IvyReport' product.
'resolve': Classpath,
'list': Address,
GenGoal.name(): GenGoal,
'unpickleable': UnpickleableResult,
'ls': File,
'cat': FileContent,
}
tasks = [
# Codegen
GenGoal.signature(),
(JavaSources, [
Select(ThriftSources),
SelectVariant(ApacheThriftJavaConfiguration, 'thrift')
], gen_apache_thrift),
(PythonSources, [
Select(ThriftSources),
SelectVariant(ApacheThriftPythonConfiguration, 'thrift')
], gen_apache_thrift),
(ScalaSources, [
Select(ThriftSources),
SelectVariant(ScroogeScalaConfiguration, 'thrift'),
SelectLiteral(scrooge_tool_address, Classpath)
], gen_scrooge_thrift),
(JavaSources, [
Select(ThriftSources),
SelectVariant(ScroogeJavaConfiguration, 'thrift'),
SelectLiteral(scrooge_tool_address, Classpath)
], gen_scrooge_thrift),
] + [
# scala dependency inference
(ScalaSources, [
Select(ScalaInferredDepsSources),
SelectDependencies(Address, ImportedJVMPackages)
], reify_scala_sources),
(ImportedJVMPackages, [
SelectProjection(FilesContent, PathGlobs,
('path_globs', ), ScalaInferredDepsSources)
], extract_scala_imports),
(Address,
[Select(JVMPackageName),
SelectDependencies(AddressFamily, Dirs)], select_package_address),
(PathGlobs, [
Select(JVMPackageName),
SelectLiteral(source_roots, SourceRoots)
], calculate_package_search_path),
] + [
# Remote dependency resolution
(Classpath, [Select(Jar)], ivy_resolve),
(Jar, [Select(ManagedJar),
SelectVariant(ManagedResolve, 'resolve')], select_rev),
] + [
# Compilers
(Classpath, [Select(ResourceSources)], isolate_resources),
(Classpath, [Select(BuildPropertiesConfiguration)], write_name_file),
(Classpath,
[Select(JavaSources),
SelectDependencies(Classpath, JavaSources)], javac),
(Classpath,
[Select(ScalaSources),
SelectDependencies(Classpath, ScalaSources)], scalac),
] + [
# TODO
(UnpickleableOutput, [], unpickleable_output),
(UnpickleableResult, [Select(UnpickleableOutput)], unpickleable_input),
] + (create_graph_tasks(address_mapper,
symbol_table_cls)) + (create_fs_tasks())
project_tree = FileSystemProjectTree(build_root)
return LocalScheduler(goals, tasks, storage, project_tree, None,
GraphValidator(symbol_table_cls)), storage
def setUp(self): super(GraphTestBase, self).setUp() self.storage = Storage.create(in_memory=True)