示例#1
0
 def __init__(self):
   # Objects indexed by their keys, i.e, content digests
   self._objects = Storage.create()
   # Memoized object Ids.
   self._id_to_key = dict()
   self._key_to_id = dict()
   self._next_id = 0
示例#2
0
文件: engine.py 项目: kwlzn/pants
  def __init__(self, scheduler, storage=None, cache=None, pool_size=None, debug=True):
    """
    :param scheduler: The local scheduler for creating execution graphs.
    :type scheduler: :class:`pants.engine.scheduler.LocalScheduler`
    :param storage: The storage instance for serializables keyed by their hashes.
    :type storage: :class:`pants.engine.storage.Storage`
    :param cache: The cache instance for storing execution results, by default it uses the same
      Storage instance if not specified.
    :type cache: :class:`pants.engine.storage.Cache`
    :param int pool_size: The number of worker processes to use; by default 2 processes per core will
                          be used.
    :param bool debug: `True` to turn on pickling error debug mode (slower); True by default.
    """
    # This is the only place where non in-memory storage is needed, create one if not specified.
    storage = storage or Storage.create(in_memory=False)
    super(LocalMultiprocessEngine, self).__init__(scheduler, storage, cache)
    self._pool_size = pool_size if pool_size and pool_size > 0 else 2 * multiprocessing.cpu_count()

    execute_step = functools.partial(_execute_step, debug)

    self._processed_queue = Queue()
    self.node_builder = scheduler.node_builder
    process_initializer = functools.partial(_process_initializer, self._storage)
    self._pool = StatefulPool(self._pool_size, process_initializer, execute_step)
    self._debug = debug
    self._pool.start()
示例#3
0
 def multiprocessing_engine(self, pool_size=None):
   storage = Storage.create(debug=True, in_memory=False)
   cache = Cache.create(storage=storage)
   with closing(LocalMultiprocessEngine(self.scheduler, storage, cache,
                                        pool_size=pool_size, debug=True)) as e:
     e.start()
     yield e
示例#4
0
  def setup_legacy_graph(path_ignore_patterns):
    """Construct and return the components necessary for LegacyBuildGraph construction.

    :param list path_ignore_patterns: A list of path ignore patterns for FileSystemProjectTree,
                                      usually taken from the `--pants-ignore` global option.
    :returns: A tuple of (scheduler, engine, symbol_table_cls, build_graph_cls).
    """

    build_root = get_buildroot()
    project_tree = FileSystemProjectTree(build_root, path_ignore_patterns)
    symbol_table_cls = LegacySymbolTable

    # Register "literal" subjects required for these tasks.
    # TODO: Replace with `Subsystems`.
    address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
                                   parser_cls=LegacyPythonCallbacksParser)

    # Create a Scheduler containing graph and filesystem tasks, with no installed goals. The
    # LegacyBuildGraph will explicitly request the products it needs.
    tasks = (
      create_legacy_graph_tasks() +
      create_fs_tasks() +
      create_graph_tasks(address_mapper, symbol_table_cls)
    )

    scheduler = LocalScheduler(dict(), tasks, project_tree)
    engine = LocalSerialEngine(scheduler, Storage.create(debug=False))

    return LegacyGraphHelper(scheduler, engine, symbol_table_cls, LegacyBuildGraph)
示例#5
0
def _process_initializer(node_builder, storage):
  """Another picklable top-level function that provides multi-processes' initial states.

  States are returned as a tuple. States are `Closable` so they can be cleaned up once
  processes are done.
  """
  return (node_builder, Storage.clone(storage))
示例#6
0
 def hybrid_engine(self, pool_size=None):
   async_nodes = (FilesystemNode,)
   storage = Storage.create(in_memory=True)
   cache = Cache.create(storage=storage)
   with closing(ThreadHybridEngine(self.scheduler, storage,
                                   threaded_node_types=async_nodes, cache=cache,
                                   pool_size=pool_size, debug=True)) as e:
     yield e
示例#7
0
文件: engine.py 项目: kwlzn/pants
def _process_initializer(storage):
  """Another picklable top-level function that provides multi-processes' initial states.

  States are returned as a tuple. States are `Closable` so they can be cleaned up once
  processes are done.
  """
  storage = Storage.clone(storage)
  return (storage, Cache.create(storage=storage))
示例#8
0
 def setUp(self):
     self.storage = Storage.create(in_memory=True)
     self.result = StepResult(state="something")
     self.request = StepRequest(
         step_id=123,
         node="some node",
         dependencies={"some dep": "some state", "another dep": "another state"},
         project_tree="some project tree",
     )
示例#9
0
def visualize_build_request(build_root, goals, subjects):
  with subsystem_instance(Native.Factory) as native_factory:
    scheduler = setup_json_scheduler(build_root, native_factory.create())

    execution_request = scheduler.build_request(goals, subjects)
    # NB: Calls `reduce` independently of `execute`, in order to render a graph before validating it.
    engine = LocalSerialEngine(scheduler, Storage.create())
    engine.reduce(execution_request)
    visualize_execution_graph(scheduler)
示例#10
0
 def setUp(self):
   self.storage = Storage.create()
   self.result = StepResult(state='something')
   self.request = StepRequest(step_id=123,
                              node='some node',
                              dependencies={'some dep': 'some state',
                                            'another dep': 'another state'},
                              inline_nodes=False,
                              project_tree='some project tree')
示例#11
0
def visualize_build_request(build_root, goals, subjects):
  scheduler = setup_json_scheduler(build_root)

  execution_request = scheduler.build_request(goals, subjects)
  # NB: Calls `reduce` independently of `execute`, in order to render a graph before validating it.
  engine = LocalSerialEngine(scheduler, Storage.create())
  try:
    engine.reduce(execution_request)
    visualize_execution_graph(scheduler, execution_request)
  finally:
    engine.close()
示例#12
0
 def setUp(self):
   """Setup cache as well as request and result."""
   self.storage = Storage.create()
   self.cache = Cache.create(storage=self.storage)
   request = StepRequest(step_id=123,
                         node='some node',
                         dependencies={'some dep': 'some state',
                                       'another dep': 'another state'},
                         inline_nodes=False,
                         project_tree='some project tree')
   self.result = StepResult(state='something')
   self.keyed_request = self.storage.key_for_request(request)
示例#13
0
 def setUp(self):
     """Setup cache as well as request and result."""
     self.storage = Storage.create(in_memory=True)
     self.cache = Cache.create(storage=self.storage)
     request = StepRequest(
         step_id=123,
         node="some node",
         dependencies={"some dep": "some state", "another dep": "another state"},
         project_tree="some project tree",
     )
     self.result = StepResult(state="something")
     self.keyed_request = self.storage.key_for_request(request)
示例#14
0
 def __init__(self, scheduler, storage=None, cache=None):
   """
   :param scheduler: The local scheduler for creating execution graphs.
   :type scheduler: :class:`pants.engine.scheduler.LocalScheduler`
   :param storage: The storage instance for serializables keyed by their hashes.
   :type storage: :class:`pants.engine.storage.Storage`
   :param cache: The cache instance for storing execution results, by default it uses the same
     Storage instance if not specified.
   :type cache: :class:`pants.engine.storage.Cache`
   """
   self._scheduler = scheduler
   self._storage = storage or Storage.create()
   self._cache = cache or Cache.create(storage)
示例#15
0
  def mk_scheduler(self,
                   tasks=None,
                   goals=None,
                   storage=None,
                   project_tree=None,
                   symbol_table_cls=EmptyTable):
    """Creates a Scheduler with "native" tasks already included, and the given additional tasks."""
    goals = goals or dict()
    tasks = tasks or []
    storage = storage or Storage.create(in_memory=True)
    project_tree = project_tree or self.mk_fs_tree()

    tasks = list(tasks) + create_fs_tasks()
    scheduler = LocalScheduler(goals, tasks, storage, project_tree)
    return scheduler, storage
示例#16
0
文件: engine.py 项目: lahosken/pants
 def __init__(self, scheduler, storage=None, cache=None, use_cache=True):
   """
   :param scheduler: The local scheduler for creating execution graphs.
   :type scheduler: :class:`pants.engine.scheduler.LocalScheduler`
   :param storage: The storage instance for serializables keyed by their hashes.
   :type storage: :class:`pants.engine.storage.Storage`
   :param cache: The cache instance for storing execution results, by default it uses the same
     Storage instance if not specified.
   :type cache: :class:`pants.engine.storage.Cache`
   :param use_cache: True to enable usage of the cache. The cache incurs a large amount of
     overhead for small tasks, and needs TODO: further improvement.
   :type use_cache: bool
   """
   self._scheduler = scheduler
   self._storage = storage or Storage.create()
   self._cache = cache or Cache.create(storage)
   self._use_cache = use_cache
示例#17
0
  def setup_legacy_graph(pants_ignore_patterns,
                         build_root=None,
                         symbol_table_cls=None,
                         build_ignore_patterns=None,
                         exclude_target_regexps=None):
    """Construct and return the components necessary for LegacyBuildGraph construction.

    :param list pants_ignore_patterns: A list of path ignore patterns for FileSystemProjectTree,
                                       usually taken from the '--pants-ignore' global option.
    :param str build_root: A path to be used as the build root. If None, then default is used.
    :param SymbolTable symbol_table_cls: A SymbolTable class to use for build file parsing, or
                                         None to use the default.
    :param list build_ignore_patterns: A list of paths ignore patterns used when searching for BUILD
                                       files, usually taken from the '--build-ignore' global option.
    :param list exclude_target_regexps: A list of regular expressions for excluding targets.
    :returns: A tuple of (scheduler, engine, symbol_table_cls, build_graph_cls).
    """

    build_root = build_root or get_buildroot()
    scm = get_scm()
    symbol_table_cls = symbol_table_cls or LegacySymbolTable

    project_tree = FileSystemProjectTree(build_root, pants_ignore_patterns)

    # Register "literal" subjects required for these tasks.
    # TODO: Replace with `Subsystems`.
    address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
                                   parser_cls=LegacyPythonCallbacksParser,
                                   build_ignore_patterns=build_ignore_patterns,
                                   exclude_target_regexps=exclude_target_regexps)

    # Create a Scheduler containing graph and filesystem tasks, with no installed goals. The
    # LegacyBuildGraph will explicitly request the products it needs.
    tasks = (
      create_legacy_graph_tasks(symbol_table_cls) +
      create_fs_tasks() +
      create_graph_tasks(address_mapper, symbol_table_cls)
    )

    scheduler = LocalScheduler(dict(), tasks, project_tree)
    # TODO: Do not use the cache yet, as it incurs a high overhead.
    engine = LocalSerialEngine(scheduler, Storage.create(), use_cache=False)
    change_calculator = EngineChangeCalculator(engine, scm) if scm else None

    return LegacyGraphHelper(scheduler, engine, symbol_table_cls, change_calculator)
示例#18
0
  def setUp(self):
    # Set up a scheduler that supports address mapping.
    symbol_table_cls = TargetTable
    self.storage = Storage.create(in_memory=True)
    address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
                                   parser_cls=JsonParser,
                                   build_pattern=r'.+\.BUILD.json$')
    tasks = create_graph_tasks(address_mapper, symbol_table_cls)

    project_tree = self.mk_fs_tree(os.path.join(os.path.dirname(__file__), 'examples/mapper_test'))
    self.build_root = project_tree.build_root
    self.scheduler, _ = self.mk_scheduler(tasks=tasks,
                                          project_tree=project_tree,
                                          storage=self.storage,
                                          symbol_table_cls=symbol_table_cls)

    self.a_b = Address.parse('a/b')
    self.a_b_target = Target(name='b',
                             dependencies=['//d:e'],
                             configurations=['//a', Struct(embedded='yes')],
                             type_alias='target')
示例#19
0
 def setUp(self):
     """Setup cache as well as request and result."""
     self.storage = Storage.create()
     self.cache = Cache.create(storage=self.storage)
     self.request = Runnable(func=_runnable, args=("this is an arg",), cacheable=True)
     self.result = "something"
示例#20
0
 def setUp(self):
     self.storage = Storage.create()
     self.result = "something"
     self.request = Runnable(func=_runnable, args=("this is an arg",), cacheable=True)
示例#21
0
 def setUp(self):
     super(GraphTestBase, self).setUp()
     self.storage = Storage.create(in_memory=True)
示例#22
0
 def setUp(self):
   self.storage = Storage.create()
   self.result = 'something'
   self.request = Runnable(func=_runnable, args=('this is an arg',))
示例#23
0
 def setUp(self):
   """Setup cache as well as request and result."""
   self.storage = Storage.create()
   self.cache = Cache.create(storage=self.storage)
   self.request = Runnable(func=_runnable, args=('this is an arg',))
   self.result = 'something'
示例#24
0
  def setUp(self):
    build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs')
    self.scheduler, self.storage = setup_json_scheduler(build_root, debug=True)
    self.cache = Cache.create(Storage.create())

    self.java = Address.parse('src/java/codegen/simple')
示例#25
0
def setup_json_scheduler(build_root, debug=True):
  """Return a build graph and scheduler configured for BLD.json files under the given build root.

  :rtype A tuple of :class:`pants.engine.scheduler.LocalScheduler`,
    :class:`pants.engine.storage.Storage`.
  """

  storage = Storage.create(debug=debug)

  symbol_table_cls = ExampleTable

  # Register "literal" subjects required for these tasks.
  # TODO: Replace with `Subsystems`.
  address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
                                                  build_pattern=r'^BLD.json$',
                                                  parser_cls=JsonParser)
  source_roots = SourceRoots(('src/java','src/scala'))
  scrooge_tool_address = Address.parse('src/scala/scrooge')

  goals = {
      'compile': Classpath,
      # TODO: to allow for running resolve alone, should split out a distinct 'IvyReport' product.
      'resolve': Classpath,
      'list': Address,
      GenGoal.name(): GenGoal,
      'unpickleable': UnpickleableResult,
      'ls': File,
      'cat': FileContent,
    }
  tasks = [
      # Codegen
      GenGoal.signature(),
      (JavaSources,
       [Select(ThriftSources),
        SelectVariant(ApacheThriftJavaConfiguration, 'thrift')],
       gen_apache_thrift),
      (PythonSources,
       [Select(ThriftSources),
        SelectVariant(ApacheThriftPythonConfiguration, 'thrift')],
       gen_apache_thrift),
      (ScalaSources,
       [Select(ThriftSources),
        SelectVariant(ScroogeScalaConfiguration, 'thrift'),
        SelectLiteral(scrooge_tool_address, Classpath)],
       gen_scrooge_thrift),
      (JavaSources,
       [Select(ThriftSources),
        SelectVariant(ScroogeJavaConfiguration, 'thrift'),
        SelectLiteral(scrooge_tool_address, Classpath)],
       gen_scrooge_thrift),
    ] + [
      # scala dependency inference
      (ScalaSources,
       [Select(ScalaInferredDepsSources),
        SelectDependencies(Address, ImportedJVMPackages)],
       reify_scala_sources),
      (ImportedJVMPackages,
       [SelectProjection(FilesContent, PathGlobs, ('path_globs',), ScalaInferredDepsSources)],
       extract_scala_imports),
      (Address,
       [Select(JVMPackageName),
        SelectDependencies(AddressFamily, Dirs)],
       select_package_address),
      (PathGlobs,
       [Select(JVMPackageName),
        SelectLiteral(source_roots, SourceRoots)],
       calculate_package_search_path),
    ] + [
      # Remote dependency resolution
      (Classpath,
       [Select(Jar)],
       ivy_resolve),
      (Jar,
       [Select(ManagedJar),
        SelectVariant(ManagedResolve, 'resolve')],
       select_rev),
    ] + [
      # Compilers
      (Classpath,
       [Select(ResourceSources)],
       isolate_resources),
      (Classpath,
       [Select(BuildPropertiesConfiguration)],
       write_name_file),
      (Classpath,
       [Select(JavaSources),
        SelectDependencies(Classpath, JavaSources)],
       javac),
      (Classpath,
       [Select(ScalaSources),
        SelectDependencies(Classpath, ScalaSources)],
       scalac),
    ] + [
      # TODO
      (UnpickleableOutput,
        [],
        unpickleable_output),
      (UnpickleableResult,
       [Select(UnpickleableOutput)],
       unpickleable_input),
    ] + (
      create_graph_tasks(address_mapper, symbol_table_cls)
    ) + (
      create_fs_tasks()
    )

  project_tree = FileSystemProjectTree(build_root)
  return LocalScheduler(goals, tasks, storage, project_tree, None, GraphValidator(symbol_table_cls)), storage
示例#26
0
 def setUp(self):
   self.storage = Storage.create()
   self.result = 'something'
   self.request = Runnable(func=_runnable, args=('this is an arg',))
示例#27
0
def setup_json_scheduler(build_root, debug=True):
    """Return a build graph and scheduler configured for BLD.json files under the given build root.

  :rtype A tuple of :class:`pants.engine.scheduler.LocalScheduler`,
    :class:`pants.engine.storage.Storage`.
  """

    storage = Storage.create(debug=debug)

    symbol_table_cls = ExampleTable

    # Register "literal" subjects required for these tasks.
    # TODO: Replace with `Subsystems`.
    address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
                                   build_pattern=r'^BLD.json$',
                                   parser_cls=JsonParser)
    source_roots = SourceRoots(('src/java', 'src/scala'))
    scrooge_tool_address = Address.parse('src/scala/scrooge')

    goals = {
        'compile': Classpath,
        # TODO: to allow for running resolve alone, should split out a distinct 'IvyReport' product.
        'resolve': Classpath,
        'list': Address,
        GenGoal.name(): GenGoal,
        'unpickleable': UnpickleableResult,
        'ls': File,
        'cat': FileContent,
    }
    tasks = [
        # Codegen
        GenGoal.signature(),
        (JavaSources, [
            Select(ThriftSources),
            SelectVariant(ApacheThriftJavaConfiguration, 'thrift')
        ], gen_apache_thrift),
        (PythonSources, [
            Select(ThriftSources),
            SelectVariant(ApacheThriftPythonConfiguration, 'thrift')
        ], gen_apache_thrift),
        (ScalaSources, [
            Select(ThriftSources),
            SelectVariant(ScroogeScalaConfiguration, 'thrift'),
            SelectLiteral(scrooge_tool_address, Classpath)
        ], gen_scrooge_thrift),
        (JavaSources, [
            Select(ThriftSources),
            SelectVariant(ScroogeJavaConfiguration, 'thrift'),
            SelectLiteral(scrooge_tool_address, Classpath)
        ], gen_scrooge_thrift),
    ] + [
        # scala dependency inference
        (ScalaSources, [
            Select(ScalaInferredDepsSources),
            SelectDependencies(Address, ImportedJVMPackages)
        ], reify_scala_sources),
        (ImportedJVMPackages, [
            SelectProjection(FilesContent, PathGlobs,
                             ('path_globs', ), ScalaInferredDepsSources)
        ], extract_scala_imports),
        (Address,
         [Select(JVMPackageName),
          SelectDependencies(AddressFamily, Dirs)], select_package_address),
        (PathGlobs, [
            Select(JVMPackageName),
            SelectLiteral(source_roots, SourceRoots)
        ], calculate_package_search_path),
    ] + [
        # Remote dependency resolution
        (Classpath, [Select(Jar)], ivy_resolve),
        (Jar, [Select(ManagedJar),
               SelectVariant(ManagedResolve, 'resolve')], select_rev),
    ] + [
        # Compilers
        (Classpath, [Select(ResourceSources)], isolate_resources),
        (Classpath, [Select(BuildPropertiesConfiguration)], write_name_file),
        (Classpath,
         [Select(JavaSources),
          SelectDependencies(Classpath, JavaSources)], javac),
        (Classpath,
         [Select(ScalaSources),
          SelectDependencies(Classpath, ScalaSources)], scalac),
    ] + [
        # TODO
        (UnpickleableOutput, [], unpickleable_output),
        (UnpickleableResult, [Select(UnpickleableOutput)], unpickleable_input),
    ] + (create_graph_tasks(address_mapper,
                            symbol_table_cls)) + (create_fs_tasks())

    project_tree = FileSystemProjectTree(build_root)
    return LocalScheduler(goals, tasks, storage, project_tree, None,
                          GraphValidator(symbol_table_cls)), storage
示例#28
0
 def setUp(self):
   super(GraphTestBase, self).setUp()
   self.storage = Storage.create(in_memory=True)