Esempio n. 1
0
def list():
  """Lists all addresses under the current build root."""

  build_root = get_buildroot()
  symbol_table_cls = LegacyTable
  address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
                                 parser_cls=LegacyPythonCallbacksParser)

  # Create a Scheduler containing only the graph tasks, with a single installed goal that
  # requests an Address.
  goal = 'list'
  tasks = create_fs_tasks(build_root) + create_graph_tasks(address_mapper, symbol_table_cls)
  scheduler = LocalScheduler({goal: Address}, symbol_table_cls, tasks)

  # Execute a request for the root.
  build_request = BuildRequest(goals=[goal], subjects=[DescendantAddresses('')])
  result = LocalSerialEngine(scheduler).execute(build_request)
  if result.error:
    raise result.error

  # Render the output.
  for state in result.root_products.values():
    if type(state) is Throw:
      raise state.exc
    elif type(state) is not Return:
      State.raise_unrecognized(dep_state)
    for address in state.value:
      print(address)
Esempio n. 2
0
def setup(options=None):
  if not options:
    options, _ = OptionsInitializer(OptionsBootstrapper()).setup()
  build_root = get_buildroot()
  cmd_line_spec_parser = CmdLineSpecParser(build_root)
  spec_roots = [cmd_line_spec_parser.parse_spec(spec) for spec in options.target_specs]

  storage = Storage.create(debug=False)
  # Ignore any dotfile below build_root except . itself
  project_tree = FileSystemProjectTree(build_root, ['.*', 'build-support/*.venv/'])
  symbol_table_cls = LegacyTable

  # Register "literal" subjects required for these tasks.
  # TODO: Replace with `Subsystems`.
  address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
                                 parser_cls=LegacyPythonCallbacksParser)

  # Create a Scheduler containing graph and filesystem tasks, with no installed goals. The ExpGraph
  # will explicitly request the products it needs.
  tasks = (
    create_legacy_graph_tasks() +
    create_fs_tasks() +
    create_graph_tasks(address_mapper, symbol_table_cls)
  )

  return (
    LocalScheduler(dict(), tasks, storage, project_tree),
    storage,
    options,
    spec_roots,
    symbol_table_cls
  )
Esempio n. 3
0
def setup(options=None):
  if not options:
    options, _ = OptionsInitializer(OptionsBootstrapper()).setup()
  build_root = get_buildroot()
  cmd_line_spec_parser = CmdLineSpecParser(build_root)
  spec_roots = [cmd_line_spec_parser.parse_spec(spec) for spec in options.target_specs]

  storage = Storage.create(debug=False)
  project_tree = FileSystemProjectTree(build_root)
  symbol_table_cls = LegacyTable

  # Register "literal" subjects required for these tasks.
  # TODO: Replace with `Subsystems`.
  address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
                                 parser_cls=LegacyPythonCallbacksParser)

  # Create a Scheduler containing graph and filesystem tasks, with no installed goals. The ExpGraph
  # will explicitly request the products it needs.
  tasks = (
    create_legacy_graph_tasks() +
    create_fs_tasks() +
    create_graph_tasks(address_mapper, symbol_table_cls)
  )

  return (
    LocalScheduler(dict(), tasks, storage, project_tree),
    storage,
    options,
    spec_roots,
    symbol_table_cls
  )
Esempio n. 4
0
  def setUp(self):
    self.work_dir = safe_mkdtemp()
    self.addCleanup(safe_rmtree, self.work_dir)
    self.build_root = os.path.join(self.work_dir, 'build_root')
    shutil.copytree(os.path.join(os.path.dirname(__file__), 'examples/mapper_test'),
                    self.build_root)

    subjects = Subjects()
    self._goal = 'list'
    symbol_table_cls = TargetTable

    project_tree_key = subjects.put(
        FileSystemProjectTree(self.build_root))
    address_mapper_key = subjects.put(
        AddressMapper(symbol_table_cls=symbol_table_cls,
                      parser_cls=JsonParser,
                      build_pattern=r'.+\.BUILD.json$'))
    tasks = (
        create_fs_tasks(project_tree_key) +
        create_graph_tasks(address_mapper_key, symbol_table_cls)
      )
    self.scheduler = LocalScheduler({self._goal: UnhydratedStruct},
                                    tasks,
                                    subjects,
                                    symbol_table_cls)

    self.a_b = Address.parse('a/b')
    self.a_b_target = Target(name='b',
                             dependencies=['//d:e'],
                             configurations=['//a', Struct(embedded='yes')],
                             type_alias='target')
Esempio n. 5
0
 def create(self, build_pattern=None, parser_cls=None, inline=False):
     symbol_table_cls = TestTable
     mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
                            build_pattern=build_pattern,
                            parser_cls=parser_cls)
     tasks = (create_fs_tasks(self._build_root) +
              create_graph_tasks(mapper, symbol_table_cls))
     return LocalScheduler({self._goal: self._product}, symbol_table_cls,
                           tasks)
Esempio n. 6
0
    def mk_scheduler(self,
                     tasks=None,
                     goals=None,
                     storage=None,
                     project_tree=None,
                     symbol_table_cls=EmptyTable):
        """Creates a Scheduler with "native" tasks already included, and the given additional tasks."""
        goals = goals or dict()
        tasks = tasks or []
        storage = storage or Storage.create(in_memory=True)
        project_tree = project_tree or self.mk_fs_tree()

        tasks = list(tasks) + create_fs_tasks()
        scheduler = LocalScheduler(goals, tasks, storage, project_tree)
        return scheduler, storage
Esempio n. 7
0
  def mk_scheduler(self,
                   tasks=None,
                   goals=None,
                   storage=None,
                   project_tree=None,
                   symbol_table_cls=EmptyTable):
    """Creates a Scheduler with "native" tasks already included, and the given additional tasks."""
    goals = goals or dict()
    tasks = tasks or []
    storage = storage or Storage.create(in_memory=True)
    project_tree = project_tree or self.mk_fs_tree()

    tasks = list(tasks) + create_fs_tasks()
    scheduler = LocalScheduler(goals, tasks, storage, project_tree)
    return scheduler, storage
Esempio n. 8
0
  def create(self, build_pattern=None, parser_cls=None, inline=False):
    subjects = Subjects()
    symbol_table_cls = TestTable

    project_tree_key = subjects.put(
        FileSystemProjectTree(self._build_root))
    address_mapper_key = subjects.put(
        AddressMapper(symbol_table_cls=symbol_table_cls,
                      build_pattern=build_pattern,
                      parser_cls=parser_cls))

    tasks = (
        create_fs_tasks(project_tree_key) +
        create_graph_tasks(address_mapper_key, symbol_table_cls)
      )
    return LocalScheduler(dict(),
                          tasks,
                          subjects,
                          symbol_table_cls)
Esempio n. 9
0
    def setUp(self):
        self.work_dir = safe_mkdtemp()
        self.addCleanup(safe_rmtree, self.work_dir)
        self.build_root = os.path.join(self.work_dir, 'build_root')
        shutil.copytree(
            os.path.join(os.path.dirname(__file__), 'examples/mapper_test'),
            self.build_root)

        self._goal = 'list'
        symbol_table_cls = TargetTable
        self.address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
                                            parser_cls=JsonParser,
                                            build_pattern=r'.+\.BUILD.json$')
        tasks = (create_fs_tasks(self.build_root) +
                 create_graph_tasks(self.address_mapper, symbol_table_cls))
        self.scheduler = LocalScheduler({self._goal: UnhydratedStruct},
                                        symbol_table_cls, tasks)

        self.a_b = Address.parse('a/b')
        self.a_b_target = Target(
            name='b',
            dependencies=['//d:e'],
            configurations=['//a', Struct(embedded='yes')])
Esempio n. 10
0
  def mk_scheduler(self,
                   tasks=None,
                   goals=None,
                   storage=None,
                   build_root_src=None,
                   symbol_table_cls=EmptyTable):
    """Creates a Scheduler with "native" tasks already included, and the given additional tasks."""
    goals = goals or dict()
    tasks = tasks or []
    storage = storage or Storage.create(in_memory=True)

    work_dir = safe_mkdtemp()
    self.addCleanup(safe_rmtree, work_dir)
    build_root = os.path.join(work_dir, 'build_root')
    if build_root_src is not None:
      shutil.copytree(build_root_src, build_root)
    else:
      os.mkdir(build_root)

    tasks = list(tasks) + create_fs_tasks()
    project_tree = FileSystemProjectTree(build_root)
    scheduler = LocalScheduler(goals, tasks, storage, project_tree)
    return scheduler, storage, build_root
Esempio n. 11
0
def setup_json_scheduler(build_root, debug=True):
  """Return a build graph and scheduler configured for BLD.json files under the given build root.

  :rtype A tuple of :class:`pants.engine.exp.scheduler.LocalScheduler`,
    :class:`pants.engine.exp.storage.Storage`.
  """

  storage = Storage.create(debug=debug)

  symbol_table_cls = ExampleTable

  # Register "literal" subjects required for these tasks.
  # TODO: Replace with `Subsystems`.
  address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
                                                  build_pattern=r'^BLD.json$',
                                                  parser_cls=JsonParser)
  source_roots = SourceRoots(('src/java','src/scala'))
  scrooge_tool_address = Address.parse('src/scala/scrooge')

  goals = {
      'compile': Classpath,
      # TODO: to allow for running resolve alone, should split out a distinct 'IvyReport' product.
      'resolve': Classpath,
      'list': Address,
      GenGoal.name(): GenGoal,
      'unpickleable': UnpickleableResult,
      'ls': File,
      'cat': FileContent,
    }
  tasks = [
      # Codegen
      GenGoal.signature(),
      (JavaSources,
       [Select(ThriftSources),
        SelectVariant(ApacheThriftJavaConfiguration, 'thrift')],
       gen_apache_thrift),
      (PythonSources,
       [Select(ThriftSources),
        SelectVariant(ApacheThriftPythonConfiguration, 'thrift')],
       gen_apache_thrift),
      (ScalaSources,
       [Select(ThriftSources),
        SelectVariant(ScroogeScalaConfiguration, 'thrift'),
        SelectLiteral(scrooge_tool_address, Classpath)],
       gen_scrooge_thrift),
      (JavaSources,
       [Select(ThriftSources),
        SelectVariant(ScroogeJavaConfiguration, 'thrift'),
        SelectLiteral(scrooge_tool_address, Classpath)],
       gen_scrooge_thrift),
    ] + [
      # scala dependency inference
      (ScalaSources,
       [Select(ScalaInferredDepsSources),
        SelectDependencies(Address, ImportedJVMPackages)],
       reify_scala_sources),
      (ImportedJVMPackages,
       [SelectProjection(FilesContent, PathGlobs, ('path_globs',), ScalaInferredDepsSources)],
       extract_scala_imports),
      (Address,
       [Select(JVMPackageName),
        SelectDependencies(AddressFamily, Dirs)],
       select_package_address),
      (PathGlobs,
       [Select(JVMPackageName),
        SelectLiteral(source_roots, SourceRoots)],
       calculate_package_search_path),
    ] + [
      # Remote dependency resolution
      (Classpath,
       [Select(Jar)],
       ivy_resolve),
      (Jar,
       [Select(ManagedJar),
        SelectVariant(ManagedResolve, 'resolve')],
       select_rev),
    ] + [
      # Compilers
      (Classpath,
       [Select(ResourceSources)],
       isolate_resources),
      (Classpath,
       [Select(BuildPropertiesConfiguration)],
       write_name_file),
      (Classpath,
       [Select(JavaSources),
        SelectDependencies(Classpath, JavaSources)],
       javac),
      (Classpath,
       [Select(ScalaSources),
        SelectDependencies(Classpath, ScalaSources)],
       scalac),
    ] + [
      # TODO
      (UnpickleableOutput,
        [],
        unpickleable_output),
      (UnpickleableResult,
       [Select(UnpickleableOutput)],
       unpickleable_input),
    ] + (
      create_graph_tasks(address_mapper, symbol_table_cls)
    ) + (
      create_fs_tasks()
    )

  project_tree = FileSystemProjectTree(build_root)
  return LocalScheduler(goals, tasks, storage, project_tree, None, GraphValidator(symbol_table_cls)), storage
Esempio n. 12
0
def setup_json_scheduler(build_root):
  """Return a build graph and scheduler configured for BLD.json files under the given build root.

  :rtype :class:`pants.engine.exp.scheduler.LocalScheduler`
  """
  symbol_table_cls = ExampleTable
  address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
                                 build_pattern=r'^BLD.json$',
                                 parser_cls=JsonParser)

  # TODO(John Sirois): once the options system is plumbed, make the tool spec configurable.
  # It could also just be pointed at the scrooge jar at that point.
  scrooge_tool_address = Address.parse('src/scala/scrooge')

  # TODO: Placeholder for the SourceRoot subsystem.
  source_roots = SourceRoots(('src/java',))

  goals = {
      'compile': Classpath,
      # TODO: to allow for running resolve alone, should split out a distinct 'IvyReport' product.
      'resolve': Classpath,
      'list': Address,
      'walk': Path,
      GenGoal.name(): GenGoal,
      'unpickleable': UnpickleableResult,
    }
  tasks = [
      # Codegen
      GenGoal.signature(),
      (JavaSources,
       [Select(ThriftSources),
        SelectVariant(ApacheThriftJavaConfiguration, 'thrift')],
       gen_apache_thrift),
      (PythonSources,
       [Select(ThriftSources),
        SelectVariant(ApacheThriftPythonConfiguration, 'thrift')],
       gen_apache_thrift),
      (ScalaSources,
       [Select(ThriftSources),
        SelectVariant(ScroogeScalaConfiguration, 'thrift'),
        SelectLiteral(scrooge_tool_address, Classpath)],
       gen_scrooge_thrift),
      (JavaSources,
       [Select(ThriftSources),
        SelectVariant(ScroogeJavaConfiguration, 'thrift'),
        SelectLiteral(scrooge_tool_address, Classpath)],
       gen_scrooge_thrift),
    ] + [
      # scala dependency inference
      (ScalaSources,
       [Select(ScalaInferredDepsSources),
        SelectDependencies(Address, ImportedJVMPackages)],
       reify_scala_sources),
      (ImportedJVMPackages,
       [SelectProjection(FilesContent, PathGlobs, ('path_globs',), ScalaInferredDepsSources)],
       extract_scala_imports),
      (Address,
       [Select(JVMPackageName),
        SelectDependencies(AddressFamily, Paths)],
       select_package_address),
      (Paths,
       [Select(JVMPackageName),
        SelectLiteral(source_roots, SourceRoots)],
       calculate_package_search_path),
    ] + [
      # Remote dependency resolution
      (Classpath,
       [Select(Jar)],
       ivy_resolve),
      (Jar,
       [Select(ManagedJar),
        SelectVariant(ManagedResolve, 'resolve')],
       select_rev),
    ] + [
      # Compilers
      (Classpath,
       [Select(ResourceSources)],
       isolate_resources),
      (Classpath,
       [Select(BuildPropertiesConfiguration)],
       write_name_file),
      (Classpath,
       [Select(JavaSources),
        SelectDependencies(Classpath, JavaSources)],
       javac),
      (Classpath,
       [Select(ScalaSources),
        SelectDependencies(Classpath, ScalaSources)],
       scalac),
    ] + [
      # TODO
      (UnpickleableOutput,
        [],
        unpickleable_output),
      (UnpickleableResult,
       [Select(UnpickleableOutput)],
       unpickleable_input),
    ] + (
      create_graph_tasks(address_mapper, symbol_table_cls)
    ) + (
      create_fs_tasks(build_root)
    )

  scheduler = LocalScheduler(goals, symbol_table_cls, tasks)
  return scheduler