Exemple #1
0
  def extern_generator_send(context_handle, func, arg):
    """Given a generator, send it the given value and return a response."""
    c = ffi.from_handle(context_handle)
    try:
      res = c.from_value(func[0]).send(c.from_value(arg[0]))
      if isinstance(res, Get):
        # Get.
        values = [res.subject]
        constraints = [constraint_for(res.product)]
        tag = 2
      elif type(res) in (tuple, list):
        # GetMulti.
        values = [g.subject for g in res]
        constraints = [constraint_for(g.product) for g in res]
        tag = 3
      else:
        # Break.
        values = [res]
        constraints = []
        tag = 0
    except Exception as e:
      # Throw.
      val = e
      val._formatted_exc = traceback.format_exc()
      values = [val]
      constraints = []
      tag = 1

    return (
        tag,
        c.vals_buf([c.to_value(v) for v in values]),
        c.vals_buf([c.to_value(v) for v in constraints])
      )
Exemple #2
0
    def extern_generator_send(context_handle, func, arg):
        """Given a generator, send it the given value and return a response."""
        c = ffi.from_handle(context_handle)
        try:
            res = c.from_value(func[0]).send(c.from_value(arg[0]))
            if isinstance(res, Get):
                # Get.
                values = [res.subject]
                constraints = [constraint_for(res.product)]
                tag = 2
            elif type(res) in (tuple, list):
                # GetMulti.
                values = [g.subject for g in res]
                constraints = [constraint_for(g.product) for g in res]
                tag = 3
            else:
                # Break.
                values = [res]
                constraints = []
                tag = 0
        except Exception as e:
            # Throw.
            val = e
            val._formatted_exc = traceback.format_exc()
            values = [val]
            constraints = []
            tag = 1

        return (tag, c.vals_buf([c.to_value(v) for v in values]),
                c.vals_buf([c.to_value(v) for v in constraints]))
Exemple #3
0
  def __init__(self, native, build_root, ignore_patterns, rule_index, root_subject_types):
    self._native = native
    # TODO: The only (?) case where we use inheritance rather than exact type unions.
    has_products_constraint = SubclassesOf(HasProducts)

    # Create the ExternContext, and the native Scheduler.
    self._scheduler = native.new_scheduler(
        build_root,
        ignore_patterns,
        Snapshot,
        _Snapshots,
        FileContent,
        FilesContent,
        Path,
        Dir,
        File,
        Link,
        has_products_constraint,
        constraint_for(Address),
        constraint_for(Variants),
        constraint_for(PathGlobs),
        constraint_for(Snapshot),
        constraint_for(_Snapshots),
        constraint_for(FilesContent),
        constraint_for(Dir),
        constraint_for(File),
        constraint_for(Link),
      )
    self._register_tasks(rule_index.tasks)
    self._register_intrinsics(rule_index.intrinsics)
    self._register_singletons(rule_index.singletons)
    self.root_subject_types = root_subject_types
Exemple #4
0
  def __init__(self, native, build_root, ignore_patterns, rule_index, root_subject_types):
    self._native = native
    # TODO: The only (?) case where we use inheritance rather than exact type unions.
    has_products_constraint = SubclassesOf(HasProducts)

    # Create the ExternContext, and the native Scheduler.
    self._tasks = native.new_tasks()
    self._register_tasks(rule_index.tasks)
    self._register_intrinsics(rule_index.intrinsics)
    self._register_singletons(rule_index.singletons)
    self.root_subject_types = root_subject_types

    self._scheduler = native.new_scheduler(
        self._tasks,
        build_root,
        ignore_patterns,
        Snapshot,
        _Snapshots,
        FileContent,
        FilesContent,
        Path,
        Dir,
        File,
        Link,
        has_products_constraint,
        constraint_for(Address),
        constraint_for(Variants),
        constraint_for(PathGlobs),
        constraint_for(Snapshot),
        constraint_for(_Snapshots),
        constraint_for(FilesContent),
        constraint_for(Dir),
        constraint_for(File),
        constraint_for(Link),
      )
Exemple #5
0
  def __init__(self,
               goals,
               tasks,
               project_tree,
               native,
               graph_lock=None):
    """
    :param goals: A dict from a goal name to a product type. A goal is just an alias for a
           particular (possibly synthetic) product.
    :param tasks: A set of (output, input selection clause, task function) triples which
           is used to compute values in the product graph.
    :param project_tree: An instance of ProjectTree for the current build root.
    :param native: An instance of engine.subsystem.native.Native.
    :param graph_lock: A re-entrant lock to use for guarding access to the internal product Graph
                       instance. Defaults to creating a new threading.RLock().
    """
    self._products_by_goal = goals
    self._project_tree = project_tree
    self._native = native
    self._product_graph_lock = graph_lock or threading.RLock()
    self._run_count = 0

    # TODO: The only (?) case where we use inheritance rather than exact type unions.
    has_products_constraint = SubclassesOf(HasProducts)

    # Create the ExternContext, and the native Scheduler.
    self._scheduler = native.new_scheduler(has_products_constraint,
                                           constraint_for(Address),
                                           constraint_for(Variants))
    self._execution_request = None

    # Validate and register all provided and intrinsic tasks.
    # TODO: This bounding of input Subject types allows for closed-world validation, but is not
    # strictly necessary for execution. We might eventually be able to remove it by only executing
    # validation below the execution roots (and thus not considering paths that aren't in use).
    select_product = lambda product: Select(product)
    root_selector_fns = {
      Address: select_product,
      AscendantAddresses: select_product,
      DescendantAddresses: select_product,
      PathGlobs: select_product,
      SiblingAddresses: select_product,
      SingleAddress: select_product,
    }
    intrinsics = create_fs_intrinsics(project_tree) + create_snapshot_intrinsics(project_tree)
    singletons = create_snapshot_singletons(project_tree)
    rule_index = RuleIndex.create(tasks, intrinsics, singletons)
    RulesetValidator(rule_index, goals, root_selector_fns).validate()
    self._register_tasks(rule_index.tasks)
    self._register_intrinsics(rule_index.intrinsics)
    self._register_singletons(rule_index.singletons)
Exemple #6
0
    def __init__(self, goals, tasks, project_tree, native, graph_lock=None):
        """
    :param goals: A dict from a goal name to a product type. A goal is just an alias for a
           particular (possibly synthetic) product.
    :param tasks: A set of (output, input selection clause, task function) triples which
           is used to compute values in the product graph.
    :param project_tree: An instance of ProjectTree for the current build root.
    :param native: An instance of engine.subsystem.native.Native.
    :param graph_lock: A re-entrant lock to use for guarding access to the internal product Graph
                       instance. Defaults to creating a new threading.RLock().
    """
        self._products_by_goal = goals
        self._project_tree = project_tree
        self._native = native
        self._product_graph_lock = graph_lock or threading.RLock()
        self._run_count = 0

        # TODO: The only (?) case where we use inheritance rather than exact type unions.
        has_products_constraint = SubclassesOf(HasProducts)

        # Create the ExternContext, and the native Scheduler.
        self._scheduler = native.new_scheduler(
            has_products_constraint, constraint_for(Address), constraint_for(Variants)
        )
        self._execution_request = None

        # Validate and register all provided and intrinsic tasks.
        # TODO: This bounding of input Subject types allows for closed-world validation, but is not
        # strictly necessary for execution. We might eventually be able to remove it by only executing
        # validation below the execution roots (and thus not considering paths that aren't in use).
        select_product = lambda product: Select(product)
        root_selector_fns = {
            Address: select_product,
            AscendantAddresses: select_product,
            DescendantAddresses: select_product,
            PathGlobs: select_product,
            SiblingAddresses: select_product,
            SingleAddress: select_product,
        }
        intrinsics = create_fs_intrinsics(project_tree) + create_snapshot_intrinsics(project_tree)
        singletons = create_snapshot_singletons(project_tree)
        rule_index = RuleIndex.create(tasks, intrinsics, singletons)
        RulesetValidator(rule_index, goals, root_selector_fns).validate()
        self._register_tasks(rule_index.tasks)
        self._register_intrinsics(rule_index.intrinsics)
        self._register_singletons(rule_index.singletons)
Exemple #7
0
    def rule_subgraph_visualization(self, root_subject_type, product_type):
        root_type_id = TypeId(self._to_id(root_subject_type))

        product_type_id = TypeConstraint(
            self._to_id(constraint_for(product_type)))
        with temporary_file_path() as path:
            self._native.lib.rule_subgraph_visualize(self._scheduler,
                                                     root_type_id,
                                                     product_type_id,
                                                     bytes(path))
            with open(path) as fd:
                for line in fd.readlines():
                    yield line.rstrip()
Exemple #8
0
  def rule_subgraph_visualization(self, root_subject_type, product_type):
    root_type_id = TypeId(self._to_id(root_subject_type))

    product_type_id = TypeConstraint(self._to_id(constraint_for(product_type)))
    with temporary_file_path() as path:
      self._native.lib.rule_subgraph_visualize(
        self._scheduler,
        root_type_id,
        product_type_id,
        bytes(path))
      with open(path) as fd:
        for line in fd.readlines():
          yield line.rstrip()
Exemple #9
0
 def _to_constraint(self, type_or_constraint):
     return TypeConstraint(self._to_id(constraint_for(type_or_constraint)))
Exemple #10
0
 def _to_constraint(self, type_or_constraint):
   return TypeConstraint(self._to_id(constraint_for(type_or_constraint)))
Exemple #11
0
    def __init__(
        self,
        native,
        project_tree,
        work_dir,
        rules,
        execution_options,
        include_trace_on_error=True,
        validate=True,
    ):
        """
    :param native: An instance of engine.native.Native.
    :param project_tree: An instance of ProjectTree for the current build root.
    :param work_dir: The pants work dir.
    :param rules: A set of Rules which is used to compute values in the graph.
    :param execution_options: Execution options for (remote) processes.
    :param include_trace_on_error: Include the trace through the graph upon encountering errors.
    :type include_trace_on_error: bool
    :param validate: True to assert that the ruleset is valid.
    """

        if execution_options.remote_execution_server and not execution_options.remote_store_server:
            raise ValueError(
                "Cannot set remote execution server without setting remote store server"
            )

        self._native = native
        self.include_trace_on_error = include_trace_on_error

        # TODO: The only (?) case where we use inheritance rather than exact type unions.
        has_products_constraint = SubclassesOf(HasProducts)

        # Validate and register all provided and intrinsic tasks.
        rule_index = RuleIndex.create(list(rules))
        self._root_subject_types = sorted(rule_index.roots)

        # Create the native Scheduler and Session.
        # TODO: This `_tasks` reference could be a local variable, since it is not used
        # after construction.
        self._tasks = native.new_tasks()
        self._register_rules(rule_index)

        self._scheduler = native.new_scheduler(
            self._tasks,
            self._root_subject_types,
            project_tree.build_root,
            work_dir,
            project_tree.ignore_patterns,
            execution_options,
            DirectoryDigest,
            Snapshot,
            FileContent,
            FilesContent,
            Path,
            Dir,
            File,
            Link,
            FallibleExecuteProcessResult,
            has_products_constraint,
            constraint_for(Address),
            constraint_for(Variants),
            constraint_for(PathGlobs),
            constraint_for(DirectoryDigest),
            constraint_for(Snapshot),
            constraint_for(FilesContent),
            constraint_for(Dir),
            constraint_for(File),
            constraint_for(Link),
            constraint_for(ExecuteProcessRequest),
            constraint_for(FallibleExecuteProcessResult),
            constraint_for(GeneratorType),
        )

        # If configured, visualize the rule graph before asserting that it is valid.
        if self.visualize_to_dir() is not None:
            rule_graph_name = 'rule_graph.dot'
            self.visualize_rule_graph_to_file(
                os.path.join(self.visualize_to_dir(), rule_graph_name))

        if validate:
            self._assert_ruleset_valid()
Exemple #12
0
    def __init__(self, native, build_root, work_dir, ignore_patterns,
                 rule_index):
        self._native = native
        # TODO: The only (?) case where we use inheritance rather than exact type unions.
        has_products_constraint = SubclassesOf(HasProducts)
        self._root_subject_types = sorted(rule_index.roots)

        # Create the ExternContext, and the native Scheduler.
        self._tasks = native.new_tasks()
        self._register_rules(rule_index)

        self._scheduler = native.new_scheduler(
            self._tasks,
            self._root_subject_types,
            build_root,
            work_dir,
            ignore_patterns,
            Snapshot,
            _Snapshots,
            FileContent,
            FilesContent,
            Path,
            Dir,
            File,
            Link,
            ExecuteProcessResult,
            has_products_constraint,
            constraint_for(Address),
            constraint_for(Variants),
            constraint_for(PathGlobs),
            constraint_for(Snapshot),
            constraint_for(_Snapshots),
            constraint_for(FilesContent),
            constraint_for(Dir),
            constraint_for(File),
            constraint_for(Link),
            constraint_for(ExecuteProcessRequest),
            constraint_for(ExecuteProcessResult),
            constraint_for(GeneratorType),
        )
Exemple #13
0
    def __init__(
        self,
        native,
        project_tree,
        work_dir,
        local_store_dir,
        rules,
        execution_options,
        include_trace_on_error=True,
        validate=True,
    ):
        """
    :param native: An instance of engine.native.Native.
    :param project_tree: An instance of ProjectTree for the current build root.
    :param work_dir: The pants work dir.
    :param local_store_dir: The directory to use for storing the engine's LMDB store in.
    :param rules: A set of Rules which is used to compute values in the graph.
    :param execution_options: Execution options for (remote) processes.
    :param include_trace_on_error: Include the trace through the graph upon encountering errors.
    :type include_trace_on_error: bool
    :param validate: True to assert that the ruleset is valid.
    """

        if execution_options.remote_execution_server and not execution_options.remote_store_server:
            raise ValueError(
                "Cannot set remote execution server without setting remote store server"
            )

        self._native = native
        self.include_trace_on_error = include_trace_on_error
        # Validate and register all provided and intrinsic tasks.
        rule_index = RuleIndex.create(list(rules))
        self._root_subject_types = sorted(rule_index.roots, key=repr)

        # Create the native Scheduler and Session.
        # TODO: This `_tasks` reference could be a local variable, since it is not used
        # after construction.
        self._tasks = native.new_tasks()
        self._register_rules(rule_index)

        self._scheduler = native.new_scheduler(
            tasks=self._tasks,
            root_subject_types=self._root_subject_types,
            build_root=project_tree.build_root,
            work_dir=work_dir,
            local_store_dir=local_store_dir,
            ignore_patterns=project_tree.ignore_patterns,
            execution_options=execution_options,
            construct_directory_digest=Digest,
            construct_snapshot=Snapshot,
            construct_file_content=FileContent,
            construct_files_content=FilesContent,
            construct_path_stat=Path,
            construct_dir=Dir,
            construct_file=File,
            construct_link=Link,
            construct_process_result=FallibleExecuteProcessResult,
            constraint_address=constraint_for(Address),
            constraint_path_globs=constraint_for(PathGlobs),
            constraint_directory_digest=constraint_for(Digest),
            constraint_snapshot=constraint_for(Snapshot),
            constraint_merge_snapshots_request=constraint_for(
                MergedDirectories),
            constraint_files_content=constraint_for(FilesContent),
            constraint_dir=constraint_for(Dir),
            constraint_file=constraint_for(File),
            constraint_link=constraint_for(Link),
            constraint_process_request=constraint_for(ExecuteProcessRequest),
            constraint_process_result=constraint_for(
                FallibleExecuteProcessResult),
            constraint_generator=constraint_for(GeneratorType),
            constraint_url_to_fetch=constraint_for(UrlToFetch),
        )

        # If configured, visualize the rule graph before asserting that it is valid.
        if self.visualize_to_dir() is not None:
            rule_graph_name = 'rule_graph.dot'
            self.visualize_rule_graph_to_file(
                os.path.join(self.visualize_to_dir(), rule_graph_name))

        if validate:
            self._assert_ruleset_valid()
Exemple #14
0
  def __init__(self, native, build_root, work_dir, ignore_patterns, rule_index):
    self._native = native
    # TODO: The only (?) case where we use inheritance rather than exact type unions.
    has_products_constraint = SubclassesOf(HasProducts)
    self._root_subject_types = sorted(rule_index.roots)

    # Create the ExternContext, and the native Scheduler.
    self._tasks = native.new_tasks()
    self._register_rules(rule_index)

    self._scheduler = native.new_scheduler(
      self._tasks,
      self._root_subject_types,
      build_root,
      work_dir,
      ignore_patterns,
      Snapshot,
      _Snapshots,
      FileContent,
      FilesContent,
      Path,
      Dir,
      File,
      Link,
      ExecuteProcessResult,
      has_products_constraint,
      constraint_for(Address),
      constraint_for(Variants),
      constraint_for(PathGlobs),
      constraint_for(Snapshot),
      constraint_for(_Snapshots),
      constraint_for(FilesContent),
      constraint_for(Dir),
      constraint_for(File),
      constraint_for(Link),
      constraint_for(ExecuteProcessRequest),
      constraint_for(ExecuteProcessResult),
    )
Exemple #15
0
  def __init__(
    self,
    native,
    project_tree,
    work_dir,
    rules,
    remote_store_server,
    remote_execution_server,
    include_trace_on_error=True,
    validate=True,
  ):
    """
    :param native: An instance of engine.native.Native.
    :param project_tree: An instance of ProjectTree for the current build root.
    :param work_dir: The pants work dir.
    :param rules: A set of Rules which is used to compute values in the graph.
    :param include_trace_on_error: Include the trace through the graph upon encountering errors.
    :type include_trace_on_error: bool
    :param validate: True to assert that the ruleset is valid.
    """

    if remote_execution_server and not remote_store_server:
      raise ValueError("Cannot set remote execution server without setting remote store server")
    # We can't currently pass Options to the rust side, so we pass empty strings for None:
    remote_execution_server = remote_execution_server or ""
    remote_store_server = remote_store_server or ""

    self._native = native
    self.include_trace_on_error = include_trace_on_error

    # TODO: The only (?) case where we use inheritance rather than exact type unions.
    has_products_constraint = SubclassesOf(HasProducts)

    # Validate and register all provided and intrinsic tasks.
    rule_index = RuleIndex.create(list(rules))
    self._root_subject_types = sorted(rule_index.roots)

    # Create the native Scheduler and Session.
    # TODO: This `_tasks` reference could be a local variable, since it is not used
    # after construction.
    self._tasks = native.new_tasks()
    self._register_rules(rule_index)

    self._scheduler = native.new_scheduler(
      self._tasks,
      self._root_subject_types,
      project_tree.build_root,
      work_dir,
      project_tree.ignore_patterns,
      remote_store_server,
      remote_execution_server,
      DirectoryDigest,
      Snapshot,
      FileContent,
      FilesContent,
      Path,
      Dir,
      File,
      Link,
      ExecuteProcessResult,
      has_products_constraint,
      constraint_for(Address),
      constraint_for(Variants),
      constraint_for(PathGlobs),
      constraint_for(DirectoryDigest),
      constraint_for(Snapshot),
      constraint_for(FilesContent),
      constraint_for(Dir),
      constraint_for(File),
      constraint_for(Link),
      constraint_for(ExecuteProcessRequest),
      constraint_for(ExecuteProcessResult),
      constraint_for(GeneratorType),
    )

    # If configured, visualize the rule graph before asserting that it is valid.
    if self.visualize_to_dir() is not None:
      rule_graph_name = 'rule_graph.dot'
      self.visualize_rule_graph_to_file(os.path.join(self.visualize_to_dir(), rule_graph_name))

    if validate:
      self._assert_ruleset_valid()