def extern_generator_send(context_handle, func, arg):
"""Given a generator, send it the given value and return a response."""
c = ffi.from_handle(context_handle)
try:
res = c.from_value(func[0]).send(c.from_value(arg[0]))
if isinstance(res, Get):
# Get.
values = [res.subject]
constraints = [constraint_for(res.product)]
tag = 2
elif type(res) in (tuple, list):
# GetMulti.
values = [g.subject for g in res]
constraints = [constraint_for(g.product) for g in res]
tag = 3
else:
# Break.
values = [res]
constraints = []
tag = 0
except Exception as e:
# Throw.
val = e
val._formatted_exc = traceback.format_exc()
values = [val]
constraints = []
tag = 1
return (
tag,
c.vals_buf([c.to_value(v) for v in values]),
c.vals_buf([c.to_value(v) for v in constraints])
)
def extern_generator_send(context_handle, func, arg):
"""Given a generator, send it the given value and return a response."""
c = ffi.from_handle(context_handle)
try:
res = c.from_value(func[0]).send(c.from_value(arg[0]))
if isinstance(res, Get):
# Get.
values = [res.subject]
constraints = [constraint_for(res.product)]
tag = 2
elif type(res) in (tuple, list):
# GetMulti.
values = [g.subject for g in res]
constraints = [constraint_for(g.product) for g in res]
tag = 3
else:
# Break.
values = [res]
constraints = []
tag = 0
except Exception as e:
# Throw.
val = e
val._formatted_exc = traceback.format_exc()
values = [val]
constraints = []
tag = 1
return (tag, c.vals_buf([c.to_value(v) for v in values]),
c.vals_buf([c.to_value(v) for v in constraints]))
def __init__(self, native, build_root, ignore_patterns, rule_index, root_subject_types):
self._native = native
# TODO: The only (?) case where we use inheritance rather than exact type unions.
has_products_constraint = SubclassesOf(HasProducts)
# Create the ExternContext, and the native Scheduler.
self._scheduler = native.new_scheduler(
build_root,
ignore_patterns,
Snapshot,
_Snapshots,
FileContent,
FilesContent,
Path,
Dir,
File,
Link,
has_products_constraint,
constraint_for(Address),
constraint_for(Variants),
constraint_for(PathGlobs),
constraint_for(Snapshot),
constraint_for(_Snapshots),
constraint_for(FilesContent),
constraint_for(Dir),
constraint_for(File),
constraint_for(Link),
)
self._register_tasks(rule_index.tasks)
self._register_intrinsics(rule_index.intrinsics)
self._register_singletons(rule_index.singletons)
self.root_subject_types = root_subject_types
def __init__(self, native, build_root, ignore_patterns, rule_index, root_subject_types):
self._native = native
# TODO: The only (?) case where we use inheritance rather than exact type unions.
has_products_constraint = SubclassesOf(HasProducts)
# Create the ExternContext, and the native Scheduler.
self._tasks = native.new_tasks()
self._register_tasks(rule_index.tasks)
self._register_intrinsics(rule_index.intrinsics)
self._register_singletons(rule_index.singletons)
self.root_subject_types = root_subject_types
self._scheduler = native.new_scheduler(
self._tasks,
build_root,
ignore_patterns,
Snapshot,
_Snapshots,
FileContent,
FilesContent,
Path,
Dir,
File,
Link,
has_products_constraint,
constraint_for(Address),
constraint_for(Variants),
constraint_for(PathGlobs),
constraint_for(Snapshot),
constraint_for(_Snapshots),
constraint_for(FilesContent),
constraint_for(Dir),
constraint_for(File),
constraint_for(Link),
)
def __init__(self,
goals,
tasks,
project_tree,
native,
graph_lock=None):
"""
:param goals: A dict from a goal name to a product type. A goal is just an alias for a
particular (possibly synthetic) product.
:param tasks: A set of (output, input selection clause, task function) triples which
is used to compute values in the product graph.
:param project_tree: An instance of ProjectTree for the current build root.
:param native: An instance of engine.subsystem.native.Native.
:param graph_lock: A re-entrant lock to use for guarding access to the internal product Graph
instance. Defaults to creating a new threading.RLock().
"""
self._products_by_goal = goals
self._project_tree = project_tree
self._native = native
self._product_graph_lock = graph_lock or threading.RLock()
self._run_count = 0
# TODO: The only (?) case where we use inheritance rather than exact type unions.
has_products_constraint = SubclassesOf(HasProducts)
# Create the ExternContext, and the native Scheduler.
self._scheduler = native.new_scheduler(has_products_constraint,
constraint_for(Address),
constraint_for(Variants))
self._execution_request = None
# Validate and register all provided and intrinsic tasks.
# TODO: This bounding of input Subject types allows for closed-world validation, but is not
# strictly necessary for execution. We might eventually be able to remove it by only executing
# validation below the execution roots (and thus not considering paths that aren't in use).
select_product = lambda product: Select(product)
root_selector_fns = {
Address: select_product,
AscendantAddresses: select_product,
DescendantAddresses: select_product,
PathGlobs: select_product,
SiblingAddresses: select_product,
SingleAddress: select_product,
}
intrinsics = create_fs_intrinsics(project_tree) + create_snapshot_intrinsics(project_tree)
singletons = create_snapshot_singletons(project_tree)
rule_index = RuleIndex.create(tasks, intrinsics, singletons)
RulesetValidator(rule_index, goals, root_selector_fns).validate()
self._register_tasks(rule_index.tasks)
self._register_intrinsics(rule_index.intrinsics)
self._register_singletons(rule_index.singletons)
def __init__(self, goals, tasks, project_tree, native, graph_lock=None):
"""
:param goals: A dict from a goal name to a product type. A goal is just an alias for a
particular (possibly synthetic) product.
:param tasks: A set of (output, input selection clause, task function) triples which
is used to compute values in the product graph.
:param project_tree: An instance of ProjectTree for the current build root.
:param native: An instance of engine.subsystem.native.Native.
:param graph_lock: A re-entrant lock to use for guarding access to the internal product Graph
instance. Defaults to creating a new threading.RLock().
"""
self._products_by_goal = goals
self._project_tree = project_tree
self._native = native
self._product_graph_lock = graph_lock or threading.RLock()
self._run_count = 0
# TODO: The only (?) case where we use inheritance rather than exact type unions.
has_products_constraint = SubclassesOf(HasProducts)
# Create the ExternContext, and the native Scheduler.
self._scheduler = native.new_scheduler(
has_products_constraint, constraint_for(Address), constraint_for(Variants)
)
self._execution_request = None
# Validate and register all provided and intrinsic tasks.
# TODO: This bounding of input Subject types allows for closed-world validation, but is not
# strictly necessary for execution. We might eventually be able to remove it by only executing
# validation below the execution roots (and thus not considering paths that aren't in use).
select_product = lambda product: Select(product)
root_selector_fns = {
Address: select_product,
AscendantAddresses: select_product,
DescendantAddresses: select_product,
PathGlobs: select_product,
SiblingAddresses: select_product,
SingleAddress: select_product,
}
intrinsics = create_fs_intrinsics(project_tree) + create_snapshot_intrinsics(project_tree)
singletons = create_snapshot_singletons(project_tree)
rule_index = RuleIndex.create(tasks, intrinsics, singletons)
RulesetValidator(rule_index, goals, root_selector_fns).validate()
self._register_tasks(rule_index.tasks)
self._register_intrinsics(rule_index.intrinsics)
self._register_singletons(rule_index.singletons)
def rule_subgraph_visualization(self, root_subject_type, product_type):
root_type_id = TypeId(self._to_id(root_subject_type))
product_type_id = TypeConstraint(
self._to_id(constraint_for(product_type)))
with temporary_file_path() as path:
self._native.lib.rule_subgraph_visualize(self._scheduler,
root_type_id,
product_type_id,
bytes(path))
with open(path) as fd:
for line in fd.readlines():
yield line.rstrip()
def rule_subgraph_visualization(self, root_subject_type, product_type):
root_type_id = TypeId(self._to_id(root_subject_type))
product_type_id = TypeConstraint(self._to_id(constraint_for(product_type)))
with temporary_file_path() as path:
self._native.lib.rule_subgraph_visualize(
self._scheduler,
root_type_id,
product_type_id,
bytes(path))
with open(path) as fd:
for line in fd.readlines():
yield line.rstrip()
def _to_constraint(self, type_or_constraint):
return TypeConstraint(self._to_id(constraint_for(type_or_constraint)))
def _to_constraint(self, type_or_constraint): return TypeConstraint(self._to_id(constraint_for(type_or_constraint)))
def __init__(
self,
native,
project_tree,
work_dir,
rules,
execution_options,
include_trace_on_error=True,
validate=True,
):
"""
:param native: An instance of engine.native.Native.
:param project_tree: An instance of ProjectTree for the current build root.
:param work_dir: The pants work dir.
:param rules: A set of Rules which is used to compute values in the graph.
:param execution_options: Execution options for (remote) processes.
:param include_trace_on_error: Include the trace through the graph upon encountering errors.
:type include_trace_on_error: bool
:param validate: True to assert that the ruleset is valid.
"""
if execution_options.remote_execution_server and not execution_options.remote_store_server:
raise ValueError(
"Cannot set remote execution server without setting remote store server"
)
self._native = native
self.include_trace_on_error = include_trace_on_error
# TODO: The only (?) case where we use inheritance rather than exact type unions.
has_products_constraint = SubclassesOf(HasProducts)
# Validate and register all provided and intrinsic tasks.
rule_index = RuleIndex.create(list(rules))
self._root_subject_types = sorted(rule_index.roots)
# Create the native Scheduler and Session.
# TODO: This `_tasks` reference could be a local variable, since it is not used
# after construction.
self._tasks = native.new_tasks()
self._register_rules(rule_index)
self._scheduler = native.new_scheduler(
self._tasks,
self._root_subject_types,
project_tree.build_root,
work_dir,
project_tree.ignore_patterns,
execution_options,
DirectoryDigest,
Snapshot,
FileContent,
FilesContent,
Path,
Dir,
File,
Link,
FallibleExecuteProcessResult,
has_products_constraint,
constraint_for(Address),
constraint_for(Variants),
constraint_for(PathGlobs),
constraint_for(DirectoryDigest),
constraint_for(Snapshot),
constraint_for(FilesContent),
constraint_for(Dir),
constraint_for(File),
constraint_for(Link),
constraint_for(ExecuteProcessRequest),
constraint_for(FallibleExecuteProcessResult),
constraint_for(GeneratorType),
)
# If configured, visualize the rule graph before asserting that it is valid.
if self.visualize_to_dir() is not None:
rule_graph_name = 'rule_graph.dot'
self.visualize_rule_graph_to_file(
os.path.join(self.visualize_to_dir(), rule_graph_name))
if validate:
self._assert_ruleset_valid()
def __init__(self, native, build_root, work_dir, ignore_patterns,
rule_index):
self._native = native
# TODO: The only (?) case where we use inheritance rather than exact type unions.
has_products_constraint = SubclassesOf(HasProducts)
self._root_subject_types = sorted(rule_index.roots)
# Create the ExternContext, and the native Scheduler.
self._tasks = native.new_tasks()
self._register_rules(rule_index)
self._scheduler = native.new_scheduler(
self._tasks,
self._root_subject_types,
build_root,
work_dir,
ignore_patterns,
Snapshot,
_Snapshots,
FileContent,
FilesContent,
Path,
Dir,
File,
Link,
ExecuteProcessResult,
has_products_constraint,
constraint_for(Address),
constraint_for(Variants),
constraint_for(PathGlobs),
constraint_for(Snapshot),
constraint_for(_Snapshots),
constraint_for(FilesContent),
constraint_for(Dir),
constraint_for(File),
constraint_for(Link),
constraint_for(ExecuteProcessRequest),
constraint_for(ExecuteProcessResult),
constraint_for(GeneratorType),
)
def __init__(
self,
native,
project_tree,
work_dir,
local_store_dir,
rules,
execution_options,
include_trace_on_error=True,
validate=True,
):
"""
:param native: An instance of engine.native.Native.
:param project_tree: An instance of ProjectTree for the current build root.
:param work_dir: The pants work dir.
:param local_store_dir: The directory to use for storing the engine's LMDB store in.
:param rules: A set of Rules which is used to compute values in the graph.
:param execution_options: Execution options for (remote) processes.
:param include_trace_on_error: Include the trace through the graph upon encountering errors.
:type include_trace_on_error: bool
:param validate: True to assert that the ruleset is valid.
"""
if execution_options.remote_execution_server and not execution_options.remote_store_server:
raise ValueError(
"Cannot set remote execution server without setting remote store server"
)
self._native = native
self.include_trace_on_error = include_trace_on_error
# Validate and register all provided and intrinsic tasks.
rule_index = RuleIndex.create(list(rules))
self._root_subject_types = sorted(rule_index.roots, key=repr)
# Create the native Scheduler and Session.
# TODO: This `_tasks` reference could be a local variable, since it is not used
# after construction.
self._tasks = native.new_tasks()
self._register_rules(rule_index)
self._scheduler = native.new_scheduler(
tasks=self._tasks,
root_subject_types=self._root_subject_types,
build_root=project_tree.build_root,
work_dir=work_dir,
local_store_dir=local_store_dir,
ignore_patterns=project_tree.ignore_patterns,
execution_options=execution_options,
construct_directory_digest=Digest,
construct_snapshot=Snapshot,
construct_file_content=FileContent,
construct_files_content=FilesContent,
construct_path_stat=Path,
construct_dir=Dir,
construct_file=File,
construct_link=Link,
construct_process_result=FallibleExecuteProcessResult,
constraint_address=constraint_for(Address),
constraint_path_globs=constraint_for(PathGlobs),
constraint_directory_digest=constraint_for(Digest),
constraint_snapshot=constraint_for(Snapshot),
constraint_merge_snapshots_request=constraint_for(
MergedDirectories),
constraint_files_content=constraint_for(FilesContent),
constraint_dir=constraint_for(Dir),
constraint_file=constraint_for(File),
constraint_link=constraint_for(Link),
constraint_process_request=constraint_for(ExecuteProcessRequest),
constraint_process_result=constraint_for(
FallibleExecuteProcessResult),
constraint_generator=constraint_for(GeneratorType),
constraint_url_to_fetch=constraint_for(UrlToFetch),
)
# If configured, visualize the rule graph before asserting that it is valid.
if self.visualize_to_dir() is not None:
rule_graph_name = 'rule_graph.dot'
self.visualize_rule_graph_to_file(
os.path.join(self.visualize_to_dir(), rule_graph_name))
if validate:
self._assert_ruleset_valid()
def __init__(self, native, build_root, work_dir, ignore_patterns, rule_index):
self._native = native
# TODO: The only (?) case where we use inheritance rather than exact type unions.
has_products_constraint = SubclassesOf(HasProducts)
self._root_subject_types = sorted(rule_index.roots)
# Create the ExternContext, and the native Scheduler.
self._tasks = native.new_tasks()
self._register_rules(rule_index)
self._scheduler = native.new_scheduler(
self._tasks,
self._root_subject_types,
build_root,
work_dir,
ignore_patterns,
Snapshot,
_Snapshots,
FileContent,
FilesContent,
Path,
Dir,
File,
Link,
ExecuteProcessResult,
has_products_constraint,
constraint_for(Address),
constraint_for(Variants),
constraint_for(PathGlobs),
constraint_for(Snapshot),
constraint_for(_Snapshots),
constraint_for(FilesContent),
constraint_for(Dir),
constraint_for(File),
constraint_for(Link),
constraint_for(ExecuteProcessRequest),
constraint_for(ExecuteProcessResult),
)
def __init__(
self,
native,
project_tree,
work_dir,
rules,
remote_store_server,
remote_execution_server,
include_trace_on_error=True,
validate=True,
):
"""
:param native: An instance of engine.native.Native.
:param project_tree: An instance of ProjectTree for the current build root.
:param work_dir: The pants work dir.
:param rules: A set of Rules which is used to compute values in the graph.
:param include_trace_on_error: Include the trace through the graph upon encountering errors.
:type include_trace_on_error: bool
:param validate: True to assert that the ruleset is valid.
"""
if remote_execution_server and not remote_store_server:
raise ValueError("Cannot set remote execution server without setting remote store server")
# We can't currently pass Options to the rust side, so we pass empty strings for None:
remote_execution_server = remote_execution_server or ""
remote_store_server = remote_store_server or ""
self._native = native
self.include_trace_on_error = include_trace_on_error
# TODO: The only (?) case where we use inheritance rather than exact type unions.
has_products_constraint = SubclassesOf(HasProducts)
# Validate and register all provided and intrinsic tasks.
rule_index = RuleIndex.create(list(rules))
self._root_subject_types = sorted(rule_index.roots)
# Create the native Scheduler and Session.
# TODO: This `_tasks` reference could be a local variable, since it is not used
# after construction.
self._tasks = native.new_tasks()
self._register_rules(rule_index)
self._scheduler = native.new_scheduler(
self._tasks,
self._root_subject_types,
project_tree.build_root,
work_dir,
project_tree.ignore_patterns,
remote_store_server,
remote_execution_server,
DirectoryDigest,
Snapshot,
FileContent,
FilesContent,
Path,
Dir,
File,
Link,
ExecuteProcessResult,
has_products_constraint,
constraint_for(Address),
constraint_for(Variants),
constraint_for(PathGlobs),
constraint_for(DirectoryDigest),
constraint_for(Snapshot),
constraint_for(FilesContent),
constraint_for(Dir),
constraint_for(File),
constraint_for(Link),
constraint_for(ExecuteProcessRequest),
constraint_for(ExecuteProcessResult),
constraint_for(GeneratorType),
)
# If configured, visualize the rule graph before asserting that it is valid.
if self.visualize_to_dir() is not None:
rule_graph_name = 'rule_graph.dot'
self.visualize_rule_graph_to_file(os.path.join(self.visualize_to_dir(), rule_graph_name))
if validate:
self._assert_ruleset_valid()