def input(self,
*,
file: Optional[str] = None,
sh: Optional[str] = None,
env: Env = None):
# we want the state *before* running the action
state = self.hashstate.state()
super().input(file=file, sh=sh, env=env)
self.run_shell_queue()
if file is not None:
self.load_deps([self.absolute(file)])
with open(self.absolute(file), "r") as f:
return f.read()
else:
log("RUNNING", sh)
out = run_shell(
sh,
shell=True,
cwd=self.cwd,
capture_output=True,
quiet=True,
inherit_env=False,
env=self.normalize(env),
).decode("utf-8")
self.memorize(state, HashState().record(sh, env).state(), out)
return out
def dep_fetcher(dep, *, initial_load=False):
if dep not in loaded_deps and in_sandbox:
if not initial_load:
raise BuildException(
f"New dep {dep} found when rerunning rule, it's likely not deterministic!"
)
if not dep.startswith(":"):
log(f"Loading dependency {dep} into sandbox")
copy_helper(
src_root=os.curdir,
dest_root=scratch_path,
src_names=[dep],
symlink=not rule.do_not_symlink,
)
# check that these deps are built! Since they may not have been checked by the PreviewExecution.
dep_rule = None
if dep not in build_state.source_files:
dep_rule = build_state.target_rule_lookup.lookup(build_state, dep)
if dep_rule not in build_state.ready:
raise MissingDependency(dep)
loaded_deps.add(dep)
return dep_rule
def enqueue_deps(build_state: BuildState, rule: Rule,
candidate_deps: Collection[str]) -> bool:
waiting_for_deps = False
with build_state.scheduling_lock:
for dep in candidate_deps:
if dep in build_state.source_files:
# nothing to do
continue
runtime_dep: Rule = build_state.target_rule_lookup.lookup(
build_state, dep)
if runtime_dep not in build_state.ready:
waiting_for_deps = True
if runtime_dep not in build_state.scheduled_but_not_ready:
# enqueue dependency
log(f"Enqueueing dependency {runtime_dep}")
build_state.scheduled_but_not_ready.add(runtime_dep)
build_state.work_queue.put(runtime_dep)
build_state.status_monitor.move(total=1)
else:
log(f"Waiting on already queued dependency {runtime_dep}")
# register task in the already queued / executing dependency
# so when it finishes we may be triggered
runtime_dep.runtime_dependents.add(rule)
rule.pending_rule_dependencies.add(runtime_dep)
return waiting_for_deps
def publish(item, board, helper, channel, web):
item_type = helper.item_type(item)
post_process(item, board, helper, web)
while True:
try:
channel.basic_publish(exchange='chan',
routing_key="%s.%s.%s" %
(chan, item_type, board),
body=json.dumps(item,
separators=(',', ':'),
ensure_ascii=False,
sort_keys=True))
if MONITORING:
distance = datetime.utcnow() - datetime.utcfromtimestamp(
helper.item_mtime(item))
monitoring.log([{
"measurement": chan,
"time": str(datetime.utcnow()),
"tags": {
"board": board
},
"fields": {
"distance": distance.total_seconds()
}
}])
break
except Exception as e:
logger.debug(traceback.format_exc())
logger.error(str(e))
channel = connect()
def dep_fetcher(dep):
if dep.startswith(":"):
if dep not in direct_lookup:
raise BuildException(f"Unable to find setup rule {dep}")
dep_rule = direct_lookup[dep]
log(f"Looking up setup rule {dep}")
if dep_rule not in ready:
raise MissingDependency(dep)
return dep_rule
def run_shell_queue(self):
for cmd, cwd, env in self.sh_queue:
log("RUNNING cmd:", cmd)
run_shell(
cmd,
shell=True,
cwd=cwd,
quiet=True,
capture_output=True,
inherit_env=False,
env=env,
)
self.sh_queue = []
def input(self, sh: Optional[str] = None, *, env: Env = None):
# we want the state *before* running the action
state = self.hashstate.state()
super().input(sh, env=env)
if self.out_of_date_deps:
raise MissingDependency(*self.out_of_date_deps)
self.run_shell_queue()
log("RUNNING", sh)
out = run_shell(
sh,
shell=True,
cwd=os.path.join(self.base, self.cwd),
capture_output=True,
quiet=True,
inherit_env=False,
env=self.normalize(env),
).decode("utf-8")
self.memorize(state, HashState().record(sh, env).state(), out)
return out
def load_deps(deps):
deps = set(deps) - loaded_deps
# check that these deps are built! Since they have not been checked by the PreviewExecution.
missing_deps = []
for dep in deps:
if dep not in build_state.source_files:
dep_rule = build_state.target_rule_lookup.lookup(
build_state, dep)
if dep_rule not in build_state.ready:
missing_deps.append(dep)
if missing_deps:
raise MissingDependency(*missing_deps)
loaded_deps.update(deps)
if in_sandbox:
log(f"Loading dependencies {deps} into sandbox")
copy_helper(
src_root=build_state.repo_root,
dest_root=scratch_path,
src_names=[dep for dep in deps if not dep.startswith(":")],
symlink=not rule.do_not_symlink,
)
def enqueue_deps(
build_state: BuildState,
rule: Optional[Rule],
candidate_deps: Collection[str],
*,
catch_failure: bool = False,
) -> bool:
waiting_for_deps = False
with build_state.scheduling_lock:
for dep in candidate_deps:
if dep in build_state.source_files:
# nothing to do
continue
if catch_failure:
runtime_dep: Rule = build_state.target_rule_lookup.try_lookup(
dep)
if runtime_dep is None:
continue
else:
runtime_dep: Rule = build_state.target_rule_lookup.lookup(
build_state, dep)
if runtime_dep not in build_state.ready:
waiting_for_deps = True
if runtime_dep not in build_state.scheduled_but_not_ready:
# enqueue dependency
log(f"Enqueueing dependency {runtime_dep}")
build_state.scheduled_but_not_ready.add(runtime_dep)
build_state.work_queue.put(runtime_dep)
build_state.status_monitor.move(total=1)
else:
log(f"Waiting on already queued dependency {runtime_dep}")
if rule:
# register task in the already queued / executing dependency
# so when it finishes we may be triggered
# if rule is None, these are deferred dependencies, so no need to register anything
log(f"Registering {rule} to wait on {runtime_dep}")
runtime_dep.runtime_dependents.add(rule)
rule.pending_rule_dependencies.add(runtime_dep)
return waiting_for_deps
def worker(build_state: BuildState, index: int):
scratch_path = Path(build_state.repo_root).joinpath(
Path(f".scratch_{index}"))
if scratch_path.exists():
rmtree(scratch_path, ignore_errors=True)
_, cache_save = make_cache_memorize(build_state.cache_directory)
_, cache_loader = make_cache_fetcher(build_state.cache_directory)
while True:
if build_state.failure is not None:
# every thread needs to clear the queue since otherwise some other thread might still be filling it up
clear_queue(build_state.work_queue)
return # some thread has failed, emergency stop
todo = build_state.work_queue.get()
if todo is None:
return
start_time = time.time()
try:
build_state.status_monitor.update(index, "Parsing: " + str(todo))
log(f"Target {todo} popped from queue by worker {index}")
# only from caches, will never run a subprocess
cache_key, deps = get_deps(build_state, todo)
if cache_key is None:
# unable to compute cache_key, potentially because not all deps are ready
log(f"Target {todo} either has unbuilt dependencies, "
f"or does not have a cached dynamic dependency resolved")
deps_ready = not enqueue_deps(build_state, todo, deps)
if deps_ready:
log("Apparently it is missing an input cache in the impl")
else:
log("Apparently it is waiting on unbuilt dependencies")
else:
log(f"All the dependencies of target {todo} are ready: {deps}")
# if the cache_key is ready, *all* the deps must be ready, not just the discoverable deps!
deps_ready = True
if deps_ready:
done = False
# check if we're already cached!
if cache_key:
if cache_loader(cache_key, todo, build_state.repo_root):
log(f"Target {todo} was loaded from the cache")
done = True
if not done:
# time to execute! but *not* inside the lock
# when we release the lock, stuff may change outside, but
# we don't care since *our* dependencies (so far) are all available
log(f"Target {todo} is not in the cache, rerunning...")
build_state.status_monitor.update(index,
"Building: " + str(todo))
try:
# if cache_key is None, we haven't finished evaluating the impl, so we
# don't know all the dependencies it could need. Therefore, we must
# run it in the working directory, so the impl can find the dependencies it needs
# Then, we run it *again*, to verify that the dependencies are accurate
in_sandbox = cache_key is not None
if not in_sandbox:
log(f"We don't know the dependencies of {todo}, "
f"so we are running the impl in the root directory to find out!"
)
cache_key = build(build_state,
todo,
deps,
scratch_path=None)
# now, if no exception has thrown, all the deps are available to the deps finder
alt_cache_key, deps = get_deps(build_state, todo)
# the alt_cache_key *MAY HAVE CHANGED*, because dynamic dependencies
# are not used for the input() cache_key [since they are only known afterwards]
# however, the alt_cache_key should match the cache_key of the subsequent run
try:
alt_cache_key = build(
build_state,
todo,
deps,
scratch_path=scratch_path,
)
except MissingDependency:
raise BuildException(
"An internal error has occurred.")
except CalledProcessError as e:
build_state.status_monitor.stop()
print(e.cmd)
print(e)
raise BuildException(
f"The dependencies for target {todo} are not fully specified, "
f"as it failed to build when provided only with them."
)
assert (cache_key == alt_cache_key
), "An internal error has occurred"
else:
log(f"We know all the dependencies of {todo}, so we can run it in a sandbox"
)
build(build_state,
todo,
deps,
scratch_path=scratch_path)
log(f"Target {todo} has been built fully!")
cache_save(cache_key, todo, scratch_path)
done = True
except MissingDependency as d:
log(f"Target {todo} failed to fully build because of the missing dynamic "
f"dependencies: {d.paths}, requeuing")
scheduled = enqueue_deps(build_state, todo, d.paths)
if not scheduled:
# due to race conditions, our dependencies are actually all ready now!
# no one else will enqueue us, so it is safe to enqueue ourselves
build_state.work_queue.put(todo)
build_state.status_monitor.move(total=1)
if scratch_path.exists():
rmtree(scratch_path, ignore_errors=True)
if done:
with build_state.scheduling_lock:
build_state.ready.add(todo)
# no one will ever add us back, since we are in `ready`
build_state.scheduled_but_not_ready.remove(todo)
# now it's time to set up our dependents
# we need to be inside the lock even if we have no dependents, in case
# we *gain* dependents from another thread which could have held the lock!
for dependent in todo.runtime_dependents:
dependent.pending_rule_dependencies.remove(todo)
if not dependent.pending_rule_dependencies:
# this guy is ready to go
build_state.work_queue.put(dependent)
build_state.status_monitor.move(total=1)
# either way, we're done with this task for now
build_state.status_monitor.move(curr=1)
build_state.work_queue.task_done()
# record timing data
run_time = time.time() - start_time
TIMINGS[str(todo)] += run_time
except Exception as e:
if not isinstance(e, BuildException):
suffix = f"\n{Style.RESET_ALL}" + traceback.format_exc()
else:
suffix = ""
build_state.status_monitor.stop()
build_state.failure = BuildException(
f"Error while executing rule {todo}: " + str(e) + suffix)
build_state.work_queue.task_done()
def worker(build_state: BuildState, index: int):
scratch_path = Path(f".scratch_{index}")
if scratch_path.exists():
rmtree(scratch_path, ignore_errors=True)
_, cache_store_files = make_cache_store(build_state.cache_directory)
_, cache_load_files = make_cache_load(build_state.cache_directory)
while True:
if build_state.failure is not None:
# every thread needs to clear the queue since otherwise some other thread might still be filling it up
clear_queue(build_state.work_queue)
return # some thread has failed, emergency stop
todo = build_state.work_queue.get()
if todo is None:
return
start_time = time.time()
try:
build_state.status_monitor.update(index, "Parsing: " + str(todo))
log(f"Target {todo} popped from queue by worker {index}")
# only from caches, will never run a subprocess
(
cache_key,
provided_value,
deps,
deferred_deps,
uses_dynamic_deps,
) = get_deps(build_state, todo, skip_cache_key=todo.do_not_cache)
if uses_dynamic_deps:
log("Target", todo, "Uses dynamic deps")
if cache_key is None:
# unable to compute cache_key, potentially because not all deps are ready
log(
f"Target {todo} either has unbuilt dependencies, "
f"or does not have a cached dynamic dependency resolved"
)
deps_ready = not enqueue_deps(
build_state,
todo,
deps,
catch_failure=uses_dynamic_deps,
)
if deps_ready:
log(f"Apparently {todo} is missing an input cache in the impl")
else:
log(f"Apparently {todo} is waiting on unbuilt dependencies")
else:
log(f"All the dependencies of target {todo} are ready: {deps}")
# if the cache_key is ready, *all* the deps must be ready, not just the discoverable deps!
# unless the cache_key is not actually cached, in which case our inputs() could be wrong, so
# we have to run in the working directory to verify
deps_ready = True
log(f"Enqueuing deferred dependencies for {todo}: {deferred_deps}")
enqueue_deps(
build_state, None, deferred_deps, catch_failure=uses_dynamic_deps
)
if deps_ready:
done = False
# check if we're already cached!
if cache_key and not todo.do_not_cache:
try:
outputs = cache_load_files(cache_key, todo, os.curdir)
log(f"Target {todo} was loaded from the cache")
done = True
except CacheMiss:
pass
if not done:
# time to execute! but *not* inside the lock
# when we release the lock, stuff may change outside, but
# we don't care since *our* dependencies (so far) are all available
log(f"Target {todo} is not in the cache, rerunning...")
build_state.status_monitor.update(index, "Building: " + str(todo))
# if cache_key is None or we use dynamic dependencies, we haven't finished evaluating
# the impl, so we don't know all the dependencies it could need. Therefore, we must
# run it in the working directory, so the impl can find the dependencies it needs
# Then, we run it *again*, to verify that the dependencies are accurate.
# The cache_key could be None due to race conditions, even if we don't use dynamic deps.
# Imagine:
# - Rule A depends on a provider of rule B, which is not yet built
# (and on C, because of B's provider)
# - We determine that rule B is not built, so A depends on B but can't complete impl,
# so A.cache_key = None
# - B is built
# - We try to enqueue the deps of A. Enqueue_deps responds that B is already built,
# so A.deps_ready=True
# - So at this stage, A.cache_key = None, A.uses_dynamic_deps=False, but A.deps is missing C!
if uses_dynamic_deps or cache_key is None:
log(
f"We don't know the dependencies of {todo}, "
f"so we are running the impl in the root directory to find out!"
)
try:
_, cache_key = build(
build_state,
todo,
scratch_path=None,
skip_cache_key=todo.do_not_cache,
)
except MissingDependency as d:
log(
f"Target {todo} failed to fully build because of the missing dynamic "
f"dependencies: {d.paths}, requeuing"
)
scheduled = enqueue_deps(build_state, todo, d.paths)
if not scheduled:
# due to race conditions, our dependencies are actually all ready now!
# no one else will enqueue us, so it is safe to enqueue ourselves
build_state.work_queue.put(todo)
build_state.status_monitor.move(total=1)
else:
# now, if no exception has thrown, all the deps are available to the deps finder
(
alt_cache_key,
provided_value,
deps,
deferred_deps,
uses_dynamic_deps,
) = get_deps(
build_state, todo, skip_cache_key=todo.do_not_cache
)
try:
provided_value, alt_cache_key_2 = build(
build_state,
todo,
scratch_path=scratch_path,
precomputed_deps=deps,
skip_cache_key=todo.do_not_cache,
)
except CalledProcessError as e:
build_state.status_monitor.stop()
print(e.cmd)
print(e)
raise BuildException(
f"The dependencies for target {todo} are not fully specified, "
f"as it failed to build when provided only with them."
)
if not todo.do_not_cache:
assert (
cache_key == alt_cache_key == alt_cache_key_2
), "An internal error has occurred"
done = True
else:
log(
f"We know all the dependencies of {todo}, so we can run it in a sandbox"
)
provided_value, alt_cache_key = build(
build_state,
todo,
scratch_path=scratch_path,
precomputed_deps=deps,
skip_cache_key=todo.do_not_cache,
)
if not todo.do_not_cache:
assert (
cache_key == alt_cache_key
), "An internal error has occurred"
done = True
if done:
log(f"Target {todo} has been built fully!")
if not todo.do_not_cache:
outputs = cache_store_files(cache_key, todo, os.curdir)
else:
outputs = []
for out in todo.outputs:
if out.endswith("/"):
outputs.extend(
os.path.join(path, filename)
for path, subdirs, files in os.walk(out)
for filename in files
)
else:
outputs.append(out)
if scratch_path.exists():
rmtree(scratch_path, ignore_errors=True)
if done:
waiting_for_deferred = enqueue_deps(
build_state, todo, deferred_deps
)
# We have to wait for deferred dependencies to build because the TransitiveOutputProvider
# needs them to complete. However, we can run the build itself before the deferred deps finish.
if waiting_for_deferred:
log(
f"Target {todo} has been built, but is waiting for deferred dependencies"
)
else:
todo.set_provided_value(
provided_value, build_state, deps, deferred_deps, outputs
)
with build_state.scheduling_lock:
build_state.ready.add(todo)
# no one will ever add us back, since we are in `ready`
build_state.scheduled_but_not_ready.remove(todo)
# now it's time to set up our dependents
# we need to be inside the lock even if we have no dependents, in case
# we *gain* dependents from another thread which could have held the lock!
for dependent in todo.runtime_dependents:
dependent.pending_rule_dependencies.remove(todo)
if not dependent.pending_rule_dependencies:
# this guy is ready to go
build_state.work_queue.put(dependent)
build_state.status_monitor.move(total=1)
# either way, we're done with this task for now
build_state.status_monitor.move(curr=1)
build_state.work_queue.task_done()
# record timing data
run_time = time.time() - start_time
TIMINGS[str(todo)] += run_time
except Exception as e:
if isinstance(e, BuildException):
suffix = f"\n{Style.RESET_ALL}"
else:
suffix = f"\n{Style.RESET_ALL}" + traceback.format_exc()
build_state.status_monitor.stop()
build_state.failure = BuildException(
f"Error while executing rule {todo}: " + str(e) + suffix
)
build_state.work_queue.task_done()
def initialize_workspace(
setup_rule_lookup: TargetLookup,
setup_target: str,
state_directory: str,
quiet: bool,
):
# we don't need the indirect lookup as we only have rule and source deps
direct_lookup: Dict[str, Rule] = setup_rule_lookup.direct_lookup
if setup_target not in direct_lookup:
raise BuildException(
f"Unknown or unspecified setup target {setup_target}")
rebuilt: Set[str] = set()
ready: Set[str] = set()
work_queue = [direct_lookup[setup_target]]
cache_fetcher, _ = make_cache_fetcher(state_directory)
cache_memorize, _ = make_cache_memorize(state_directory)
status_monitor = create_status_monitor(1, quiet)
status_monitor.move(total=1)
while work_queue:
todo = work_queue.pop()
log(f"Popping setup rule {todo} off work queue")
hashstate = HashState()
ctx = WorkspaceExecutionContext(hashstate)
unchecked_rules = []
for dep in todo.deps:
hashstate.record(dep)
if dep.startswith(":"):
if dep not in direct_lookup:
raise BuildException(f"Unable to find setup rule {dep}")
dep_rule = direct_lookup[dep]
if dep_rule not in ready:
unchecked_rules.append(dep_rule)
continue
ctx.deps[dep] = dep_rule.provided_value
setattr(ctx.deps, dep[1:], dep_rule.provided_value)
else:
try:
hashstate.update(hash_file(dep))
except FileNotFoundError:
raise BuildException(f"Source file {dep} not found.")
if unchecked_rules:
for dep in unchecked_rules:
if dep not in work_queue:
log(f"Setup rule {todo} is enqueuing {dep}")
status_monitor.move(total=1)
work_queue.append(dep)
else:
log(f"Setup rule {todo} is waiting on {dep}, which is already enqueued"
)
dep.runtime_dependents.add(todo)
todo.pending_rule_dependencies.add(dep)
else:
# our dependent rules are ready, now we need to see if we need to rerun
todo.provided_value = todo.impl(ctx)
if todo.name is None:
raise BuildException(
f"All setup rules must have names, but {todo} does not.")
try:
ok = cache_fetcher("workspace", todo.name) == hashstate.state()
if not ok:
log(f"State mismatch for rule {todo}, need to rerun")
except CacheMiss:
log(f"State not found for rule {todo}, need to run for first time"
)
ok = False
for dep in todo.deps:
if dep.startswith(":"):
if direct_lookup[dep] in rebuilt:
log(f"Dependency {dep} of setup rule {todo} was rebuilt, so we must rebuild {todo} as well"
)
ok = True
for out in todo.outputs:
if not os.path.exists(out):
log(f"Output {out} is missing for setup rule {todo}, forcing rerun"
)
ok = False
break
if not ok:
# we need to fully run
ctx.run_shell_queue()
rebuilt.add(todo)
cache_memorize("workspace", todo.name, hashstate.state())
# either way, now we can trigger our dependents
ready.add(todo)
for dep in todo.runtime_dependents:
dep.pending_rule_dependencies.remove(todo)
if not dep.pending_rule_dependencies:
work_queue.append(dep)
status_monitor.move(total=1)
status_monitor.move(curr=1)
def get_deps(build_state: BuildState, rule: Rule):
"""
Use static dependencies and caches to try and identify as *many*
needed dependencies as possible, without *any* spurious dependencies.
"""
hashstate = HashState()
cache_fetcher, _ = make_cache_fetcher(build_state.cache_directory)
dep_fetcher = make_dep_fetcher(build_state)
ctx = PreviewContext(
build_state.repo_root,
rule.location,
hashstate,
dep_fetcher,
cache_fetcher,
)
log(f"Looking for static dependencies of {rule}")
for dep in rule.deps:
if dep not in build_state.source_files:
dep_rule = build_state.target_rule_lookup.lookup(build_state, dep)
if dep_rule not in build_state.ready:
log(f"Static dependency {dep} of {dep_rule} is not ready, skipping impl"
)
# static deps are not yet ready
break
ctx.deps[dep] = dep_rule.provided_value
if dep.startswith(":"):
setattr(ctx.deps, dep[1:], dep_rule.provided_value)
continue
hashstate.update(dep.encode("utf-8"))
try:
hashstate.update(dep_fetcher(dep, get_hash=True))
except MissingDependency:
# get static deps before running the impl!
# this means that a source file is *missing*, but the error will be thrown in enqueue_deps
break
else:
ok = False
try:
log(f"Running impl of {rule} to discover dynamic dependencies")
rule.provided_value = rule.impl(ctx)
log(f"Impl of {rule} completed with discovered deps: {ctx.inputs}")
for out in rule.outputs:
# needed so that if we ask for another output, we don't panic if it's not in the cache
hashstate.record(out)
ok = True
except CacheMiss:
log(f"Cache miss while running impl of {rule}")
pass # stops context execution
except MissingDependency as e:
log(f"Dependencies {e.paths} were unavailable while running impl of {rule}"
)
pass # dep already added to ctx.inputs
except Exception as e:
print(
"Error occurred during PreviewExecution. This may be normal, if a cached file that has not "
"yet been reported / processed has been changed. However, it may also be an internal error, so "
"it is being logged here. If it is an internal error, please contact the maintainer."
)
print(repr(e))
# if `ok`, hash loaded dynamic dependencies
if ok:
log(f"Runtime dependencies resolved for {rule}, now checking dynamic dependencies"
)
for input_path in ctx.inputs:
if input_path.startswith(":"):
if input_path not in build_state.ready:
ok = False
log(f"Dynamic rule dependency {input_path} is not yet ready"
)
break
else:
hashstate.update(input_path.encode("utf-8"))
try:
data = dep_fetcher(input_path, get_hash=True)
except MissingDependency as e:
# this dependency was not needed for deps calculation
# but is not verified to be up-to-date
ok = False
log(f"Dynamic dependencies {e.paths} were not needed for the impl, but are not up to date"
)
break
else:
hashstate.update(data)
return (
hashstate.state() if ok else None,
ctx.inputs + rule.deps,
)
return None, rule.deps
def initialize_workspace(
setup_rule_lookup: TargetLookup,
setup_targets: List[str],
state_directory: str,
quiet: bool,
):
# we don't need the indirect lookup as we only have rule and source deps
direct_lookup: Dict[str, Rule] = setup_rule_lookup.direct_lookup
work_queue = []
for setup_target in setup_targets:
if setup_target not in direct_lookup:
raise BuildException(
f"Unknown or unspecified setup target {setup_target}")
work_queue.append(direct_lookup[setup_target])
rebuilt: Set[str] = set()
ready: Set[str] = set()
cache_load_string, _ = make_cache_load(state_directory)
cache_store_string, _ = make_cache_store(state_directory)
if work_queue:
status_monitor = create_status_monitor(1, quiet)
status_monitor.move(total=len(work_queue))
def dep_fetcher(dep):
if dep.startswith(":"):
if dep not in direct_lookup:
raise BuildException(f"Unable to find setup rule {dep}")
dep_rule = direct_lookup[dep]
log(f"Looking up setup rule {dep}")
if dep_rule not in ready:
raise MissingDependency(dep)
return dep_rule
while work_queue:
todo = work_queue.pop()
log(f"Popping setup rule {todo} off work queue")
try:
if todo.name is None:
raise BuildException(
f"All setup rules must have names, but {todo} does not.")
hashstate = HashState()
ctx = WorkspaceExecutionContext(hashstate, dep_fetcher)
unchecked_rules = []
try:
todo.set_provided_value(
todo.impl(ctx),
None,
ctx.inputs,
ctx.deferred_inputs,
[], # todo: implement output providers for setup rules
)
if ctx.out_of_date_deps:
raise MissingDependency(*ctx.out_of_date_deps)
except MissingDependency as e:
unchecked_rules = [direct_lookup[x] for x in e.paths]
if unchecked_rules:
for dep in unchecked_rules:
if dep not in work_queue:
log(f"Setup rule {todo} is enqueuing {dep}")
status_monitor.move(total=1)
work_queue.append(dep)
else:
log(f"Setup rule {todo} is waiting on {dep}, which is already enqueued"
)
dep.runtime_dependents.add(todo)
todo.pending_rule_dependencies.add(dep)
else:
log(f"Setup rule {todo} ran with inputs {ctx.inputs + ctx.deferred_inputs}"
)
for dep in ctx.inputs + ctx.deferred_inputs:
if dep.startswith(":"):
continue
try:
hashstate.record(dep)
hashstate.update(hash_file(dep))
except FileNotFoundError:
raise BuildException(f"Source file {dep} not found.")
try:
ok = cache_load_string("workspace",
todo.name) == hashstate.state()
if not ok:
log(f"State mismatch for rule {todo}, need to rerun")
except CacheMiss:
log(f"State not found for rule {todo}, need to run for first time"
)
ok = False
for dep in ctx.inputs + ctx.deferred_inputs:
if dep.startswith(":"):
if direct_lookup[dep] in rebuilt:
log(f"Dependency {dep} of setup rule {todo} was rebuilt, so we must rebuild {todo} as well"
)
ok = False
for out in todo.outputs:
if not os.path.exists(out):
log(f"Output {out} is missing for setup rule {todo}, forcing rerun"
)
ok = False
break
if not ok:
# we need to fully run
log(f"Fully running setup rule {todo}")
ctx.run_shell_queue()
rebuilt.add(todo)
cache_store_string("workspace", todo.name,
hashstate.state())
# either way, now we can trigger our dependents
ready.add(todo)
for dep in todo.runtime_dependents:
dep.pending_rule_dependencies.remove(todo)
if not dep.pending_rule_dependencies:
work_queue.append(dep)
status_monitor.move(total=1)
status_monitor.move(curr=1)
except Exception as e:
if not isinstance(e, BuildException):
suffix = f"\n{Style.RESET_ALL}" + traceback.format_exc()
else:
suffix = ""
status_monitor.stop()
raise BuildException(f"Error while executing rule {todo}: " +
str(e) + suffix)
def get_deps(build_state: BuildState, rule: Rule, *, skip_cache_key: bool):
"""
Use static dependencies and caches to try and identify as *many*
needed dependencies as possible, without *any* spurious dependencies.
"""
hashstate = HashState()
cache_load_string, _ = make_cache_load(build_state.cache_directory)
dep_fetcher = make_dep_fetcher(build_state)
ctx = PreviewContext(
rule.location,
build_state.macros,
hashstate,
dep_fetcher,
cache_load_string,
)
ok = False
provided_value = None
try:
log(f"Running impl of {rule} to discover dependencies")
if not skip_cache_key:
for out in rule.outputs:
# needed so that if we ask for another output, we don't panic if it's not in the cache
hashstate.record(out)
provided_value = rule.impl(ctx)
log(f"Impl of {rule} completed with deps: {ctx.inputs}")
ok = True
except CacheMiss:
log(f"Cache miss while running impl of {rule}")
pass # stops context execution
except MissingDependency as e:
log(f"Dependencies {e.paths} were unavailable while running impl of {rule}"
)
pass # dep already added to ctx.inputs
except Exception as e:
print(
"Error occurred during PreviewExecution. This may be normal, if a cached file that has not "
"yet been reported / processed has been changed. However, it may also be an internal error, so "
"it is being logged here. If it is an internal error, please contact the maintainer."
)
print(repr(e))
if not ctx.uses_dynamic_inputs:
raise
# if `ok`, hash loaded dynamic dependencies
if ok:
log(f"Inputs and dependencies resolved for {rule}")
for input_path in ctx.inputs:
if input_path.startswith(":"):
input_dep = build_state.target_rule_lookup.try_lookup(
input_path)
if input_dep is None or input_dep not in build_state.ready:
ok = False
log(f"Rule dependency {input_path} is not yet ready (or does not exist)"
)
break
else:
if not skip_cache_key:
hashstate.update(input_path.encode("utf-8"))
try:
data = dep_fetcher(input_path,
"rule" if skip_cache_key else "hash")
except MissingDependency as e:
# this dependency was not needed for deps calculation
# but is not verified to be up-to-date
ok = False
log(f"Dependencies {e.paths} were not needed for the impl, but are not up to date"
)
break
else:
if not skip_cache_key:
hashstate.update(data)
hashstate.record("done")
return (
hashstate.state() if ok else None,
provided_value,
ctx.inputs,
ctx.deferred_inputs,
ctx.uses_dynamic_inputs,
)