def __run_job(self, job):
try:
spirit = TaskLoader.init(job["spirit_id"], task_root=self.task_dir)
except TaskLoadError:
trace = traceback.format_exc()
print("Could not load job %s" % job)
print(trace)
self.__finish_job(job, FinishState.LOAD_ERROR, message=trace)
except Exception:
trace = traceback.format_exc()
print("Aborted job %s with error %s" % (job, trace))
self.__finish_job(job, FinishState.WORKER_ERROR, message=trace)
else:
# Run spirit with runner, and send different finish states
# depending on if there was an execution error, unit test error, abort, or success
try:
dep_input = TaskLoader.load_dependencies(
spirit, self.config, default_driver=self.default_driver)
if spirit.stored_in() is None:
writer = self.default_driver.write(spirit, self.config)
else:
writer = spirit.stored_in().write(spirit, self.config)
except Exception:
trace = traceback.format_exc()
print("Aborted spirit %s with error %s" % (spirit, trace))
self.__finish_job(job, FinishState.LOAD_ERROR, message=trace)
else:
do_heartbeat = True
try:
print("Run spirit %s" % spirit)
def do_thread():
timeout = 0.1 * self.config.get("watchdog.timeout")
transaction_id = job["transaction_id"]
while do_heartbeat:
self.remote.heartbeat(transaction_id)
time.sleep(timeout)
heartbeat_thread = threading.Thread(target=do_thread)
heartbeat_thread.start()
spirit.executed_by().run(
PathFinder.get_task_path(spirit.name(),
task_root=self.task_dir),
spirit.parameters, dep_input, writer)
except Exception:
trace = traceback.format_exc()
print("Aborted spirit %s with error %s" % (spirit, trace))
self.__finish_job(job,
FinishState.EXEC_ERROR,
message=trace)
else:
print("Completed spirit %s" % spirit)
self.__finish_job(job, FinishState.SUCCESS)
do_heartbeat = False
def test_recursive_branches(self):
r2 = TaskLoader.init(("testing.recursive_dependency", {
"n": 4,
"m": 5
}))
r3 = TaskLoader.init(("testing.parameter_requires", {
"requires": [r2.spirit_id(), r2.spirit_id()]
}))
self.assertEqual(
len(DependencyExplorer.involved_spirits(r3.spirit_id())), 6)
def test_recursive(self):
r1 = TaskLoader.init(("testing.recursive_dependency", {
"n": 10,
"m": 1
}))
r2 = TaskLoader.init(("testing.recursive_dependency", {
"n": 4,
"m": 5
}))
with self.assertRaises(CyclicDependencyException):
DependencyExplorer.build_graph(r1.spirit_id())
self.assertEqual(
len(DependencyExplorer.involved_spirits(r2.spirit_id())), 5)
def delete_corrupt(self):
self.logger.notice("Delete corrupt requested")
# Delete all casks that are not registered in the meta db,
# is a pipe (now) or cannot be initiated from its still
existing_casks = [spirit for spirit in self.__cask_spirits() if not TaskLoader.spirit_is_pipe(spirit)]
self.delete_all(whitelist=existing_casks)
def enforce_expired(spirit):
cask_dt = self.__get_cask_datetime(spirit)
return not TaskLoader.spirit_is_pipe(spirit) and (
cask_dt is None or
(scheduling_info.age_requirement is not None
and cask_dt + datetime.timedelta(
seconds=scheduling_info.age_requirement) < now))
def remove_cask_spirit(self, handle, params, body):
try:
spirit_id = body.get("spirit_id", None)
except Exception:
return handle.error(400)
if spirit_id is None:
return handle.error(400)
try:
spirit = TaskLoader.init(spirit_id)
handle.server.env.gc.delete_spirit(spirit)
except Exception as e:
handle.server.env.logger.claim("CoreHandler").error(e)
return handle.error(500)
handle.json({"status": "ok"})
def add(self, scheduling_info, persistent=False):
"""Add a scheduling rule to the backlog
Only for once instance of the daemon if persistent=False
Or in the meta db if persistent=True"""
if TaskLoader.init(scheduling_info.spirit_id).inherits("DefaultPipe"):
raise ValueError("Target cannot be a pipe")
if (persistent or scheduling_info.reoccurring
) and scheduling_info.age_requirement == 0:
raise ValueError(
"Reoccurring target cannot have age requirement 0")
if persistent:
scheduling_info.reoccurring = True
scheduling_info = self.env.meta.add_scheduled_spirit(
scheduling_info)
heapq.heappush(self.backlog, scheduling_info)
def __cask_spirits(self):
return [
TaskLoader.init(cask["spirit_id"], none_on_error=True)
for cask in self.env.meta.get_all_casks()
]
def test_not_found(self):
with self.assertRaises(TaskLoadError):
TaskLoader.load("testing.not_existing")
def test_valid_definition(self):
ValidSpirit = TaskLoader.load("testing.valid_definition")
aSpirit = ValidSpirit()
self.assertTrue(aSpirit.a_test_function())
def test_corrupt_definition(self):
with self.assertRaises(TaskLoadError):
TaskLoader.load("testing.corrupt_definition")
def involved_spirits(cls, target_spirit_id, **kwargs):
return list(
cls.__explore(TaskLoader.init(target_spirit_id),
**kwargs)[0].keys())
def build_graph(cls, target_spirit_id, **kwargs):
return cls.__explore(TaskLoader.init(target_spirit_id), **kwargs)[2]
def __explore(cls,
current_spirit,
all_nodes=None,
downward_used_spirits=None,
downward_occurrences=None,
**kwargs):
enforce_func = kwargs.get("enforce_func", None)
skip_pipes = kwargs.get("skip_pipes", True)
if all_nodes is None:
all_nodes = dict()
if downward_used_spirits is None:
downward_used_spirits = set()
if downward_occurrences is None:
downward_occurrences = dict()
# Check if current spirit has already been used
if current_spirit in downward_used_spirits:
raise CyclicDependencyException(current_spirit.spirit_id())
up_downward_used_spirits = set(downward_used_spirits)
up_downward_used_spirits.add(current_spirit)
# Check if the task exceeds its occurrence count
up_downward_occurrences = dict(downward_occurrences)
curr_task_id = current_spirit.name()
if curr_task_id not in up_downward_occurrences:
up_downward_occurrences[curr_task_id] = current_spirit.occurrences(
)
up_downward_occurrences[curr_task_id] -= 1
if up_downward_occurrences[curr_task_id] < 0:
raise CyclicDependencyException(curr_task_id, task=True)
parents = set()
roots = set()
for dep in current_spirit.requires():
dep_spirit = TaskLoader.init(dep)
_, dep_nodes, dep_roots = cls.__explore(
dep_spirit,
all_nodes=all_nodes,
downward_used_spirits=up_downward_used_spirits,
downward_occurrences=up_downward_occurrences,
**kwargs)
if dep_nodes is not None:
parents.update(dep_nodes)
roots.update(dep_roots)
has_parents = len(parents) > 0
# Default is all enforced
build_enforced = enforce_func is None or enforce_func(current_spirit)
is_pipe = TaskLoader.spirit_is_pipe(current_spirit)
# No prune possible if upward needs to be built or build for this spirit is enforced.
if skip_pipes and is_pipe:
if has_parents:
return all_nodes, parents, roots
else:
return all_nodes, None, []
elif has_parents or build_enforced:
curr_node = cls.__get_node(all_nodes, current_spirit)
for parent in parents:
curr_node.add_parent(parent)
# No parents? This is the first with enforced build.
# Therefore, it is the first root of this path.
if not has_parents:
roots = [curr_node]
return all_nodes, [curr_node], roots
else:
return all_nodes, None, []
def setUp(self):
self.t1 = TaskLoader.init(("testing.parameter_requires_pipe", {
"requires": [],
"id": 1
}))
self.t2 = TaskLoader.init(("testing.parameter_requires", {
"requires": [self.t1.spirit_id()],
"id": 2
}))
self.t3 = TaskLoader.init(("testing.parameter_requires", {
"requires": [self.t2.spirit_id()],
"id": 3
}))
self.t4 = TaskLoader.init(("testing.parameter_requires_pipe", {
"requires": [self.t3.spirit_id()],
"id": 4
}))
self.t5 = TaskLoader.init(("testing.parameter_requires", {
"requires": [self.t4.spirit_id()],
"id": 5
}))
self.t6 = TaskLoader.init(("testing.parameter_requires", {
"requires": [],
"id": 6
}))
self.t7 = TaskLoader.init(("testing.parameter_requires_pipe", {
"requires": [self.t6.spirit_id()],
"id": 7
}))
self.t8 = TaskLoader.init(("testing.parameter_requires", {
"requires": [],
"id": 8
}))
self.t9 = TaskLoader.init(("testing.parameter_requires", {
"requires": [self.t7.spirit_id(),
self.t8.spirit_id()],
"id":
9
}))
self.t10 = TaskLoader.init(("testing.parameter_requires", {
"requires": [self.t7.spirit_id()],
"id": 10
}))
self.t11 = TaskLoader.init(("testing.parameter_requires", {
"requires":
[self.t5.spirit_id(),
self.t9.spirit_id(),
self.t10.spirit_id()],
"id":
11
}))
self.ua = TaskLoader.init(("testing.parameter_requires", {
"requires": [],
"id": "A"
}))
self.ub = TaskLoader.init(("testing.parameter_requires", {
"requires": [],
"id": "B"
}))
self.uc = TaskLoader.init(("testing.parameter_requires", {
"requires": [],
"id": "C"
}))
self.up1 = TaskLoader.init(("testing.parameter_requires_pipe", {
"requires": [self.ua.spirit_id(),
self.ub.spirit_id()],
"id":
"P1"
}))
self.up2 = TaskLoader.init(("testing.parameter_requires_pipe", {
"requires": [self.up1.spirit_id(),
self.uc.spirit_id()],
"id":
"P2"
}))
self.ud = TaskLoader.init(("testing.parameter_requires", {
"requires": [self.up2.spirit_id()],
"id": "D"
}))
def it(self):
return PipeIterator(
self.spirit.pipe_iterator(
TaskLoader.load_dependencies(self.spirit, self.config)))
def test_missing_definition(self):
with self.assertRaises(TaskLoadError):
TaskLoader.load("testing.missing_definition")
