def test_get_job_map_find_duplicates(self):
A = make_job('A')
another_A = make_job('A')
with self.assertRaises(DependencyDuplicateError) as call:
DependencySolver._get_job_map([A, another_A])
self.assertIs(call.exception.job, A)
self.assertIs(call.exception.duplicate_job, another_A)
def test_get_job_map_find_duplicates(self):
A = make_job('A')
another_A = make_job('A')
with self.assertRaises(DependencyDuplicateError) as call:
DependencySolver._get_job_map([A, another_A])
self.assertIs(call.exception.job, A)
self.assertIs(call.exception.duplicate_job, another_A)
def test_dependency_cycle_self(self):
# This tests dependency loops
# A -> A
A = make_job(id='A', depends='A')
job_list = [A]
with self.assertRaises(DependencyCycleError) as call:
DependencySolver.resolve_dependencies(job_list)
self.assertEqual(call.exception.job_list, [A, A])
def test_duplicate_error(self):
A = make_job('A')
another_A = make_job('A')
job_list = [A, another_A]
with self.assertRaises(DependencyDuplicateError) as call:
DependencySolver.resolve_dependencies(job_list)
self.assertIs(call.exception.job, A)
self.assertIs(call.exception.duplicate_job, another_A)
def test_dependency_cycle_self(self):
# This tests dependency loops
# A -> A
A = make_job(name='A', depends='A')
job_list = [A]
with self.assertRaises(DependencyCycleError) as call:
DependencySolver.resolve_dependencies(job_list)
self.assertEqual(call.exception.job_list, [A, A])
def test_duplicate_error(self):
A = make_job('A')
another_A = make_job('A')
job_list = [A, another_A]
with self.assertRaises(DependencyDuplicateError) as call:
DependencySolver.resolve_dependencies(job_list)
self.assertIs(call.exception.job, A)
self.assertIs(call.exception.duplicate_job, another_A)
def test_dependency_cycle_via_resource(self):
# This tests dependency loops
# A -> R -> A
A = make_job(id='A', requires='R.key == "value"')
R = make_job(id='R', depends='A', plugin="resource")
job_list = [A, R]
with self.assertRaises(DependencyCycleError) as call:
DependencySolver.resolve_dependencies(job_list)
self.assertEqual(call.exception.job_list, [A, R, A])
def test_dependency_cycle_via_resource(self):
# This tests dependency loops
# A -> R -> A
A = make_job(name='A', requires='R.key == "value"')
R = make_job(name='R', depends='A', plugin="resource")
job_list = [A, R]
with self.assertRaises(DependencyCycleError) as call:
DependencySolver.resolve_dependencies(job_list)
self.assertEqual(call.exception.job_list, [A, R, A])
def test_missing_direct_dependency(self):
# This tests missing dependencies
# A -> (inexisting B)
A = make_job(name='A', depends='B')
job_list = [A]
with self.assertRaises(DependencyMissingError) as call:
DependencySolver.resolve_dependencies(job_list)
self.assertIs(call.exception.job, A)
self.assertEqual(call.exception.missing_job_name, 'B')
self.assertEqual(call.exception.dep_type,
call.exception.DEP_TYPE_DIRECT)
def test_missing_resource_dependency(self):
# This tests missing resource dependencies
# A ~> (inexisting R)
A = make_job(name='A', requires='R.attr == "value"')
job_list = [A]
with self.assertRaises(DependencyMissingError) as call:
DependencySolver.resolve_dependencies(job_list)
self.assertIs(call.exception.job, A)
self.assertEqual(call.exception.missing_job_name, 'R')
self.assertEqual(call.exception.dep_type,
call.exception.DEP_TYPE_RESOURCE)
def test_dependency_cycle_longer(self):
# This tests dependency loops
# A -> B -> C -> D -> B
A = make_job(name='A', depends='B')
B = make_job(name='B', depends='C')
C = make_job(name='C', depends='D')
D = make_job(name='D', depends='B')
job_list = [A, B, C, D]
with self.assertRaises(DependencyCycleError) as call:
DependencySolver.resolve_dependencies(job_list)
self.assertEqual(call.exception.job_list, [B, C, D, B])
def test_dependency_cycle_longer(self):
# This tests dependency loops
# A -> B -> C -> D -> B
A = make_job(id='A', depends='B')
B = make_job(id='B', depends='C')
C = make_job(id='C', depends='D')
D = make_job(id='D', depends='B')
job_list = [A, B, C, D]
with self.assertRaises(DependencyCycleError) as call:
DependencySolver.resolve_dependencies(job_list)
self.assertEqual(call.exception.job_list, [B, C, D, B])
def test_missing_resource_dependency(self):
# This tests missing resource dependencies
# A ~> (inexisting R)
A = make_job(id='A', requires='R.attr == "value"')
job_list = [A]
with self.assertRaises(DependencyMissingError) as call:
DependencySolver.resolve_dependencies(job_list)
self.assertIs(call.exception.job, A)
self.assertEqual(call.exception.missing_job_id, 'R')
self.assertEqual(call.exception.dep_type,
call.exception.DEP_TYPE_RESOURCE)
def test_missing_direct_dependency(self):
# This tests missing dependencies
# A -> (inexisting B)
A = make_job(id='A', depends='B')
job_list = [A]
with self.assertRaises(DependencyMissingError) as call:
DependencySolver.resolve_dependencies(job_list)
self.assertIs(call.exception.job, A)
self.assertEqual(call.exception.missing_job_id, 'B')
self.assertEqual(call.exception.dep_type,
call.exception.DEP_TYPE_DIRECT)
def test_resource_deps(self):
# This tests resource deps
# A ~> R
A = make_job(name='A', requires='R.foo == "bar"')
R = make_job(name='R', plugin='resource')
job_list = [A, R]
expected = [R, A]
observed = DependencySolver.resolve_dependencies(job_list)
self.assertEqual(expected, observed)
def test_resource_deps(self):
# This tests resource deps
# A ~> R
A = make_job(id='A', requires='R.foo == "bar"')
R = make_job(id='R', plugin='resource')
job_list = [A, R]
expected = [R, A]
observed = DependencySolver.resolve_dependencies(job_list)
self.assertEqual(expected, observed)
def test_direct_deps(self):
# This tests the following simple job chain
# A -> B -> C
A = make_job(id='A', depends='B')
B = make_job(id='B', depends='C')
C = make_job(id='C')
job_list = [A, B, C]
expected = [C, B, A]
observed = DependencySolver.resolve_dependencies(job_list)
self.assertEqual(expected, observed)
def test_direct_deps(self):
# This tests the following simple job chain
# A -> B -> C
A = make_job(name='A', depends='B')
B = make_job(name='B', depends='C')
C = make_job(name='C')
job_list = [A, B, C]
expected = [C, B, A]
observed = DependencySolver.resolve_dependencies(job_list)
self.assertEqual(expected, observed)
def test_visiting_blackend_node(self):
# This tests a visit to already visited job
# A
# B -> A
# A will be visited twice
A = make_job(id='A')
B = make_job(id='B', depends='A')
job_list = [A, B]
expected = [A, B]
observed = DependencySolver.resolve_dependencies(job_list)
self.assertEqual(expected, observed)
def test_visiting_blackend_node(self):
# This tests a visit to already visited job
# A
# B -> A
# A will be visited twice
A = make_job(name='A')
B = make_job(name='B', depends='A')
job_list = [A, B]
expected = [A, B]
observed = DependencySolver.resolve_dependencies(job_list)
self.assertEqual(expected, observed)
def test_independent_groups_deps(self):
# This tests two independent job chains
# A1 -> B1
# A2 -> B2
A1 = make_job(name='A1', depends='B1')
B1 = make_job(name='B1',)
A2 = make_job(name='A2', depends='B2')
B2 = make_job(name='B2')
job_list = [A1, B1, A2, B2]
expected = [B1, A1, B2, A2]
observed = DependencySolver.resolve_dependencies(job_list)
self.assertEqual(expected, observed)
def test_independent_groups_deps(self):
# This tests two independent job chains
# A1 -> B1
# A2 -> B2
A1 = make_job(id='A1', depends='B1')
B1 = make_job(id='B1',)
A2 = make_job(id='A2', depends='B2')
B2 = make_job(id='B2')
job_list = [A1, B1, A2, B2]
expected = [B1, A1, B2, A2]
observed = DependencySolver.resolve_dependencies(job_list)
self.assertEqual(expected, observed)
def __init__(self, job_list):
# Start by making a copy of job_list as we may modify it below
job_list = job_list[:]
while True:
try:
# Construct a solver with the job list as passed by the caller.
# This will do a little bit of validation and might raise
# DepdendencyDuplicateError if there are any duplicates at this
# stage.
#
# There's a single case that is handled here though, if both
# jobs are identical this problem is silently fixed. This
# should not happen in normal circumstances but is non the less
# harmless (as long as both jobs are perfectly identical)
#
# Since this problem can happen any number of times (many
# duplicates) this is performed in a loop. The loop breaks when
# we cannot solve the problem _OR_ when no error occurs.
DependencySolver(job_list)
except DependencyDuplicateError as exc:
# If both jobs are identical then silently fix the problem by
# removing one of the jobs (here the second one we've seen but
# it's not relevant as they are possibly identical) and try
# again
if exc.job == exc.duplicate_job:
job_list.remove(exc.duplicate_job)
continue
else:
# If the jobs differ report this back to the caller
raise
else:
# If there are no problems then break the loop
break
self._job_list = job_list
self._job_state_map = {
job.name: JobState(job)
for job in self._job_list
}
self._desired_job_list = []
self._run_list = []
self._resource_map = {}
# Temporary directory used as 'scratch space' for running jobs. Removed
# entirely when session is terminated. Internally this is exposed as
# $CHECKBOX_DATA to script environment.
self._session_dir = None
# Directory used to store jobs IO logs.
self._jobs_io_log_dir = None
def __init__(self, job_list):
"""
Initialize a new SessionState with a given list of jobs.
The jobs are all of the jobs that the session knows about.
"""
# Start by making a copy of job_list as we may modify it below
job_list = job_list[:]
while True:
try:
# Construct a solver with the job list as passed by the caller.
# This will do a little bit of validation and might raise
# DepdendencyDuplicateError if there are any duplicates at this
# stage.
#
# There's a single case that is handled here though, if both
# jobs are identical this problem is silently fixed. This
# should not happen in normal circumstances but is non the less
# harmless (as long as both jobs are perfectly identical)
#
# Since this problem can happen any number of times (many
# duplicates) this is performed in a loop. The loop breaks when
# we cannot solve the problem _OR_ when no error occurs.
DependencySolver(job_list)
except DependencyDuplicateError as exc:
# If both jobs are identical then silently fix the problem by
# removing one of the jobs (here the second one we've seen but
# it's not relevant as they are possibly identical) and try
# again
if exc.job == exc.duplicate_job:
job_list.remove(exc.duplicate_job)
continue
else:
# If the jobs differ report this back to the caller
raise
else:
# If there are no problems then break the loop
break
self._job_list = job_list
self._job_state_map = {job.name: JobState(job)
for job in self._job_list}
self._desired_job_list = []
self._run_list = []
self._resource_map = {}
self._metadata = SessionMetaData()
super(SessionState, self).__init__()
def update_desired_job_list(self, desired_job_list):
"""
Update the set of desired jobs (that ought to run)
This method can be used by the UI to recompute the dependency graph.
The argument 'desired_job_list' is a list of jobs that should run.
Those jobs must be a sub-collection of the job_list argument that was
passed to the constructor.
It never fails although it may reduce the actual permitted
desired_job_list to an empty list. It returns a list of problems (all
instances of DependencyError class), one for each job that had to be
removed.
"""
# Remember a copy of original desired job list. We may modify this list
# so let's not mess up data passed by the caller.
self._desired_job_list = list(desired_job_list)
# Reset run list just in case desired_job_list is empty
self._run_list = []
# Try to solve the dependency graph. This is done in a loop as may need
# to remove a problematic job and re-try. The loop provides a stop
# condition as we will eventually run out of jobs.
problems = []
while self._desired_job_list:
# XXX: it might be more efficient to incorporate this 'recovery
# mode' right into the solver, this way we'd probably save some
# resources or runtime complexity.
try:
self._run_list = DependencySolver.resolve_dependencies(
self._job_list, self._desired_job_list)
except DependencyError as exc:
# When a dependency error is detected remove the affected job
# form _desired_job_list and try again.
self._desired_job_list.remove(exc.affected_job)
# Remember each problem, this can be presented by the UI
problems.append(exc)
continue
else:
# Don't iterate the loop if there was no exception
break
# Update all job readiness state
self._recompute_job_readiness()
# Return all dependency problems to the caller
return problems
def update_desired_job_list(self, desired_job_list):
"""
Update the set of desired jobs (that ought to run)
This method can be used by the UI to recompute the dependency graph.
The argument 'desired_job_list' is a list of jobs that should run.
Those jobs must be a sub-collection of the job_list argument that was
passed to the constructor.
It never fails although it may reduce the actual permitted
desired_job_list to an empty list. It returns a list of problems (all
instances of DependencyError class), one for each job that had to be
removed.
"""
# Remember a copy of original desired job list. We may modify this list
# so let's not mess up data passed by the caller.
self._desired_job_list = list(desired_job_list)
# Reset run list just in case desired_job_list is empty
self._run_list = []
# Try to solve the dependency graph. This is done in a loop as may need
# to remove a problematic job and re-try. The loop provides a stop
# condition as we will eventually run out of jobs.
problems = []
while self._desired_job_list:
# XXX: it might be more efficient to incorporate this 'recovery
# mode' right into the solver, this way we'd probably save some
# resources or runtime complexity.
try:
self._run_list = DependencySolver.resolve_dependencies(
self._job_list, self._desired_job_list)
except DependencyError as exc:
# When a dependency error is detected remove the affected job
# form _desired_job_list and try again.
self._desired_job_list.remove(exc.affected_job)
# Remember each problem, this can be presented by the UI
problems.append(exc)
continue
else:
# Don't iterate the loop if there was no exception
break
# Update all job readiness state
self._recompute_job_readiness()
# Return all dependency problems to the caller
return problems
def test_get_dependency_set_empty(self):
A = make_job('A')
expected = set()
observed = DependencySolver._get_dependency_set(A)
self.assertEqual(expected, observed)
def test_get_dependency_set_empty(self):
A = make_job('A')
expected = set()
observed = DependencySolver._get_dependency_set(A)
self.assertEqual(expected, observed)
def test_get_dependency_set_direct_two(self):
A = make_job('A', depends='B, C')
expected = set([("direct", 'B'), ("direct", 'C')])
observed = DependencySolver._get_dependency_set(A)
self.assertEqual(expected, observed)
def test_get_job_map_produces_map(self):
A = make_job('A')
B = make_job('B')
expected = {'A': A, 'B': B}
observed = DependencySolver._get_job_map([A, B])
self.assertEqual(expected, observed)
def test_empty(self):
observed = DependencySolver.resolve_dependencies([])
expected = []
self.assertEqual(expected, observed)
def test_get_dependency_set_direct_two(self):
A = make_job('A', depends='B, C')
expected = set([("direct", 'B'), ("direct", 'C')])
observed = DependencySolver._get_dependency_set(A)
self.assertEqual(expected, observed)
def test_get_job_map_produces_map(self):
A = make_job('A')
B = make_job('B')
expected = {'A': A, 'B': B}
observed = DependencySolver._get_job_map([A, B])
self.assertEqual(expected, observed)
def test_empty(self):
observed = DependencySolver.resolve_dependencies([])
expected = []
self.assertEqual(expected, observed)