示例#1
0
 def test_get_job_map_find_duplicates(self):
     A = make_job('A')
     another_A = make_job('A')
     with self.assertRaises(DependencyDuplicateError) as call:
         DependencySolver._get_job_map([A, another_A])
     self.assertIs(call.exception.job, A)
     self.assertIs(call.exception.duplicate_job, another_A)
示例#2
0
 def test_get_job_map_find_duplicates(self):
     A = make_job('A')
     another_A = make_job('A')
     with self.assertRaises(DependencyDuplicateError) as call:
         DependencySolver._get_job_map([A, another_A])
     self.assertIs(call.exception.job, A)
     self.assertIs(call.exception.duplicate_job, another_A)
示例#3
0
 def test_dependency_cycle_self(self):
     # This tests dependency loops
     # A -> A
     A = make_job(id='A', depends='A')
     job_list = [A]
     with self.assertRaises(DependencyCycleError) as call:
         DependencySolver.resolve_dependencies(job_list)
     self.assertEqual(call.exception.job_list, [A, A])
示例#4
0
 def test_duplicate_error(self):
     A = make_job('A')
     another_A = make_job('A')
     job_list = [A, another_A]
     with self.assertRaises(DependencyDuplicateError) as call:
         DependencySolver.resolve_dependencies(job_list)
     self.assertIs(call.exception.job, A)
     self.assertIs(call.exception.duplicate_job, another_A)
示例#5
0
 def test_dependency_cycle_self(self):
     # This tests dependency loops
     # A -> A
     A = make_job(name='A', depends='A')
     job_list = [A]
     with self.assertRaises(DependencyCycleError) as call:
         DependencySolver.resolve_dependencies(job_list)
     self.assertEqual(call.exception.job_list, [A, A])
示例#6
0
 def test_duplicate_error(self):
     A = make_job('A')
     another_A = make_job('A')
     job_list = [A, another_A]
     with self.assertRaises(DependencyDuplicateError) as call:
         DependencySolver.resolve_dependencies(job_list)
     self.assertIs(call.exception.job, A)
     self.assertIs(call.exception.duplicate_job, another_A)
示例#7
0
 def test_dependency_cycle_via_resource(self):
     # This tests dependency loops
     # A -> R -> A
     A = make_job(id='A', requires='R.key == "value"')
     R = make_job(id='R', depends='A', plugin="resource")
     job_list = [A, R]
     with self.assertRaises(DependencyCycleError) as call:
         DependencySolver.resolve_dependencies(job_list)
     self.assertEqual(call.exception.job_list, [A, R, A])
示例#8
0
 def test_dependency_cycle_via_resource(self):
     # This tests dependency loops
     # A -> R -> A
     A = make_job(name='A', requires='R.key == "value"')
     R = make_job(name='R', depends='A', plugin="resource")
     job_list = [A, R]
     with self.assertRaises(DependencyCycleError) as call:
         DependencySolver.resolve_dependencies(job_list)
     self.assertEqual(call.exception.job_list, [A, R, A])
示例#9
0
 def test_missing_direct_dependency(self):
     # This tests missing dependencies
     # A -> (inexisting B)
     A = make_job(name='A', depends='B')
     job_list = [A]
     with self.assertRaises(DependencyMissingError) as call:
         DependencySolver.resolve_dependencies(job_list)
     self.assertIs(call.exception.job, A)
     self.assertEqual(call.exception.missing_job_name, 'B')
     self.assertEqual(call.exception.dep_type,
                      call.exception.DEP_TYPE_DIRECT)
示例#10
0
 def test_missing_resource_dependency(self):
     # This tests missing resource dependencies
     # A ~> (inexisting R)
     A = make_job(name='A', requires='R.attr == "value"')
     job_list = [A]
     with self.assertRaises(DependencyMissingError) as call:
         DependencySolver.resolve_dependencies(job_list)
     self.assertIs(call.exception.job, A)
     self.assertEqual(call.exception.missing_job_name, 'R')
     self.assertEqual(call.exception.dep_type,
                      call.exception.DEP_TYPE_RESOURCE)
示例#11
0
 def test_dependency_cycle_longer(self):
     # This tests dependency loops
     # A -> B -> C -> D -> B
     A = make_job(name='A', depends='B')
     B = make_job(name='B', depends='C')
     C = make_job(name='C', depends='D')
     D = make_job(name='D', depends='B')
     job_list = [A, B, C, D]
     with self.assertRaises(DependencyCycleError) as call:
         DependencySolver.resolve_dependencies(job_list)
     self.assertEqual(call.exception.job_list, [B, C, D, B])
示例#12
0
 def test_dependency_cycle_longer(self):
     # This tests dependency loops
     # A -> B -> C -> D -> B
     A = make_job(id='A', depends='B')
     B = make_job(id='B', depends='C')
     C = make_job(id='C', depends='D')
     D = make_job(id='D', depends='B')
     job_list = [A, B, C, D]
     with self.assertRaises(DependencyCycleError) as call:
         DependencySolver.resolve_dependencies(job_list)
     self.assertEqual(call.exception.job_list, [B, C, D, B])
示例#13
0
 def test_missing_resource_dependency(self):
     # This tests missing resource dependencies
     # A ~> (inexisting R)
     A = make_job(id='A', requires='R.attr == "value"')
     job_list = [A]
     with self.assertRaises(DependencyMissingError) as call:
         DependencySolver.resolve_dependencies(job_list)
     self.assertIs(call.exception.job, A)
     self.assertEqual(call.exception.missing_job_id, 'R')
     self.assertEqual(call.exception.dep_type,
                      call.exception.DEP_TYPE_RESOURCE)
示例#14
0
 def test_missing_direct_dependency(self):
     # This tests missing dependencies
     # A -> (inexisting B)
     A = make_job(id='A', depends='B')
     job_list = [A]
     with self.assertRaises(DependencyMissingError) as call:
         DependencySolver.resolve_dependencies(job_list)
     self.assertIs(call.exception.job, A)
     self.assertEqual(call.exception.missing_job_id, 'B')
     self.assertEqual(call.exception.dep_type,
                      call.exception.DEP_TYPE_DIRECT)
示例#15
0
 def test_resource_deps(self):
     # This tests resource deps
     # A ~> R
     A = make_job(name='A', requires='R.foo == "bar"')
     R = make_job(name='R', plugin='resource')
     job_list = [A, R]
     expected = [R, A]
     observed = DependencySolver.resolve_dependencies(job_list)
     self.assertEqual(expected, observed)
示例#16
0
 def test_resource_deps(self):
     # This tests resource deps
     # A ~> R
     A = make_job(id='A', requires='R.foo == "bar"')
     R = make_job(id='R', plugin='resource')
     job_list = [A, R]
     expected = [R, A]
     observed = DependencySolver.resolve_dependencies(job_list)
     self.assertEqual(expected, observed)
示例#17
0
 def test_direct_deps(self):
     # This tests the following simple job chain
     # A -> B -> C
     A = make_job(id='A', depends='B')
     B = make_job(id='B', depends='C')
     C = make_job(id='C')
     job_list = [A, B, C]
     expected = [C, B, A]
     observed = DependencySolver.resolve_dependencies(job_list)
     self.assertEqual(expected, observed)
示例#18
0
 def test_direct_deps(self):
     # This tests the following simple job chain
     # A -> B -> C
     A = make_job(name='A', depends='B')
     B = make_job(name='B', depends='C')
     C = make_job(name='C')
     job_list = [A, B, C]
     expected = [C, B, A]
     observed = DependencySolver.resolve_dependencies(job_list)
     self.assertEqual(expected, observed)
示例#19
0
 def test_visiting_blackend_node(self):
     # This tests a visit to already visited job
     # A
     # B -> A
     # A will be visited twice
     A = make_job(id='A')
     B = make_job(id='B', depends='A')
     job_list = [A, B]
     expected = [A, B]
     observed = DependencySolver.resolve_dependencies(job_list)
     self.assertEqual(expected, observed)
示例#20
0
 def test_visiting_blackend_node(self):
     # This tests a visit to already visited job
     # A
     # B -> A
     # A will be visited twice
     A = make_job(name='A')
     B = make_job(name='B', depends='A')
     job_list = [A, B]
     expected = [A, B]
     observed = DependencySolver.resolve_dependencies(job_list)
     self.assertEqual(expected, observed)
示例#21
0
 def test_independent_groups_deps(self):
     # This tests two independent job chains
     # A1 -> B1
     # A2 -> B2
     A1 = make_job(name='A1', depends='B1')
     B1 = make_job(name='B1',)
     A2 = make_job(name='A2', depends='B2')
     B2 = make_job(name='B2')
     job_list = [A1, B1, A2, B2]
     expected = [B1, A1, B2, A2]
     observed = DependencySolver.resolve_dependencies(job_list)
     self.assertEqual(expected, observed)
示例#22
0
 def test_independent_groups_deps(self):
     # This tests two independent job chains
     # A1 -> B1
     # A2 -> B2
     A1 = make_job(id='A1', depends='B1')
     B1 = make_job(id='B1',)
     A2 = make_job(id='A2', depends='B2')
     B2 = make_job(id='B2')
     job_list = [A1, B1, A2, B2]
     expected = [B1, A1, B2, A2]
     observed = DependencySolver.resolve_dependencies(job_list)
     self.assertEqual(expected, observed)
示例#23
0
    def __init__(self, job_list):
        # Start by making a copy of job_list as we may modify it below
        job_list = job_list[:]
        while True:
            try:
                # Construct a solver with the job list as passed by the caller.
                # This will do a little bit of validation and might raise
                # DepdendencyDuplicateError if there are any duplicates at this
                # stage.
                #
                # There's a single case that is handled here though, if both
                # jobs are identical this problem is silently fixed. This
                # should not happen in normal circumstances but is non the less
                # harmless (as long as both jobs are perfectly identical)
                #
                # Since this problem can happen any number of times (many
                # duplicates) this is performed in a loop. The loop breaks when
                # we cannot solve the problem _OR_ when no error occurs.
                DependencySolver(job_list)
            except DependencyDuplicateError as exc:
                # If both jobs are identical then silently fix the problem by
                # removing one of the jobs (here the second one we've seen but
                # it's not relevant as they are possibly identical) and try
                # again
                if exc.job == exc.duplicate_job:
                    job_list.remove(exc.duplicate_job)
                    continue
                else:
                    # If the jobs differ report this back to the caller
                    raise
            else:
                # If there are no problems then break the loop
                break
        self._job_list = job_list
        self._job_state_map = {
            job.name: JobState(job)
            for job in self._job_list
        }
        self._desired_job_list = []
        self._run_list = []
        self._resource_map = {}
        # Temporary directory used as 'scratch space' for running jobs. Removed
        # entirely when session is terminated. Internally this is exposed as
        # $CHECKBOX_DATA to script environment.
        self._session_dir = None

        # Directory used to store jobs IO logs.
        self._jobs_io_log_dir = None
示例#24
0
    def __init__(self, job_list):
        """
        Initialize a new SessionState with a given list of jobs.

        The jobs are all of the jobs that the session knows about.
        """
        # Start by making a copy of job_list as we may modify it below
        job_list = job_list[:]
        while True:
            try:
                # Construct a solver with the job list as passed by the caller.
                # This will do a little bit of validation and might raise
                # DepdendencyDuplicateError if there are any duplicates at this
                # stage.
                #
                # There's a single case that is handled here though, if both
                # jobs are identical this problem is silently fixed. This
                # should not happen in normal circumstances but is non the less
                # harmless (as long as both jobs are perfectly identical)
                #
                # Since this problem can happen any number of times (many
                # duplicates) this is performed in a loop. The loop breaks when
                # we cannot solve the problem _OR_ when no error occurs.
                DependencySolver(job_list)
            except DependencyDuplicateError as exc:
                # If both jobs are identical then silently fix the problem by
                # removing one of the jobs (here the second one we've seen but
                # it's not relevant as they are possibly identical) and try
                # again
                if exc.job == exc.duplicate_job:
                    job_list.remove(exc.duplicate_job)
                    continue
                else:
                    # If the jobs differ report this back to the caller
                    raise
            else:
                # If there are no problems then break the loop
                break
        self._job_list = job_list
        self._job_state_map = {job.name: JobState(job)
                               for job in self._job_list}
        self._desired_job_list = []
        self._run_list = []
        self._resource_map = {}
        self._metadata = SessionMetaData()
        super(SessionState, self).__init__()
示例#25
0
    def update_desired_job_list(self, desired_job_list):
        """
        Update the set of desired jobs (that ought to run)

        This method can be used by the UI to recompute the dependency graph.
        The argument 'desired_job_list' is a list of jobs that should run.
        Those jobs must be a sub-collection of the job_list argument that was
        passed to the constructor.

        It never fails although it may reduce the actual permitted
        desired_job_list to an empty list. It returns a list of problems (all
        instances of DependencyError class), one for each job that had to be
        removed.
        """
        # Remember a copy of original desired job list. We may modify this list
        # so let's not mess up data passed by the caller.
        self._desired_job_list = list(desired_job_list)
        # Reset run list just in case desired_job_list is empty
        self._run_list = []
        # Try to solve the dependency graph. This is done in a loop as may need
        # to remove a problematic job and re-try. The loop provides a stop
        # condition as we will eventually run out of jobs.
        problems = []
        while self._desired_job_list:
            # XXX: it might be more efficient to incorporate this 'recovery
            # mode' right into the solver, this way we'd probably save some
            # resources or runtime complexity.
            try:
                self._run_list = DependencySolver.resolve_dependencies(
                    self._job_list, self._desired_job_list)
            except DependencyError as exc:
                # When a dependency error is detected remove the affected job
                # form _desired_job_list and try again.
                self._desired_job_list.remove(exc.affected_job)
                # Remember each problem, this can be presented by the UI
                problems.append(exc)
                continue
            else:
                # Don't iterate the loop if there was no exception
                break
        # Update all job readiness state
        self._recompute_job_readiness()
        # Return all dependency problems to the caller
        return problems
示例#26
0
    def update_desired_job_list(self, desired_job_list):
        """
        Update the set of desired jobs (that ought to run)

        This method can be used by the UI to recompute the dependency graph.
        The argument 'desired_job_list' is a list of jobs that should run.
        Those jobs must be a sub-collection of the job_list argument that was
        passed to the constructor.

        It never fails although it may reduce the actual permitted
        desired_job_list to an empty list. It returns a list of problems (all
        instances of DependencyError class), one for each job that had to be
        removed.
        """
        # Remember a copy of original desired job list. We may modify this list
        # so let's not mess up data passed by the caller.
        self._desired_job_list = list(desired_job_list)
        # Reset run list just in case desired_job_list is empty
        self._run_list = []
        # Try to solve the dependency graph. This is done in a loop as may need
        # to remove a problematic job and re-try. The loop provides a stop
        # condition as we will eventually run out of jobs.
        problems = []
        while self._desired_job_list:
            # XXX: it might be more efficient to incorporate this 'recovery
            # mode' right into the solver, this way we'd probably save some
            # resources or runtime complexity.
            try:
                self._run_list = DependencySolver.resolve_dependencies(
                    self._job_list, self._desired_job_list)
            except DependencyError as exc:
                # When a dependency error is detected remove the affected job
                # form _desired_job_list and try again.
                self._desired_job_list.remove(exc.affected_job)
                # Remember each problem, this can be presented by the UI
                problems.append(exc)
                continue
            else:
                # Don't iterate the loop if there was no exception
                break
        # Update all job readiness state
        self._recompute_job_readiness()
        # Return all dependency problems to the caller
        return problems
示例#27
0
 def test_get_dependency_set_empty(self):
     A = make_job('A')
     expected = set()
     observed = DependencySolver._get_dependency_set(A)
     self.assertEqual(expected, observed)
示例#28
0
 def test_get_dependency_set_empty(self):
     A = make_job('A')
     expected = set()
     observed = DependencySolver._get_dependency_set(A)
     self.assertEqual(expected, observed)
示例#29
0
 def test_get_dependency_set_direct_two(self):
     A = make_job('A', depends='B, C')
     expected = set([("direct", 'B'), ("direct", 'C')])
     observed = DependencySolver._get_dependency_set(A)
     self.assertEqual(expected, observed)
示例#30
0
 def test_get_job_map_produces_map(self):
     A = make_job('A')
     B = make_job('B')
     expected = {'A': A, 'B': B}
     observed = DependencySolver._get_job_map([A, B])
     self.assertEqual(expected, observed)
示例#31
0
 def test_empty(self):
     observed = DependencySolver.resolve_dependencies([])
     expected = []
     self.assertEqual(expected, observed)
示例#32
0
 def test_get_dependency_set_direct_two(self):
     A = make_job('A', depends='B, C')
     expected = set([("direct", 'B'), ("direct", 'C')])
     observed = DependencySolver._get_dependency_set(A)
     self.assertEqual(expected, observed)
示例#33
0
 def test_get_job_map_produces_map(self):
     A = make_job('A')
     B = make_job('B')
     expected = {'A': A, 'B': B}
     observed = DependencySolver._get_job_map([A, B])
     self.assertEqual(expected, observed)
示例#34
0
 def test_empty(self):
     observed = DependencySolver.resolve_dependencies([])
     expected = []
     self.assertEqual(expected, observed)