def test_obsolete_observation_failure_or_found(self): context = ExecutionContext() observation = jc.Observation() observation.add_error(ValueError('not the error')) failure_verifier = TestObsoleteObservationFailureVerifier( 'Verify', 'NotFound') comment = failure_verifier._error_not_found_comment(observation) failure_result = jp.PredicateResult(valid=False, comment=comment) # We've already established this result is what we expect bad_observation_result = failure_verifier(context, observation) success_pred_result = jp.PredicateResult(valid=True) good_observation_result = _makeObservationVerifyResult( valid=True, good_results=[success_pred_result], observation=observation) success_verifier = FakeObservationVerifier( 'Found', dnf_verifier=[], result=good_observation_result) builder = jc.ObservationVerifierBuilder(title='Observation Verifier') builder.EXPECT(failure_verifier).OR(success_verifier) verifier = builder.build() expect = jc.ObservationVerifyResult( valid=True, observation=observation, bad_results=bad_observation_result.bad_results, good_results=good_observation_result.good_results, failed_constraints=[]) got = verifier(context, observation) self.assertEqual(expect, got)
def test_result_observation_verifier_conjunction_failure_aborts_early(self): context = ExecutionContext() builder = jc.ObservationVerifierBuilder(title='Test') verifiers = [] results = [] pred_results = [jp.PredicateResult(False, comment='Result %d' % i) for i in range(3)] for i in range(3): result = _makeObservationVerifyResult( valid=False, bad_results=[pred_results[i]]) fake_verifier = FakeObservationVerifier( title=i, dnf_verifier=[], result=result) verifiers.append(fake_verifier) results.append(result) builder.AND(fake_verifier) # verify build can work multiple times self.assertEqual(builder.build(), builder.build()) verifier = builder.build() self.assertEqual([verifiers], verifier.dnf_verifiers) expect = _makeObservationVerifyResult( False, bad_results=[pred_results[0]]) global _called_verifiers _called_verifiers = [] got = verifier(context, jc.Observation()) self.assertEqual(expect, got) self.assertEqual(verifiers[:1], _called_verifiers)
def __call__(self, context, value): if not isinstance(value, HttpError): return jp.PredicateResult( valid=False, comment='{0} != HttpType'.format(value.__class__.__name__)) message = ['HttpError'] sep = 'with' # Separator between confirmation phrases. if self.__http_code is not None: if self.__http_code != value.resp.status: return HttpErrorPredicateResult( False, value, comment='HTTP Status={0} != {1}'.format(value.resp.status, self.__http_code)) message.append('{0} HTTP Status={1}'.format(sep, self.__http_code)) sep = 'and' if self.__content_regex is not None: if not re.search(self.__content_regex, value.content): return HttpErrorPredicateResult( False, value, comment='Response content does not match "{0}"'.format( self.__content_regex)) message.append('{0} content matches "{1}"'.format( sep, self.__content_regex)) return HttpErrorPredicateResult(True, value, comment=' '.join(message))
def test_obsolete_observation_failure_not_ok(self): error_text = 'the error' context = ExecutionContext() observation = jc.Observation() error = ValueError('not the error') observation.add_error(error) failure_verifier = TestObsoleteObservationFailureVerifier( 'Test', error_text) comment = failure_verifier._error_not_found_comment(observation) failure_pred_result = jp.PredicateResult(valid=False, comment=comment) expect_failure = jc.ObservationVerifyResult( valid=False, observation=observation, bad_results=[jp.ObjectResultMapAttempt(observation, failure_pred_result)], good_results=[], failed_constraints=[], comment=comment) self.assertEqual(expect_failure, failure_verifier(context, observation)) builder = jc.ObservationVerifierBuilder(title='Test Verifier') builder.EXPECT(failure_verifier) verifier = builder.build() expect = jc.ObservationVerifyResult( valid=False, observation=observation, bad_results=expect_failure.bad_results, good_results=[], failed_constraints=[]) got = verifier(context, observation) self.assertEqual(expect, got)
def test_result_observation_verifier_disjunction_success_aborts_early(self): context = ExecutionContext() builder = jc.ObservationVerifierBuilder(title='Test') verifiers = [] results = [] pred_results = [jp.PredicateResult(False, comment='Result %d' % i) for i in range(2)] for i in range(2): result = _makeObservationVerifyResult( valid=True, good_results=[pred_results[i]]) fake_verifier = FakeObservationVerifier( title=i, dnf_verifier=[], result=result) verifiers.append(fake_verifier) results.append(result) builder.OR(fake_verifier) verifier = builder.build() self.assertEqual([verifiers[0:1], verifiers[1:2]], verifier.dnf_verifiers) expect = _makeObservationVerifyResult(True, good_results=[pred_results[0]]) global _called_verifiers _called_verifiers = [] got = verifier(context, jc.Observation()) self.assertEqual(expect, got) self.assertEqual(verifiers[:1], _called_verifiers)
def test_result_observation_verifier_disjunction_failure(self): context = ExecutionContext() observation = jc.Observation() builder = jc.ObservationVerifierBuilder(title='Test') verifiers = [] results = [] pred_results = [jp.PredicateResult(False, comment='Result %d' % i) for i in range(2)] for i in range(2): result = _makeObservationVerifyResult(observation=observation, valid=False, bad_results=[pred_results[i]]) fake_verifier = FakeObservationVerifier( title=i, dnf_verifier=[], result=result) verifiers.append(fake_verifier) results.append(result) builder.OR(fake_verifier) verifier = builder.build() self.assertEqual([verifiers[0:1], verifiers[1:2]], verifier.dnf_verifiers) expect = _makeObservationVerifyResult( False, observation=observation, bad_results=pred_results) global _called_verifiers _called_verifiers = [] got = verifier(context, observation) self.assertEqual(expect, got) self.assertEqual(verifiers, _called_verifiers)
def test_result_observation_verifier_conjunction_ok(self): context = ExecutionContext() builder = jc.ObservationVerifierBuilder(title='Test') verifiers = [] pred_results = [] for i in range(3): this_result = jp.PredicateResult(True, comment='Pred {0}'.format(i)) pred_results.append(this_result) result = _makeObservationVerifyResult( valid=True, good_results=[this_result]) fake_verifier = FakeObservationVerifier( title=i, dnf_verifier=[], result=result) verifiers.append(fake_verifier) builder.AND(fake_verifier) # verify build can work multiple times self.assertEqual(builder.build(), builder.build()) verifier = builder.build() self.assertEqual([verifiers], verifier.dnf_verifiers) expect = _makeObservationVerifyResult(True, good_results=pred_results) global _called_verifiers _called_verifiers = [] got = verifier(context, jc.Observation()) self.assertEqual(expect, got) self.assertEqual(verifiers, _called_verifiers)
def __call__(self, context, value): """Implements ValuePredicate interface.""" if not isinstance(value, CliAgentRunError): return jp.JsonError('Expected program to fail, but it did not') ok = value.match_regex(self.__error_regex) return jp.PredicateResult(ok is not None)
def __do_test_observation_failure_verifier_with_error(self, klass): context = ExecutionContext() valid = klass == IOError error = klass('Could not connect') observation = jc.Observation() observation.add_error(error) verifier = TestIoErrorFailureVerifier('Test') result = verifier(context, observation) self.assertEqual(valid, result.valid) if valid: attempt = jp.ObjectResultMapAttempt( observation, jc.ObservationFailedError([error], valid=valid)) self.assertFalse(result.bad_results) self.assertEqual([attempt], result.good_results) self.assertEqual('HAVE Could not connect', result.comment) else: attempt = jp.ObjectResultMapAttempt( observation, jp.PredicateResult(valid=False, comment='Expected error was not found.')) self.assertFalse(result.good_results) self.assertEqual([attempt], result.bad_results) self.assertEqual('Expected error was not found.', result.comment)
def __call__(self, context, value): observation = value valid = False error = None comment = None for error in observation.errors: comment = self._error_comment_or_none(error) if comment != None: valid = True break if valid: result = ObservationFailedError(valid=True, failures=[error]) map_attempt = jp.ObjectResultMapAttempt(observation, result) good_results = [map_attempt] bad_results = [] else: comment = self._error_not_found_comment(observation) result = jp.PredicateResult(valid=False, comment=comment) map_attempt = jp.ObjectResultMapAttempt(observation, result) good_results = [] bad_results = [map_attempt] return ov.ObservationVerifyResult(valid=valid, observation=observation, good_results=good_results, bad_results=bad_results, failed_constraints=[], comment=comment)
def test_exception_types_differ(self): ex_predicate = jp.ExceptionMatchesPredicate(ValueError, regex='the error') self.assertEqual( jp.PredicateResult(False, comment='Expected ValueError, got KeyError.'), ex_predicate(ExecutionContext(), KeyError('the error')))
def test_observation(self): observation = jc.Observation() self.assertEqual([], observation.errors) self.assertEqual([], observation.objects) expect = jc.Observation() self.assertEqual(expect, observation) observation.add_error(jp.PredicateResult(False)) self.assertNotEqual(expect, observation) expect.add_error(jp.PredicateResult(False)) self.assertEqual(expect, observation) observation = jc.Observation() expect = jc.Observation() observation.add_object('A') self.assertNotEqual(expect, observation) self.assertEqual(['A'], observation.objects) expect.add_object('A') self.assertEqual(expect, observation) observation.add_object('B') self.assertNotEqual(expect, observation) self.assertEqual(['A', 'B'], observation.objects) expect = jc.Observation() expect.add_all_objects(['A', 'B']) self.assertEqual(expect, observation) observation.add_object(['C']) self.assertEqual(observation.objects, ['A', 'B', ['C']]) expect = jc.Observation() expect.add_all_objects(['A', 'B', ['C']]) self.assertEqual(expect, observation) expect = observation observation = jc.Observation() observation.extend(expect) self.assertEqual(expect, observation)
def test_observation_failure_verifier_without_error(self): observation = jc.Observation() verifier = TestIoErrorFailureVerifier('Test') result = verifier(observation) self.assertFalse(result.valid) # Because has no error self.assertFalse(result.good_results) attempt_list = result.bad_results self.assertEqual([ jp.ObjectResultMapAttempt( observation, jp.PredicateResult(valid=False, comment='Observation had no errors.')) ], result.bad_results)
def _makeObservationVerifyResult( valid, observation=None, good_results=None, bad_results=None, failed_constraints=None): default_result = jp.PredicateResult(valid=valid) good_results = good_results or ([default_result] if valid else []) bad_results = bad_results or ([] if valid else [default_result]) failed_constraints = failed_constraints or [] observation = observation or jc.Observation() good_attempt_results = [jp.ObjectResultMapAttempt(observation, result) for result in good_results] bad_attempt_results = [jp.ObjectResultMapAttempt(observation, result) for result in bad_results] return jc.ObservationVerifyResult( valid=valid, observation=observation, good_results=good_attempt_results, bad_results=bad_attempt_results, failed_constraints=failed_constraints)
def test_contract_observation_failure(self): observation = jc.Observation() observation.add_error( jp.PredicateResult(False, comment='Observer Failed')) fake_observer = FakeObserver(observation) eq_A = jp.STR_EQ('A') verifier = jc.ValueObservationVerifier('Has A', constraints=[eq_A]) clause = jc.ContractClause('TestClause', fake_observer, verifier) contract = jc.Contract() contract.add_clause(clause) expect_result = jc.contract.ContractVerifyResult( False, [clause.verify()]) result = contract.verify() self.assertEqual(expect_result, result) self.assertFalse(result)
def test_contract_observation_failure(self): context = ExecutionContext() observation = jc.Observation() observation.add_error( jp.PredicateResult(False, comment='Observer Failed')) fake_observer = FakeObserver(observation) eq_A = jp.LIST_MATCHES([jp.STR_EQ('A')]) verifier = jc.ValueObservationVerifierBuilder('Has A').EXPECT( eq_A).build() clause = jc.ContractClause('TestClause', fake_observer, verifier) contract = jc.Contract() contract.add_clause(clause) expect_result = jc.contract.ContractVerifyResult( False, [clause.verify(context)]) result = contract.verify(context) self.assertEqual(expect_result, result) self.assertFalse(result)
def __call__(self, context, value): if not isinstance(value, HttpAgentError): return jp.PredicateResult( valid=False, comment='{0} != HttpAgentError'.format(value.__class__.__name__)) return self.__pred(context, value.http_result)
def test_exception_messages_differ(self): ex_predicate = jp.ExceptionMatchesPredicate(ValueError, regex='the error') self.assertEqual(jp.PredicateResult(False, comment='Errors differ.'), ex_predicate(ExecutionContext(), ValueError()))
def test_exception_matches(self): ex_predicate = jp.ExceptionMatchesPredicate(ValueError, regex='the error') self.assertEqual( jp.PredicateResult(True, comment='Error matches.'), ex_predicate(ExecutionContext(), ValueError('the error')))