def test_obsolete_observation_failure_or_found(self):
context = ExecutionContext()
observation = jc.Observation()
observation.add_error(ValueError('not the error'))
failure_verifier = TestObsoleteObservationFailureVerifier(
'Verify', 'NotFound')
comment = failure_verifier._error_not_found_comment(observation)
failure_result = jp.PredicateResult(valid=False, comment=comment)
# We've already established this result is what we expect
bad_observation_result = failure_verifier(context, observation)
success_pred_result = jp.PredicateResult(valid=True)
good_observation_result = _makeObservationVerifyResult(
valid=True,
good_results=[success_pred_result],
observation=observation)
success_verifier = FakeObservationVerifier(
'Found', dnf_verifier=[], result=good_observation_result)
builder = jc.ObservationVerifierBuilder(title='Observation Verifier')
builder.EXPECT(failure_verifier).OR(success_verifier)
verifier = builder.build()
expect = jc.ObservationVerifyResult(
valid=True, observation=observation,
bad_results=bad_observation_result.bad_results,
good_results=good_observation_result.good_results,
failed_constraints=[])
got = verifier(context, observation)
self.assertEqual(expect, got)
def test_result_observation_verifier_conjunction_failure_aborts_early(self):
context = ExecutionContext()
builder = jc.ObservationVerifierBuilder(title='Test')
verifiers = []
results = []
pred_results = [jp.PredicateResult(False, comment='Result %d' % i)
for i in range(3)]
for i in range(3):
result = _makeObservationVerifyResult(
valid=False, bad_results=[pred_results[i]])
fake_verifier = FakeObservationVerifier(
title=i, dnf_verifier=[], result=result)
verifiers.append(fake_verifier)
results.append(result)
builder.AND(fake_verifier)
# verify build can work multiple times
self.assertEqual(builder.build(), builder.build())
verifier = builder.build()
self.assertEqual([verifiers], verifier.dnf_verifiers)
expect = _makeObservationVerifyResult(
False, bad_results=[pred_results[0]])
global _called_verifiers
_called_verifiers = []
got = verifier(context, jc.Observation())
self.assertEqual(expect, got)
self.assertEqual(verifiers[:1], _called_verifiers)
def __call__(self, context, value):
if not isinstance(value, HttpError):
return jp.PredicateResult(
valid=False,
comment='{0} != HttpType'.format(value.__class__.__name__))
message = ['HttpError']
sep = 'with' # Separator between confirmation phrases.
if self.__http_code is not None:
if self.__http_code != value.resp.status:
return HttpErrorPredicateResult(
False, value,
comment='HTTP Status={0} != {1}'.format(value.resp.status,
self.__http_code))
message.append('{0} HTTP Status={1}'.format(sep, self.__http_code))
sep = 'and'
if self.__content_regex is not None:
if not re.search(self.__content_regex, value.content):
return HttpErrorPredicateResult(
False, value,
comment='Response content does not match "{0}"'.format(
self.__content_regex))
message.append('{0} content matches "{1}"'.format(
sep, self.__content_regex))
return HttpErrorPredicateResult(True, value, comment=' '.join(message))
def test_obsolete_observation_failure_not_ok(self):
error_text = 'the error'
context = ExecutionContext()
observation = jc.Observation()
error = ValueError('not the error')
observation.add_error(error)
failure_verifier = TestObsoleteObservationFailureVerifier(
'Test', error_text)
comment = failure_verifier._error_not_found_comment(observation)
failure_pred_result = jp.PredicateResult(valid=False, comment=comment)
expect_failure = jc.ObservationVerifyResult(
valid=False, observation=observation,
bad_results=[jp.ObjectResultMapAttempt(observation,
failure_pred_result)],
good_results=[], failed_constraints=[],
comment=comment)
self.assertEqual(expect_failure, failure_verifier(context, observation))
builder = jc.ObservationVerifierBuilder(title='Test Verifier')
builder.EXPECT(failure_verifier)
verifier = builder.build()
expect = jc.ObservationVerifyResult(
valid=False, observation=observation,
bad_results=expect_failure.bad_results,
good_results=[], failed_constraints=[])
got = verifier(context, observation)
self.assertEqual(expect, got)
def test_result_observation_verifier_disjunction_success_aborts_early(self):
context = ExecutionContext()
builder = jc.ObservationVerifierBuilder(title='Test')
verifiers = []
results = []
pred_results = [jp.PredicateResult(False, comment='Result %d' % i)
for i in range(2)]
for i in range(2):
result = _makeObservationVerifyResult(
valid=True, good_results=[pred_results[i]])
fake_verifier = FakeObservationVerifier(
title=i, dnf_verifier=[], result=result)
verifiers.append(fake_verifier)
results.append(result)
builder.OR(fake_verifier)
verifier = builder.build()
self.assertEqual([verifiers[0:1], verifiers[1:2]], verifier.dnf_verifiers)
expect = _makeObservationVerifyResult(True, good_results=[pred_results[0]])
global _called_verifiers
_called_verifiers = []
got = verifier(context, jc.Observation())
self.assertEqual(expect, got)
self.assertEqual(verifiers[:1], _called_verifiers)
def test_result_observation_verifier_disjunction_failure(self):
context = ExecutionContext()
observation = jc.Observation()
builder = jc.ObservationVerifierBuilder(title='Test')
verifiers = []
results = []
pred_results = [jp.PredicateResult(False, comment='Result %d' % i)
for i in range(2)]
for i in range(2):
result = _makeObservationVerifyResult(observation=observation,
valid=False, bad_results=[pred_results[i]])
fake_verifier = FakeObservationVerifier(
title=i, dnf_verifier=[], result=result)
verifiers.append(fake_verifier)
results.append(result)
builder.OR(fake_verifier)
verifier = builder.build()
self.assertEqual([verifiers[0:1], verifiers[1:2]], verifier.dnf_verifiers)
expect = _makeObservationVerifyResult(
False, observation=observation, bad_results=pred_results)
global _called_verifiers
_called_verifiers = []
got = verifier(context, observation)
self.assertEqual(expect, got)
self.assertEqual(verifiers, _called_verifiers)
def test_result_observation_verifier_conjunction_ok(self):
context = ExecutionContext()
builder = jc.ObservationVerifierBuilder(title='Test')
verifiers = []
pred_results = []
for i in range(3):
this_result = jp.PredicateResult(True, comment='Pred {0}'.format(i))
pred_results.append(this_result)
result = _makeObservationVerifyResult(
valid=True, good_results=[this_result])
fake_verifier = FakeObservationVerifier(
title=i, dnf_verifier=[], result=result)
verifiers.append(fake_verifier)
builder.AND(fake_verifier)
# verify build can work multiple times
self.assertEqual(builder.build(), builder.build())
verifier = builder.build()
self.assertEqual([verifiers], verifier.dnf_verifiers)
expect = _makeObservationVerifyResult(True, good_results=pred_results)
global _called_verifiers
_called_verifiers = []
got = verifier(context, jc.Observation())
self.assertEqual(expect, got)
self.assertEqual(verifiers, _called_verifiers)
def __call__(self, context, value):
"""Implements ValuePredicate interface."""
if not isinstance(value, CliAgentRunError):
return jp.JsonError('Expected program to fail, but it did not')
ok = value.match_regex(self.__error_regex)
return jp.PredicateResult(ok is not None)
def __do_test_observation_failure_verifier_with_error(self, klass):
context = ExecutionContext()
valid = klass == IOError
error = klass('Could not connect')
observation = jc.Observation()
observation.add_error(error)
verifier = TestIoErrorFailureVerifier('Test')
result = verifier(context, observation)
self.assertEqual(valid, result.valid)
if valid:
attempt = jp.ObjectResultMapAttempt(
observation, jc.ObservationFailedError([error], valid=valid))
self.assertFalse(result.bad_results)
self.assertEqual([attempt], result.good_results)
self.assertEqual('HAVE Could not connect', result.comment)
else:
attempt = jp.ObjectResultMapAttempt(
observation,
jp.PredicateResult(valid=False,
comment='Expected error was not found.'))
self.assertFalse(result.good_results)
self.assertEqual([attempt], result.bad_results)
self.assertEqual('Expected error was not found.', result.comment)
def __call__(self, context, value):
observation = value
valid = False
error = None
comment = None
for error in observation.errors:
comment = self._error_comment_or_none(error)
if comment != None:
valid = True
break
if valid:
result = ObservationFailedError(valid=True, failures=[error])
map_attempt = jp.ObjectResultMapAttempt(observation, result)
good_results = [map_attempt]
bad_results = []
else:
comment = self._error_not_found_comment(observation)
result = jp.PredicateResult(valid=False, comment=comment)
map_attempt = jp.ObjectResultMapAttempt(observation, result)
good_results = []
bad_results = [map_attempt]
return ov.ObservationVerifyResult(valid=valid,
observation=observation,
good_results=good_results,
bad_results=bad_results,
failed_constraints=[],
comment=comment)
def test_exception_types_differ(self):
ex_predicate = jp.ExceptionMatchesPredicate(ValueError,
regex='the error')
self.assertEqual(
jp.PredicateResult(False,
comment='Expected ValueError, got KeyError.'),
ex_predicate(ExecutionContext(), KeyError('the error')))
def test_observation(self):
observation = jc.Observation()
self.assertEqual([], observation.errors)
self.assertEqual([], observation.objects)
expect = jc.Observation()
self.assertEqual(expect, observation)
observation.add_error(jp.PredicateResult(False))
self.assertNotEqual(expect, observation)
expect.add_error(jp.PredicateResult(False))
self.assertEqual(expect, observation)
observation = jc.Observation()
expect = jc.Observation()
observation.add_object('A')
self.assertNotEqual(expect, observation)
self.assertEqual(['A'], observation.objects)
expect.add_object('A')
self.assertEqual(expect, observation)
observation.add_object('B')
self.assertNotEqual(expect, observation)
self.assertEqual(['A', 'B'], observation.objects)
expect = jc.Observation()
expect.add_all_objects(['A', 'B'])
self.assertEqual(expect, observation)
observation.add_object(['C'])
self.assertEqual(observation.objects, ['A', 'B', ['C']])
expect = jc.Observation()
expect.add_all_objects(['A', 'B', ['C']])
self.assertEqual(expect, observation)
expect = observation
observation = jc.Observation()
observation.extend(expect)
self.assertEqual(expect, observation)
def test_observation_failure_verifier_without_error(self):
observation = jc.Observation()
verifier = TestIoErrorFailureVerifier('Test')
result = verifier(observation)
self.assertFalse(result.valid) # Because has no error
self.assertFalse(result.good_results)
attempt_list = result.bad_results
self.assertEqual([
jp.ObjectResultMapAttempt(
observation,
jp.PredicateResult(valid=False,
comment='Observation had no errors.'))
], result.bad_results)
def _makeObservationVerifyResult(
valid, observation=None,
good_results=None, bad_results=None, failed_constraints=None):
default_result = jp.PredicateResult(valid=valid)
good_results = good_results or ([default_result] if valid else [])
bad_results = bad_results or ([] if valid else [default_result])
failed_constraints = failed_constraints or []
observation = observation or jc.Observation()
good_attempt_results = [jp.ObjectResultMapAttempt(observation, result)
for result in good_results]
bad_attempt_results = [jp.ObjectResultMapAttempt(observation, result)
for result in bad_results]
return jc.ObservationVerifyResult(
valid=valid, observation=observation,
good_results=good_attempt_results,
bad_results=bad_attempt_results,
failed_constraints=failed_constraints)
def test_contract_observation_failure(self):
observation = jc.Observation()
observation.add_error(
jp.PredicateResult(False, comment='Observer Failed'))
fake_observer = FakeObserver(observation)
eq_A = jp.STR_EQ('A')
verifier = jc.ValueObservationVerifier('Has A', constraints=[eq_A])
clause = jc.ContractClause('TestClause', fake_observer, verifier)
contract = jc.Contract()
contract.add_clause(clause)
expect_result = jc.contract.ContractVerifyResult(
False, [clause.verify()])
result = contract.verify()
self.assertEqual(expect_result, result)
self.assertFalse(result)
def test_contract_observation_failure(self):
context = ExecutionContext()
observation = jc.Observation()
observation.add_error(
jp.PredicateResult(False, comment='Observer Failed'))
fake_observer = FakeObserver(observation)
eq_A = jp.LIST_MATCHES([jp.STR_EQ('A')])
verifier = jc.ValueObservationVerifierBuilder('Has A').EXPECT(
eq_A).build()
clause = jc.ContractClause('TestClause', fake_observer, verifier)
contract = jc.Contract()
contract.add_clause(clause)
expect_result = jc.contract.ContractVerifyResult(
False, [clause.verify(context)])
result = contract.verify(context)
self.assertEqual(expect_result, result)
self.assertFalse(result)
def __call__(self, context, value):
if not isinstance(value, HttpAgentError):
return jp.PredicateResult(
valid=False,
comment='{0} != HttpAgentError'.format(value.__class__.__name__))
return self.__pred(context, value.http_result)
def test_exception_messages_differ(self):
ex_predicate = jp.ExceptionMatchesPredicate(ValueError,
regex='the error')
self.assertEqual(jp.PredicateResult(False, comment='Errors differ.'),
ex_predicate(ExecutionContext(), ValueError()))
def test_exception_matches(self):
ex_predicate = jp.ExceptionMatchesPredicate(ValueError,
regex='the error')
self.assertEqual(
jp.PredicateResult(True, comment='Error matches.'),
ex_predicate(ExecutionContext(), ValueError('the error')))
