def test_observation_failure_ok(self): error_text = 'the error' context = ExecutionContext() observation = jc.Observation() error = ValueError(error_text) observation.add_error(error) ex_pred = jp.ExceptionMatchesPredicate(ValueError, error_text) ex_result = ex_pred(context, error) ex_observation_predicate_result = jc.ObservationPredicateResult( True, observation, jp.LIST_MATCHES([ex_pred]), jp.LIST_MATCHES([ex_pred])(context, [error])) expect_failure = jc.ObservationVerifyResult( valid=True, observation=observation, good_results=[ex_observation_predicate_result], bad_results=[], failed_constraints=[]) builder = jc.ValueObservationVerifierBuilder(title='Test For Error') builder.EXPECT(jc.ObservationErrorPredicate(jp.LIST_MATCHES([ex_pred ]))) verifier = builder.build() self.assertEqual(expect_failure, verifier(context, observation))
def test_result_observation_verifier_disjunction_failure(self): builder = jc.ObservationVerifierBuilder(title='Test') verifiers = [] results = [] for i in range(2): result = _makeObservationVerifyResult(valid=False) fake_verifier = FakeObservationVerifier(title=i, dnf_verifier=[], result=result) verifiers.append(fake_verifier) results.append(result) builder.append_verifier(fake_verifier, new_term=True) verifier = builder.build() self.assertEqual([verifiers[0:1], verifiers[1:2]], verifier.dnf_verifiers) expect = _makeObservationVerifyResult(False, bad_results=[results]) global _called_verifiers _called_verifiers = [] got = verifier(jc.Observation()) self.assertEqual(expect, got) self.assertEqual(verifiers, _called_verifiers)
def test_result_observation_verifier_disjunction_success_aborts_early(self): context = ExecutionContext() builder = jc.ObservationVerifierBuilder(title='Test') verifiers = [] results = [] pred_results = [jp.PredicateResult(False, comment='Result %d' % i) for i in range(2)] for i in range(2): result = _makeObservationVerifyResult( valid=True, good_results=[pred_results[i]]) fake_verifier = FakeObservationVerifier( title=i, dnf_verifier=[], result=result) verifiers.append(fake_verifier) results.append(result) builder.OR(fake_verifier) verifier = builder.build() self.assertEqual([verifiers[0:1], verifiers[1:2]], verifier.dnf_verifiers) expect = _makeObservationVerifyResult(True, good_results=[pred_results[0]]) global _called_verifiers _called_verifiers = [] got = verifier(context, jc.Observation()) self.assertEqual(expect, got) self.assertEqual(verifiers[:1], _called_verifiers)
def test_result_observation_verifier_conjunction_ok(self): context = ExecutionContext() builder = jc.ObservationVerifierBuilder(title='Test') verifiers = [] pred_results = [] for i in range(3): this_result = jp.PredicateResult(True, comment='Pred {0}'.format(i)) pred_results.append(this_result) result = _makeObservationVerifyResult( valid=True, good_results=[this_result]) fake_verifier = FakeObservationVerifier( title=i, dnf_verifier=[], result=result) verifiers.append(fake_verifier) builder.AND(fake_verifier) # verify build can work multiple times self.assertEqual(builder.build(), builder.build()) verifier = builder.build() self.assertEqual([verifiers], verifier.dnf_verifiers) expect = _makeObservationVerifyResult(True, good_results=pred_results) global _called_verifiers _called_verifiers = [] got = verifier(context, jc.Observation()) self.assertEqual(expect, got) self.assertEqual(verifiers, _called_verifiers)
def test_result_observation_verifier_conjunction_failure_aborts_early(self): context = ExecutionContext() builder = jc.ObservationVerifierBuilder(title='Test') verifiers = [] results = [] pred_results = [jp.PredicateResult(False, comment='Result %d' % i) for i in range(3)] for i in range(3): result = _makeObservationVerifyResult( valid=False, bad_results=[pred_results[i]]) fake_verifier = FakeObservationVerifier( title=i, dnf_verifier=[], result=result) verifiers.append(fake_verifier) results.append(result) builder.AND(fake_verifier) # verify build can work multiple times self.assertEqual(builder.build(), builder.build()) verifier = builder.build() self.assertEqual([verifiers], verifier.dnf_verifiers) expect = _makeObservationVerifyResult( False, bad_results=[pred_results[0]]) global _called_verifiers _called_verifiers = [] got = verifier(context, jc.Observation()) self.assertEqual(expect, got) self.assertEqual(verifiers[:1], _called_verifiers)
def test_result_observation_verifier_conjunction_failure_aborts_early( self): builder = jc.ObservationVerifierBuilder(title='Test') verifiers = [] results = [] for i in range(3): result = _makeObservationVerifyResult(valid=False) fake_verifier = FakeObservationVerifier(title=i, dnf_verifier=[], result=result) verifiers.append(fake_verifier) results.append(result) builder.append_verifier(fake_verifier) # verify build can work multiple times self.assertEqual(builder.build(), builder.build()) verifier = builder.build() self.assertEqual([verifiers], verifier.dnf_verifiers) expect = _makeObservationVerifyResult(False, bad_results=[results[0]]) global _called_verifiers _called_verifiers = [] got = verifier(jc.Observation()) self.assertEqual(expect, got) self.assertEqual(verifiers[:1], _called_verifiers)
def test_object_observation_verifier_multiple_constraint_found(self): pred_list = [ jp.PathPredicate('a', jp.STR_EQ('A')), jp.PathPredicate('b', jp.STR_EQ('B')) ] # This is our object verifier for these tests. verifier = jc.ValueObservationVerifier(title='Find Both', constraints=pred_list) test_cases = [('dict', _LETTER_DICT), ('array', _DICT_ARRAY), ('multi', _MULTI_ARRAY)] for test in test_cases: observation = jc.Observation() builder = jc.ObservationVerifyResultBuilder(observation) obj = test[1] if isinstance(test, list): observation.add_all_objects(obj) else: observation.add_object(obj) for pred in pred_list: builder.add_path_predicate_result(pred(observation.objects)) # All of these tests succeed. verify_results = builder.build(True) try: self._try_verify(verifier, observation, True, verify_results) except: print 'testing {0}'.format(test[0]) raise
def test_object_observation_verifier_one_constraint_not_found(self): context = ExecutionContext() pred_list = [jp.PathPredicate('a', jp.STR_EQ('NOT_FOUND'))] # This is our object verifier for these tests. verifier = (jc.ValueObservationVerifierBuilder( 'Cannot find one').contains_match(pred_list).build()) test_cases = [('array', _DICT_ARRAY), ('dict', _LETTER_DICT), ('array', _DICT_ARRAY), ('multi', _MULTI_ARRAY)] for test in test_cases: observation = jc.Observation() builder = jc.ObservationVerifyResultBuilder(observation) obj = test[1] if isinstance(test, list): observation.add_all_objects(obj) else: observation.add_object(obj) for pred in pred_list: builder.add_path_predicate_result( pred(context, observation.objects)) # None of these tests succeed. verify_results = builder.build(False) self.assertEqual(pred_list, verify_results.failed_constraints) try: self._try_verify(context, verifier, observation, False, verify_results) except: print 'testing {0}'.format(test[0]) raise
def mixed_exclude_helper(self, strict): context = ExecutionContext() observation = jc.Observation() observation.add_object('A') observation.add_object('B') observation.add_object('C') fake_observer = FakeObserver(observation) # We dont expect to see B in the list. # This can be interpreted two ways -- strictly or not strictly. # Strictly means no results should ever contain B. # Non strict means some result should not contain B. builder = jc.ValueObservationVerifierBuilder('Test Excludes', strict=strict) builder.excludes_path_value(None, 'B') clause = jc.ContractClause('TestClause', fake_observer, builder.build()) contract = jc.Contract() contract.add_clause(clause) # Doesnt matter whether strict or not since this is checking cardinality # over the entire list via the excludes clause. expect_result = jc.contract.ContractVerifyResult( False, [clause.verify(context)]) result = contract.verify(context) self.assertEqual(expect_result, result) self.assertEqual(False, result.valid)
def test_result_observation_verifier_disjunction_success_aborts_early( self): context = ExecutionContext() builder = jc.ObservationVerifierBuilder(title='Test') verifiers = [] results = [] for i in range(2): result = _makeObservationVerifyResult(valid=True) fake_verifier = FakeObservationVerifier(title=i, dnf_verifier=[], result=result) verifiers.append(fake_verifier) results.append(result) builder.append_verifier(fake_verifier, new_term=True) verifier = builder.build() self.assertEqual([verifiers[0:1], verifiers[1:2]], verifier.dnf_verifiers) expect = _makeObservationVerifyResult(True, bad_results=[results[0]]) global _called_verifiers _called_verifiers = [] got = verifier(context, jc.Observation()) self.assertEqual(expect, got) self.assertEqual(verifiers[:1], _called_verifiers)
def test_result_observation_verifier_conjunction_ok(self): context = ExecutionContext() builder = jc.ObservationVerifierBuilder(title='Test') verifiers = [] results = [] for i in range(3): result = _makeObservationVerifyResult(valid=True) fake_verifier = FakeObservationVerifier(title=i, dnf_verifier=[], result=result) verifiers.append(fake_verifier) results.append(result) builder.append_verifier(fake_verifier) # verify build can work multiple times self.assertEqual(builder.build(), builder.build()) verifier = builder.build() self.assertEqual([verifiers], verifier.dnf_verifiers) expect = _makeObservationVerifyResult(True, good_results=results) global _called_verifiers _called_verifiers = [] got = verifier(context, jc.Observation()) self.assertEqual(expect, got) self.assertEqual(verifiers, _called_verifiers)
def __do_test_observation_failure_verifier_with_error(self, klass): context = ExecutionContext() valid = klass == IOError error = klass('Could not connect') observation = jc.Observation() observation.add_error(error) verifier = TestIoErrorFailureVerifier('Test') result = verifier(context, observation) self.assertEqual(valid, result.valid) if valid: attempt = jp.ObjectResultMapAttempt( observation, jc.ObservationFailedError([error], valid=valid)) self.assertFalse(result.bad_results) self.assertEqual([attempt], result.good_results) self.assertEqual('HAVE Could not connect', result.comment) else: attempt = jp.ObjectResultMapAttempt( observation, jp.PredicateResult(valid=False, comment='Expected error was not found.')) self.assertFalse(result.good_results) self.assertEqual([attempt], result.bad_results) self.assertEqual('Expected error was not found.', result.comment)
def test_result_observation_verifier_disjunction_failure(self): context = ExecutionContext() observation = jc.Observation() builder = jc.ObservationVerifierBuilder(title='Test') verifiers = [] results = [] pred_results = [jp.PredicateResult(False, comment='Result %d' % i) for i in range(2)] for i in range(2): result = _makeObservationVerifyResult(observation=observation, valid=False, bad_results=[pred_results[i]]) fake_verifier = FakeObservationVerifier( title=i, dnf_verifier=[], result=result) verifiers.append(fake_verifier) results.append(result) builder.OR(fake_verifier) verifier = builder.build() self.assertEqual([verifiers[0:1], verifiers[1:2]], verifier.dnf_verifiers) expect = _makeObservationVerifyResult( False, observation=observation, bad_results=pred_results) global _called_verifiers _called_verifiers = [] got = verifier(context, observation) self.assertEqual(expect, got) self.assertEqual(verifiers, _called_verifiers)
def test_observation_failure_or_found(self): context = ExecutionContext() observation = jc.Observation() observation.add_object(_LETTER_DICT) failure_predicate = jc.ObservationErrorPredicate( jp.ExceptionMatchesPredicate(ValueError, regex='an error')) failure_result = failure_predicate(context, observation) self.assertFalse(failure_result) good_predicate = jc.ObservationValuePredicate( jp.PathPredicate('a', jp.STR_EQ('A'))) builder = jc.ObservationVerifierBuilder('TestAddConstraint') verifier = ( builder.EXPECT(failure_predicate).OR(good_predicate).build()) expect = jc.ObservationVerifyResult( valid=True, observation=observation, bad_results=[failure_result], good_results=[good_predicate(context, observation)], failed_constraints=[]) got = verifier(context, observation) self.assertEqual(expect, got)
def test_obsolete_observation_failure_ok(self): error_text = 'the error' context = ExecutionContext() observation = jc.Observation() error = ValueError(error_text) observation.add_error(error) failure_verifier = TestObsoleteObservationFailureVerifier( 'Test', error_text) failure_pred_result = jc.ObservationFailedError([error], valid=True) expect_failure = jc.ObservationVerifyResult( valid=True, observation=observation, good_results=[jp.ObjectResultMapAttempt(observation, failure_pred_result)], bad_results=[], failed_constraints=[], comment=_TEST_FOUND_ERROR_COMMENT) got = failure_verifier(context, observation) self.assertEqual(expect_failure, got) builder = jc.ObservationVerifierBuilder(title='Test') builder.EXPECT(failure_verifier) verifier = builder.build() expect = jc.ObservationVerifyResult( valid=True, observation=observation, good_results=expect_failure.good_results, bad_results=[], failed_constraints=[]) got = verifier(context, observation) self.assertEqual(expect, got)
def test_obsolete_observation_failure_not_ok(self): error_text = 'the error' context = ExecutionContext() observation = jc.Observation() error = ValueError('not the error') observation.add_error(error) failure_verifier = TestObsoleteObservationFailureVerifier( 'Test', error_text) comment = failure_verifier._error_not_found_comment(observation) failure_pred_result = jp.PredicateResult(valid=False, comment=comment) expect_failure = jc.ObservationVerifyResult( valid=False, observation=observation, bad_results=[jp.ObjectResultMapAttempt(observation, failure_pred_result)], good_results=[], failed_constraints=[], comment=comment) self.assertEqual(expect_failure, failure_verifier(context, observation)) builder = jc.ObservationVerifierBuilder(title='Test Verifier') builder.EXPECT(failure_verifier) verifier = builder.build() expect = jc.ObservationVerifyResult( valid=False, observation=observation, bad_results=expect_failure.bad_results, good_results=[], failed_constraints=[]) got = verifier(context, observation) self.assertEqual(expect, got)
def test_obsolete_observation_failure_or_found(self): context = ExecutionContext() observation = jc.Observation() observation.add_error(ValueError('not the error')) failure_verifier = TestObsoleteObservationFailureVerifier( 'Verify', 'NotFound') comment = failure_verifier._error_not_found_comment(observation) failure_result = jp.PredicateResult(valid=False, comment=comment) # We've already established this result is what we expect bad_observation_result = failure_verifier(context, observation) success_pred_result = jp.PredicateResult(valid=True) good_observation_result = _makeObservationVerifyResult( valid=True, good_results=[success_pred_result], observation=observation) success_verifier = FakeObservationVerifier( 'Found', dnf_verifier=[], result=good_observation_result) builder = jc.ObservationVerifierBuilder(title='Observation Verifier') builder.EXPECT(failure_verifier).OR(success_verifier) verifier = builder.build() expect = jc.ObservationVerifyResult( valid=True, observation=observation, bad_results=bad_observation_result.bad_results, good_results=good_observation_result.good_results, failed_constraints=[]) got = verifier(context, observation) self.assertEqual(expect, got)
def test_assertVerifyResults_ok(self): observation = jc.Observation() verify_results = jc.ObservationVerifyResult(valid=True, observation=observation, good_results=[], bad_results=[], failed_constraints=[]) self.assertVerifyResults(verify_results)
def test_observation_strict_vs_nonstrict(self): aA = jp.PathEqPredicate('a', 'A') bB = jp.PathEqPredicate('b', 'B') unstrict_object_list = [_NUMBER_DICT, _LETTER_DICT, _MIXED_DICT] unstrict_observation = jc.Observation() unstrict_observation.add_all_objects(unstrict_object_list) strict_object_list = [_LETTER_DICT, {'a': 'A', 'b': 'B', 'x': 'X'}] strict_observation = jc.Observation() strict_observation.add_all_objects(strict_object_list) none_object_list = [_NUMBER_DICT, _MIXED_DICT] none_observation = jc.Observation() none_observation.add_all_objects(none_object_list) # pylint: disable=bad-whitespace test_cases = [ # Name jc.Observation Strict, Unstrict #--------------------------------------------------- ('Strict', strict_observation, True, True), ('Unstrict', unstrict_observation, False, True), ('None', none_observation, False, False) ] # For each of the cases, test it with strict and non-strict verification. context = ExecutionContext() for test in test_cases: name = test[0] observation = test[1] # For this case, check it strict (2) and unstrict (3). for index in [2, 3]: test_strict = index == 2 expected = test[index] verifier = jc.ValueObservationVerifier(title='Verifier', constraints=[aA, bB], strict=test_strict) verify_result = verifier(context, observation) try: self.assertEqual(expected, verify_result.__nonzero__()) except Exception as e: print '*** FAILED case={0}:\n{1}'.format(name, e) print 'GOT {0}'.format(verify_result) raise
def test_assertVerifyResults_failed(self): observation = jc.Observation() verify_results = jc.ObservationVerifyResult(valid=False, observation=observation, good_results=[], bad_results=[], failed_constraints=[]) self.assertRaises(AssertionError, self.assertVerifyResults, verify_results)
def test_object_observation_verifier_with_conditional(self): # We need strict True here because we want each object to pass # the constraint test. Otherwise, if any object passes, then the whole # observation would pass. This causes a problem when we say that # we dont ever want to see 'name' unless it has a particular 'value'. # Without strict test, we'd allow this to occur as long as another object # satisfied that constraint. # When we use 'excludes', it applies to the whole observation since this # is normally the intent. However, here we are excluding values under # certain context -- "If the 'name' field is 'NAME' then it must contain # a value field 'VALUE'". Excluding name='NAME' everywhere would # not permit the context where value='VALUE' which we want to permit. verifier_builder = jc.ValueObservationVerifierBuilder( title='Test Conditional', strict=True) name_eq_pred = jp.PathEqPredicate('name', 'NAME') value_eq_pred = jp.PathEqPredicate('value', 'VALUE') conditional = jp.IF(name_eq_pred, value_eq_pred) pred_list = [jp.PathPredicate('', conditional)] verifier_builder.add_constraint(conditional) match_name_value_obj = {'name': 'NAME', 'value': 'VALUE'} match_value_not_name_obj = {'name': 'GOOD', 'value': 'VALUE'} match_neither_obj = {'name': 'GOOD', 'value': 'GOOD'} match_name_not_value_obj = {'name': 'NAME', 'value': 'BAD'} # bad test_cases = [(True, [match_name_value_obj, match_neither_obj]), (True, [match_value_not_name_obj, match_neither_obj]), (False, [match_neither_obj, match_name_not_value_obj])] context = ExecutionContext() verifier = verifier_builder.build() for test in test_cases: observation = jc.Observation() result_builder = jc.ObservationVerifyResultBuilder(observation) expect_valid = test[0] obj_list = test[1] observation.add_all_objects(obj_list) result_builder.add_observation_predicate_result( jc.ObservationValuePredicate(conditional)(context, observation)) # All of these tests succeed. verify_results = result_builder.build(expect_valid) try: self._try_verify(context, verifier, observation, expect_valid, verify_results) except: print 'testing {0}'.format(obj_list) raise
def _makeObservationVerifyResult(valid, good_results=None, bad_results=None, failed_constraints=None): good_results = good_results or [] bad_results = bad_results or [] failed_constraints = failed_constraints or [] return jc.ObservationVerifyResult(valid=valid, observation=jc.Observation(), good_results=[], bad_results=[], failed_constraints=[])
def test_observation_failure_verifier_without_error(self): observation = jc.Observation() verifier = TestIoErrorFailureVerifier('Test') result = verifier(observation) self.assertFalse(result.valid) # Because has no error self.assertFalse(result.good_results) attempt_list = result.bad_results self.assertEqual([ jp.ObjectResultMapAttempt( observation, jp.PredicateResult(valid=False, comment='Observation had no errors.')) ], result.bad_results)
def test_clause_failure(self): observation = jc.Observation() observation.add_object('B') fake_observer = FakeObserver(observation) eq_A = jp.STR_EQ('A') verifier = jc.ValueObservationVerifier('Has A', constraints=[eq_A]) clause = jc.ContractClause('TestClause', fake_observer, verifier) expect_result = jc.contract.ContractClauseVerifyResult( False, clause, verifier(observation)) result = clause.verify() self.assertEqual(expect_result, result) self.assertFalse(result)
def test_clause_success(self): context = ExecutionContext() observation = jc.Observation() observation.add_object('A') fake_observer = FakeObserver(observation) eq_A = jp.STR_EQ('A') verifier = jc.ValueObservationVerifier('Has A', constraints=[eq_A]) clause = jc.ContractClause('TestClause', fake_observer, verifier) expect_result = jc.contract.ContractClauseVerifyResult( True, clause, verifier(context, observation)) result = clause.verify(context) self.assertEqual(expect_result, result) self.assertTrue(result)
def test_clause_failure(self): context = ExecutionContext() observation = jc.Observation() observation.add_object('B') fake_observer = FakeObserver(observation) eq_A = jp.LIST_MATCHES([jp.STR_EQ('A')]) verifier = jc.ValueObservationVerifierBuilder('Has A').EXPECT( eq_A).build() clause = jc.ContractClause('TestClause', fake_observer, verifier) expect_result = jc.contract.ContractClauseVerifyResult( False, clause, verifier(context, observation)) result = clause.verify(context) self.assertEqual(expect_result, result) self.assertFalse(result)
def test_contract_success(self): observation = jc.Observation() observation.add_object('A') fake_observer = FakeObserver(observation) eq_A = jp.STR_EQ('A') verifier = jc.ValueObservationVerifier('Has A', constraints=[eq_A]) clause = jc.ContractClause('TestClause', fake_observer, verifier) contract = jc.Contract() contract.add_clause(clause) expect_result = jc.contract.ContractVerifyResult( True, [clause.verify()]) result = contract.verify() self.assertEqual(expect_result, result) self.assertTrue(result)
def test_object_observer_map(self): # Test no filter. context = ExecutionContext() observer = jc.ObjectObserver() observation = jc.Observation() expected = jc.Observation() expected.add_object(_NUMBER_DICT) observer.filter_all_objects_to_observation(context, [_NUMBER_DICT], observation) self.assertEqual([_NUMBER_DICT], observation.objects) self.assertEqual(expected, observation) pred_list = [ jp.PathEqPredicate('a', 'A'), jp.PathEqPredicate('b', 'B') ] conjunction = jp.AND(pred_list) observer = jc.ObjectObserver(conjunction) observation = jc.Observation() expected = jc.Observation() expected.add_object(_LETTER_DICT) observer.filter_all_objects_to_observation(context, [_LETTER_DICT], observation) self.assertEqual([_LETTER_DICT], observation.objects) self.assertEqual(expected, observation) observation = jc.Observation() expected = jc.Observation() expected.add_object(_LETTER_DICT) expected.add_object(_LETTER_DICT) observer.filter_all_objects_to_observation( context, [_LETTER_DICT, _NUMBER_DICT, _LETTER_DICT], observation) self.assertEqual([_LETTER_DICT, _LETTER_DICT], observation.objects) observation = jc.Observation() expected = jc.Observation() observer.filter_all_objects_to_observation(context, [_NUMBER_DICT], observation) self.assertEqual([], observation.objects) # Note that filtering doesnt observe errors. self.assertEqual(expected, observation)
def test_result_builder_add_bad_result(self): observation = jc.Observation() observation.add_object('A') pred = jp.PathPredicate(None, jp.STR_EQ('B')) builder = jc.ObservationVerifyResultBuilder(observation) map_pred = jp.MapPredicate(pred) map_result = map_pred(observation.objects) builder.add_map_result(map_result) verify_results = builder.build(False) self.assertFalse(verify_results) self.assertEqual(observation, verify_results.observation) self.assertEqual([], verify_results.good_results) self.assertEqual([pred], verify_results.failed_constraints) self.assertEqual(map_result.bad_object_result_mappings, verify_results.bad_results)
def _makeObservationVerifyResult( valid, observation=None, good_results=None, bad_results=None, failed_constraints=None): default_result = jp.PredicateResult(valid=valid) good_results = good_results or ([default_result] if valid else []) bad_results = bad_results or ([] if valid else [default_result]) failed_constraints = failed_constraints or [] observation = observation or jc.Observation() good_attempt_results = [jp.ObjectResultMapAttempt(observation, result) for result in good_results] bad_attempt_results = [jp.ObjectResultMapAttempt(observation, result) for result in bad_results] return jc.ObservationVerifyResult( valid=valid, observation=observation, good_results=good_attempt_results, bad_results=bad_attempt_results, failed_constraints=failed_constraints)