def run_test_case(self, test_case, context=None, **kwargs):
"""Run the specified test operation from start to finish.
Args:
test_case: [OperationContract] To test.
context: [ExecutionContext] The citest execution context to run in.
timeout_ok: [bool] Whether an AgentOperationStatus timeout implies
a test failure. If it is ok to timeout, then we'll still verify the
contracts, but skip the final status check if there is no final
status yet.
max_retries: [int] Number of independent retries permitted on
individual operations if the operation status fails. A value of 0
indicates that a test should only be given a single attempt.
retry_interval_secs: [int] The number of seconds to wait between retries.
max_wait_secs: [int] How long to wait for status completion.
Default=Determined by operation in the test case.
"""
if context is None:
context = ExecutionContext()
# This is complicated because of all the individual parts
# that we want to ensure excute, so we'll break it up into
# helper functions based on scope and use the context to
# pass back some shared state variables since they make sense
# to communicate in the context anyway.
#
# This particular method is responsible for the logging context
# and the post-execution cleanup, if any.
#
# It will delegate to a helper function for the execution and
# pre/post hooks
#
# To get the context relation, we'll peek inside the execution
# context to see how the status and validation turned out.
JournalLogger.begin_context('Test "{0}"'.format(test_case.title))
try:
self._do_run_test_case_with_hooks(test_case, context, **kwargs)
finally:
try:
if test_case.cleanup:
attempt_info = context.get(self.CONTEXT_KEY_ATTEMPT_INFO, None)
if attempt_info is None or attempt_info.status is None:
self.logger.info('Skipping operation cleanup because'
' operation could not be performed at all.')
else:
self.logger.info('Invoking injected operation cleanup.')
test_case.cleanup(context)
finally:
verify_results = context.get(
self.CONTEXT_KEY_CONTRACT_VERIFY_RESULTS, None)
if verify_results is None:
context_relation = 'ERROR'
else:
final_status_ok = context.get(self.CONTEXT_KEY_FINAL_STATUS_OK, False)
context_relation = ('VALID' if (final_status_ok and verify_results)
else 'INVALID')
JournalLogger.end_context(relation=context_relation)
def test_inspect_not_found_ok(self):
context = ExecutionContext()
# Return a 404 Not found
# The string we return just needs to end with " was not found",
# which is what gcloud currently returns (subject to change)
# and all we test for.
error_response = st.CliResponseType(-
1, '',
'ERROR: (gcloud.preview.managed-instance-groups.describe)'
' The thing you requested was not found')
gcloud = fake_gcloud_agent.FakeGCloudAgent(
'PROJECT', 'ZONE', default_response=error_response)
contract_builder = gt.GCloudContractBuilder(gcloud)
context.add_internal('test', 'arg2')
extra_args = ['arg1', lambda x: x['test'], 'arg3']
expect_extra_args = ['arg1', 'arg2', 'arg3']
c1 = contract_builder.new_clause_builder('TITLE')
verifier = c1.inspect_resource(
'instances', 'test_name', extra_args=extra_args, no_resource_ok=True)
verifier.contains_path_value('field', 'value')
self.assertTrue(isinstance(verifier, jc.ValueObservationVerifierBuilder))
contract = contract_builder.build()
verification_result = contract.verify(context)
self.assertTrue(verification_result,
JsonSnapshotHelper.ValueToEncodedJson(verification_result))
command = gcloud.build_gcloud_command_args(
'instances', ['describe', 'test_name'] + expect_extra_args,
project='PROJECT', zone='ZONE')
self.assertEquals(command, gcloud.last_run_params)
def test_path_conjunction(self):
context = ExecutionContext()
text = 'x=X, a=A, b=B, z=Z'
aA = jp.STR_SUBSTR('a=A')
bB = jp.STR_SUBSTR('b=B')
conjunction = jp.AND([aA, bB])
pred = PathPredicate('value', conjunction)
source = {'value': text, 'wrong': 'a=A', 'another': 'b=B'}
conjunction_result = conjunction(context, text)
builder = PathPredicateResultBuilder(source, pred)
builder.add_result_candidate(
PathValue('value', text),
conjunction_result.clone_with_source(source=source,
base_target_path='value',
base_value_path='value'))
expect = builder.build(True)
pred_result = pred(context, source)
self.assertEqual(expect, pred_result)
def test_contract_observation_failure(self):
context = ExecutionContext()
observation = jc.Observation()
observation.add_error(
jp.PredicateResult(False, comment='Observer Failed'))
fake_observer = FakeObserver(observation)
eq_A = jp.STR_EQ('A')
verifier = jc.ValueObservationVerifier('Has A', constraints=[eq_A])
clause = jc.ContractClause('TestClause', fake_observer, verifier)
contract = jc.Contract()
contract.add_clause(clause)
expect_result = jc.contract.ContractVerifyResult(
False, [clause.verify(context)])
result = contract.verify(context)
self.assertEqual(expect_result, result)
self.assertFalse(result)
def test_contract_mixed_object_failure_ok(self):
context = ExecutionContext()
observation = jc.Observation()
observation.add_object('A')
observation.add_object('B')
fake_observer = FakeObserver(observation)
eq_A = jp.STR_EQ('A')
verifier = jc.ValueObservationVerifier('Has A', constraints=[eq_A])
clause = jc.ContractClause('TestClause', fake_observer, verifier)
contract = jc.Contract()
contract.add_clause(clause)
expect_result = jc.contract.ContractVerifyResult(
True, [clause.verify(context)])
result = contract.verify(context)
self.assertEqual(expect_result, result)
self.assertTrue(result)
def test_dict_nested_subset_exact_values(self):
context = ExecutionContext()
small = {'first': 'Apple', 'second': 'Banana'}
small_nested = {'outer': small}
# In dictionary we want exact matches of strings, not substrings.
nested_operand = {'first': 'A'}
operand = {'outer': nested_operand}
subset_pred = jp.DICT_SUBSET(operand)
self.assertEqual(
jp.PathValueResult(valid=False,
pred=jp.STR_EQ('A'),
path_value=PathValue(
jp.PATH_SEP.join(['outer', 'first']),
'Apple'),
source=small_nested,
target_path=jp.PATH_SEP.join(['outer',
'first'])),
subset_pred(context, small_nested))
def test_dict_match_strict_bad(self):
context = ExecutionContext()
source = {'n': 10, 'extra': 'EXTRA'}
pred = jp.NUM_LE(20)
want = {'n': pred}
match_pred = jp.DICT_MATCHES(want, strict=True)
result = match_pred(context, source)
expect = (jp.KeyedPredicateResultBuilder(match_pred).add_result(
'n', self._match_dict_attribute_result(
context, pred, 'n', source)).add_result(
'extra',
jp.UnexpectedPathError(source=source,
target_path='extra',
path_value=jp.PathValue(
'extra',
'EXTRA'))).build(False))
self.assertFalse(result)
self.assertEquals(expect, result)
def test_not_fail(self):
context = ExecutionContext()
aA = jp.PathEqPredicate('a', 'A')
not_aA = jc.NOT(aA)
expect = jc.SequencedPredicateResult(
valid=False, pred=not_aA, results=[aA(context, _LETTER_DICT)])
result = not_aA(context, _LETTER_DICT)
self.assertFalse(result)
self.assertEqual(expect, result)
bB = jp.PathEqPredicate('b', 'B')
bB_or_aA = jc.OR([bB, aA])
not_bB_or_aA = jc.NOT(bB_or_aA)
expect = jc.SequencedPredicateResult(
valid=False,
pred=not_bB_or_aA,
results=[bB_or_aA(context, _LETTER_DICT)])
result = not_bB_or_aA(context, _LETTER_DICT)
self.assertFalse(result)
self.assertEqual(expect, result)
def test_dict_match_multi_bad(self):
context = ExecutionContext()
source = {'a': 'testing', 'n': 10}
n_pred = jp.NUM_NE(10)
a_pred = jp.STR_SUBSTR('test')
want = {'n': n_pred, 'a': a_pred}
result = jp.DICT_MATCHES(want)(context, source)
expect = (jp.KeyedPredicateResultBuilder(
jp.DICT_MATCHES(want)).add_result(
'n',
self._match_dict_attribute_result(
context, n_pred, 'n', source)).add_result(
'a',
self._match_dict_attribute_result(
context, a_pred, 'a', source)).build(False))
self.assertFalse(result)
self.assertTrue(result.results['a'])
self.assertFalse(result.results['n'])
self.assertEquals(expect, result)
def test_multiple_required(self):
context = ExecutionContext()
observation = jc.Observation()
observation.add_object('A')
observation.add_object('B')
observation.add_object('C')
fake_observer = FakeObserver(observation)
eq_A_or_B = jp.OR([jp.STR_EQ('A'), jp.STR_EQ('B')])
builder = jc.ValueObservationVerifierBuilder('Test Multiple')
builder.contains_path_pred(None, eq_A_or_B, min=2)
clause = jc.ContractClause('TestClause', fake_observer, builder.build())
contract = jc.Contract()
contract.add_clause(clause)
expect_result = jc.contract.ContractVerifyResult(
True, [clause.verify(context)])
result = contract.verify(context)
self.assertEqual(expect_result, result)
self.assertEqual(True, result.valid)
def test_result_builder_add_mixed_results(self):
context = ExecutionContext()
observation = jc.Observation()
observation.add_object('GOOD')
observation.add_object('BAD')
pred = jp.PathPredicate(None, jp.STR_EQ('GOOD'))
builder = jc.ObservationVerifyResultBuilder(observation)
map_pred = jp.MapPredicate(pred)
map_result = map_pred(context, observation.objects)
builder.add_map_result(map_result)
verify_results = builder.build(False)
self.assertFalse(verify_results)
self.assertEqual(observation, verify_results.observation)
self.assertEqual(map_result.good_object_result_mappings,
verify_results.good_results)
self.assertEqual([], verify_results.failed_constraints)
self.assertEqual(map_result.bad_object_result_mappings,
verify_results.bad_results)
def test_not_success(self):
context = ExecutionContext()
a1 = jp.PathEqPredicate('a', '1')
not_a1 = jc.NOT(a1)
expect = jc.SequencedPredicateResult(
valid=True, pred=not_a1, results=[a1(context, _LETTER_DICT)])
result = not_a1(context, _LETTER_DICT)
self.assertTrue(result)
self.assertEqual(expect, result)
b2 = jp.PathEqPredicate('b', '2')
b2_or_a1 = jc.OR([b2, a1])
not_b2_or_a1 = jc.NOT(b2_or_a1)
expect = jc.SequencedPredicateResult(
valid=True,
pred=not_b2_or_a1,
results=[b2_or_a1(context, _LETTER_DICT)])
result = not_b2_or_a1(context, _LETTER_DICT)
self.assertTrue(result)
self.assertEqual(expect, result)
def test_object_observation_verifier_some_but_not_all_constraints_found(
self):
context = ExecutionContext()
pred_list = [
jp.PathPredicate('a', jp.STR_EQ('NOT_FOUND')),
jp.PathPredicate('b', jp.STR_EQ('B'))
]
# This is our object verifier for these tests.
verifier = (jc.ValueObservationVerifierBuilder(
'Find one of two').contains_match(pred_list).build())
test_cases = [('array', _DICT_ARRAY), ('dict', _LETTER_DICT),
('array', _DICT_ARRAY), ('multi', _MULTI_ARRAY)]
for test in test_cases:
observation = jc.Observation()
builder = jc.ObservationVerifyResultBuilder(observation)
obj = test[1]
if isinstance(test, list):
observation.add_all_objects(obj)
else:
observation.add_object(obj)
for pred in pred_list:
pred_result = jp.PathPredicate('', pred)(context,
observation.objects)
builder.add_path_predicate_result(
pred(context, observation.objects))
# None of these tests succeed.
verify_results = builder.build(False)
try:
self._try_verify(context, verifier, observation, False,
verify_results)
except:
print 'testing {0}'.format(test[0])
raise
def setUpClass(cls):
runner = citest.base.TestRunner.global_runner()
bindings = runner.bindings
cls.FIRST_CERT = 'first-cert-%s' % (bindings['TEST_APP'])
cls.SECOND_CERT = 'second-cert-%s' % (bindings['TEST_APP'])
runner = citest.base.TestRunner.global_runner()
bindings = runner.bindings
scenario = runner.get_shared_data(GoogleHttpLoadBalancerTestScenario)
managed_region = runner.bindings['TEST_GCE_REGION']
title = 'Check Quota for {0}'.format(scenario.__class__.__name__)
verify_results = gcp.verify_quota(
title,
scenario.gcp_observer,
project_quota=GoogleHttpLoadBalancerTestScenario.
MINIMUM_PROJECT_QUOTA,
regions=[(managed_region,
GoogleHttpLoadBalancerTestScenario.MINIMUM_REGION_QUOTA)
])
if not verify_results:
raise RuntimeError(
'Insufficient Quota: {0}'.format(verify_results))
# No predicates against this agent, context is empty.
context = ExecutionContext()
compute_agent = (gcp.GcpComputeAgent.make_agent(
scopes=SCOPES, credentials_path=bindings['GCE_CREDENTIALS_PATH']))
compute_agent.invoke_resource(
context,
'insert',
resource_type='sslCertificates',
project=bindings['GOOGLE_PRIMARY_MANAGED_PROJECT_ID'],
body=cls.make_ssl_cert(cls.FIRST_CERT))
compute_agent.invoke_resource(
context,
'insert',
resource_type='sslCertificates',
project=bindings['GOOGLE_PRIMARY_MANAGED_PROJECT_ID'],
body=cls.make_ssl_cert(cls.SECOND_CERT))
def test_collect_from_dict_identity(self):
context = ExecutionContext()
source = _LETTER_DICT
pred = PathPredicate('')
values = pred(context, source)
builder = PathPredicateResultBuilder(source, pred)
builder.add_result_candidate(
PathValue('', source),
PathValueResult(source=source,
target_path='',
path_value=PathValue('', source),
valid=True))
self.assertEqual(builder.build(True), values)
self.assertEqual([PathValue('', source)], values.path_values)
self.assertEqual([], values.path_failures)
pred = PathPredicate('/')
values = pred(context, source)
self.assertEqual([PathValue('', source)], values.path_values)
self.assertEqual([], values.path_failures)
def test_collect_enumerated_terminal_list(self):
# """Enumerated path to a value that is a list."""
context = ExecutionContext()
array = ['A', 'B', 'C']
source = {'a': array}
pred = PathPredicate('a' + PATH_SEP)
values = pred(context, source)
self.assertEqual([
PathValue('a[0]', 'A'),
PathValue('a[1]', 'B'),
PathValue('a[2]', 'C')
], values.path_values)
self.assertEqual([], values.path_failures)
pred = PathPredicate('a')
values = pred(context, source)
self.assertEqual([
PathValue('a[0]', 'A'),
PathValue('a[1]', 'B'),
PathValue('a[2]', 'C')
], values.path_values)
self.assertEqual([], values.path_failures)
def test_inspect_indirect(self):
context = ExecutionContext(test_id='TESTID', test_project='PROJECT')
default_variables = {'project': lambda x: x.get('test_project', 'UNKNOWN')}
service = MyFakeGcpService(['Hello, World'])
agent = TestGcpAgent.make_test_agent(
service=service, default_variables=default_variables)
contract_builder = gt.GcpContractBuilder(agent)
c1 = contract_builder.new_clause_builder('TITLE')
verifier = c1.inspect_resource(
'regions', resource_id=lambda x: x['test_id'],
no_resource_ok=True)
verifier.contains_path_value('', 'Hello, World')
self.assertTrue(isinstance(verifier, jc.ValueObservationVerifierBuilder))
contract = contract_builder.build()
verification_result = contract.verify(context)
self.assertTrue(verification_result,
JsonSnapshotHelper.ValueToEncodedJson(verification_result))
self.assertEquals({'project': 'PROJECT', 'region': 'TESTID'},
agent.service.last_get_args)
def test_dict_nested_subset_exact_keys(self):
context = ExecutionContext()
small_dict = {'first':'Apple', 'second':'Banana'}
small_nested_dict = {'outer':small_dict}
# In dictionary we want exact key matches, not subkeys.
operand = {'out':{'first':'Apple'}}
subset_pred = jp.DICT_SUBSET(operand)
self.assertEqual(
jp.MissingPathError(source=small_nested_dict, target_path='out',
path_value=PathValue('', small_nested_dict)),
subset_pred(context, small_nested_dict))
# In dictionary we want exact key matches, not subkeys.
nested_operand = {'fir':'Apple'}
operand = {'outer':nested_operand}
subset_pred = jp.DICT_SUBSET(operand)
self.assertEqual(
jp.MissingPathError(source=small_nested_dict,
target_path=jp.PATH_SEP.join(['outer', 'fir']),
path_value=PathValue('outer', small_dict)),
subset_pred(context, small_nested_dict))
def _do_init_bindings(self):
"""Hook to initialize custom test bindings so journaling is scoped."""
context = ExecutionContext()
config = self.agent.deployed_config
enabled = config.get('spinnaker.gcs.enabled', False)
if not enabled:
raise ValueError('spinnaker.gcs.enabled is not True')
self.BUCKET = config['spinnaker.gcs.bucket']
self.BASE_PATH = config['spinnaker.gcs.rootFolder']
self.TEST_APP = self.bindings['TEST_APP']
self.TEST_PIPELINE_NAME = 'My {app} Pipeline'.format(app=self.TEST_APP)
self.TEST_PIPELINE_ID = '{app}-pipeline-id'.format(app=self.TEST_APP)
self.gcs_observer = gcp.GcpStorageAgent.make_agent(
credentials_path=self.bindings['GCE_CREDENTIALS_PATH'],
scopes=gcp.gcp_storage_agent.STORAGE_FULL_SCOPE)
metadata = self.gcs_observer.inspect_bucket(context, self.BUCKET)
self.versioning_enabled = (metadata.get('versioning',
{}).get('enabled', False))
if not self.versioning_enabled:
self.logger.info('bucket=%s versioning enabled=%s', self.BUCKET,
self.versioning_enabled)
def collect_resource_usage(self, gcp_agent, scanner, apis=None):
"""Returns current GcpResourceUsage"""
def extract_quota(region_info):
return {info['metric']: info['usage'] for info in region_info['quotas']}
apis = apis or ['compute']
item_filter = lambda item: True
context = ExecutionContext()
project_quota = extract_quota(gcp_agent.invoke_resource(
context, 'get', 'projects'))
region_info = gcp_agent.invoke_resource(context, 'list', 'regions')
region_quota = {elem['name']: extract_quota(elem)
for elem in region_info['items']}
api_resource_list_map = {}
for api in apis:
resources, errs = scanner.list_api(api, item_filter=item_filter)
api_resource_list_map[api] = GcpApiResourceList(resources, errs)
return GcpResourceUsage(project_quota, region_quota, api_resource_list_map)
def test_collect_filter_good(self):
context = ExecutionContext()
source = {'outer': [_LETTER_DICT, _NUMBER_DICT]}
filter_pred = TestEqualsPredicate(2)
pred = PathPredicate(PATH_SEP.join(['outer', 'b']), pred=filter_pred)
# This is a precise test against the exact result returned.
builder = PathPredicateResultBuilder(source, pred)
path_0 = PATH_SEP.join(['outer[0]', 'b'])
path_1 = PATH_SEP.join(['outer[1]', 'b'])
bad_result = PathValueResult(source=source,
target_path=pred.path,
path_value=PathValue(path_0, 'B'),
valid=False,
pred=filter_pred)
good_result = PathValueResult(source=source,
target_path=pred.path,
path_value=PathValue(path_1, 2),
valid=True,
pred=filter_pred)
builder.add_result_candidate(PathValue(path_0, 'B'), bad_result)
builder.add_result_candidate(PathValue(path_1, 2), good_result)
expect = builder.build(True)
pred_result = pred(context, source)
self.assertEqual([PathValue(PATH_SEP.join(['outer[1]', 'b']), 2)],
pred_result.path_values)
self.assertEqual(expect, pred_result)
self.assertEqual([
jp.PathPredicateResultCandidate(PathValue(path_0, 'B'), bad_result)
], pred_result.invalid_candidates)
self.assertEqual([
jp.PathPredicateResultCandidate(PathValue(path_1, 2), good_result)
], pred_result.valid_candidates)
self.assertEqual([], pred_result.path_failures)
def test_gcloud_list(self):
context = ExecutionContext()
standard_params = ['--format', 'json', '--project', 'PROJECT']
gcloud = fake_gcloud_agent.FakeGCloudAgent('PROJECT', 'ZONE')
gcloud.list_resources(context, 'instances')
self.assertEqual(
gcloud.last_run_params,
['-q', 'compute'] + standard_params + ['instances', 'list'])
gcloud.list_resources(context, 'managed-instance-groups')
self.assertEqual(
gcloud.last_run_params,
['-q', 'compute'] + standard_params
+ ['instance-groups', 'managed', 'list', '--zone', 'ZONE'])
gcloud.list_resources(context, 'unmanaged-instance-groups')
self.assertEqual(
gcloud.last_run_params,
['-q', 'compute'] + standard_params
+ ['instance-groups', 'unmanaged', 'list', '--zone', 'ZONE'])
gcloud.describe_resource(context, 'instances', 'NAME')
self.assertEqual(gcloud.last_run_params,
['-q', 'compute'] + standard_params
+ ['instances', 'describe', 'NAME', '--zone', 'ZONE'])
gcloud.describe_resource(context, 'managed-instance-groups', 'NAME')
self.assertEqual(gcloud.last_run_params,
['-q', 'compute'] + standard_params
+ ['instance-groups', 'managed',
'describe', 'NAME', '--zone', 'ZONE'])
gcloud.describe_resource(context, 'unmanaged-instance-groups', 'NAME')
self.assertEqual(gcloud.last_run_params,
['-q', 'compute'] + standard_params
+ ['instance-groups', 'unmanaged',
'describe', 'NAME', '--zone', 'ZONE'])
def test_obsolete_observation_failure_ok(self):
error_text = 'the error'
context = ExecutionContext()
observation = jc.Observation()
error = ValueError(error_text)
observation.add_error(error)
failure_verifier = TestObsoleteObservationFailureVerifier(
'Test', error_text)
failure_pred_result = jc.ObservationFailedError([error], valid=True)
expect_failure = jc.ObservationVerifyResult(
valid=True,
observation=observation,
good_results=[
jp.ObjectResultMapAttempt(observation, failure_pred_result)
],
bad_results=[],
failed_constraints=[],
comment=_TEST_FOUND_ERROR_COMMENT)
got = failure_verifier(context, observation)
self.assertEqual(expect_failure, got)
builder = jc.ObservationVerifierBuilder(title='Test')
builder.EXPECT(failure_verifier)
verifier = builder.build()
expect = jc.ObservationVerifyResult(
valid=True,
observation=observation,
good_results=expect_failure.good_results,
bad_results=[],
failed_constraints=[])
got = verifier(context, observation)
self.assertEqual(expect, got)
def test_observation_failure_ok(self):
error_text = 'the error'
context = ExecutionContext()
observation = jc.Observation()
error = ValueError(error_text)
observation.add_error(error)
exception_pred = jp.ExceptionMatchesPredicate(
ValueError, regex=error_text)
builder = jc.ObservationVerifierBuilder(title='Test')
builder.EXPECT(jc.ObservationErrorPredicate(jp.LIST_MATCHES([exception_pred])))
failure_verifier = builder.build()
observation_predicate_result = jc.ObservationPredicateResult(
True, observation, jp.LIST_MATCHES([exception_pred]),
jp.LIST_MATCHES([exception_pred])(context, [error]))
expect_failure = jc.ObservationVerifyResult(
True, observation,
good_results=[observation_predicate_result],
bad_results=[], failed_constraints=[])
got = failure_verifier(context, observation)
self.assertEqual(expect_failure, got)
def test_obsolete_observation_failure_not_ok(self):
error_text = 'the error'
context = ExecutionContext()
observation = jc.Observation()
error = ValueError('not the error')
observation.add_error(error)
failure_verifier = TestObsoleteObservationFailureVerifier(
'Test', error_text)
comment = failure_verifier._error_not_found_comment(observation)
failure_pred_result = jp.PredicateResult(valid=False, comment=comment)
expect_failure = jc.ObservationVerifyResult(
valid=False,
observation=observation,
bad_results=[
jp.ObjectResultMapAttempt(observation, failure_pred_result)
],
good_results=[],
failed_constraints=[],
comment=comment)
self.assertEqual(expect_failure,
failure_verifier(context, observation))
builder = jc.ObservationVerifierBuilder(title='Test Verifier')
builder.EXPECT(failure_verifier)
verifier = builder.build()
expect = jc.ObservationVerifyResult(
valid=False,
observation=observation,
bad_results=expect_failure.bad_results,
good_results=[],
failed_constraints=[])
got = verifier(context, observation)
self.assertEqual(expect, got)
def test_object_observation_verifier_multiple_constraint_found(self):
context = ExecutionContext()
pred_list = [
jp.PathPredicate('a', jp.STR_EQ('A')),
jp.PathPredicate('b', jp.STR_EQ('B'))
]
# This is our object verifier for these tests.
verifier = (jc.ValueObservationVerifierBuilder('Find Both').EXPECT(
pred_list[0]).AND(pred_list[1]).build())
test_cases = [('dict', _LETTER_DICT), ('array', _DICT_ARRAY),
('multi', _MULTI_ARRAY)]
for test in test_cases:
observation = jc.Observation()
builder = jc.ObservationVerifyResultBuilder(observation)
obj = test[1]
if isinstance(test, list):
observation.add_all_objects(obj)
else:
observation.add_object(obj)
for pred in pred_list:
builder.add_observation_predicate_result(
jc.ObservationValuePredicate(pred)(context, observation))
# All of these tests succeed.
verify_results = builder.build(True)
self.assertEqual([], verify_results.failed_constraints)
try:
self._try_verify(context, verifier, observation, True,
verify_results)
except:
print 'testing {0}'.format(test[0])
raise
def test_result_observation_verifier_disjunction_failure(self):
context = ExecutionContext()
observation = jc.Observation()
builder = jc.ObservationVerifierBuilder(title='Test')
verifiers = []
results = []
pred_results = [
jp.PredicateResult(False, comment='Result %d' % i)
for i in range(2)
]
for i in range(2):
result = _makeObservationVerifyResult(
observation=observation,
valid=False,
bad_results=[pred_results[i]])
fake_verifier = FakeObservationVerifier(title=i,
dnf_verifier=[],
result=result)
verifiers.append(fake_verifier)
results.append(result)
builder.OR(fake_verifier)
verifier = builder.build()
self.assertEqual([verifiers[0:1], verifiers[1:2]],
verifier.dnf_verifiers)
expect = _makeObservationVerifyResult(False,
observation=observation,
bad_results=pred_results)
global _called_verifiers
_called_verifiers = []
got = verifier(context, observation)
self.assertEqual(expect, got)
self.assertEqual(verifiers, _called_verifiers)
def tearDownClass(cls):
runner = citest.base.TestRunner.global_runner()
bindings = runner.bindings
context = ExecutionContext()
compute_agent = (
gcp.GcpComputeAgent
.make_agent(scopes=SCOPES,
credentials_path=bindings['GCE_CREDENTIALS_PATH']))
compute_agent.invoke_resource(context,
'delete',
resource_type='sslCertificates',
project=bindings['GOOGLE_PRIMARY_MANAGED_PROJECT_ID'],
sslCertificate=cls.FIRST_CERT)
compute_agent.invoke_resource(context,
'delete',
resource_type='sslCertificates',
project=bindings['GOOGLE_PRIMARY_MANAGED_PROJECT_ID'],
sslCertificate=cls.SECOND_CERT)
# Delete the orphaned, updated health check.
compute_agent.invoke_resource(context,
'delete',
resource_type='httpHealthChecks',
project=bindings['GOOGLE_PRIMARY_MANAGED_PROJECT_ID'],
httpHealthCheck=cls.UPDATED_HC)
class HelperClass(object):
"""Need to share state between these helper methods and outer scope.
This class provides the functions we are going to inject, along
with state we can check in the outer scope.
"""
cleanup_calls = 0
execution_context = ExecutionContext() if with_context else None
@staticmethod
def status_extractor(status, context):
if with_context:
# Verify this is the context we injected.
self.assertEquals(HelperClass.execution_context, context)
self.assertIsNotNone(context.get('OperationStatus', None))
self.assertIsNone(context.get('GOT_STATUS', None))
context['GOT_STATUS'] = status
@staticmethod
def cleanup(context):
self.assertIsNotNone(context.get('OperationStatus', None))
self.assertEquals(context.get('OperationStatus', None),
context.get('GOT_STATUS', None))
HelperClass.cleanup_calls += 1
def test_map_predicate_good_and_bad_min_indirect(self):
context = ExecutionContext(min=2)
aA = jp.PathPredicate('a', jp.STR_EQ('A'))
expect_result = jp.MapPredicateResult(
valid=False,
pred=aA,
obj_list=[_NUMBER_DICT, _LETTER_DICT],
all_results=[aA(context, _NUMBER_DICT),
aA(context, _LETTER_DICT)],
good_map=[
jp.ObjectResultMapAttempt(_LETTER_DICT,
aA(context, _LETTER_DICT))
],
bad_map=[
jp.ObjectResultMapAttempt(_NUMBER_DICT,
aA(context, _NUMBER_DICT))
])
self._try_map(context,
aA, [_NUMBER_DICT, _LETTER_DICT],
False,
expect_result,
min=lambda x: x['min'])
def test_cardinality_bounds_1(self):
context = ExecutionContext()
for min in range(0, 3):
for max in range(0, 3):
if min > max:
continue
predicate = jp.CardinalityPredicate(_AorX, min=min, max=max)
expect_ok = min <= 1 and max >= 1
source = _CAB
result = predicate(context, source)
all_results = [
jp.SequencedPredicateResult(False, _AorX, [
jp.PathValueResult(_CAB,
'',
PathValue('[0]', 'C'),
valid=False,
pred=_eq_A),
jp.PathValueResult(_CAB,
'',
PathValue('[0]', 'C'),
valid=False,
pred=_eq_X)
]),
jp.SequencedPredicateResult(True, _AorX, [
jp.PathValueResult(_CAB,
'',
PathValue('[1]', 'A'),
valid=True,
pred=_eq_A)
]),
jp.SequencedPredicateResult(False, _AorX, [
jp.PathValueResult(_CAB,
'',
PathValue('[2]', 'B'),
valid=False,
pred=_eq_A),
jp.PathValueResult(_CAB,
'',
PathValue('[2]', 'B'),
valid=False,
pred=_eq_X)
])
]
builder = jp.PathPredicateResultBuilder(pred=jp.PathPredicate(
'', _AorX),
source=_CAB)
builder.add_result_candidate(PathValue('[0]', 'C'),
all_results[0])
builder.add_result_candidate(PathValue('[1]', 'A'),
all_results[1])
builder.add_result_candidate(PathValue('[2]', 'B'),
all_results[2])
expect_path_result = builder.build(True)
if expect_ok:
self.assertEqual(
jp.ConfirmedCardinalityResult(predicate,
expect_path_result),
result)
elif max == 0:
self.assertEqual(
jp.UnexpectedValueCardinalityResult(
predicate, expect_path_result), result)
else:
self.assertEqual(
jp.FailedCardinalityRangeResult(
predicate, expect_path_result), result)
def run_test_case(self, test_case, context=None, **kwargs):
"""Run the specified test operation from start to finish.
Args:
test_case: [OperationContract] To test.
context: [ExecutionContext] The citest execution context to run in.
timeout_ok: [bool] Whether an AgentOperationStatus timeout implies
a test failure. If it is ok to timeout, then we'll still verify the
contracts, but skip the final status check if there is no final
status yet.
max_retries: [int] Number of independent retries permitted on
individual operations if the operation status fails. A value of 0
indicates that a test should only be given a single attempt.
retry_interval_secs: [int] The number of seconds to wait between retries.
poll_every_secs: [int] Number of seconds between wait polls. Default=1.
"""
if context is None:
context = ExecutionContext()
timeout_ok = kwargs.pop('timeout_ok', False)
max_retries = kwargs.pop('max_retries', 0)
retry_interval_secs = kwargs.pop('retry_interval_secs', 5)
poll_every_secs = kwargs.pop('poll_every_secs', 1)
full_trace = kwargs.pop('full_trace', False) # Deprecated
if kwargs:
raise TypeError('Unrecognized arguments {0}'.format(kwargs.keys()))
self.log_start_test(test_case.title)
if max_retries < 0:
raise ValueError(
'max_retries={max} cannot be negative'.format(max=max_retries))
if retry_interval_secs < 0:
raise ValueError(
'retry_interval_secs={secs} cannot be negative'.format(
secs=retry_interval_secs))
execution_trace = OperationContractExecutionTrace(test_case)
verify_results = None
final_status_ok = None
context_relation = None
attempt_info = None
status = None
try:
JournalLogger.begin_context('Test "{0}"'.format(test_case.title))
JournalLogger.delegate(
"store", test_case.operation,
_title='Operation "{0}" Specification'.format(
test_case.operation.title))
max_tries = 1 + max_retries
# We attempt the operation on the agent multiple times until the agent
# thinks that it succeeded. But we will only verify once the agent thinks
# it succeeded. We do not give multiple chances to satisfy the
# verification.
for i in range(max_tries):
context.clear_key('OperationStatus')
context.clear_key('AttemptInfo')
attempt_info = execution_trace.new_attempt()
status = None
status = test_case.operation.execute(agent=self.testing_agent)
status.wait(poll_every_secs=poll_every_secs)
summary = status.error or ('Operation status OK' if status.finished_ok
else 'Operation status Unknown')
# Write the status (and attempt_info) into the execution_context
# to make it available to contract verifiers. For example, to
# make specific details in the status (e.g. new resource names)
# available to downstream validators for their consideration.
context.set_internal('AttemptInfo', attempt_info)
context.set_internal('OperationStatus', status)
attempt_info.set_status(status, summary)
if test_case.status_extractor:
test_case.status_extractor(status, context)
if not status.exception_details:
execution_trace.set_operation_summary('Completed test.')
break
if max_tries - i > 1:
self.logger.warning(
'Got an exception: %s.\nTrying again in %r secs...',
status.exception_details, retry_interval_secs)
time.sleep(retry_interval_secs)
elif max_tries > 1:
execution_trace.set_operation_summary('Gave up retrying operation.')
self.logger.error('Giving up retrying test.')
# We're always going to verify the contract, even if the request itself
# failed. We set the verification on the attempt here, but do not assert
# anything. We'll assert below outside this try/catch handler.
verify_results = test_case.contract.verify(context)
execution_trace.set_verify_results(verify_results)
final_status_ok = self.verify_final_status_ok(
status, timeout_ok=timeout_ok,
final_attempt=attempt_info,
execution_trace=execution_trace)
context_relation = ('VALID' if (final_status_ok and verify_results)
else 'INVALID')
except BaseException as ex:
context_relation = 'ERROR'
execution_trace.set_exception(ex)
if attempt_info is None:
execution_trace.set_exception(ex, traceback_module.format_exc())
elif not attempt_info.completed:
# Exception happened during the attempt as opposed to during our
# verification afterwards.
attempt_info.set_exception(ex, traceback_module.format_exc())
try:
self.logger.error('Test failed with exception: %s', ex)
self.logger.error('Last status was:\n%s', str(status))
self.logger.debug('Exception was at:\n%s',
traceback_module.format_exc())
except BaseException as unexpected:
self.logger.error(
'Unexpected error %s\nHandling original exception %s',
unexpected, ex)
self.logger.debug('Unexpected exception was at:\n%s',
traceback_module.format_exc())
raise
finally:
try:
context.set_internal('ContractVerifyResults', verify_results)
self.log_end_test(test_case.title)
self.report(execution_trace)
if test_case.cleanup:
if status is None:
self.logger.info('Skipping operation cleanup because'
' operation could not be performed at all.')
else:
self.logger.info('Invoking injected operation cleanup.')
test_case.cleanup(context)
finally:
JournalLogger.end_context(relation=context_relation)
if not final_status_ok:
self.raise_final_status_not_ok(status, attempt_info)
if verify_results is not None:
self.assertVerifyResults(verify_results)
def __try_delete_all(self, agent, resource_type, results_to_delete,
aggregated):
"""Implements the actual delete heuristics.
Args:
agent: [GcpAgent] The agent to delete the resources.
resource_type: [string] The resource type to delete.
results_to_delete: [string] The listing results to be deleted.
These may be the ids or may be tuples (params, result) if
the listing was an aggreatedList.
aggregated: [bool] Indicates whether results_to_delete were aggregated.
"""
context = ExecutionContext()
result_by_code = {}
for elem in results_to_delete:
params = {}
name = elem
if aggregated:
name = elem[1]
try:
param_name, param_value = elem[0].split('/', 1)
except ValueError as vex:
print('Ignoring error {0}'.format(vex))
print(' type={0}, name={1}: ELEM[0] was {2!r}'.format(
resource_type, name, elem[0]))
continue
if param_name[-1] == 's':
param_name = param_name[:-1]
# Just because the aggregation returned a parameter
# does not mean the delete API takes it. Confirm before adding.
if (agent.resource_type_to_discovery_info(resource_type).get(
'methods', {}).get('delete',
{}).get('parameters',
{}).get(param_name)):
params[param_name] = param_value
name = elem[1] if aggregated else elem
try:
if self.__dry_run:
variables = agent.resource_method_to_variables(
'delete', resource_type, resource_id=name, **params)
args_str = ','.join([
' {0}={1!r}'.format(key, value)
for key, value in variables.items()
])
print('[dry run] delete "{type}" {name} {args}'.format(
type=resource_type, name=name, args=args_str))
else:
agent.invoke_resource(context,
'delete',
resource_type,
resource_id=name,
**params)
print('Deleted "{type}" {name}'.format(type=resource_type,
name=name))
if HTTPStatus.OK in result_by_code:
result_by_code[HTTPStatus.OK].append(elem)
else:
result_by_code[HTTPStatus.OK] = [elem]
except HttpError as http_error:
if http_error.resp.status in result_by_code:
result_by_code[http_error.resp.status].append(elem)
else:
result_by_code[http_error.resp.status] = [elem]
if http_error.resp.status == HTTPStatus.NOT_FOUND:
print(' - "{type}" "{name}" was already deleted'.format(
type=resource_type, name=name))
else:
print(' Ignoring error deleting "{type}" "{name}": {msg}'.
format(type=resource_type, name=name,
msg=http_error))
except ValueError as value_error:
# NOTE(ewiseblatt): 20170928
# This is a quick fix because instanceGroupManagers.aggregatedList
# is returning some regions but the delete only takes zones. The
# region results are missing the zone value. Ignore those errors.
# This isnt the best place to handle this, but is the easiest for
# now and I dont have time to devise a cleaner solution right now.
print('Ignoring error with "delete {0} {1} {2}": {3}'.format(
resource_type, name, params, value_error))
if -1 in result_by_code:
result_by_code[-1].append(elem)
else:
result_by_code[-1] = [elem]
return result_by_code
