def test_result_observation_verifier_conjunction_failure_aborts_early(
self):
builder = jc.ObservationVerifierBuilder(title='Test')
verifiers = []
results = []
for i in range(3):
result = _makeObservationVerifyResult(valid=False)
fake_verifier = FakeObservationVerifier(title=i,
dnf_verifier=[],
result=result)
verifiers.append(fake_verifier)
results.append(result)
builder.append_verifier(fake_verifier)
# verify build can work multiple times
self.assertEqual(builder.build(), builder.build())
verifier = builder.build()
self.assertEqual([verifiers], verifier.dnf_verifiers)
expect = _makeObservationVerifyResult(False, bad_results=[results[0]])
global _called_verifiers
_called_verifiers = []
got = verifier(jc.Observation())
self.assertEqual(expect, got)
self.assertEqual(verifiers[:1], _called_verifiers)
def test_obsolete_observation_failure_not_ok(self):
error_text = 'the error'
context = ExecutionContext()
observation = jc.Observation()
error = ValueError('not the error')
observation.add_error(error)
failure_verifier = TestObsoleteObservationFailureVerifier(
'Test', error_text)
comment = failure_verifier._error_not_found_comment(observation)
failure_pred_result = jp.PredicateResult(valid=False, comment=comment)
expect_failure = jc.ObservationVerifyResult(
valid=False, observation=observation,
bad_results=[jp.ObjectResultMapAttempt(observation,
failure_pred_result)],
good_results=[], failed_constraints=[],
comment=comment)
self.assertEqual(expect_failure, failure_verifier(context, observation))
builder = jc.ObservationVerifierBuilder(title='Test Verifier')
builder.EXPECT(failure_verifier)
verifier = builder.build()
expect = jc.ObservationVerifyResult(
valid=False, observation=observation,
bad_results=expect_failure.bad_results,
good_results=[], failed_constraints=[])
got = verifier(context, observation)
self.assertEqual(expect, got)
def test_obsolete_observation_failure_or_found(self):
context = ExecutionContext()
observation = jc.Observation()
observation.add_error(ValueError('not the error'))
failure_verifier = TestObsoleteObservationFailureVerifier(
'Verify', 'NotFound')
comment = failure_verifier._error_not_found_comment(observation)
failure_result = jp.PredicateResult(valid=False, comment=comment)
# We've already established this result is what we expect
bad_observation_result = failure_verifier(context, observation)
success_pred_result = jp.PredicateResult(valid=True)
good_observation_result = _makeObservationVerifyResult(
valid=True,
good_results=[success_pred_result],
observation=observation)
success_verifier = FakeObservationVerifier(
'Found', dnf_verifier=[], result=good_observation_result)
builder = jc.ObservationVerifierBuilder(title='Observation Verifier')
builder.EXPECT(failure_verifier).OR(success_verifier)
verifier = builder.build()
expect = jc.ObservationVerifyResult(
valid=True, observation=observation,
bad_results=bad_observation_result.bad_results,
good_results=good_observation_result.good_results,
failed_constraints=[])
got = verifier(context, observation)
self.assertEqual(expect, got)
def test_result_observation_verifier_disjunction_failure(self):
context = ExecutionContext()
observation = jc.Observation()
builder = jc.ObservationVerifierBuilder(title='Test')
verifiers = []
results = []
pred_results = [jp.PredicateResult(False, comment='Result %d' % i)
for i in range(2)]
for i in range(2):
result = _makeObservationVerifyResult(observation=observation,
valid=False, bad_results=[pred_results[i]])
fake_verifier = FakeObservationVerifier(
title=i, dnf_verifier=[], result=result)
verifiers.append(fake_verifier)
results.append(result)
builder.OR(fake_verifier)
verifier = builder.build()
self.assertEqual([verifiers[0:1], verifiers[1:2]], verifier.dnf_verifiers)
expect = _makeObservationVerifyResult(
False, observation=observation, bad_results=pred_results)
global _called_verifiers
_called_verifiers = []
got = verifier(context, observation)
self.assertEqual(expect, got)
self.assertEqual(verifiers, _called_verifiers)
def test_obsolete_observation_failure_ok(self):
error_text = 'the error'
context = ExecutionContext()
observation = jc.Observation()
error = ValueError(error_text)
observation.add_error(error)
failure_verifier = TestObsoleteObservationFailureVerifier(
'Test', error_text)
failure_pred_result = jc.ObservationFailedError([error], valid=True)
expect_failure = jc.ObservationVerifyResult(
valid=True, observation=observation,
good_results=[jp.ObjectResultMapAttempt(observation,
failure_pred_result)],
bad_results=[], failed_constraints=[],
comment=_TEST_FOUND_ERROR_COMMENT)
got = failure_verifier(context, observation)
self.assertEqual(expect_failure, got)
builder = jc.ObservationVerifierBuilder(title='Test')
builder.EXPECT(failure_verifier)
verifier = builder.build()
expect = jc.ObservationVerifyResult(
valid=True, observation=observation,
good_results=expect_failure.good_results,
bad_results=[], failed_constraints=[])
got = verifier(context, observation)
self.assertEqual(expect, got)
def test_result_observation_verifier_conjunction_failure_aborts_early(self):
context = ExecutionContext()
builder = jc.ObservationVerifierBuilder(title='Test')
verifiers = []
results = []
pred_results = [jp.PredicateResult(False, comment='Result %d' % i)
for i in range(3)]
for i in range(3):
result = _makeObservationVerifyResult(
valid=False, bad_results=[pred_results[i]])
fake_verifier = FakeObservationVerifier(
title=i, dnf_verifier=[], result=result)
verifiers.append(fake_verifier)
results.append(result)
builder.AND(fake_verifier)
# verify build can work multiple times
self.assertEqual(builder.build(), builder.build())
verifier = builder.build()
self.assertEqual([verifiers], verifier.dnf_verifiers)
expect = _makeObservationVerifyResult(
False, bad_results=[pred_results[0]])
global _called_verifiers
_called_verifiers = []
got = verifier(context, jc.Observation())
self.assertEqual(expect, got)
self.assertEqual(verifiers[:1], _called_verifiers)
def test_observation_failure_ok(self):
error_text = 'the error'
context = ExecutionContext()
observation = jc.Observation()
error = ValueError(error_text)
observation.add_error(error)
exception_pred = jp.ExceptionMatchesPredicate(ValueError,
regex=error_text)
builder = jc.ObservationVerifierBuilder(title='Test')
builder.EXPECT(
jc.ObservationErrorPredicate(jp.LIST_MATCHES([exception_pred])))
failure_verifier = builder.build()
observation_predicate_result = jc.ObservationPredicateResult(
True, observation, jp.LIST_MATCHES([exception_pred]),
jp.LIST_MATCHES([exception_pred])(context, [error]))
expect_failure = jc.ObservationVerifyResult(
True,
observation,
good_results=[observation_predicate_result],
bad_results=[],
failed_constraints=[])
got = failure_verifier(context, observation)
self.assertEqual(expect_failure, got)
def test_observation_failure_or_found(self):
context = ExecutionContext()
observation = jc.Observation()
observation.add_object(_LETTER_DICT)
failure_predicate = jc.ObservationErrorPredicate(
jp.ExceptionMatchesPredicate(ValueError, regex='an error'))
failure_result = failure_predicate(context, observation)
self.assertFalse(failure_result)
good_predicate = jc.ObservationValuePredicate(
jp.PathPredicate('a', jp.STR_EQ('A')))
builder = jc.ObservationVerifierBuilder('TestAddConstraint')
verifier = (
builder.EXPECT(failure_predicate).OR(good_predicate).build())
expect = jc.ObservationVerifyResult(
valid=True,
observation=observation,
bad_results=[failure_result],
good_results=[good_predicate(context, observation)],
failed_constraints=[])
got = verifier(context, observation)
self.assertEqual(expect, got)
def test_result_observation_verifier_conjunction_ok(self):
context = ExecutionContext()
builder = jc.ObservationVerifierBuilder(title='Test')
verifiers = []
pred_results = []
for i in range(3):
this_result = jp.PredicateResult(True, comment='Pred {0}'.format(i))
pred_results.append(this_result)
result = _makeObservationVerifyResult(
valid=True, good_results=[this_result])
fake_verifier = FakeObservationVerifier(
title=i, dnf_verifier=[], result=result)
verifiers.append(fake_verifier)
builder.AND(fake_verifier)
# verify build can work multiple times
self.assertEqual(builder.build(), builder.build())
verifier = builder.build()
self.assertEqual([verifiers], verifier.dnf_verifiers)
expect = _makeObservationVerifyResult(True, good_results=pred_results)
global _called_verifiers
_called_verifiers = []
got = verifier(context, jc.Observation())
self.assertEqual(expect, got)
self.assertEqual(verifiers, _called_verifiers)
def test_result_observation_verifier_disjunction_success_aborts_early(
self):
context = ExecutionContext()
builder = jc.ObservationVerifierBuilder(title='Test')
verifiers = []
results = []
for i in range(2):
result = _makeObservationVerifyResult(valid=True)
fake_verifier = FakeObservationVerifier(title=i,
dnf_verifier=[],
result=result)
verifiers.append(fake_verifier)
results.append(result)
builder.append_verifier(fake_verifier, new_term=True)
verifier = builder.build()
self.assertEqual([verifiers[0:1], verifiers[1:2]],
verifier.dnf_verifiers)
expect = _makeObservationVerifyResult(True, bad_results=[results[0]])
global _called_verifiers
_called_verifiers = []
got = verifier(context, jc.Observation())
self.assertEqual(expect, got)
self.assertEqual(verifiers[:1], _called_verifiers)
def get_url_path(self, path, allow_http_error_status=None):
"""Perform the observation using HTTP GET on a path.
Args:
path [string]: The path relative to the agent's baseUrl.
allow_http_error_status: [int] If not None then we allow this HTTP error
status as a valid response without further constraints. For example
404 would mean that we permit a 404 error, otherwise we may expect
other constraints on the observed path as a normal clause would
specify.
"""
self.observer = HttpObjectObserver(self.__agent, path)
if allow_http_error_status:
error_verifier = HttpObservationFailureVerifier(
'Got HTTP {0} Error'.format(allow_http_error_status),
allow_http_error_status)
disjunction_builder = jc.ObservationVerifierBuilder(
'Get url {0} or {1}'.format(path, allow_http_error_status))
disjunction_builder.append_verifier(error_verifier)
observation_builder = jc.ValueObservationVerifierBuilder(
'Get url {0}'.format(path))
disjunction_builder.append_verifier_builder(observation_builder,
new_term=True)
self.verifier_builder.append_verifier_builder(disjunction_builder,
new_term=True)
else:
observation_builder = jc.ValueObservationVerifierBuilder(
'Get ' + path, strict=self.__strict)
self.verifier_builder.append_verifier_builder(observation_builder)
return observation_builder
def test_result_observation_verifier_conjunction_ok(self):
context = ExecutionContext()
builder = jc.ObservationVerifierBuilder(title='Test')
verifiers = []
results = []
for i in range(3):
result = _makeObservationVerifyResult(valid=True)
fake_verifier = FakeObservationVerifier(title=i,
dnf_verifier=[],
result=result)
verifiers.append(fake_verifier)
results.append(result)
builder.append_verifier(fake_verifier)
# verify build can work multiple times
self.assertEqual(builder.build(), builder.build())
verifier = builder.build()
self.assertEqual([verifiers], verifier.dnf_verifiers)
expect = _makeObservationVerifyResult(True, good_results=results)
global _called_verifiers
_called_verifiers = []
got = verifier(context, jc.Observation())
self.assertEqual(expect, got)
self.assertEqual(verifiers, _called_verifiers)
def get_resources(self, type, extra_args=None, no_resource_ok=False):
"""Observe resources of a particular type.
This ultimately calls a "kubectl ... get |type| |extra_args|"
Args:
no_resource_ok: Whether or not the resource is required.
If the resource is not required, "not found" is treated as a valid
check. Because resource deletion is asynchronous, there is no
explicit API here to confirm that a resource does not exist.
"""
self.observer = self.__factory.new_get_resources(type,
extra_args=extra_args)
if no_resource_ok:
# Unfortunately gcloud does not surface the actual 404 but prints an
# error message saying that it was not found.
error_verifier = cli_agent.CliAgentObservationFailureVerifier(
title='Not Found Permitted', error_regex='.* not found')
disjunction_builder = jc.ObservationVerifierBuilder(
'Get {0} {1} or Not Found'.format(type, extra_args))
disjunction_builder.append_verifier(error_verifier)
get_builder = jc.ValueObservationVerifierBuilder(
'Get {0} {1}'.format(type, extra_args), strict=self.__strict)
disjunction_builder.append_verifier_builder(get_builder,
new_term=True)
self.verifier_builder.append_verifier_builder(disjunction_builder,
new_term=True)
else:
get_builder = jc.ValueObservationVerifierBuilder(
'Get {0} {1}'.format(type, extra_args), strict=self.__strict)
self.verifier_builder.append_verifier_builder(get_builder)
return get_builder
def test_result_observation_verifier_disjunction_failure(self):
builder = jc.ObservationVerifierBuilder(title='Test')
verifiers = []
results = []
for i in range(2):
result = _makeObservationVerifyResult(valid=False)
fake_verifier = FakeObservationVerifier(title=i,
dnf_verifier=[],
result=result)
verifiers.append(fake_verifier)
results.append(result)
builder.append_verifier(fake_verifier, new_term=True)
verifier = builder.build()
self.assertEqual([verifiers[0:1], verifiers[1:2]],
verifier.dnf_verifiers)
expect = _makeObservationVerifyResult(False, bad_results=[results])
global _called_verifiers
_called_verifiers = []
got = verifier(jc.Observation())
self.assertEqual(expect, got)
self.assertEqual(verifiers, _called_verifiers)
def test_result_observation_verifier_disjunction_success_aborts_early(self):
context = ExecutionContext()
builder = jc.ObservationVerifierBuilder(title='Test')
verifiers = []
results = []
pred_results = [jp.PredicateResult(False, comment='Result %d' % i)
for i in range(2)]
for i in range(2):
result = _makeObservationVerifyResult(
valid=True, good_results=[pred_results[i]])
fake_verifier = FakeObservationVerifier(
title=i, dnf_verifier=[], result=result)
verifiers.append(fake_verifier)
results.append(result)
builder.OR(fake_verifier)
verifier = builder.build()
self.assertEqual([verifiers[0:1], verifiers[1:2]], verifier.dnf_verifiers)
expect = _makeObservationVerifyResult(True, good_results=[pred_results[0]])
global _called_verifiers
_called_verifiers = []
got = verifier(context, jc.Observation())
self.assertEqual(expect, got)
self.assertEqual(verifiers[:1], _called_verifiers)
def inspect_resource(self,
type,
name,
extra_args=None,
no_resource_ok=False):
"""Observe the details of a specific instance.
This ultimately calls a "gcloud ... |type| |name| describe |extra_args|"
Args:
type: The gcloud resource type (e.g. instances)
name: The GCE resource name
extra_args: Additional parameters to pass to gcloud.
no_resource_ok: Whether or not the resource is required.
If the resource is not required, a 404 is treated as a valid check.
Because resource deletion is asynchronous, there is no explicit
API here to confirm that a resource does not exist.
Returns:
A js.ValueObservationVerifier that will collect the requested resource
when its verify() method is run.
"""
self.observer = self.__factory.new_inspect_resource(
type, name, extra_args)
if no_resource_ok:
# Unfortunately gcloud does not surface the actual 404 but prints an
# error message saying that it was not found.
error_verifier = cli_agent.CliAgentObservationFailureVerifier(
title='404 Permitted', error_regex='.* was not found.*')
disjunction_builder = jc.ObservationVerifierBuilder(
'Inspect {0} {1} or 404'.format(type, name))
disjunction_builder.append_verifier(error_verifier)
inspect_builder = jc.ValueObservationVerifierBuilder(
'Inspect {0} {1}'.format(type, name), strict=self.__strict)
disjunction_builder.append_verifier_builder(inspect_builder,
new_term=True)
self.verifier_builder.append_verifier_builder(disjunction_builder,
new_term=True)
else:
inspect_builder = jc.ValueObservationVerifierBuilder(
'Inspect {0} {1}'.format(type, name), strict=self.__strict)
self.verifier_builder.append_verifier_builder(inspect_builder)
return inspect_builder
def collect_resources(self,
aws_module,
command,
args=None,
filter=None,
no_resources_ok=False):
"""Collect the AWS resources of a particular type.
Args:
aws_module: The aws program module name we're looking in (e.g. 'ec2')
command: The aws command name to run (e.g. 'describe-instances')
args: An array of strings containing the remaining aws command parameters.
filter: If provided, a filter to use for refining the collection.
no_resources_ok: Whether or not the resource is required.
If the resource is not required, 'resource not found' error is
considered successful.
"""
args = args or []
cmd = self.__aws.build_aws_command_args(command,
args,
aws_module=aws_module,
profile=self.__aws.profile)
self.observer = AwsObjectObserver(self.__aws, cmd)
if no_resources_ok:
error_verifier = cli_agent.CliAgentObservationFailureVerifier(
title='"Not Found" permitted.',
error_regex=
'(?:.* operation: Cannot find .*)|(?:.*\(.*NotFound\).*)')
disjunction_builder = jc.ObservationVerifierBuilder(
'Collect {0} or Not Found'.format(command))
disjunction_builder.append_verifier(error_verifier)
collect_builder = jc.ValueObservationVerifierBuilder(
'Collect {0}'.format(command), strict=self.__strict)
disjunction_builder.append_verifier_builder(collect_builder,
new_term=True)
self.verifier_builder.append_verifier_builder(disjunction_builder,
new_term=True)
else:
collect_builder = jc.ValueObservationVerifierBuilder(
'Collect {0}'.format(command), strict=self.__strict)
self.verifier_builder.append_verifier_builder(collect_builder)
return collect_builder
def collect_resources(self,
command,
args=None,
filter=None,
no_resources_ok=False):
"""Collect the OpenStack resources of a particular type.
Args:
command: The openstack command name to run (e.g. 'image', 'server')
args: An array of strings containing the remaining openstack command
parameters.
filter: If provided, a filter to use for refining the collection.
no_resources_ok: Whether or not the resource is required.
If the resource is not required, 'resource not found' error is
considered successful.
"""
args = args or []
cmd = self.__os.build_os_command_args(command,
args,
os_cloud=self.__os.os_cloud)
self.observer = OsObjectObserver(self.__os, cmd)
if no_resources_ok:
error_verifier = cli_agent.CliAgentObservationFailureVerifier(
title='"Not Found" permitted.',
error_regex=
'(?:.* operation: Cannot find .*)|(?:.*\(.*NotFound\)|(Error .*).*)'
)
disjunction_builder = jc.ObservationVerifierBuilder(
'Collect {0} or Not Found'.format(command))
disjunction_builder.append_verifier(error_verifier)
collect_builder = jc.ValueObservationVerifierBuilder(
'Collect {0}'.format(command), strict=self.__strict)
disjunction_builder.append_verifier_builder(collect_builder,
new_term=True)
self.verifier_builder.append_verifier_builder(disjunction_builder,
new_term=True)
else:
collect_builder = jc.ValueObservationVerifierBuilder(
'Collect {0}'.format(command), strict=self.__strict)
self.verifier_builder.append_verifier_builder(collect_builder)
return collect_builder
def show_resource(self, command, resource_name, no_resources_ok=False):
"""Show the OpenStack resource of a particular type.
Args:
command: The openstack command name to run
(e.g. 'security group', 'server').
resource_name: Name of the OpenStack resource
(e.g. Name of a security group or an image).
no_resources_ok: Whether or not the resource is required.
If the resource is not required, 'resource not found' error is
considered successful.
"""
args = ['show', resource_name, '--format', 'json']
cmd = self.__os.build_os_command_args(command,
args,
os_cloud=self.__os.os_cloud)
self.observer = OsObjectObserver(self.__os, cmd)
if no_resources_ok:
error_verifier = cli_agent.CliAgentObservationFailureVerifier(
title='"Not Found" permitted.',
error_regex=
'(?:.* operation: Cannot find .*)|(?:.*\(.*NotFound\)|(Error .*).*)'
)
disjunction_builder = jc.ObservationVerifierBuilder(
'Collect {0} or Not Found'.format(command))
disjunction_builder.append_verifier(error_verifier)
collect_builder = jc.ValueObservationVerifierBuilder(
'Collect {0}'.format(command), strict=self.__strict)
disjunction_builder.append_verifier_builder(collect_builder,
new_term=True)
self.verifier_builder.append_verifier_builder(disjunction_builder,
new_term=True)
else:
collect_builder = jc.ValueObservationVerifierBuilder(
'Collect {0}'.format(command), strict=self.__strict)
self.verifier_builder.append_verifier_builder(collect_builder)
return collect_builder
def destroy_app(self):
contract = jc.Contract()
app_path = os.path.join('/default/applications/name', self.TEST_APP)
obs_builder = jc.ObservationVerifierBuilder('Removed Application')
obs_builder.append_verifier(
st.HttpObservationFailureVerifier('Not Found', 404))
f50_observer = st.HttpObjectObserver(self.agent, app_path)
f50_clause = jc.ContractClause(title='Deleted Application',
observer=f50_observer,
verifier=obs_builder.build())
contract.add_clause(f50_clause)
gcs_builder = gcp.GoogleCloudStorageContractBuilder(self.gcs_observer)
(gcs_builder.new_clause_builder('Deleted File').list(
self.BUCKET_ROOT,
'applications').excludes_path_value('name', self.TEST_APP))
for clause in gcs_builder.build().clauses:
contract.add_clause(clause)
path = os.path.join('/default/applications/name/', self.TEST_APP)
return st.OperationContract(self.new_delete_operation(
title='delete_app', data=None, path=path),
contract=contract)
