def collect_resources(self,
aws_module,
command,
args=None,
filter=None,
no_resources_ok=False):
"""Collect the AWS resources of a particular type.
Args:
aws_module: The aws program module name we're looking in (e.g. 'ec2')
command: The aws command name to run (e.g. 'describe-instances')
args: An array of strings containing the remaining aws command parameters.
filter: If provided, a filter to use for refining the collection.
no_resources_ok: Whether or not the resource is required.
If the resource is not required, 'resource not found' error is
considered successful.
"""
args = args or []
cmd = self.__aws.build_aws_command_args(command,
args,
aws_module=aws_module,
profile=self.__aws.profile)
self.observer = AwsObjectObserver(self.__aws, cmd)
if no_resources_ok:
error_verifier = cli_agent.CliAgentObservationFailureVerifier(
title='"Not Found" permitted.',
error_regex=
'(?:.* operation: Cannot find .*)|(?:.*\(.*NotFound\).*)')
disjunction_builder = jc.ObservationVerifierBuilder(
'Collect {0} or Not Found'.format(command))
disjunction_builder.append_verifier(error_verifier)
collect_builder = jc.ValueObservationVerifierBuilder(
'Collect {0}'.format(command), strict=self.__strict)
disjunction_builder.append_verifier_builder(collect_builder,
new_term=True)
self.verifier_builder.append_verifier_builder(disjunction_builder,
new_term=True)
else:
collect_builder = jc.ValueObservationVerifierBuilder(
'Collect {0}'.format(command), strict=self.__strict)
self.verifier_builder.append_verifier_builder(collect_builder)
return collect_builder
def collect_resources(self,
command,
args=None,
filter=None,
no_resources_ok=False):
"""Collect the OpenStack resources of a particular type.
Args:
command: The openstack command name to run (e.g. 'image', 'server')
args: An array of strings containing the remaining openstack command
parameters.
filter: If provided, a filter to use for refining the collection.
no_resources_ok: Whether or not the resource is required.
If the resource is not required, 'resource not found' error is
considered successful.
"""
args = args or []
cmd = self.__os.build_os_command_args(command,
args,
os_cloud=self.__os.os_cloud)
self.observer = OsObjectObserver(self.__os, cmd)
if no_resources_ok:
error_verifier = cli_agent.CliAgentObservationFailureVerifier(
title='"Not Found" permitted.',
error_regex=
'(?:.* operation: Cannot find .*)|(?:.*\(.*NotFound\)|(Error .*).*)'
)
disjunction_builder = jc.ObservationVerifierBuilder(
'Collect {0} or Not Found'.format(command))
disjunction_builder.append_verifier(error_verifier)
collect_builder = jc.ValueObservationVerifierBuilder(
'Collect {0}'.format(command), strict=self.__strict)
disjunction_builder.append_verifier_builder(collect_builder,
new_term=True)
self.verifier_builder.append_verifier_builder(disjunction_builder,
new_term=True)
else:
collect_builder = jc.ValueObservationVerifierBuilder(
'Collect {0}'.format(command), strict=self.__strict)
self.verifier_builder.append_verifier_builder(collect_builder)
return collect_builder
def test_verifier_builder_add_constraint(self):
aA = jp.PathPredicate('a', jp.STR_EQ('A'))
bB = jp.PathPredicate('b', jp.STR_EQ('B'))
builder = jc.ValueObservationVerifierBuilder('TestAddConstraint')
builder.add_constraint(aA)
builder.add_constraint(bB)
verifier = builder.build()
self.assertEqual('TestAddConstraint', verifier.title)
self.assertEqual([aA, bB], verifier.constraints)
def list_resource(self, resource_type, **kwargs):
"""Observe resources of a particular type."""
self.observer = GcpObjectObserver(self.__gcp_agent.list_resource,
resource_type=resource_type,
**kwargs)
observation_builder = jc.ValueObservationVerifierBuilder(
'List ' + resource_type, strict=self.__strict)
self.verifier_builder.append_verifier_builder(observation_builder)
return observation_builder
def test_verifier_builder_contains_pred_list(self):
aA = jp.PathPredicate('a', jp.STR_EQ('A'))
bB = jp.PathPredicate('b', jp.STR_EQ('B'))
builder = jc.ValueObservationVerifierBuilder('TestContainsGroup')
builder.contains_path_pred('a', jp.STR_EQ('A'))
builder.contains_path_pred('b', jp.STR_EQ('B'))
verifier = builder.build()
count_aA = jp.CardinalityPredicate(aA, 1, None)
count_bB = jp.CardinalityPredicate(bB, 1, None)
self.assertEqual([count_aA, count_bB], verifier.constraints)
def test_object_observation_verifier_with_conditional(self):
# We need strict True here because we want each object to pass
# the constraint test. Otherwise, if any object passes, then the whole
# observation would pass. This causes a problem when we say that
# we dont ever want to see 'name' unless it has a particular 'value'.
# Without strict test, we'd allow this to occur as long as another object
# satisfied that constraint.
# When we use 'excludes', it applies to the whole observation since this
# is normally the intent. However, here we are excluding values under
# certain context -- "If the 'name' field is 'NAME' then it must contain
# a value field 'VALUE'". Excluding name='NAME' everywhere would
# not permit the context where value='VALUE' which we want to permit.
verifier_builder = jc.ValueObservationVerifierBuilder(
title='Test Conditional', strict=True)
name_eq_pred = jp.PathEqPredicate('name', 'NAME')
value_eq_pred = jp.PathEqPredicate('value', 'VALUE')
conditional = jp.IF(name_eq_pred, value_eq_pred)
pred_list = [jp.PathPredicate('', conditional)]
verifier_builder.add_constraint(conditional)
match_name_value_obj = {'name': 'NAME', 'value': 'VALUE'}
match_value_not_name_obj = {'name': 'GOOD', 'value': 'VALUE'}
match_neither_obj = {'name': 'GOOD', 'value': 'GOOD'}
match_name_not_value_obj = {'name': 'NAME', 'value': 'BAD'} # bad
test_cases = [(True, [match_name_value_obj, match_neither_obj]),
(True, [match_value_not_name_obj, match_neither_obj]),
(False, [match_neither_obj, match_name_not_value_obj])]
context = ExecutionContext()
verifier = verifier_builder.build()
for test in test_cases:
observation = jc.Observation()
result_builder = jc.ObservationVerifyResultBuilder(observation)
expect_valid = test[0]
obj_list = test[1]
observation.add_all_objects(obj_list)
result_builder.add_observation_predicate_result(
jc.ObservationValuePredicate(conditional)(context,
observation))
# All of these tests succeed.
verify_results = result_builder.build(expect_valid)
try:
self._try_verify(context, verifier, observation, expect_valid,
verify_results)
except:
print 'testing {0}'.format(obj_list)
raise
def list_resources(self, type, extra_args=None):
"""Observe resources of a particular type.
This ultimately calls a "gcloud ... |type| list |extra_args|"
"""
self.observer = self.__factory.new_list_resources(type, extra_args)
observation_builder = jc.ValueObservationVerifierBuilder(
'List ' + type, strict=self.__strict)
self.verifier_builder.append_verifier_builder(observation_builder)
return observation_builder
def test_verifier_builder_add_constraint(self):
aA = jp.PathPredicate('a', jp.STR_EQ('A'))
bB = jp.PathPredicate('b', jp.STR_EQ('B'))
builder = jc.ValueObservationVerifierBuilder('TestAddConstraint')
builder.EXPECT(aA).AND(bB)
verifier = builder.build()
self.assertEqual('TestAddConstraint', verifier.title)
self.assertEqual([[
jc.ObservationValuePredicate(aA),
jc.ObservationValuePredicate(bB)
]], verifier.dnf_verifiers)
def show_resource(self, command, resource_name, no_resources_ok=False):
"""Show the OpenStack resource of a particular type.
Args:
command: The openstack command name to run
(e.g. 'security group', 'server').
resource_name: Name of the OpenStack resource
(e.g. Name of a security group or an image).
no_resources_ok: Whether or not the resource is required.
If the resource is not required, 'resource not found' error is
considered successful.
"""
args = ['show', resource_name, '--format', 'json']
cmd = self.__os.build_os_command_args(command,
args,
os_cloud=self.__os.os_cloud)
self.observer = OsObjectObserver(self.__os, cmd)
if no_resources_ok:
error_verifier = cli_agent.CliAgentObservationFailureVerifier(
title='"Not Found" permitted.',
error_regex=
'(?:.* operation: Cannot find .*)|(?:.*\(.*NotFound\)|(Error .*).*)'
)
disjunction_builder = jc.ObservationVerifierBuilder(
'Collect {0} or Not Found'.format(command))
disjunction_builder.append_verifier(error_verifier)
collect_builder = jc.ValueObservationVerifierBuilder(
'Collect {0}'.format(command), strict=self.__strict)
disjunction_builder.append_verifier_builder(collect_builder,
new_term=True)
self.verifier_builder.append_verifier_builder(disjunction_builder,
new_term=True)
else:
collect_builder = jc.ValueObservationVerifierBuilder(
'Collect {0}'.format(command), strict=self.__strict)
self.verifier_builder.append_verifier_builder(collect_builder)
return collect_builder
def get_url_path(self,
path,
allow_http_error_status=None,
observer_factory=None):
"""Perform the observation using HTTP GET on a path.
Args:
path [string]: The path relative to the agent's baseUrl.
allow_http_error_status: [int] If not None then we allow this HTTP error
status as a valid response without further constraints. For example
404 would mean that we permit a 404 error, otherwise we may expect
other constraints on the observed path as a normal clause would
specify.
observer_factory: [callable] ObjectObserver class to use if overriding
This is so you can use HttpResponseObserver instead of
HttpObjectObserver to use at HttpResponse metadata rather than payload.
"""
observer_factory = observer_factory or self.__observer_factory
self.observer = observer_factory(self.__agent, path)
if allow_http_error_status:
error_verifier = HttpObservationFailureVerifier(
'Got HTTP {0} Error'.format(allow_http_error_status),
allow_http_error_status)
disjunction_builder = jc.ObservationVerifierBuilder(
'Get url {0} or {1}'.format(path, allow_http_error_status))
disjunction_builder.append_verifier(error_verifier)
observation_builder = jc.ValueObservationVerifierBuilder(
'Get url {0}'.format(path))
disjunction_builder.append_verifier_builder(observation_builder,
new_term=True)
self.verifier_builder.append_verifier_builder(disjunction_builder,
new_term=True)
else:
observation_builder = jc.ValueObservationVerifierBuilder(
'Get ' + path, strict=self.__strict)
self.verifier_builder.append_verifier_builder(observation_builder)
return observation_builder
def get_marathon_resources(self, type, extra_args=None):
"""Observe resources of a particular type.
This ultimately calls a "dcos marathon |type| |extra_args|"
"""
self.observer = self.__factory.new_get_marathon_resources(
type, action='list', extra_args=extra_args)
get_builder = jc.ValueObservationVerifierBuilder('Get {0} {1}'.format(
type, extra_args),
strict=self.__strict)
self.verifier_builder.append_verifier_builder(get_builder)
return get_builder
def call_method(self, method, *pos_args, **kwargs):
"""Call a method with the provided arguments.
Args:
method: The method to call is probably on the instance returned by
agent.make_client().
pos_args: [list] Positional arguments for method.
kwargs: [kwargs] Depends on the resource being collected.
"""
self.observer = AwsObjectObserver(self.__aws_agent.call_method, method,
*pos_args, **kwargs)
observation_builder = jc.ValueObservationVerifierBuilder(
'Call ' + method.__name__, strict=self.__strict)
self.verifier_builder.append_verifier_builder(observation_builder)
return observation_builder
def call_method_and_extract_singular_response(self, method, *pos_args,
**kwargs):
"""Call the method and return the one result it resturns.
method: The method to call is probably on the instance returned by
agent.make_client().
pos_args: [list] Positional arguments for method.
kwargs: [kwargs] Depends on the resource being collected.
"""
self.observer = AwsObjectObserver(
self.__aws_agent.call_method_and_extract_singular_response, method,
*pos_args, **kwargs)
observation_builder = jc.ValueObservationVerifierBuilder(
'Get result from ' + method.__name__, strict=self.__strict)
self.verifier_builder.append_verifier_builder(observation_builder)
return observation_builder
def test_clause_failure(self):
context = ExecutionContext()
observation = jc.Observation()
observation.add_object('B')
fake_observer = FakeObserver(observation)
eq_A = jp.LIST_MATCHES([jp.STR_EQ('A')])
verifier = jc.ValueObservationVerifierBuilder('Has A').EXPECT(
eq_A).build()
clause = jc.ContractClause('TestClause', fake_observer, verifier)
expect_result = jc.contract.ContractClauseVerifyResult(
False, clause, verifier(context, observation))
result = clause.verify(context)
self.assertEqual(expect_result, result)
self.assertFalse(result)
def test_observation_strict_vs_nonstrict(self):
aA = jp.PathEqPredicate('a', 'A')
bB = jp.PathEqPredicate('b', 'B')
unstrict_object_list = [_NUMBER_DICT, _LETTER_DICT, _MIXED_DICT]
unstrict_observation = jc.Observation()
unstrict_observation.add_all_objects(unstrict_object_list)
strict_object_list = [_LETTER_DICT, {'a': 'A', 'b': 'B', 'x': 'X'}]
strict_observation = jc.Observation()
strict_observation.add_all_objects(strict_object_list)
none_object_list = [_NUMBER_DICT, _MIXED_DICT]
none_observation = jc.Observation()
none_observation.add_all_objects(none_object_list)
# pylint: disable=bad-whitespace
test_cases = [
# Name jc.Observation Strict, Unstrict
#---------------------------------------------------
('Strict', strict_observation, True, True),
('Unstrict', unstrict_observation, False, True),
('None', none_observation, False, False)
]
# For each of the cases, test it with strict and non-strict verification.
context = ExecutionContext()
for test in test_cases:
name = test[0]
observation = test[1]
# For this case, check it strict (2) and unstrict (3).
for index in [2, 3]:
test_strict = index == 2
expected = test[index]
aA_bB = jp.LIST_MATCHES([aA, bB], strict=test_strict)
verifier = (jc.ValueObservationVerifierBuilder(
'verifier').EXPECT(aA_bB).build())
verify_result = verifier(context, observation)
try:
self.assertEqual(expected, verify_result.__nonzero__())
except Exception as e:
print '*** FAILED case={0}:\n{1}'.format(name, e)
print 'GOT {0}'.format(verify_result)
raise
def test_contract_mixed_clause_failure_not_ok(self):
context = ExecutionContext()
observation = jc.Observation()
observation.add_object('A')
fake_observer = FakeObserver(observation)
verifier = (
jc.ValueObservationVerifierBuilder('Has A and B').contains_match(
[jp.STR_EQ('A'), jp.STR_EQ('B')]).build())
clause = jc.ContractClause('TestClause', fake_observer, verifier)
contract = jc.Contract()
contract.add_clause(clause)
expect_result = jc.contract.ContractVerifyResult(
False, [clause.verify(context)])
result = contract.verify(context)
self.assertEqual(expect_result, result)
self.assertFalse(result)
def test_contract_success(self):
context = ExecutionContext()
observation = jc.Observation()
observation.add_object('A')
fake_observer = FakeObserver(observation)
eq_A = jp.LIST_MATCHES([jp.STR_EQ('A')])
verifier = jc.ValueObservationVerifierBuilder('Has A').EXPECT(
eq_A).build()
clause = jc.ContractClause('TestClause', fake_observer, verifier)
contract = jc.Contract()
contract.add_clause(clause)
expect_result = jc.contract.ContractVerifyResult(
True, [clause.verify(context)])
result = contract.verify(context)
self.assertEqual(expect_result, result)
self.assertTrue(result)
def test_contract_observation_failure(self):
context = ExecutionContext()
observation = jc.Observation()
observation.add_error(
jp.PredicateResult(False, comment='Observer Failed'))
fake_observer = FakeObserver(observation)
eq_A = jp.LIST_MATCHES([jp.STR_EQ('A')])
verifier = jc.ValueObservationVerifierBuilder('Has A').EXPECT(
eq_A).build()
clause = jc.ContractClause('TestClause', fake_observer, verifier)
contract = jc.Contract()
contract.add_clause(clause)
expect_result = jc.contract.ContractVerifyResult(
False, [clause.verify(context)])
result = contract.verify(context)
self.assertEqual(expect_result, result)
self.assertFalse(result)
def test_multiple_required(self):
context = ExecutionContext()
observation = jc.Observation()
observation.add_object('A')
observation.add_object('B')
observation.add_object('C')
fake_observer = FakeObserver(observation)
eq_A_or_B = jp.OR([jp.STR_EQ('A'), jp.STR_EQ('B')])
builder = jc.ValueObservationVerifierBuilder('Test Multiple')
builder.contains_path_pred(None, eq_A_or_B, min=2)
clause = jc.ContractClause('TestClause', fake_observer, builder.build())
contract = jc.Contract()
contract.add_clause(clause)
expect_result = jc.contract.ContractVerifyResult(
True, [clause.verify(context)])
result = contract.verify(context)
self.assertEqual(expect_result, result)
self.assertEqual(True, result.valid)
def list_bucket(self, bucket, path_prefix, with_versions=False, **kwargs):
"""List the bucket contents at a given path.
Args:
bucket: [string] The name of the bucket to list.
path_prefix: [string] The object path prefix within the bucket.
with_versions: [boolean] If False then just latest version,
otherwise return all versions of all files.
"""
self.observer = GcpObjectObserver(self.gcp_agent.list_bucket,
bucket=bucket,
path_prefix=path_prefix,
with_versions=with_versions,
**kwargs)
title = ('List {0} (with_versions={1}, {2})'.format(
'/'.join([bucket, path_prefix]), with_versions, kwargs))
observation_builder = jc.ValueObservationVerifierBuilder(
title, strict=self.__strict)
self.verifier_builder.append_verifier_builder(observation_builder)
return observation_builder
def test_object_observation_verifier_some_but_not_all_constraints_found(
self):
context = ExecutionContext()
pred_list = [
jp.PathPredicate('a', jp.STR_EQ('NOT_FOUND')),
jp.PathPredicate('b', jp.STR_EQ('B'))
]
# This is our object verifier for these tests.
verifier = (jc.ValueObservationVerifierBuilder(
'Find one of two').contains_match(pred_list).build())
test_cases = [('array', _DICT_ARRAY), ('dict', _LETTER_DICT),
('array', _DICT_ARRAY), ('multi', _MULTI_ARRAY)]
for test in test_cases:
observation = jc.Observation()
builder = jc.ObservationVerifyResultBuilder(observation)
obj = test[1]
if isinstance(test, list):
observation.add_all_objects(obj)
else:
observation.add_object(obj)
for pred in pred_list:
pred_result = jp.PathPredicate('', pred)(context,
observation.objects)
builder.add_path_predicate_result(
pred(context, observation.objects))
# None of these tests succeed.
verify_results = builder.build(False)
try:
self._try_verify(context, verifier, observation, False,
verify_results)
except:
print 'testing {0}'.format(test[0])
raise
def retrieve_content(self, bucket, path, transform=None):
"""Retrieve actual contents of file at path.
Args:
bucket: [string] The gcs bucket containing the path.
path: [string] The path within the gcs bucket to retreive.
no_resource_ok: [bool] True if a 404/resource not found is ok.
"""
self.observer = GcpObjectObserver(self.gcp_agent.retrieve_content,
bucket=bucket,
path=path,
transform=transform)
# Construct the rule <user supplied specification>
# Here we return the user supplied specification clause as before.
# But we dont need the outer disjunction, so we also add it directly
# as the verifier.
retrieve_builder = jc.ValueObservationVerifierBuilder(
'Retrieve bucket={0} path={1}'.format(bucket, path),
strict=self.__strict)
self.verifier_builder.append_verifier_builder(retrieve_builder)
return retrieve_builder
def test_object_observation_verifier_multiple_constraint_found(self):
context = ExecutionContext()
pred_list = [
jp.PathPredicate('a', jp.STR_EQ('A')),
jp.PathPredicate('b', jp.STR_EQ('B'))
]
# This is our object verifier for these tests.
verifier = (jc.ValueObservationVerifierBuilder('Find Both').EXPECT(
pred_list[0]).AND(pred_list[1]).build())
test_cases = [('dict', _LETTER_DICT), ('array', _DICT_ARRAY),
('multi', _MULTI_ARRAY)]
for test in test_cases:
observation = jc.Observation()
builder = jc.ObservationVerifyResultBuilder(observation)
obj = test[1]
if isinstance(test, list):
observation.add_all_objects(obj)
else:
observation.add_object(obj)
for pred in pred_list:
builder.add_observation_predicate_result(
jc.ObservationValuePredicate(pred)(context, observation))
# All of these tests succeed.
verify_results = builder.build(True)
self.assertEqual([], verify_results.failed_constraints)
try:
self._try_verify(context, verifier, observation, True,
verify_results)
except:
print 'testing {0}'.format(test[0])
raise
