def collect_resources(self, aws_module, command, args=None, filter=None, no_resources_ok=False): """Collect the AWS resources of a particular type. Args: aws_module: The aws program module name we're looking in (e.g. 'ec2') command: The aws command name to run (e.g. 'describe-instances') args: An array of strings containing the remaining aws command parameters. filter: If provided, a filter to use for refining the collection. no_resources_ok: Whether or not the resource is required. If the resource is not required, 'resource not found' error is considered successful. """ args = args or [] cmd = self.__aws.build_aws_command_args(command, args, aws_module=aws_module, profile=self.__aws.profile) self.observer = AwsObjectObserver(self.__aws, cmd) if no_resources_ok: error_verifier = cli_agent.CliAgentObservationFailureVerifier( title='"Not Found" permitted.', error_regex= '(?:.* operation: Cannot find .*)|(?:.*\(.*NotFound\).*)') disjunction_builder = jc.ObservationVerifierBuilder( 'Collect {0} or Not Found'.format(command)) disjunction_builder.append_verifier(error_verifier) collect_builder = jc.ValueObservationVerifierBuilder( 'Collect {0}'.format(command), strict=self.__strict) disjunction_builder.append_verifier_builder(collect_builder, new_term=True) self.verifier_builder.append_verifier_builder(disjunction_builder, new_term=True) else: collect_builder = jc.ValueObservationVerifierBuilder( 'Collect {0}'.format(command), strict=self.__strict) self.verifier_builder.append_verifier_builder(collect_builder) return collect_builder
def collect_resources(self, command, args=None, filter=None, no_resources_ok=False): """Collect the OpenStack resources of a particular type. Args: command: The openstack command name to run (e.g. 'image', 'server') args: An array of strings containing the remaining openstack command parameters. filter: If provided, a filter to use for refining the collection. no_resources_ok: Whether or not the resource is required. If the resource is not required, 'resource not found' error is considered successful. """ args = args or [] cmd = self.__os.build_os_command_args(command, args, os_cloud=self.__os.os_cloud) self.observer = OsObjectObserver(self.__os, cmd) if no_resources_ok: error_verifier = cli_agent.CliAgentObservationFailureVerifier( title='"Not Found" permitted.', error_regex= '(?:.* operation: Cannot find .*)|(?:.*\(.*NotFound\)|(Error .*).*)' ) disjunction_builder = jc.ObservationVerifierBuilder( 'Collect {0} or Not Found'.format(command)) disjunction_builder.append_verifier(error_verifier) collect_builder = jc.ValueObservationVerifierBuilder( 'Collect {0}'.format(command), strict=self.__strict) disjunction_builder.append_verifier_builder(collect_builder, new_term=True) self.verifier_builder.append_verifier_builder(disjunction_builder, new_term=True) else: collect_builder = jc.ValueObservationVerifierBuilder( 'Collect {0}'.format(command), strict=self.__strict) self.verifier_builder.append_verifier_builder(collect_builder) return collect_builder
def test_verifier_builder_add_constraint(self): aA = jp.PathPredicate('a', jp.STR_EQ('A')) bB = jp.PathPredicate('b', jp.STR_EQ('B')) builder = jc.ValueObservationVerifierBuilder('TestAddConstraint') builder.add_constraint(aA) builder.add_constraint(bB) verifier = builder.build() self.assertEqual('TestAddConstraint', verifier.title) self.assertEqual([aA, bB], verifier.constraints)
def list_resource(self, resource_type, **kwargs): """Observe resources of a particular type.""" self.observer = GcpObjectObserver(self.__gcp_agent.list_resource, resource_type=resource_type, **kwargs) observation_builder = jc.ValueObservationVerifierBuilder( 'List ' + resource_type, strict=self.__strict) self.verifier_builder.append_verifier_builder(observation_builder) return observation_builder
def test_verifier_builder_contains_pred_list(self): aA = jp.PathPredicate('a', jp.STR_EQ('A')) bB = jp.PathPredicate('b', jp.STR_EQ('B')) builder = jc.ValueObservationVerifierBuilder('TestContainsGroup') builder.contains_path_pred('a', jp.STR_EQ('A')) builder.contains_path_pred('b', jp.STR_EQ('B')) verifier = builder.build() count_aA = jp.CardinalityPredicate(aA, 1, None) count_bB = jp.CardinalityPredicate(bB, 1, None) self.assertEqual([count_aA, count_bB], verifier.constraints)
def test_object_observation_verifier_with_conditional(self): # We need strict True here because we want each object to pass # the constraint test. Otherwise, if any object passes, then the whole # observation would pass. This causes a problem when we say that # we dont ever want to see 'name' unless it has a particular 'value'. # Without strict test, we'd allow this to occur as long as another object # satisfied that constraint. # When we use 'excludes', it applies to the whole observation since this # is normally the intent. However, here we are excluding values under # certain context -- "If the 'name' field is 'NAME' then it must contain # a value field 'VALUE'". Excluding name='NAME' everywhere would # not permit the context where value='VALUE' which we want to permit. verifier_builder = jc.ValueObservationVerifierBuilder( title='Test Conditional', strict=True) name_eq_pred = jp.PathEqPredicate('name', 'NAME') value_eq_pred = jp.PathEqPredicate('value', 'VALUE') conditional = jp.IF(name_eq_pred, value_eq_pred) pred_list = [jp.PathPredicate('', conditional)] verifier_builder.add_constraint(conditional) match_name_value_obj = {'name': 'NAME', 'value': 'VALUE'} match_value_not_name_obj = {'name': 'GOOD', 'value': 'VALUE'} match_neither_obj = {'name': 'GOOD', 'value': 'GOOD'} match_name_not_value_obj = {'name': 'NAME', 'value': 'BAD'} # bad test_cases = [(True, [match_name_value_obj, match_neither_obj]), (True, [match_value_not_name_obj, match_neither_obj]), (False, [match_neither_obj, match_name_not_value_obj])] context = ExecutionContext() verifier = verifier_builder.build() for test in test_cases: observation = jc.Observation() result_builder = jc.ObservationVerifyResultBuilder(observation) expect_valid = test[0] obj_list = test[1] observation.add_all_objects(obj_list) result_builder.add_observation_predicate_result( jc.ObservationValuePredicate(conditional)(context, observation)) # All of these tests succeed. verify_results = result_builder.build(expect_valid) try: self._try_verify(context, verifier, observation, expect_valid, verify_results) except: print 'testing {0}'.format(obj_list) raise
def list_resources(self, type, extra_args=None): """Observe resources of a particular type. This ultimately calls a "gcloud ... |type| list |extra_args|" """ self.observer = self.__factory.new_list_resources(type, extra_args) observation_builder = jc.ValueObservationVerifierBuilder( 'List ' + type, strict=self.__strict) self.verifier_builder.append_verifier_builder(observation_builder) return observation_builder
def test_verifier_builder_add_constraint(self): aA = jp.PathPredicate('a', jp.STR_EQ('A')) bB = jp.PathPredicate('b', jp.STR_EQ('B')) builder = jc.ValueObservationVerifierBuilder('TestAddConstraint') builder.EXPECT(aA).AND(bB) verifier = builder.build() self.assertEqual('TestAddConstraint', verifier.title) self.assertEqual([[ jc.ObservationValuePredicate(aA), jc.ObservationValuePredicate(bB) ]], verifier.dnf_verifiers)
def show_resource(self, command, resource_name, no_resources_ok=False): """Show the OpenStack resource of a particular type. Args: command: The openstack command name to run (e.g. 'security group', 'server'). resource_name: Name of the OpenStack resource (e.g. Name of a security group or an image). no_resources_ok: Whether or not the resource is required. If the resource is not required, 'resource not found' error is considered successful. """ args = ['show', resource_name, '--format', 'json'] cmd = self.__os.build_os_command_args(command, args, os_cloud=self.__os.os_cloud) self.observer = OsObjectObserver(self.__os, cmd) if no_resources_ok: error_verifier = cli_agent.CliAgentObservationFailureVerifier( title='"Not Found" permitted.', error_regex= '(?:.* operation: Cannot find .*)|(?:.*\(.*NotFound\)|(Error .*).*)' ) disjunction_builder = jc.ObservationVerifierBuilder( 'Collect {0} or Not Found'.format(command)) disjunction_builder.append_verifier(error_verifier) collect_builder = jc.ValueObservationVerifierBuilder( 'Collect {0}'.format(command), strict=self.__strict) disjunction_builder.append_verifier_builder(collect_builder, new_term=True) self.verifier_builder.append_verifier_builder(disjunction_builder, new_term=True) else: collect_builder = jc.ValueObservationVerifierBuilder( 'Collect {0}'.format(command), strict=self.__strict) self.verifier_builder.append_verifier_builder(collect_builder) return collect_builder
def get_url_path(self, path, allow_http_error_status=None, observer_factory=None): """Perform the observation using HTTP GET on a path. Args: path [string]: The path relative to the agent's baseUrl. allow_http_error_status: [int] If not None then we allow this HTTP error status as a valid response without further constraints. For example 404 would mean that we permit a 404 error, otherwise we may expect other constraints on the observed path as a normal clause would specify. observer_factory: [callable] ObjectObserver class to use if overriding This is so you can use HttpResponseObserver instead of HttpObjectObserver to use at HttpResponse metadata rather than payload. """ observer_factory = observer_factory or self.__observer_factory self.observer = observer_factory(self.__agent, path) if allow_http_error_status: error_verifier = HttpObservationFailureVerifier( 'Got HTTP {0} Error'.format(allow_http_error_status), allow_http_error_status) disjunction_builder = jc.ObservationVerifierBuilder( 'Get url {0} or {1}'.format(path, allow_http_error_status)) disjunction_builder.append_verifier(error_verifier) observation_builder = jc.ValueObservationVerifierBuilder( 'Get url {0}'.format(path)) disjunction_builder.append_verifier_builder(observation_builder, new_term=True) self.verifier_builder.append_verifier_builder(disjunction_builder, new_term=True) else: observation_builder = jc.ValueObservationVerifierBuilder( 'Get ' + path, strict=self.__strict) self.verifier_builder.append_verifier_builder(observation_builder) return observation_builder
def get_marathon_resources(self, type, extra_args=None): """Observe resources of a particular type. This ultimately calls a "dcos marathon |type| |extra_args|" """ self.observer = self.__factory.new_get_marathon_resources( type, action='list', extra_args=extra_args) get_builder = jc.ValueObservationVerifierBuilder('Get {0} {1}'.format( type, extra_args), strict=self.__strict) self.verifier_builder.append_verifier_builder(get_builder) return get_builder
def call_method(self, method, *pos_args, **kwargs): """Call a method with the provided arguments. Args: method: The method to call is probably on the instance returned by agent.make_client(). pos_args: [list] Positional arguments for method. kwargs: [kwargs] Depends on the resource being collected. """ self.observer = AwsObjectObserver(self.__aws_agent.call_method, method, *pos_args, **kwargs) observation_builder = jc.ValueObservationVerifierBuilder( 'Call ' + method.__name__, strict=self.__strict) self.verifier_builder.append_verifier_builder(observation_builder) return observation_builder
def call_method_and_extract_singular_response(self, method, *pos_args, **kwargs): """Call the method and return the one result it resturns. method: The method to call is probably on the instance returned by agent.make_client(). pos_args: [list] Positional arguments for method. kwargs: [kwargs] Depends on the resource being collected. """ self.observer = AwsObjectObserver( self.__aws_agent.call_method_and_extract_singular_response, method, *pos_args, **kwargs) observation_builder = jc.ValueObservationVerifierBuilder( 'Get result from ' + method.__name__, strict=self.__strict) self.verifier_builder.append_verifier_builder(observation_builder) return observation_builder
def test_clause_failure(self): context = ExecutionContext() observation = jc.Observation() observation.add_object('B') fake_observer = FakeObserver(observation) eq_A = jp.LIST_MATCHES([jp.STR_EQ('A')]) verifier = jc.ValueObservationVerifierBuilder('Has A').EXPECT( eq_A).build() clause = jc.ContractClause('TestClause', fake_observer, verifier) expect_result = jc.contract.ContractClauseVerifyResult( False, clause, verifier(context, observation)) result = clause.verify(context) self.assertEqual(expect_result, result) self.assertFalse(result)
def test_observation_strict_vs_nonstrict(self): aA = jp.PathEqPredicate('a', 'A') bB = jp.PathEqPredicate('b', 'B') unstrict_object_list = [_NUMBER_DICT, _LETTER_DICT, _MIXED_DICT] unstrict_observation = jc.Observation() unstrict_observation.add_all_objects(unstrict_object_list) strict_object_list = [_LETTER_DICT, {'a': 'A', 'b': 'B', 'x': 'X'}] strict_observation = jc.Observation() strict_observation.add_all_objects(strict_object_list) none_object_list = [_NUMBER_DICT, _MIXED_DICT] none_observation = jc.Observation() none_observation.add_all_objects(none_object_list) # pylint: disable=bad-whitespace test_cases = [ # Name jc.Observation Strict, Unstrict #--------------------------------------------------- ('Strict', strict_observation, True, True), ('Unstrict', unstrict_observation, False, True), ('None', none_observation, False, False) ] # For each of the cases, test it with strict and non-strict verification. context = ExecutionContext() for test in test_cases: name = test[0] observation = test[1] # For this case, check it strict (2) and unstrict (3). for index in [2, 3]: test_strict = index == 2 expected = test[index] aA_bB = jp.LIST_MATCHES([aA, bB], strict=test_strict) verifier = (jc.ValueObservationVerifierBuilder( 'verifier').EXPECT(aA_bB).build()) verify_result = verifier(context, observation) try: self.assertEqual(expected, verify_result.__nonzero__()) except Exception as e: print '*** FAILED case={0}:\n{1}'.format(name, e) print 'GOT {0}'.format(verify_result) raise
def test_contract_mixed_clause_failure_not_ok(self): context = ExecutionContext() observation = jc.Observation() observation.add_object('A') fake_observer = FakeObserver(observation) verifier = ( jc.ValueObservationVerifierBuilder('Has A and B').contains_match( [jp.STR_EQ('A'), jp.STR_EQ('B')]).build()) clause = jc.ContractClause('TestClause', fake_observer, verifier) contract = jc.Contract() contract.add_clause(clause) expect_result = jc.contract.ContractVerifyResult( False, [clause.verify(context)]) result = contract.verify(context) self.assertEqual(expect_result, result) self.assertFalse(result)
def test_contract_success(self): context = ExecutionContext() observation = jc.Observation() observation.add_object('A') fake_observer = FakeObserver(observation) eq_A = jp.LIST_MATCHES([jp.STR_EQ('A')]) verifier = jc.ValueObservationVerifierBuilder('Has A').EXPECT( eq_A).build() clause = jc.ContractClause('TestClause', fake_observer, verifier) contract = jc.Contract() contract.add_clause(clause) expect_result = jc.contract.ContractVerifyResult( True, [clause.verify(context)]) result = contract.verify(context) self.assertEqual(expect_result, result) self.assertTrue(result)
def test_contract_observation_failure(self): context = ExecutionContext() observation = jc.Observation() observation.add_error( jp.PredicateResult(False, comment='Observer Failed')) fake_observer = FakeObserver(observation) eq_A = jp.LIST_MATCHES([jp.STR_EQ('A')]) verifier = jc.ValueObservationVerifierBuilder('Has A').EXPECT( eq_A).build() clause = jc.ContractClause('TestClause', fake_observer, verifier) contract = jc.Contract() contract.add_clause(clause) expect_result = jc.contract.ContractVerifyResult( False, [clause.verify(context)]) result = contract.verify(context) self.assertEqual(expect_result, result) self.assertFalse(result)
def test_multiple_required(self): context = ExecutionContext() observation = jc.Observation() observation.add_object('A') observation.add_object('B') observation.add_object('C') fake_observer = FakeObserver(observation) eq_A_or_B = jp.OR([jp.STR_EQ('A'), jp.STR_EQ('B')]) builder = jc.ValueObservationVerifierBuilder('Test Multiple') builder.contains_path_pred(None, eq_A_or_B, min=2) clause = jc.ContractClause('TestClause', fake_observer, builder.build()) contract = jc.Contract() contract.add_clause(clause) expect_result = jc.contract.ContractVerifyResult( True, [clause.verify(context)]) result = contract.verify(context) self.assertEqual(expect_result, result) self.assertEqual(True, result.valid)
def list_bucket(self, bucket, path_prefix, with_versions=False, **kwargs): """List the bucket contents at a given path. Args: bucket: [string] The name of the bucket to list. path_prefix: [string] The object path prefix within the bucket. with_versions: [boolean] If False then just latest version, otherwise return all versions of all files. """ self.observer = GcpObjectObserver(self.gcp_agent.list_bucket, bucket=bucket, path_prefix=path_prefix, with_versions=with_versions, **kwargs) title = ('List {0} (with_versions={1}, {2})'.format( '/'.join([bucket, path_prefix]), with_versions, kwargs)) observation_builder = jc.ValueObservationVerifierBuilder( title, strict=self.__strict) self.verifier_builder.append_verifier_builder(observation_builder) return observation_builder
def test_object_observation_verifier_some_but_not_all_constraints_found( self): context = ExecutionContext() pred_list = [ jp.PathPredicate('a', jp.STR_EQ('NOT_FOUND')), jp.PathPredicate('b', jp.STR_EQ('B')) ] # This is our object verifier for these tests. verifier = (jc.ValueObservationVerifierBuilder( 'Find one of two').contains_match(pred_list).build()) test_cases = [('array', _DICT_ARRAY), ('dict', _LETTER_DICT), ('array', _DICT_ARRAY), ('multi', _MULTI_ARRAY)] for test in test_cases: observation = jc.Observation() builder = jc.ObservationVerifyResultBuilder(observation) obj = test[1] if isinstance(test, list): observation.add_all_objects(obj) else: observation.add_object(obj) for pred in pred_list: pred_result = jp.PathPredicate('', pred)(context, observation.objects) builder.add_path_predicate_result( pred(context, observation.objects)) # None of these tests succeed. verify_results = builder.build(False) try: self._try_verify(context, verifier, observation, False, verify_results) except: print 'testing {0}'.format(test[0]) raise
def retrieve_content(self, bucket, path, transform=None): """Retrieve actual contents of file at path. Args: bucket: [string] The gcs bucket containing the path. path: [string] The path within the gcs bucket to retreive. no_resource_ok: [bool] True if a 404/resource not found is ok. """ self.observer = GcpObjectObserver(self.gcp_agent.retrieve_content, bucket=bucket, path=path, transform=transform) # Construct the rule <user supplied specification> # Here we return the user supplied specification clause as before. # But we dont need the outer disjunction, so we also add it directly # as the verifier. retrieve_builder = jc.ValueObservationVerifierBuilder( 'Retrieve bucket={0} path={1}'.format(bucket, path), strict=self.__strict) self.verifier_builder.append_verifier_builder(retrieve_builder) return retrieve_builder
def test_object_observation_verifier_multiple_constraint_found(self): context = ExecutionContext() pred_list = [ jp.PathPredicate('a', jp.STR_EQ('A')), jp.PathPredicate('b', jp.STR_EQ('B')) ] # This is our object verifier for these tests. verifier = (jc.ValueObservationVerifierBuilder('Find Both').EXPECT( pred_list[0]).AND(pred_list[1]).build()) test_cases = [('dict', _LETTER_DICT), ('array', _DICT_ARRAY), ('multi', _MULTI_ARRAY)] for test in test_cases: observation = jc.Observation() builder = jc.ObservationVerifyResultBuilder(observation) obj = test[1] if isinstance(test, list): observation.add_all_objects(obj) else: observation.add_object(obj) for pred in pred_list: builder.add_observation_predicate_result( jc.ObservationValuePredicate(pred)(context, observation)) # All of these tests succeed. verify_results = builder.build(True) self.assertEqual([], verify_results.failed_constraints) try: self._try_verify(context, verifier, observation, True, verify_results) except: print 'testing {0}'.format(test[0]) raise