def test_validate_elements_of_collection_with_elements_referenced_by_out_of_range_json_pointer(self): metadata = """ <Schema xmlns="http://docs.oasis-open.org/odata/ns/edm" Namespace="N"> <EntityType Name="Outer" Abstract="false"> <NavigationProperty Name="MyCollection" Type="Collection(ReferencedEntity)" ContainsTarget="true"> <Annotation Term="OData.Permissions" EnumMember="OData.Permission/Read"/> </NavigationProperty> </EntityType> <EntityType Name="ReferencedEntity" Abstract="false"> <Property Name="name" Type="Edm.String"/> </EntityType> </Schema> """ resource = \ { "@odata.id": "resource.id", "@odata.type": "N.Outer", "MyCollection": [ { "@odata.id": "/outer#MyCollection/0" }, { "@odata.id": "/outer#MyCollection/2" # pointer out of range } ] } metadata_manager = MetadataManager(["qualifier"]) metadata_container = metadata_manager.read_metadata_from_strings("Unknown", metadata) discovery_container = DiscoveryContainer() configuration = Configuration(**dict(UseSSL='True', ApiEndpoint=API_ENDPOINT)) api_explorer = ApiExplorer(metadata_container, configuration) with mock.patch('cts_core.commons.api_caller.ApiCaller.get_resource') as get_resource: get_resource.side_effect = [(Link('url1', 'netloc'), RequestStatus.SUCCESS, ReturnCodes.OK, resource, Link('url1', 'netloc')), (Link('url2', 'netloc'), RequestStatus.SUCCESS, ReturnCodes.OK, resource, Link('url2', 'netloc')), (Link('url3', 'netloc'), RequestStatus.SUCCESS, ReturnCodes.OK, resource, Link('url3', 'netloc'))] with StdoutCapture() as output: api_explorer.discover("/outer", "N.Outer", discovery_container) self.assertEqual(3, get_resource.call_count) self.assertIn( 'ERROR::JSON pointer exception while dereferencing /MyCollection/2', ';'.join(output)) requirements = [ ] validator = MetadataGetValidator(metadata_container, requirements) with StdoutCapture() as output: self.assertEqual(ValidationStatus.PASSED, validator.validate(discovery_container)) self.assertEqual(0, len(re.findall('ERROR::', ';'.join(output))))
def test_should_pass_if_entity_type_declared_on_ignore_list_with_asterix(self): metadata_manager = MetadataManager(['qualifier'], ignore_types=["N.*"]) self.metadata_container = metadata_manager.read_metadata_from_strings('Unknown', METADATA) self.discovery_container = DiscoveryContainer() self.discovery_container.add_resource(ApiResource('/redfish/resource1', 'netloc', RESOURCE_WITH_INVALID_PROPERTY_VALUE, '#N.R')) requirements = [] validator = MetadataGetValidator(self.metadata_container, requirements) self.assertEqual(ValidationStatus.PASSED, validator.validate(self.discovery_container))
def test_should_pass_if_property_of_incorrect_type_declared_on_ignore_list(self): metadata_manager = MetadataManager(["qualifier"], ignore_types=["N.ObjectWithNumericProperty"]) self.metadata_container = metadata_manager.read_metadata_from_strings("Unknown", METADATA) self.discovery_container = DiscoveryContainer() self.discovery_container.add_resource(ApiResource("/redfish/resource1", 'netloc', RESOURCE_WITH_INVALID_PROPERTY_VALUE, "#N.R")) requirements = [ ] validator = MetadataGetValidator(self.metadata_container, requirements) self.assertEqual(ValidationStatus.PASSED, validator.validate(self.discovery_container))
def test_should_pass_when_unknown_complex_type_mapped_on_known_type(self): metadata_manager = MetadataManager(["qualifier"], map_types={"A" : "B", "unknown.complex.type" : "N.ObjectWithNumericProperty"}) self.metadata_container = metadata_manager.read_metadata_from_strings("Unknown", METADATA) self.discovery_container = DiscoveryContainer() self.discovery_container.add_resource(ApiResource("/redfish/resource1", 'netloc', RESOURCE_WITH_UNKNOWN_COMPLEX_TYPE, "#N.R")) requirements = [ ] validator = MetadataGetValidator(self.metadata_container, requirements) self.assertEqual(ValidationStatus.PASSED, validator.validate(self.discovery_container))
def test_should_pass_with_correct_resource(self): metadata_manager = MetadataManager(["qualifier"]) self.metadata_container = metadata_manager.read_metadata_from_strings("Unknown", METADATA) self.discovery_container = DiscoveryContainer() self.discovery_container.add_resource(ApiResource("/redfish/resource1", 'netloc', RESOURCE_CORRECT, "#N.R")) requirements = [ ] validator = MetadataGetValidator(self.metadata_container, requirements) with StdoutCapture() as output: self.assertEqual(ValidationStatus.PASSED, validator.validate(self.discovery_container))
def test_should_pass_if_entity_type_declared_on_ignore_list(self): metadata_manager = MetadataManager(["qualifier"], ignore_types=["N.R"]) self.metadata_container = metadata_manager.read_metadata_from_strings( "Unknown", METADATA) self.discovery_container = DiscoveryContainer() self.discovery_container.add_resource( ApiResource("/redfish/resource1", 'netloc', RESOURCE_WITH_INVALID_PROPERTY_VALUE, "#N.R")) requirements = [] validator = MetadataGetValidator(self.metadata_container, requirements) self.assertEqual(ValidationStatus.PASSED, validator.validate(self.discovery_container))
def test_should_pass_with_correct_resource(self): metadata_manager = MetadataManager(["qualifier"]) self.metadata_container = metadata_manager.read_metadata_from_strings( "Unknown", METADATA) self.discovery_container = DiscoveryContainer() self.discovery_container.add_resource( ApiResource("/redfish/resource1", 'netloc', RESOURCE_CORRECT, "#N.R")) requirements = [] validator = MetadataGetValidator(self.metadata_container, requirements) with StdoutCapture() as output: self.assertEqual(ValidationStatus.PASSED, validator.validate(self.discovery_container))
def test_should_fail_if_property_of_incorrect_type(self): metadata_manager = MetadataManager(["qualifier"]) self.metadata_container = metadata_manager.read_metadata_from_strings("Unknown", METADATA) self.discovery_container = DiscoveryContainer() self.discovery_container.add_resource(ApiResource("/redfish/resource1", 'netloc', RESOURCE_WITH_INVALID_PROPERTY_VALUE, "#N.R")) requirements = [ ] validator = MetadataGetValidator(self.metadata_container, requirements) with StdoutCapture() as output: self.assertEqual(ValidationStatus.FAILED, validator.validate(self.discovery_container)) self.assertIn('Property resource.id->ObjectWithNumber is expected to be json object', '\n'.join(output))
def test_should_fail_bacause_of_unknown_complex_type(self): metadata_manager = MetadataManager(["qualifier"]) self.metadata_container = metadata_manager.read_metadata_from_strings("Unknown", METADATA) self.discovery_container = DiscoveryContainer() self.discovery_container.add_resource(ApiResource("/redfish/resource1", 'netloc', RESOURCE_WITH_UNKNOWN_COMPLEX_TYPE, "#N.R")) requirements = [ ] validator = MetadataGetValidator(self.metadata_container, requirements) with StdoutCapture() as output: self.assertEqual(ValidationStatus.FAILED, validator.validate(self.discovery_container)) self.assertIn('Reference to unknown type unknown.complex.type', '\n'.join(output))
def test_should_fail_bacause_of_unknown_entity_type(self): metadata_manager = MetadataManager(["qualifier"]) self.metadata_container = metadata_manager.read_metadata_from_strings("Unknown", METADATA) self.discovery_container = DiscoveryContainer() self.discovery_container.add_resource(ApiResource("/redfish/resource1", 'netloc', RESOURCE_WITH_UNKNOWN_ENTITY_TYPE, "#N.R")) requirements = [ ] validator = MetadataGetValidator(self.metadata_container, requirements) with StdoutCapture() as output: self.assertEqual(ValidationStatus.FAILED, validator.validate(self.discovery_container)) self.assertIn("ERROR::Unknown entity type 'unknown.entity.type'", output.raw)
def test_should_fail_if_non_nullable_property_is_null(self): metadata_manager = MetadataManager(["qualifier"]) self.metadata_container = metadata_manager.read_metadata_from_strings("Unknown", METADATA) self.discovery_container = DiscoveryContainer() self.discovery_container.add_resource(ApiResource("/redfish/resource1", 'netloc', RESOURCE_WITH_NULL_ON_NON_NULLABLE, "#N.R")) requirements = [ ] validator = MetadataGetValidator(self.metadata_container, requirements) with StdoutCapture() as output: self.assertEqual(ValidationStatus.FAILED, validator.validate(self.discovery_container)) self.assertIn('ERROR::Property resource.id->CapacityGiB cannot be null', '\n'.join(output)) self.assertIn('ERROR::Property resource.id->ObjectWithNumber->Number cannot be null', '\n'.join(output))
def test_should_validate_dynamic_properties_at_entity_level(self): metadata_manager = MetadataManager(["qualifier"]) self.metadata_container = metadata_manager.read_metadata_from_strings("Unknown", METADATA) self.discovery_container = DiscoveryContainer() self.discovery_container.add_resource(ApiResource("/redfish/resource1", 'netloc', RESOURCE_WITH_INCOMPLETE_DYNAMIC_PROPERTY, "#N.R")) requirements = [ ] validator = MetadataGetValidator(self.metadata_container, requirements) with StdoutCapture() as output: self.assertEqual(ValidationStatus.FAILED, validator.validate(self.discovery_container)) self.assertIn('ERROR::Required property resource.id->aaaaaaaa->Obligatory not present in the resource', '\n'.join(output))
def test_should_pass_when_unknown_entity_type_mapped_to_known_one(self): metadata_manager = MetadataManager(["qualifier"], map_types={ "A": "B", "unknown.entity.type": "N.R" }) self.metadata_container = metadata_manager.read_metadata_from_strings( "Unknown", METADATA) self.discovery_container = DiscoveryContainer() self.discovery_container.add_resource( ApiResource("/redfish/resource1", 'netloc', RESOURCE_WITH_UNKNOWN_ENTITY_TYPE, "#N.R")) requirements = [] validator = MetadataGetValidator(self.metadata_container, requirements) self.assertEqual(ValidationStatus.PASSED, validator.validate(self.discovery_container))
def test_should_fail_bacause_of_unknown_complex_type(self): metadata_manager = MetadataManager(["qualifier"]) self.metadata_container = metadata_manager.read_metadata_from_strings( "Unknown", METADATA) self.discovery_container = DiscoveryContainer() self.discovery_container.add_resource( ApiResource("/redfish/resource1", 'netloc', RESOURCE_WITH_UNKNOWN_COMPLEX_TYPE, "#N.R")) requirements = [] validator = MetadataGetValidator(self.metadata_container, requirements) with StdoutCapture() as output: self.assertEqual(ValidationStatus.FAILED, validator.validate(self.discovery_container)) self.assertIn('Reference to unknown type unknown.complex.type', '\n'.join(output))
def test_should_fail_bacause_of_unknown_entity_type(self): metadata_manager = MetadataManager(["qualifier"]) self.metadata_container = metadata_manager.read_metadata_from_strings( "Unknown", METADATA) self.discovery_container = DiscoveryContainer() self.discovery_container.add_resource( ApiResource("/redfish/resource1", 'netloc', RESOURCE_WITH_UNKNOWN_ENTITY_TYPE, "#N.R")) requirements = [] validator = MetadataGetValidator(self.metadata_container, requirements) with StdoutCapture() as output: self.assertEqual(ValidationStatus.FAILED, validator.validate(self.discovery_container)) self.assertIn("ERROR::Unknown entity type 'unknown.entity.type'", output.raw)
def run(self): print "TEST_CASE::API crawling" api_explorer = ApiExplorer(self.metadata_container, self.configuration) discovery_container, status = api_explorer.discover( MetadataConstants.SERVICE_ROOT_URI, MetadataConstants.SERVICE_ROOT) print "STATUS::{status}".format(status=status) MetadataGetValidator( self.metadata_container, RequiredTypes_PSME_2_1).validate(discovery_container)
def test_should_fail_if_property_of_incorrect_type(self): metadata_manager = MetadataManager(["qualifier"]) self.metadata_container = metadata_manager.read_metadata_from_strings( "Unknown", METADATA) self.discovery_container = DiscoveryContainer() self.discovery_container.add_resource( ApiResource("/redfish/resource1", 'netloc', RESOURCE_WITH_INVALID_PROPERTY_VALUE, "#N.R")) requirements = [] validator = MetadataGetValidator(self.metadata_container, requirements) with StdoutCapture() as output: self.assertEqual(ValidationStatus.FAILED, validator.validate(self.discovery_container)) self.assertIn( 'Property resource.id->ObjectWithNumber is expected to be json object', '\n'.join(output))
def test_should_suggest_other_names_for_additionals(self): metadata_manager = MetadataManager(["qualifier"]) self.metadata_container = metadata_manager.read_metadata_from_strings("Unknown", METADATA) self.discovery_container = DiscoveryContainer() self.discovery_container.add_resource(ApiResource("/redfish/resource1", 'netloc', RESOURCE_CORRECT, "#N.R")) requirements = [] validator = MetadataGetValidator(self.metadata_container, requirements) with StdoutCapture() as output: self.assertEqual(ValidationStatus.PASSED, validator.validate(self.discovery_container)) self.assertIn('Schema defines other, not used, but similar properties: [OptionalProperty1, OptionalProperty3]', '\n'.join(output)) self.assertIn( 'Schema defines other, not used, but similar properties: [OptionalPropertyInComplex1, OptionalPropertyInComplex3]', '\n'.join(output))
def run(self): print "TEST_CASE::API crawling" api_explorer = ApiExplorer(self.metadata_container, self.configuration) discovery_container, status = api_explorer.discover(MetadataConstants.SERVICE_ROOT_URI, MetadataConstants.SERVICE_ROOT) print "STATUS::{status}".format(status=status) requirements = [ Requirement(MetadataConstants.SERVICE_ROOT, min=1, max=1) ] MetadataGetValidator(self.metadata_container, requirements).validate(discovery_container)
def test_should_validate_dynamic_properties_at_entity_level(self): metadata_manager = MetadataManager(["qualifier"]) self.metadata_container = metadata_manager.read_metadata_from_strings( "Unknown", METADATA) self.discovery_container = DiscoveryContainer() self.discovery_container.add_resource( ApiResource("/redfish/resource1", 'netloc', RESOURCE_WITH_INCOMPLETE_DYNAMIC_PROPERTY, "#N.R")) requirements = [] validator = MetadataGetValidator(self.metadata_container, requirements) with StdoutCapture() as output: self.assertEqual(ValidationStatus.FAILED, validator.validate(self.discovery_container)) self.assertIn( 'ERROR::Required property resource.id->aaaaaaaa->Obligatory not present in the resource', '\n'.join(output))
def test_should_pass_when_unknown_complex_type_mapped_on_known_type(self): metadata_manager = MetadataManager(["qualifier"], map_types={ "A": "B", "unknown.complex.type": "N.ObjectWithNumericProperty" }) self.metadata_container = metadata_manager.read_metadata_from_strings( METADATA) self.discovery_container = DiscoveryContainer() self.discovery_container.add_resource( ApiResource("/redfish/resource1", 'netloc', RESOURCE_WITH_UNKNOWN_COMPLEX_TYPE, "#N.R")) requirements = [] validator = MetadataGetValidator(self.metadata_container, requirements) self.assertEqual(ValidationStatus.PASSED, validator.validate(self.discovery_container))
def run(self): if self.metadata_container is None: return print "TEST_CASE::API crawling" api_explorer = ApiExplorer(self.metadata_container, self.configuration) discovery_container, status = api_explorer.discover(MetadataConstants.SERVICE_ROOT_URI, MetadataConstants.SERVICE_ROOT) print "STATUS::{status}".format(status=status) MetadataGetValidator(self.metadata_container, RequiredTypes_Storage_2_4).validate(discovery_container) performance_status = self._measuring_request_responses(api_explorer) print("STATUS::{status}".format(status=performance_status))
def add_resource(self, resource, check_resource_against_metadata=None): """ :type resource: cts_core.discovery.api_resource.ApiResource :return: cts_framework.commons.enums.RequestStatus """ if resource is None: return if resource.odata_id is None: return self[resource.url] = resource odata_type = resource.odata_type if odata_type: # do not add the resource to any table if it is of an unknown type self.register_url_pattern(resource.url, odata_type) resource.expected_odata_type = odata_type new_row = json_normalize(resource.body) new_row["url"] = resource.url if odata_type in self._dataframes: # delete the deprecated definition of the resource if it exists if len(self._dataframes[odata_type] [self._dataframes[odata_type]["url"] == resource.url].index): self._dataframes[odata_type] = \ self._dataframes[odata_type][self._dataframes[odata_type]["url"] != resource.url] self._dataframes[odata_type] = \ concat([self._dataframes[odata_type], new_row], axis=0, ignore_index=True) else: self._dataframes[odata_type] = new_row if not check_resource_against_metadata: return RequestStatus.SUCCESS # this method is called in ApiCaller.get_resource() method, hence RequestStatus as a return type if self.metadata_container: status = MetadataGetValidator( self.metadata_container).validate_single_entity(resource) if status == ValidationStatus.PASSED: print "DEBUG::Metadata validation for resource {} passed".format( resource.url) return RequestStatus.SUCCESS else: cts_error( "Resource {odata_id:id} did not pass metadata validation", odata_id=resource.url) return RequestStatus.FAILED else: cts_warning("Metadata not known, skipping resource validation") return RequestStatus.SUCCESS
def test_should_fail_if_non_nullable_property_is_null(self): metadata_manager = MetadataManager(["qualifier"]) self.metadata_container = metadata_manager.read_metadata_from_strings( "Unknown", METADATA) self.discovery_container = DiscoveryContainer() self.discovery_container.add_resource( ApiResource("/redfish/resource1", 'netloc', RESOURCE_WITH_NULL_ON_NON_NULLABLE, "#N.R")) requirements = [] validator = MetadataGetValidator(self.metadata_container, requirements) with StdoutCapture() as output: self.assertEqual(ValidationStatus.FAILED, validator.validate(self.discovery_container)) self.assertIn( 'ERROR::Property resource.id->CapacityGiB cannot be null', '\n'.join(output)) self.assertIn( 'ERROR::Property resource.id->ObjectWithNumber->Number cannot be null', '\n'.join(output))
def test_should_suggest_other_names_for_additionals(self): metadata_manager = MetadataManager(["qualifier"]) self.metadata_container = metadata_manager.read_metadata_from_strings( "Unknown", METADATA) self.discovery_container = DiscoveryContainer() self.discovery_container.add_resource( ApiResource("/redfish/resource1", 'netloc', RESOURCE_CORRECT, "#N.R")) requirements = [] validator = MetadataGetValidator(self.metadata_container, requirements) with StdoutCapture() as output: self.assertEqual(ValidationStatus.PASSED, validator.validate(self.discovery_container)) self.assertIn( 'Schema defines other, not used, but similar properties: [OptionalProperty1, OptionalProperty3]', '\n'.join(output)) self.assertIn( 'Schema defines other, not used, but similar properties: [OptionalPropertyInComplex1, OptionalPropertyInComplex3]', '\n'.join(output))
def run(self): if self.metadata_container is None: return print "TEST_CASE::API crawling" api_explorer = ApiExplorer(self.metadata_container, self.configuration) discovery_container, status = api_explorer.discover( MetadataConstants.SERVICE_ROOT_URI, MetadataConstants.SERVICE_ROOT) print "STATUS::{status}".format(status=status) MetadataGetValidator( self.metadata_container, RequiredTypes_PSME_2_3, skip_check_predicate=SkipCheckPredicate_PSME_2_3).validate( discovery_container)
def test_validate_elements_of_collection_with_elements_referenced_by_out_of_range_json_pointer( self): metadata = """ <Schema xmlns="http://docs.oasis-open.org/odata/ns/edm" Namespace="N"> <EntityType Name="Outer" Abstract="false"> <NavigationProperty Name="MyCollection" Type="Collection(ReferencedEntity)" ContainsTarget="true"> <Annotation Term="OData.Permissions" EnumMember="OData.Permission/Read"/> </NavigationProperty> </EntityType> <EntityType Name="ReferencedEntity" Abstract="false"> <Property Name="name" Type="Edm.String"/> </EntityType> </Schema> """ resource = \ { "@odata.id": "resource.id", "@odata.type": "N.Outer", "MyCollection": [ { "@odata.id": "/outer#MyCollection/0" }, { "@odata.id": "/outer#MyCollection/2" # pointer out of range } ] } metadata_manager = MetadataManager(["qualifier"]) metadata_container = metadata_manager.read_metadata_from_strings( "Unknown", metadata) discovery_container = DiscoveryContainer() configuration = Configuration( **dict(UseSSL='True', ApiEndpoint=API_ENDPOINT)) api_explorer = ApiExplorer(metadata_container, configuration) with mock.patch('cts_core.commons.api_caller.ApiCaller.get_resource' ) as get_resource: get_resource.side_effect = [ (Link('url1', 'netloc'), RequestStatus.SUCCESS, ReturnCodes.OK, resource, Link('url1', 'netloc')), (Link('url2', 'netloc'), RequestStatus.SUCCESS, ReturnCodes.OK, resource, Link('url2', 'netloc')), (Link('url3', 'netloc'), RequestStatus.SUCCESS, ReturnCodes.OK, resource, Link('url3', 'netloc')) ] with StdoutCapture() as output: api_explorer.discover("/outer", "N.Outer", discovery_container) self.assertEqual(3, get_resource.call_count) self.assertIn( 'ERROR::JSON pointer exception while dereferencing /MyCollection/2', ';'.join(output)) requirements = [] validator = MetadataGetValidator(metadata_container, requirements) with StdoutCapture() as output: self.assertEqual(ValidationStatus.PASSED, validator.validate(discovery_container)) self.assertEqual(0, len(re.findall('ERROR::', ';'.join(output))))