Beispiel #1
0
    def expected(self):
        for pluginXbrlMethod in pluginClassMethods(
                "ModelTestcaseVariation.ExpectedResult"):
            expected = pluginXbrlMethod(self)
            if expected:
                return expected
        # default behavior without plugins
        if self.localName == "testcase":
            return self.document.basename[:4]  #starts with PASS or FAIL
        elif self.localName == "testGroup":  #w3c testcase
            instanceTestElement = XmlUtil.descendant(self, None,
                                                     "instanceTest")
            if instanceTestElement is not None:  # take instance first
                return XmlUtil.descendantAttr(instanceTestElement, None,
                                              "expected", "validity")
            else:
                schemaTestElement = XmlUtil.descendant(self, None,
                                                       "schemaTest")
                if schemaTestElement is not None:
                    return XmlUtil.descendantAttr(schemaTestElement, None,
                                                  "expected", "validity")
        resultElement = XmlUtil.descendant(self, None, "result")
        if resultElement is not None:
            expected = resultElement.get("expected")
            if expected and resultElement.get(
                    "nonStandardErrorCodes") == "true":
                # if @expected and @nonStandardErrorCodes then use expected instead of error codes
                return expected
        errorElement = XmlUtil.descendant(self, None, "error")
        if errorElement is not None:
            _errorText = XmlUtil.text(errorElement)
            if ' ' in _errorText:  # list of tokens
                return _errorText
            return ModelValue.qname(errorElement,
                                    _errorText)  # turn into a QName
        if resultElement is not None:
            if expected:
                return expected
            for assertElement in XmlUtil.children(resultElement, None,
                                                  "assert"):
                num = assertElement.get("num")
                if num == "99999":  # inline test, use name as expected
                    return assertElement.get("name")
                if len(num) == 5:
                    return "EFM.{0}.{1}.{2}".format(num[0], num[1:3], num[3:6])
            asserTests = {}
            for atElt in XmlUtil.children(resultElement, None,
                                          "assertionTests"):
                try:
                    asserTests[atElt.get("assertionID")] = (
                        _INT(atElt.get("countSatisfied")),
                        _INT(atElt.get("countNotSatisfied")))
                except ValueError:
                    pass
            if asserTests:
                return asserTests
        elif self.get("result"):
            return self.get("result")

        return None
Beispiel #2
0
 def readMeFirstUris(self):
     try:
         return self._readMeFirstUris
     except AttributeError:
         self._readMeFirstUris = []
         # first look if any plugin method to get readme first URIs
         if not any(
                 pluginXbrlMethod(self)
                 for pluginXbrlMethod in pluginClassMethods(
                     "ModelTestcaseVariation.ReadMeFirstUris")):
             if self.localName == "testGroup":  #w3c testcase
                 instanceTestElement = XmlUtil.descendant(
                     self, None, "instanceTest")
                 if instanceTestElement is not None:  # take instance first
                     self._readMeFirstUris.append(
                         XmlUtil.descendantAttr(
                             instanceTestElement, None, "instanceDocument",
                             "{http://www.w3.org/1999/xlink}href"))
                 else:
                     schemaTestElement = XmlUtil.descendant(
                         self, None, "schemaTest")
                     if schemaTestElement is not None:
                         self._readMeFirstUris.append(
                             XmlUtil.descendantAttr(
                                 schemaTestElement, None, "schemaDocument",
                                 "{http://www.w3.org/1999/xlink}href"))
             elif self.localName == "test-case":  #xpath testcase
                 inputFileElement = XmlUtil.descendant(
                     self, None, "input-file")
                 if inputFileElement is not None:  # take instance first
                     self._readMeFirstUris.append("TestSources/" +
                                                  inputFileElement.text +
                                                  ".xml")
             else:
                 # default built-in method for readme first uris
                 for anElement in self.iterdescendants():
                     if isinstance(anElement,
                                   ModelObject) and anElement.get(
                                       "readMeFirst") == "true":
                         if anElement.get(
                                 "{http://www.w3.org/1999/xlink}href"):
                             uri = anElement.get(
                                 "{http://www.w3.org/1999/xlink}href")
                         else:
                             uri = XmlUtil.innerText(anElement)
                         if anElement.get("name"):
                             self._readMeFirstUris.append(
                                 (ModelValue.qname(anElement,
                                                   anElement.get("name")),
                                  uri))
                         elif anElement.get("dts"):
                             self._readMeFirstUris.append(
                                 (anElement.get("dts"), uri))
                         else:
                             self._readMeFirstUris.append(uri)
         if not self._readMeFirstUris:  # provide a dummy empty instance document
             self._readMeFirstUris.append(
                 os.path.join(self.modelXbrl.modelManager.cntlr.configDir,
                              "empty-instance.xml"))
         return self._readMeFirstUris
 def readMeFirstUris(self):
     try:
         return self._readMeFirstUris
     except AttributeError:
         self._readMeFirstUris = []
         # first look if any plugin method to get readme first URIs
         if not any(
                 pluginXbrlMethod(self)
                 for pluginXbrlMethod in pluginClassMethods(
                     "ModelTestcaseVariation.ReadMeFirstUris")):
             if self.localName == "testGroup":  #w3c testcase
                 instanceTestElement = XmlUtil.descendant(
                     self, None, "instanceTest")
                 if instanceTestElement is not None:  # take instance first
                     self._readMeFirstUris.append(
                         XmlUtil.descendantAttr(
                             instanceTestElement, None, "instanceDocument",
                             "{http://www.w3.org/1999/xlink}href"))
                 else:
                     schemaTestElement = XmlUtil.descendant(
                         self, None, "schemaTest")
                     if schemaTestElement is not None:
                         self._readMeFirstUris.append(
                             XmlUtil.descendantAttr(
                                 schemaTestElement, None, "schemaDocument",
                                 "{http://www.w3.org/1999/xlink}href"))
             elif self.localName == "test-case":  #xpath testcase
                 inputFileElement = XmlUtil.descendant(
                     self, None, "input-file")
                 if inputFileElement is not None:  # take instance first
                     self._readMeFirstUris.append(
                         "TestSources/" + inputFileElement.text + ".xml")
             else:
                 # default built-in method for readme first uris
                 for anElement in self.iterdescendants():
                     if isinstance(anElement,
                                   ModelObject) and anElement.get(
                                       "readMeFirst") == "true":
                         if anElement.get(
                                 "{http://www.w3.org/1999/xlink}href"):
                             uri = anElement.get(
                                 "{http://www.w3.org/1999/xlink}href")
                         else:
                             uri = XmlUtil.innerText(anElement)
                         if anElement.get("name"):
                             self._readMeFirstUris.append((ModelValue.qname(
                                 anElement, anElement.get("name")), uri))
                         elif anElement.get("dts"):
                             self._readMeFirstUris.append(
                                 (anElement.get("dts"), uri))
                         else:
                             self._readMeFirstUris.append(uri)
         if not self._readMeFirstUris:  # provide a dummy empty instance document
             self._readMeFirstUris.append(
                 os.path.join(self.modelXbrl.modelManager.cntlr.configDir,
                              "empty-instance.xml"))
         return self._readMeFirstUris
Beispiel #4
0
 def expected(self):
     for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ExpectedResult"):
         expected = pluginXbrlMethod(self)
         if expected:
             return expected
     # default behavior without plugins
     if self.localName == "testcase":
         return self.document.basename[:4]   #starts with PASS or FAIL
     elif self.localName == "testGroup":  #w3c testcase
         instanceTestElement = XmlUtil.descendant(self, None, "instanceTest")
         if instanceTestElement is not None: # take instance first
             return XmlUtil.descendantAttr(instanceTestElement, None, "expected", "validity")
         else:
             schemaTestElement = XmlUtil.descendant(self, None, "schemaTest")
             if schemaTestElement is not None:
                 return XmlUtil.descendantAttr(schemaTestElement, None, "expected", "validity")
     resultElement = XmlUtil.descendant(self, None, "result")
     if resultElement is not None:
         expected = resultElement.get("expected")
         if expected and resultElement.get("nonStandardErrorCodes") == "true":
             # if @expected and @nonStandardErrorCodes then use expected instead of error codes
             return expected
     errorElement = XmlUtil.descendant(self, None, "error")
     resultElement = XmlUtil.descendant(self, None, "result")
     if errorElement is not None and not errorElement.get("nonStandardErrorCodes"):
         _errorText = XmlUtil.text(errorElement)
         if ' ' in _errorText: # list of tokens
             return _errorText
         return ModelValue.qname(errorElement, _errorText)  # turn into a QName
     if resultElement is not None:
         if expected:
             return expected
         for assertElement in XmlUtil.children(resultElement, None, "assert"):
             num = assertElement.get("num")
             if num == "99999": # inline test, use name as expected
                 return assertElement.get("name")
             if len(num) == 5:
                 return "EFM.{0}.{1}.{2}".format(num[0],num[1:3],num[3:6])
         asserTests = {}
         for atElt in XmlUtil.children(resultElement, None, "assertionTests"):
             try:
                 asserTests[atElt.get("assertionID")] = (_INT(atElt.get("countSatisfied")),_INT(atElt.get("countNotSatisfied")))
             except ValueError:
                 pass
         if asserTests:
             return asserTests
     elif self.get("result"):
         return self.get("result")
             
     return None
 def blockedMessageCodes(self):
     blockedCodesRegex = XmlUtil.descendantAttr(
         self, None, "results", "blockedMessageCodes")  # DQC 4/5 test suite
     if not blockedCodesRegex:
         ignoredCodes = XmlUtil.descendants(
             self, None, "ignore-error")  # ESEF test suite
         if ignoredCodes:
             blockedCodesRegex = "|".join(".*" + c.stringValue
                                          for c in ignoredCodes)
     return blockedCodesRegex
 def expected(self):
     for pluginXbrlMethod in pluginClassMethods(u"ModelTestcaseVariation.ExpectedResult"):
         expected = pluginXbrlMethod(self)
         if expected:
             return expected
     # default behavior without plugins
     if self.localName == u"testcase":
         return self.document.basename[:4]   #starts with PASS or FAIL
     elif self.localName == u"testGroup":  #w3c testcase
         instanceTestElement = XmlUtil.descendant(self, None, u"instanceTest")
         if instanceTestElement is not None: # take instance first
             return XmlUtil.descendantAttr(instanceTestElement, None, u"expected", u"validity")
         else:
             schemaTestElement = XmlUtil.descendant(self, None, u"schemaTest")
             if schemaTestElement is not None:
                 return XmlUtil.descendantAttr(schemaTestElement, None, u"expected", u"validity")
     errorElement = XmlUtil.descendant(self, None, u"error")
     if errorElement is not None:
         return ModelValue.qname(errorElement, XmlUtil.text(errorElement))
     resultElement = XmlUtil.descendant(self, None, u"result")
     if resultElement is not None:
         expected = resultElement.get(u"expected")
         if expected:
             return expected
         for assertElement in XmlUtil.children(resultElement, None, u"assert"):
             num = assertElement.get(u"num")
             if len(num) == 5:
                 return u"EFM.{0}.{1}.{2}".format(num[0],num[1:3],num[3:6])
         asserTests = {}
         for atElt in XmlUtil.children(resultElement, None, u"assertionTests"):
             try:
                 asserTests[atElt.get(u"assertionID")] = (_INT(atElt.get(u"countSatisfied")),_INT(atElt.get(u"countNotSatisfied")))
             except ValueError:
                 pass
         if asserTests:
             return asserTests
     elif self.get(u"result"):
         return self.get(u"result")
             
     return None
 def blockedMessageCodes(self):
     return XmlUtil.descendantAttr(self, None, "results", "blockedMessageCodes")
Beispiel #8
0
def checkDTS(val, modelDocument, visited):
    global targetNamespaceDatePattern, efmFilenamePattern, roleTypePattern, arcroleTypePattern, \
            arcroleDefinitionPattern, namePattern, linkroleDefinitionBalanceIncomeSheet, \
            namespacesConflictPattern
    if targetNamespaceDatePattern is None:
        targetNamespaceDatePattern = re.compile(r"/([12][0-9]{3})-([01][0-9])-([0-3][0-9])|"
                                            r"/([12][0-9]{3})([01][0-9])([0-3][0-9])|")
        efmFilenamePattern = re.compile(r"^[a-z0-9][a-z0-9_\.\-]*(\.xsd|\.xml)$")
        roleTypePattern = re.compile(r"^.*/role/[^/\s]+$")
        arcroleTypePattern = re.compile(r"^.*/arcrole/[^/\s]+$")
        arcroleDefinitionPattern = re.compile(r"^.*[^\\s]+.*$")  # at least one non-whitespace character
        namePattern = re.compile("[][()*+?\\\\/^{}|@#%^=~`\"';:,<>&$\u00a3\u20ac]") # u20ac=Euro, u00a3=pound sterling 
        linkroleDefinitionBalanceIncomeSheet = re.compile(r"[^-]+-\s+Statement\s+-\s+.*(income|balance|financial\W+position)",
                                                          re.IGNORECASE)
        namespacesConflictPattern = re.compile(r"http://(xbrl\.us|fasb\.org|xbrl\.sec\.gov)/(dei|us-types|us-roles|rr)/([0-9]{4}-[0-9]{2}-[0-9]{2})$")
    nonDomainItemNameProblemPattern = re.compile(
        r"({0})|(FirstQuarter|SecondQuarter|ThirdQuarter|FourthQuarter|[1-4]Qtr|Qtr[1-4]|ytd|YTD|HalfYear)(?:$|[A-Z\W])"
        .format(re.sub(r"\W", "", (val.entityRegistrantName or "").title())))
    
        
    visited.append(modelDocument)
    definesLabelLinkbase = False
    for referencedDocument in modelDocument.referencesDocument.items():
        #6.07.01 no includes
        if referencedDocument[1] == "include":
            val.modelXbrl.error(("EFM.6.07.01", "GFM.1.03.01", "SBR.NL.2.2.0.18"),
                _("Taxonomy schema %(schema)s includes %(include)s, only import is allowed"),
                modelObject=modelDocument,
                    schema=os.path.basename(modelDocument.uri), 
                    include=os.path.basename(referencedDocument[0].uri))
        if referencedDocument[0] not in visited:
            checkDTS(val, referencedDocument[0], visited)
            
    if val.disclosureSystem.standardTaxonomiesDict is None:
        pass

    if val.validateEFM: 
        if modelDocument.uri in val.disclosureSystem.standardTaxonomiesDict:
            if modelDocument.targetNamespace:
                # check for duplicates of us-types, dei, and rr taxonomies
                match = namespacesConflictPattern.match(modelDocument.targetNamespace)
                if match is not None:
                    val.standardNamespaceConflicts[match.group(2)].add(modelDocument)
        else:
            if len(modelDocument.basename) > 32:
                val.modelXbrl.error("EFM.5.01.01.tooManyCharacters",
                    _("Document file name %(filename)s must not exceed 32 characters."),
                    modelObject=modelDocument, filename=modelDocument.basename)
            if not efmFilenamePattern.match(modelDocument.basename):
                val.modelXbrl.error("EFM.5.01.01",
                    _("Document file name %(filename)s must start with a-z or 0-9, contain lower case letters, ., -, _, and end with .xsd or .xml."),
                    modelObject=modelDocument, filename=modelDocument.basename)
    
    if (modelDocument.type == ModelDocument.Type.SCHEMA and 
        modelDocument.targetNamespace not in val.disclosureSystem.baseTaxonomyNamespaces and
        modelDocument.uri.startswith(val.modelXbrl.uriDir)):
        
        # check schema contents types
        if val.validateSBRNL:
            definesLinkroles = False
            definesArcroles = False
            definesLinkParts = False
            definesAbstractItems = False
            definesNonabstractItems = False
            definesConcepts = False
            definesTuples = False
            definesPresentationTuples = False
            definesSpecificationTuples = False
            definesTypes = False
            definesEnumerations = False
            definesDimensions = False
            definesDomains = False
            definesHypercubes = False
                
        # 6.7.3 check namespace for standard authority
        targetNamespaceAuthority = UrlUtil.authority(modelDocument.targetNamespace) 
        if targetNamespaceAuthority in val.disclosureSystem.standardAuthorities:
            val.modelXbrl.error(("EFM.6.07.03", "GFM.1.03.03"),
                _("Taxonomy schema %(schema)s namespace %(targetNamespace)s is a disallowed authority"),
                modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace, targetNamespaceAuthority=targetNamespaceAuthority)
            
        # 6.7.4 check namespace format
        if modelDocument.targetNamespace is None or not modelDocument.targetNamespace.startswith("http://"):
            match = None
        elif val.validateEFMorGFM:
            targetNamespaceDate = modelDocument.targetNamespace[len(targetNamespaceAuthority):]
            match = targetNamespaceDatePattern.match(targetNamespaceDate)
        else:
            match = None
        if match is not None:
            try:
                if match.lastindex == 3:
                    date = datetime.date(int(match.group(1)),int(match.group(2)),int(match.group(3)))
                elif match.lastindex == 6:
                    date = datetime.date(int(match.group(4)),int(match.group(5)),int(match.group(6)))
                else:
                    match = None
            except ValueError:
                match = None
        if match is None:
            val.modelXbrl.error(("EFM.6.07.04", "GFM.1.03.04"),
                _("Taxonomy schema %(schema)s namespace %(targetNamespace)s must have format http://{authority}/{versionDate}"),
                modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace)
        elif val.fileNameDate and date > val.fileNameDate:
            val.modelXbrl.info(("EFM.6.07.06", "GFM.1.03.06"),
                _("Warning: Taxonomy schema %(schema)s namespace %(targetNamespace)s has date later than document name date %(docNameDate)s"),
                modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace,
                docNameDate=val.fileNameDate)

        if modelDocument.targetNamespace is not None:
            # 6.7.5 check prefix for _
            authority = UrlUtil.authority(modelDocument.targetNamespace)
            if not re.match(r"(http://|https://|ftp://|urn:)\w+",authority):
                val.modelXbrl.error(("EFM.6.07.05", "GFM.1.03.05"),
                    _("Taxonomy schema %(schema)s namespace %(targetNamespace)s must be a valid URL with a valid authority for the namespace."),
                    modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace)
            prefix = XmlUtil.xmlnsprefix(modelDocument.xmlRootElement,modelDocument.targetNamespace)
            if not prefix:
                val.modelXbrl.error(("EFM.6.07.07", "GFM.1.03.07"),
                    _("Taxonomy schema %(schema)s namespace %(targetNamespace)s missing prefix for the namespace."),
                    modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace)
            elif "_" in prefix:
                val.modelXbrl.error(("EFM.6.07.07", "GFM.1.03.07"),
                    _("Taxonomy schema %(schema)s namespace %(targetNamespace)s prefix %(prefix)s must not have an '_'"),
                    modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace, prefix=prefix)

            if val.validateSBRNL:
                genrlSpeclRelSet = val.modelXbrl.relationshipSet(XbrlConst.generalSpecial)
            for modelConcept in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/2001/XMLSchema}element"):
                if isinstance(modelConcept,ModelConcept):
                    # 6.7.16 name not duplicated in standard taxonomies
                    name = modelConcept.get("name")
                    if name is None: 
                        name = ""
                        if modelConcept.get("ref") is not None:
                            continue    # don't validate ref's here
                    for c in val.modelXbrl.nameConcepts.get(name, []):
                        if c.modelDocument != modelDocument:
                            if (val.validateEFMorGFM and
                                  not c.modelDocument.uri.startswith(val.modelXbrl.uriDir)):
                                val.modelXbrl.error(("EFM.6.07.16", "GFM.1.03.18"),
                                    _("Concept %(concept)s is also defined in standard taxonomy schema schema %(standardSchema)s"),
                                    modelObject=c, concept=modelConcept.qname, standardSchema=os.path.basename(c.modelDocument.uri))
                            elif val.validateSBRNL:
                                if not (genrlSpeclRelSet.isRelated(modelConcept, "child", c) or genrlSpeclRelSet.isRelated(c, "child", modelConcept)):
                                    val.modelXbrl.error("SBR.NL.2.2.2.02",
                                        _("Concept %(concept)s is also defined in standard taxonomy schema %(standardSchema)s without a general-special relationship"),
                                        modelObject=c, concept=modelConcept.qname, standardSchema=os.path.basename(c.modelDocument.uri))
                    ''' removed RH 2011-12-23 corresponding set up of table in ValidateFiling
                    if val.validateSBRNL and name in val.nameWordsTable:
                        if not any( any( genrlSpeclRelSet.isRelated(c, "child", modelConcept)
                                         for c in val.modelXbrl.nameConcepts.get(partialWordName, []))
                                    for partialWordName in val.nameWordsTable[name]):
                            val.modelXbrl.error("SBR.NL.2.3.2.01",
                                _("Concept %(specialName)s is appears to be missing a general-special relationship to %(generalNames)s"),
                                modelObject=c, specialName=modelConcept.qname, generalNames=', or to '.join(val.nameWordsTable[name]))
                    '''

                    # 6.7.17 id properly formed
                    id = modelConcept.id
                    requiredId = (prefix if prefix is not None else "") + "_" + name
                    if val.validateEFMorGFM and id != requiredId:
                        val.modelXbrl.error(("EFM.6.07.17", "GFM.1.03.19"),
                            _("Concept %(concept)s id %(id)s should be $(requiredId)s"),
                            modelObject=modelConcept, concept=modelConcept.qname, id=id, requiredId=requiredId)
                        
                    # 6.7.18 nillable is true
                    nillable = modelConcept.get("nillable")
                    if nillable != "true":
                        val.modelXbrl.error(("EFM.6.07.18", "GFM.1.03.20"),
                            _("Taxonomy schema %(schema)s element %(concept)s nillable %(nillable)s should be 'true'"),
                            modelObject=modelConcept, schema=os.path.basename(modelDocument.uri),
                            concept=name, nillable=nillable)
        
                    # 6.7.19 not tuple
                    if modelConcept.isTuple:
                        if val.validateEFMorGFM:
                            val.modelXbrl.error(("EFM.6.07.19", "GFM.1.03.21"),
                                _("Concept %(concept)s is a tuple"),
                                modelObject=modelConcept, concept=modelConcept.qname)
                        
                    # 6.7.20 no typed domain ref
                    if modelConcept.isTypedDimension:
                        val.modelXbrl.error(("EFM.6.07.20", "GFM.1.03.22"),
                            _("Concept %(concept)s has typedDomainRef %(typedDomainRef)s"),
                            modelObject=modelConcept, concept=modelConcept.qname,
                            typedDomainRef=modelConcept.typedDomainElement.qname if modelConcept.typedDomainElement is not None else modelConcept.typedDomainRef)
                        
                    # 6.7.21 abstract must be duration
                    isDuration = modelConcept.periodType == "duration"
                    if modelConcept.abstract == "true" and not isDuration:
                        val.modelXbrl.error(("EFM.6.07.21", "GFM.1.03.23"),
                            _("Taxonomy schema %(schema)s element %(concept)s is abstract but period type is not duration"),
                            modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=name)
                        
                    # 6.7.22 abstract must be stringItemType
                    ''' removed SEC EFM v.17, Edgar release 10.4, and GFM 2011-04-08
                    if modelConcept.abstract == "true" and modelConcept.typeQname != XbrlConst. qnXbrliStringItemType:
                        val.modelXbrl.error(("EFM.6.07.22", "GFM.1.03.24"),
                            _("Concept %(concept)s  is abstract but type is not xbrli:stringItemType"),
                            modelObject=modelConcept, concept=modelConcept.qname)
					'''
                    substititutionGroupQname = modelConcept.substitutionGroupQname
                    # 6.7.23 Axis must be subs group dimension
                    if name.endswith("Axis") ^ (substititutionGroupQname == XbrlConst.qnXbrldtDimensionItem):
                        val.modelXbrl.error(("EFM.6.07.23", "GFM.1.03.25"),
                            _("Concept %(concept)s must end in Axis to be in xbrldt:dimensionItem substitution group"),
                            modelObject=modelConcept, concept=modelConcept.qname)

                    # 6.7.24 Table must be subs group hypercube
                    if name.endswith("Table") ^ (substititutionGroupQname == XbrlConst.qnXbrldtHypercubeItem):
                        val.modelXbrl.error(("EFM.6.07.24", "GFM.1.03.26"),
                            _("Concept %(concept)s must end in Table to be in xbrldt:hypercubeItem substitution group"),
                            modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=modelConcept.qname)

                    # 6.7.25 if neither hypercube or dimension, substitution group must be item
                    if substititutionGroupQname not in (None,
                                                        XbrlConst.qnXbrldtDimensionItem, 
                                                        XbrlConst.qnXbrldtHypercubeItem,
                                                        XbrlConst.qnXbrliItem):                           
                        val.modelXbrl.error(("EFM.6.07.25", "GFM.1.03.27"),
                            _("Concept %(concept)s has disallowed substitution group %(substitutionGroup)s"),
                            modelObject=modelConcept, concept=modelConcept.qname,
                            substitutionGroup=modelConcept.substitutionGroupQname)
                        
                    # 6.7.26 Table must be subs group hypercube
                    if name.endswith("LineItems") and modelConcept.abstract != "true":
                        val.modelXbrl.error(("EFM.6.07.26", "GFM.1.03.28"),
                            _("Concept %(concept)s is a LineItems but not abstract"),
                            modelObject=modelConcept, concept=modelConcept.qname)

                    # 6.7.27 type domainMember must end with Domain or Member
                    conceptType = modelConcept.type
                    isDomainItemType = conceptType is not None and conceptType.isDomainItemType
                    endsWithDomainOrMember = name.endswith("Domain") or name.endswith("Member")
                    if isDomainItemType != endsWithDomainOrMember:
                        val.modelXbrl.error(("EFM.6.07.27", "GFM.1.03.29"),
                            _("Concept %(concept)s must end with Domain or Member for type of domainItemType"),
                            modelObject=modelConcept, concept=modelConcept.qname)

                    # 6.7.28 domainItemType must be duration
                    if isDomainItemType and not isDuration:
                        val.modelXbrl.error(("EFM.6.07.28", "GFM.1.03.30"),
                            _("Concept %(concept)s is a domainItemType and must be periodType duration"),
                            modelObject=modelConcept, concept=modelConcept.qname)
                        
                    # 6.8.5 semantic check, check LC3 name
                    if not name[0].isupper():
                        val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.05.firstLetter", "GFM.2.03.05.firstLetter"),
                            _("Concept %(concept)s name must start with a capital letter"),
                            modelObject=modelConcept, concept=modelConcept.qname)
                    if namePattern.search(name):
                        val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.05.disallowedCharacter", "GFM.2.03.05.disallowedCharacter"),
                            _("Concept %(concept)s has disallowed name character"),
                            modelObject=modelConcept, concept=modelConcept.qname)
                    if len(name) > 200:
                        val.modelXbrl.log("ERROR-SEMANTIC", "EFM.6.08.05.nameLength",
                            _("Concept %(concept)s name length %(namelength)s exceeds 200 characters"),
                            modelObject=modelConcept, concept=modelConcept.qname, namelength=len(name))
                        
                    if val.validateEFM:
                        label = modelConcept.label(lang="en-US", fallbackToQname=False)
                        if label:
                            # allow Joe's Bar, N.A.  to be JoesBarNA -- remove ', allow A. as not article "a"
                            lc3name = ''.join(re.sub(r"['.-]", "", (w[0] or w[2] or w[3] or w[4])).title()
                                              for w in re.findall(r"((\w+')+\w+)|(A[.-])|([.-]A(?=\W|$))|(\w+)", label) # EFM implies this should allow - and . re.findall(r"[\w\-\.]+", label)
                                              if w[4].lower() not in ("the", "a", "an"))
                            if not(name == lc3name or 
                                   (name and lc3name and lc3name[0].isdigit() and name[1:] == lc3name and (name[0].isalpha() or name[0] == '_'))):
                                val.modelXbrl.log("WARNING-SEMANTIC", "EFM.6.08.05.LC3",
                                    _("Concept %(concept)s should match expected LC3 composition %(lc3name)s"),
                                    modelObject=modelConcept, concept=modelConcept.qname, lc3name=lc3name)
                                
                    if conceptType is not None:
                        # 6.8.6 semantic check
                        if not isDomainItemType and conceptType.qname != XbrlConst.qnXbrliDurationItemType:
                            nameProblems = nonDomainItemNameProblemPattern.findall(name)
                            if any(any(t) for t in nameProblems):  # list of tuples with possibly nonempty strings
                                val.modelXbrl.log("WARNING-SEMANTIC", ("EFM.6.08.06", "GFM.2.03.06"),
                                    _("Concept %(concept)s should not contain company or period information, found: %(matches)s"),
                                    modelObject=modelConcept, concept=modelConcept.qname, 
                                    matches=", ".join(''.join(t) for t in nameProblems))
                        
                        if conceptType.qname == XbrlConst.qnXbrliMonetaryItemType:
                            if not modelConcept.balance:
                                # 6.8.11 may not appear on a income or balance statement
                                if any(linkroleDefinitionBalanceIncomeSheet.match(roleType.definition)
                                       for rel in val.modelXbrl.relationshipSet(XbrlConst.parentChild).toModelObject(modelConcept)
                                       for roleType in val.modelXbrl.roleTypes.get(rel.linkrole,())):
                                    val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.11", "GFM.2.03.11"),
                                        _("Concept %(concept)s must have a balance because it appears in a statement of income or balance sheet"),
                                        modelObject=modelConcept, concept=modelConcept.qname)
                                # 6.11.5 semantic check, must have a documentation label
                                stdLabel = modelConcept.label(lang="en-US", fallbackToQname=False)
                                defLabel = modelConcept.label(preferredLabel=XbrlConst.documentationLabel, lang="en-US", fallbackToQname=False)
                                if not defLabel or ( # want different words than std label
                                    stdLabel and re.findall(r"\w+", stdLabel) == re.findall(r"\w+", defLabel)):
                                    val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.11.05", "GFM.2.04.04"),
                                        _("Concept %(concept)s is monetary without a balance and must have a documentation label that disambiguates its sign"),
                                        modelObject=modelConcept, concept=modelConcept.qname)
                        
                        # 6.8.16 semantic check
                        if conceptType.qname == XbrlConst.qnXbrliDateItemType and modelConcept.periodType != "duration":
                            val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.16", "GFM.2.03.16"),
                                _("Concept %(concept)s of type xbrli:dateItemType must have periodType duration"),
                                modelObject=modelConcept, concept=modelConcept.qname)
                        
                        # 6.8.17 semantic check
                        if conceptType.qname == XbrlConst.qnXbrliStringItemType and modelConcept.periodType != "duration":
                            val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.17", "GFM.2.03.17"),
                                _("Concept %(concept)s of type xbrli:stringItemType must have periodType duration"),
                                modelObject=modelConcept, concept=modelConcept.qname)
                        
                    if val.validateSBRNL:
                        if modelConcept.isTuple:
                            if modelConcept.substitutionGroupQname.localName == "presentationTuple" and modelConcept.substitutionGroupQname.namespaceURI.endswith("/basis/sbr/xbrl/xbrl-syntax-extension"): # namespace may change each year
                                definesPresentationTuples = True
                            elif modelConcept.substitutionGroupQname.localName == "specificationTuple" and modelConcept.substitutionGroupQname.namespaceURI.endswith("/basis/sbr/xbrl/xbrl-syntax-extension"): # namespace may change each year
                                definesSpecificationTuples = True
                            else:
                                definesTuples = True
                            definesConcepts = True
                            if modelConcept.isAbstract:
                                val.modelXbrl.error("SBR.NL.2.2.2.03",
                                    _("Concept %(concept)s is an abstract tuple"),
                                    modelObject=modelConcept, concept=modelConcept.qname)
                            if tupleCycle(val,modelConcept):
                                val.modelXbrl.error("SBR.NL.2.2.2.07",
                                    _("Tuple %(concept)s has a tuple cycle"),
                                    modelObject=modelConcept, concept=modelConcept.qname)
                            if modelConcept.get("nillable") != "false" and modelConcept.isRoot:
                                val.modelXbrl.error("SBR.NL.2.2.2.17", #don't want default, just what was really there
                                    _("Tuple %(concept)s must have nillable='false'"),
                                    modelObject=modelConcept, concept=modelConcept.qname)
                        elif modelConcept.isItem:
                            definesConcepts = True
                        if modelConcept.abstract == "true":
                            if modelConcept.isRoot:
                                if modelConcept.get("nillable") != "false": #don't want default, just what was really there
                                    val.modelXbrl.error("SBR.NL.2.2.2.16",
                                        _("Abstract root concept %(concept)s must have nillable='false'"),
                                    modelObject=modelConcept, concept=modelConcept.qname)
                                if modelConcept.typeQname != XbrlConst.qnXbrliStringItemType:
                                    val.modelXbrl.error("SBR.NL.2.2.2.21",
                                        _("Abstract root concept %(concept)s must have type='xbrli:stringItemType'"),
                                    modelObject=modelConcept, concept=modelConcept.qname)
                            if modelConcept.balance:
                                val.modelXbrl.error("SBR.NL.2.2.2.22",
                                    _("Abstract concept %(concept)s must not have a balance attribute"),
                                    modelObject=modelConcept, concept=modelConcept.qname)
                            if modelConcept.isHypercubeItem:
                                definesHypercubes = True
                            elif modelConcept.isDimensionItem:
                                definesDimensions = True
                            elif substititutionGroupQname and substititutionGroupQname.localName in ("domainItem","domainMemberItem"):
                                definesDomains = True
                            elif modelConcept.isItem:
                                definesAbstractItems = True
                        else:   # not abstract
                            if modelConcept.isItem:
                                definesNonabstractItems = True
                                if not (modelConcept.label(preferredLabel=XbrlConst.documentationLabel,fallbackToQname=False,lang="nl") or
                                        val.modelXbrl.relationshipSet(XbrlConst.conceptReference).fromModelObject(c) or
                                        modelConcept.genLabel(role=XbrlConst.genDocumentationLabel,lang="nl") or
                                        val.modelXbrl.relationshipSet(XbrlConst.elementReference).fromModelObject(c)):
                                    val.modelXbrl.error("SBR.NL.2.2.2.28",
                                        _("Concept %(concept)s must have a documentation label or reference"),
                                        modelObject=modelConcept, concept=modelConcept.qname)
                        if modelConcept.balance and not modelConcept.instanceOfType(XbrlConst.qnXbrliMonetaryItemType):
                            val.modelXbrl.error("SBR.NL.2.2.2.24",
                                _("Non-monetary concept %(concept)s must not have a balance attribute"),
                                modelObject=modelConcept, concept=modelConcept.qname)
                        if modelConcept.isLinkPart:
                            definesLinkParts = True
                            val.modelXbrl.error("SBR.NL.2.2.5.01",
                                _("Link:part concept %(concept)s is not allowed"),
                                modelObject=modelConcept, concept=modelConcept.qname)
                            if not modelConcept.genLabel(fallbackToQname=False,lang="nl"):
                                val.modelXbrl.error("SBR.NL.2.2.5.02",
                                    _("Link part definition %(concept)s must have a generic label in language 'nl'"),
                                    modelObject=modelConcept, concept=modelConcept.qname)
        # 6.7.8 check for embedded linkbase
        for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}linkbase"):
            if isinstance(e,ModelObject):
                val.modelXbrl.error(("EFM.6.07.08", "GFM.1.03.08"),
                    _("Taxonomy schema %(schema)s contains an embedded linkbase"),
                    modelObject=e, schema=os.path.basename(modelDocument.uri))
                break

        requiredUsedOns = {XbrlConst.qnLinkPresentationLink,
                           XbrlConst.qnLinkCalculationLink,
                           XbrlConst.qnLinkDefinitionLink}
        
        standardUsedOns = {XbrlConst.qnLinkLabel, XbrlConst.qnLinkReference, XbrlConst.qnLinkFootnote,
                           XbrlConst.qnLinkDefinitionArc, XbrlConst.qnLinkCalculationArc, XbrlConst.qnLinkPresentationArc, 
                           XbrlConst.qnLinkLabelArc, XbrlConst.qnLinkReferenceArc 
                           # XbrlConst.qnLinkFootnoteArc note: not in EFM table for 6.8.3 and causes tests 6.5.30 to fail
                           }

        # 6.7.9 role types authority
        for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}roleType"):
            if isinstance(e,ModelObject):
                roleURI = e.get("roleURI")
                if targetNamespaceAuthority != UrlUtil.authority(roleURI):
                    val.modelXbrl.error(("EFM.6.07.09", "GFM.1.03.09"),
                        _("RoleType %(roleType)s does not match authority %(targetNamespaceAuthority)s"),
                        modelObject=e, roleType=roleURI, targetNamespaceAuthority=targetNamespaceAuthority)
                # 6.7.9 end with .../role/lc3 name
                if not roleTypePattern.match(roleURI):
                    val.modelXbrl.warning(("EFM.6.07.09.roleEnding", "GFM.1.03.09"),
                        "RoleType %(roleType)s should end with /role/{LC3name}",
                        modelObject=e, roleType=roleURI)
                    
                # 6.7.10 only one role type declaration in DTS
                modelRoleTypes = val.modelXbrl.roleTypes.get(roleURI)
                if modelRoleTypes is not None:
                    modelRoleType = modelRoleTypes[0]
                    definition = modelRoleType.definitionNotStripped
                    usedOns = modelRoleType.usedOns
                    if len(modelRoleTypes) > 1:
                        val.modelXbrl.error(("EFM.6.07.10", "GFM.1.03.10"),
                            _("RoleType %(roleType)s is defined in multiple taxonomies"),
                            modelObject=modelRoleTypes, roleType=roleURI, numberOfDeclarations=len(modelRoleTypes))
                    elif len(modelRoleTypes) == 1:
                        # 6.7.11 used on's for pre, cal, def if any has a used on
                        if not usedOns.isdisjoint(requiredUsedOns) and len(requiredUsedOns - usedOns) > 0:
                            val.modelXbrl.error(("EFM.6.07.11", "GFM.1.03.11"),
                                _("RoleType %(roleType)s missing used on %(usedOn)s"),
                                modelObject=e, roleType=roleURI, usedOn=requiredUsedOns - usedOns)
                            
                        # 6.7.12 definition match pattern
                        if (val.disclosureSystem.roleDefinitionPattern is not None and
                            (definition is None or not val.disclosureSystem.roleDefinitionPattern.match(definition))):
                            val.modelXbrl.error(("EFM.6.07.12", "GFM.1.03.12-14"),
                                _("RoleType %(roleType)s definition \"%(definition)s\" must match {Sortcode} - {Type} - {Title}"),
                                modelObject=e, roleType=roleURI, definition=definition)

                    if usedOns & standardUsedOns: # semantics check
                        val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.03", "GFM.2.03.03"),
                            _("RoleType %(roleuri)s is defined using role types already defined by standard roles for: %(qnames)s"),
                            modelObject=e, roleuri=roleURI, qnames=', '.join(str(qn) for qn in usedOns & standardUsedOns))

                    if val.validateSBRNL:
                        if usedOns & XbrlConst.standardExtLinkQnames or XbrlConst.qnGenLink in usedOns:
                            definesLinkroles = True
                            if not e.genLabel():
                                val.modelXbrl.error("SBR.NL.2.2.3.03",
                                    _("Link RoleType %(roleType)s missing a generic standard label"),
                                    modelObject=e, roleType=roleURI)
                            nlLabel = e.genLabel(lang="nl")
                            if definition != nlLabel:
                                val.modelXbrl.error("SBR.NL.2.2.3.04",
                                    _("Link RoleType %(roleType)s definition does not match NL standard generic label, \ndefinition: %(definition)s \nNL label: %(label)s"),
                                    modelObject=e, roleType=roleURI, definition=definition, label=nlLabel)
                        if definition and (definition[0].isspace() or definition[-1].isspace()):
                            val.modelXbrl.error("SBR.NL.2.2.3.07",
                                _('Link RoleType %(roleType)s definition has leading or trailing spaces: "%(definition)s"'),
                                modelObject=e, roleType=roleURI, definition=definition)

        # 6.7.13 arcrole types authority
        for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}arcroleType"):
            if isinstance(e,ModelObject):
                arcroleURI = e.get("arcroleURI")
                if targetNamespaceAuthority != UrlUtil.authority(arcroleURI):
                    val.modelXbrl.error(("EFM.6.07.13", "GFM.1.03.15"),
                        _("ArcroleType %(arcroleType)s does not match authority %(targetNamespaceAuthority)s"),
                        modelObject=e, arcroleType=arcroleURI, targetNamespaceAuthority=targetNamespaceAuthority)
                # 6.7.13 end with .../arcrole/lc3 name
                if not arcroleTypePattern.match(arcroleURI):
                    val.modelXbrl.warning(("EFM.6.07.13.arcroleEnding", "GFM.1.03.15"),
                        _("ArcroleType %(arcroleType)s should end with /arcrole/{LC3name}"),
                        modelObject=e, arcroleType=arcroleURI)
                    
                # 6.7.14 only one arcrole type declaration in DTS
                modelRoleTypes = val.modelXbrl.arcroleTypes[arcroleURI]
                if len(modelRoleTypes) > 1:
                    val.modelXbrl.error(("EFM.6.07.14", "GFM.1.03.16"),
                        _("ArcroleType %(arcroleType)s is defined in multiple taxonomies"),
                        modelObject=e, arcroleType=arcroleURI, numberOfDeclarations=len(modelRoleTypes) )
                    
                # 6.7.15 definition match pattern
                definition = modelRoleTypes[0].definition
                if definition is None or not arcroleDefinitionPattern.match(definition):
                    val.modelXbrl.error(("EFM.6.07.15", "GFM.1.03.17"),
                        _("ArcroleType %(arcroleType)s definition must be non-empty"),
                        modelObject=e, arcroleType=arcroleURI)
    
                # semantic checks
                usedOns = modelRoleTypes[0].usedOns
                if usedOns & standardUsedOns: # semantics check
                    val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.03", "GFM.2.03.03"),
                        _("ArcroleType %(arcroleuri)s is defined using role types already defined by standard arcroles for: %(qnames)s"),
                        modelObject=e, arcroleuri=arcroleURI, qnames=', '.join(str(qn) for qn in usedOns & standardUsedOns))

                if val.validateSBRNL:
                    definesArcroles = True
                    val.modelXbrl.error("SBR.NL.2.2.4.01",
                        _("Arcrole type definition is not allowed: %(arcroleURI)s"),
                        modelObject=e, arcroleURI=arcroleURI)
                    
        if val.validateSBRNL:
            for appinfoElt in modelDocument.xmlRootElement.iter(tag="{http://www.w3.org/2001/XMLSchema}appinfo"):
                for nonLinkElt in appinfoElt.iterdescendants():
                    if isinstance(nonLinkElt, ModelObject) and nonLinkElt.namespaceURI != XbrlConst.link:
                        val.modelXbrl.error("SBR.NL.2.2.11.05",
                            _("Appinfo contains disallowed non-link element %(element)s"),
                            modelObject=nonLinkElt, element=nonLinkElt.qname)

            for cplxTypeElt in modelDocument.xmlRootElement.iter(tag="{http://www.w3.org/2001/XMLSchema}complexType"):
                choiceElt = cplxTypeElt.find("{http://www.w3.org/2001/XMLSchema}choice")
                if choiceElt is not None:
                    val.modelXbrl.error("SBR.NL.2.2.11.09",
                        _("ComplexType contains disallowed xs:choice element"),
                        modelObject=choiceElt)
                    
            for cplxContentElt in modelDocument.xmlRootElement.iter(tag="{http://www.w3.org/2001/XMLSchema}complexContent"):
                if XmlUtil.descendantAttr(cplxContentElt, "http://www.w3.org/2001/XMLSchema", ("extension","restriction"), "base") != "sbr:placeholder":
                    val.modelXbrl.error("SBR.NL.2.2.11.10",
                        _("ComplexContent is disallowed"),
                        modelObject=cplxContentElt)

            definesTypes = (modelDocument.xmlRootElement.find("{http://www.w3.org/2001/XMLSchema}complexType") is not None or
                            modelDocument.xmlRootElement.find("{http://www.w3.org/2001/XMLSchema}simpleType") is not None)
            if (definesLinkroles + definesArcroles + definesLinkParts +
                definesAbstractItems + definesNonabstractItems + 
                definesTuples + definesPresentationTuples + definesSpecificationTuples + definesTypes +
                definesEnumerations + definesDimensions + definesDomains + 
                definesHypercubes) != 1:
                schemaContents = []
                if definesLinkroles: schemaContents.append(_("linkroles"))
                if definesArcroles: schemaContents.append(_("arcroles"))
                if definesLinkParts: schemaContents.append(_("link parts"))
                if definesAbstractItems: schemaContents.append(_("abstract items"))
                if definesNonabstractItems: schemaContents.append(_("nonabstract items"))
                if definesTuples: schemaContents.append(_("tuples"))
                if definesPresentationTuples: schemaContents.append(_("sbrPresentationTuples"))
                if definesSpecificationTuples: schemaContents.append(_("sbrSpecificationTuples"))
                if definesTypes: schemaContents.append(_("types"))
                if definesEnumerations: schemaContents.append(_("enumerations"))
                if definesDimensions: schemaContents.append(_("dimensions"))
                if definesDomains: schemaContents.append(_("domains"))
                if definesHypercubes: schemaContents.append(_("hypercubes"))
                if schemaContents:
                    val.modelXbrl.error("SBR.NL.2.2.1.01",
                        _("Taxonomy schema may only define one of these: %(contents)s"),
                        modelObject=modelDocument, contents=', '.join(schemaContents))
                elif not any(refDoc.inDTS and refDoc.targetNamespace not in val.disclosureSystem.baseTaxonomyNamespaces
                             for refDoc in modelDocument.referencesDocument.keys()): # no linkbase ref or includes
                    val.modelXbrl.error("SBR.NL.2.2.1.01",
                        _("Taxonomy schema must be a DTS entrypoint OR define linkroles OR arcroles OR link:parts OR context fragments OR abstract items OR tuples OR non-abstract elements OR types OR enumerations OR dimensions OR domains OR hypercubes"),
                        modelObject=modelDocument)
            if definesConcepts ^ any(  # xor so either concepts and no label LB or no concepts and has label LB
                       (refDoc.type == ModelDocument.Type.LINKBASE and
                        XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link, "labelLink") is not None)
                       for refDoc in modelDocument.referencesDocument.keys()): # no label linkbase
                val.modelXbrl.error("SBR.NL.2.2.1.02",
                    _("A schema that defines concepts MUST have a linked 2.1 label linkbase"),
                    modelObject=modelDocument)
            if (definesNonabstractItems or definesTuples) and not any(  # was xor but changed to and not per RH 1/11/12
                       (refDoc.type == ModelDocument.Type.LINKBASE and
                       (XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link, "referenceLink") is not None or
                        XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link, "label", "{http://www.w3.org/1999/xlink}role", "http://www.xbrl.org/2003/role/documentation" ) is not None))
                        for refDoc in modelDocument.referencesDocument.keys()):
                val.modelXbrl.error("SBR.NL.2.2.1.03",
                    _("A schema that defines non-abstract items MUST have a linked (2.1) reference linkbase AND/OR a label linkbase with @xlink:role=documentation"),
                    modelObject=modelDocument)

        if val.fileNameBasePart:
            #6.3.3 filename check
            expectedFilename = "{0}-{1}.xsd".format(val.fileNameBasePart, val.fileNameDatePart)
            if modelDocument.basename != expectedFilename and not ( # skip if an edgar testcase
               re.match("e[0-9]{8}(gd|ng)", val.fileNameBasePart) and re.match("e.*-[0-9]{8}.*", modelDocument.basename)):
                val.modelXbrl.error(("EFM.6.03.03", "GFM.1.01.01"),
                    _('Invalid schema file name: %(filename)s, expected %(expectedFilename)s'),
                    modelObject=modelDocument, filename=modelDocument.basename, expectedFilename=expectedFilename)

    elif modelDocument.type == ModelDocument.Type.LINKBASE:
        # if it is part of the submission (in same directory) check name
        if modelDocument.filepath.startswith(val.modelXbrl.modelDocument.filepathdir):
            #6.3.3 filename check
            extLinkElt = XmlUtil.descendant(modelDocument.xmlRootElement, XbrlConst.link, "*", "{http://www.w3.org/1999/xlink}type", "extended")
            if val.fileNameBasePart:
                if extLinkElt is not None:
                    if extLinkElt.localName in extLinkEltFileNameEnding: 
                        expectedFilename = "{0}-{1}_{2}.xml".format(val.fileNameBasePart, val.fileNameDatePart, 
                                                                    extLinkEltFileNameEnding[extLinkElt.localName])
                        if modelDocument.basename != expectedFilename and not ( # skip if an edgar testcase
                            re.match("e[0-9]{8}(gd|ng)", val.fileNameBasePart) and re.match("e.*-[0-9]{8}.*", modelDocument.basename)):
                            val.modelXbrl.error(("EFM.6.03.03", "GFM.1.01.01"),
                                _('Invalid linkbase file name: %(filename)s, expected %(expectedFilename)s'),
                                modelObject=modelDocument, filename=modelDocument.basename, expectedFilename=expectedFilename)
                else: # no ext link element
                    val.modelXbrl.error(("EFM.6.03.03.noLinkElement", "GFM.1.01.01.noLinkElement"),
                        _('Invalid linkbase file name: %(filename)s, has no extended link element, cannot determine link type.'),
                        modelObject=modelDocument, filename=modelDocument.basename)
            
    visited.remove(modelDocument)
 def blockedMessageCodes(self):
     return XmlUtil.descendantAttr(self, None, "results", "blockedMessageCodes")
Beispiel #10
0
def checkFilingDTS(val, modelDocument, visited):
    global targetNamespaceDatePattern, efmFilenamePattern, roleTypePattern, arcroleTypePattern, \
            arcroleDefinitionPattern, namePattern, linkroleDefinitionBalanceIncomeSheet, \
            namespacesConflictPattern
    if targetNamespaceDatePattern is None:
        targetNamespaceDatePattern = re.compile(
            r"/([12][0-9]{3})-([01][0-9])-([0-3][0-9])|"
            r"/([12][0-9]{3})([01][0-9])([0-3][0-9])|")
        efmFilenamePattern = re.compile(
            r"^[a-z0-9][a-zA-Z0-9_\.\-]*(\.xsd|\.xml)$")
        roleTypePattern = re.compile(r"^.*/role/[^/\s]+$")
        arcroleTypePattern = re.compile(r"^.*/arcrole/[^/\s]+$")
        arcroleDefinitionPattern = re.compile(
            r"^.*[^\\s]+.*$")  # at least one non-whitespace character
        namePattern = re.compile(
            "[][()*+?\\\\/^{}|@#%^=~`\"';:,<>&$\u00a3\u20ac]"
        )  # u20ac=Euro, u00a3=pound sterling
        linkroleDefinitionBalanceIncomeSheet = re.compile(
            r"[^-]+-\s+Statement\s+-\s+.*(income|balance|financial\W+position)",
            re.IGNORECASE)
        namespacesConflictPattern = re.compile(
            r"http://(xbrl\.us|fasb\.org|xbrl\.sec\.gov)/(dei|us-types|us-roles|rr)/([0-9]{4}-[0-9]{2}-[0-9]{2})$"
        )

    visited.append(modelDocument)
    for referencedDocument, modelDocumentReference in modelDocument.referencesDocument.items(
    ):
        #6.07.01 no includes
        if "include" in modelDocumentReference.referenceTypes:
            val.modelXbrl.error(
                "SBR.NL.2.2.0.18",
                _("Taxonomy schema %(schema)s includes %(include)s, only import is allowed"
                  ),
                modelObject=modelDocumentReference.referringModelObject,
                schema=os.path.basename(modelDocument.uri),
                include=os.path.basename(referencedDocument.uri))
        if referencedDocument not in visited:
            checkFilingDTS(val, referencedDocument, visited)

    if val.disclosureSystem.standardTaxonomiesDict is None:
        pass

    if (modelDocument.type == ModelDocument.Type.SCHEMA
            and modelDocument.targetNamespace
            not in val.disclosureSystem.baseTaxonomyNamespaces
            and modelDocument.uri.startswith(val.modelXbrl.uriDir)):

        # check schema contents types
        definesLinkroles = False
        definesArcroles = False
        definesLinkParts = False
        definesAbstractItems = False
        definesNonabstractItems = False
        definesConcepts = False
        definesTuples = False
        definesPresentationTuples = False
        definesSpecificationTuples = False
        definesTypes = False
        definesEnumerations = False
        definesDimensions = False
        definesDomains = False
        definesHypercubes = False

        genrlSpeclRelSet = val.modelXbrl.relationshipSet(
            XbrlConst.generalSpecial)
        for modelConcept in modelDocument.xmlRootElement.iterdescendants(
                tag="{http://www.w3.org/2001/XMLSchema}element"):
            if isinstance(modelConcept, ModelConcept):
                # 6.7.16 name not duplicated in standard taxonomies
                name = modelConcept.get("name")
                if name is None:
                    name = ""
                    if modelConcept.get("ref") is not None:
                        continue  # don't validate ref's here
                for c in val.modelXbrl.nameConcepts.get(name, []):
                    if c.modelDocument != modelDocument:
                        if not (genrlSpeclRelSet.isRelated(
                                modelConcept, "child", c)
                                or genrlSpeclRelSet.isRelated(
                                    c, "child", modelConcept)):
                            val.modelXbrl.error(
                                "SBR.NL.2.2.2.02",
                                _("Concept %(concept)s is also defined in standard taxonomy schema %(standardSchema)s without a general-special relationship"
                                  ),
                                modelObject=c,
                                concept=modelConcept.qname,
                                standardSchema=os.path.basename(
                                    c.modelDocument.uri))
                ''' removed RH 2011-12-23 corresponding set up of table in ValidateFiling
                if val.validateSBRNL and name in val.nameWordsTable:
                    if not any( any( genrlSpeclRelSet.isRelated(c, "child", modelConcept)
                                     for c in val.modelXbrl.nameConcepts.get(partialWordName, []))
                                for partialWordName in val.nameWordsTable[name]):
                        val.modelXbrl.error("SBR.NL.2.3.2.01",
                            _("Concept %(specialName)s is appears to be missing a general-special relationship to %(generalNames)s"),
                            modelObject=c, specialName=modelConcept.qname, generalNames=', or to '.join(val.nameWordsTable[name]))
                '''

                if modelConcept.isTuple:
                    if modelConcept.substitutionGroupQname.localName == "presentationTuple" and modelConcept.substitutionGroupQname.namespaceURI.endswith(
                            "/basis/sbr/xbrl/xbrl-syntax-extension"
                    ):  # namespace may change each year
                        definesPresentationTuples = True
                    elif modelConcept.substitutionGroupQname.localName == "specificationTuple" and modelConcept.substitutionGroupQname.namespaceURI.endswith(
                            "/basis/sbr/xbrl/xbrl-syntax-extension"
                    ):  # namespace may change each year
                        definesSpecificationTuples = True
                    else:
                        definesTuples = True
                    definesConcepts = True
                    if modelConcept.isAbstract:
                        val.modelXbrl.error(
                            "SBR.NL.2.2.2.03",
                            _("Concept %(concept)s is an abstract tuple"),
                            modelObject=modelConcept,
                            concept=modelConcept.qname)
                    if tupleCycle(val, modelConcept):
                        val.modelXbrl.error(
                            "SBR.NL.2.2.2.07",
                            _("Tuple %(concept)s has a tuple cycle"),
                            modelObject=modelConcept,
                            concept=modelConcept.qname)
                    if modelConcept.get(
                            "nillable") != "false" and modelConcept.isRoot:
                        val.modelXbrl.error(
                            "SBR.NL.2.2.2.17",  #don't want default, just what was really there
                            _("Tuple %(concept)s must have nillable='false'"),
                            modelObject=modelConcept,
                            concept=modelConcept.qname)
                elif modelConcept.isItem:
                    definesConcepts = True
                if modelConcept.abstract == "true":
                    if modelConcept.isRoot:
                        if modelConcept.get(
                                "nillable"
                        ) != "false":  #don't want default, just what was really there
                            val.modelXbrl.error(
                                "SBR.NL.2.2.2.16",
                                _("Abstract root concept %(concept)s must have nillable='false'"
                                  ),
                                modelObject=modelConcept,
                                concept=modelConcept.qname)
                        if modelConcept.typeQname != XbrlConst.qnXbrliStringItemType:
                            val.modelXbrl.error(
                                "SBR.NL.2.2.2.21",
                                _("Abstract root concept %(concept)s must have type='xbrli:stringItemType'"
                                  ),
                                modelObject=modelConcept,
                                concept=modelConcept.qname)
                    if modelConcept.balance:
                        val.modelXbrl.error(
                            "SBR.NL.2.2.2.22",
                            _("Abstract concept %(concept)s must not have a balance attribute"
                              ),
                            modelObject=modelConcept,
                            concept=modelConcept.qname)
                    if modelConcept.isHypercubeItem:
                        definesHypercubes = True
                    elif modelConcept.isDimensionItem:
                        definesDimensions = True
                    elif modelConcept.substitutionGroupQname and modelConcept.substitutionGroupQname.localName in (
                            "domainItem", "domainMemberItem"):
                        definesDomains = True
                    elif modelConcept.isItem:
                        definesAbstractItems = True
                else:  # not abstract
                    if modelConcept.isItem:
                        definesNonabstractItems = True
                        if not (modelConcept.label(
                                preferredLabel=XbrlConst.documentationLabel,
                                fallbackToQname=False,
                                lang="nl") or val.modelXbrl.relationshipSet(
                                    XbrlConst.conceptReference
                                ).fromModelObject(c) or modelConcept.genLabel(
                                    role=XbrlConst.genDocumentationLabel,
                                    lang="nl")
                                or val.modelXbrl.relationshipSet(
                                    XbrlConst.elementReference).
                                fromModelObject(c)):
                            val.modelXbrl.error(
                                "SBR.NL.2.2.2.28",
                                _("Concept %(concept)s must have a documentation label or reference"
                                  ),
                                modelObject=modelConcept,
                                concept=modelConcept.qname)
                if modelConcept.balance and not modelConcept.instanceOfType(
                        XbrlConst.qnXbrliMonetaryItemType):
                    val.modelXbrl.error(
                        "SBR.NL.2.2.2.24",
                        _("Non-monetary concept %(concept)s must not have a balance attribute"
                          ),
                        modelObject=modelConcept,
                        concept=modelConcept.qname)
                if modelConcept.isLinkPart:
                    definesLinkParts = True
                    val.modelXbrl.error(
                        "SBR.NL.2.2.5.01",
                        _("Link:part concept %(concept)s is not allowed"),
                        modelObject=modelConcept,
                        concept=modelConcept.qname)
                    if not modelConcept.genLabel(fallbackToQname=False,
                                                 lang="nl"):
                        val.modelXbrl.error(
                            "SBR.NL.2.2.5.02",
                            _("Link part definition %(concept)s must have a generic label in language 'nl'"
                              ),
                            modelObject=modelConcept,
                            concept=modelConcept.qname)

        # 6.7.9 role types authority
        for e in modelDocument.xmlRootElement.iterdescendants(
                tag="{http://www.xbrl.org/2003/linkbase}roleType"):
            if isinstance(e, ModelObject):
                roleURI = e.get("roleURI")
                # 6.7.10 only one role type declaration in DTS
                modelRoleTypes = val.modelXbrl.roleTypes.get(roleURI)
                if modelRoleTypes is not None:
                    modelRoleType = modelRoleTypes[0]
                    definition = modelRoleType.definitionNotStripped
                    usedOns = modelRoleType.usedOns
                    if usedOns & XbrlConst.standardExtLinkQnames or XbrlConst.qnGenLink in usedOns:
                        definesLinkroles = True
                        if not e.genLabel():
                            val.modelXbrl.error(
                                "SBR.NL.2.2.3.03",
                                _("Link RoleType %(roleType)s missing a generic standard label"
                                  ),
                                modelObject=e,
                                roleType=roleURI)
                        nlLabel = e.genLabel(lang="nl")
                        if definition != nlLabel:
                            val.modelXbrl.error(
                                "SBR.NL.2.2.3.04",
                                _("Link RoleType %(roleType)s definition does not match NL standard generic label, \ndefinition: %(definition)s \nNL label: %(label)s"
                                  ),
                                modelObject=e,
                                roleType=roleURI,
                                definition=definition,
                                label=nlLabel)
                    if definition and (definition[0].isspace()
                                       or definition[-1].isspace()):
                        val.modelXbrl.error(
                            "SBR.NL.2.2.3.07",
                            _('Link RoleType %(roleType)s definition has leading or trailing spaces: "%(definition)s"'
                              ),
                            modelObject=e,
                            roleType=roleURI,
                            definition=definition)

        # 6.7.13 arcrole types authority
        for e in modelDocument.xmlRootElement.iterdescendants(
                tag="{http://www.xbrl.org/2003/linkbase}arcroleType"):
            if isinstance(e, ModelObject):
                arcroleURI = e.get("arcroleURI")
                definesArcroles = True
                val.modelXbrl.error(
                    "SBR.NL.2.2.4.01",
                    _("Arcrole type definition is not allowed: %(arcroleURI)s"
                      ),
                    modelObject=e,
                    arcroleURI=arcroleURI)

        for appinfoElt in modelDocument.xmlRootElement.iter(
                tag="{http://www.w3.org/2001/XMLSchema}appinfo"):
            for nonLinkElt in appinfoElt.iterdescendants():
                if isinstance(nonLinkElt, ModelObject
                              ) and nonLinkElt.namespaceURI != XbrlConst.link:
                    val.modelXbrl.error(
                        "SBR.NL.2.2.11.05",
                        _("Appinfo contains disallowed non-link element %(element)s"
                          ),
                        modelObject=nonLinkElt,
                        element=nonLinkElt.qname)

        for cplxTypeElt in modelDocument.xmlRootElement.iter(
                tag="{http://www.w3.org/2001/XMLSchema}complexType"):
            choiceElt = cplxTypeElt.find(
                "{http://www.w3.org/2001/XMLSchema}choice")
            if choiceElt is not None:
                val.modelXbrl.error(
                    "SBR.NL.2.2.11.09",
                    _("ComplexType contains disallowed xs:choice element"),
                    modelObject=choiceElt)

        for cplxContentElt in modelDocument.xmlRootElement.iter(
                tag="{http://www.w3.org/2001/XMLSchema}complexContent"):
            if XmlUtil.descendantAttr(
                    cplxContentElt, "http://www.w3.org/2001/XMLSchema",
                ("extension", "restriction"), "base") != "sbr:placeholder":
                val.modelXbrl.error("SBR.NL.2.2.11.10",
                                    _("ComplexContent is disallowed"),
                                    modelObject=cplxContentElt)

        for typeEltTag in ("{http://www.w3.org/2001/XMLSchema}complexType",
                           "{http://www.w3.org/2001/XMLSchema}simpleType"):
            for typeElt in modelDocument.xmlRootElement.iter(tag=typeEltTag):
                definesTypes = True
                name = typeElt.get("name")
                if name:
                    if not name[0].islower() or not name.isalnum():
                        val.modelXbrl.error(
                            "SBR.NL.3.2.8.09",
                            _("Type name attribute must be lower camelcase: %(name)s."
                              ),
                            modelObject=typeElt,
                            name=name)

        for enumElt in modelDocument.xmlRootElement.iter(
                tag="{http://www.w3.org/2001/XMLSchema}enumeration"):
            definesEnumerations = True
            if any(not valueElt.genLabel(lang="nl")
                   for valueElt in enumElt.iter(
                       tag="{http://www.w3.org/2001/XMLSchema}value")):
                val.modelXbrl.error(
                    "SBR.NL.2.2.7.05",
                    _("Enumeration element has value(s) without generic label."
                      ),
                    modelObject=enumElt)

        if (definesLinkroles + definesArcroles + definesLinkParts +
                definesAbstractItems + definesNonabstractItems + definesTuples
                + definesPresentationTuples + definesSpecificationTuples +
                definesTypes + definesEnumerations + definesDimensions +
                definesDomains + definesHypercubes) != 1:
            schemaContents = []
            if definesLinkroles: schemaContents.append(_("linkroles"))
            if definesArcroles: schemaContents.append(_("arcroles"))
            if definesLinkParts: schemaContents.append(_("link parts"))
            if definesAbstractItems: schemaContents.append(_("abstract items"))
            if definesNonabstractItems:
                schemaContents.append(_("nonabstract items"))
            if definesTuples: schemaContents.append(_("tuples"))
            if definesPresentationTuples:
                schemaContents.append(_("sbrPresentationTuples"))
            if definesSpecificationTuples:
                schemaContents.append(_("sbrSpecificationTuples"))
            if definesTypes: schemaContents.append(_("types"))
            if definesEnumerations: schemaContents.append(_("enumerations"))
            if definesDimensions: schemaContents.append(_("dimensions"))
            if definesDomains: schemaContents.append(_("domains"))
            if definesHypercubes: schemaContents.append(_("hypercubes"))
            if schemaContents:
                if not ((definesTuples or definesPresentationTuples
                         or definesSpecificationTuples)
                        and not (definesLinkroles or definesArcroles
                                 or definesLinkParts or definesAbstractItems
                                 or definesTypes or definesDimensions
                                 or definesDomains or definesHypercubes)):
                    val.modelXbrl.error(
                        "SBR.NL.2.2.1.01",
                        _("Taxonomy schema may only define one of these: %(contents)s"
                          ),
                        modelObject=modelDocument,
                        contents=', '.join(schemaContents))
            elif not any(refDoc.inDTS and refDoc.targetNamespace not in
                         val.disclosureSystem.baseTaxonomyNamespaces
                         for refDoc in modelDocument.referencesDocument.keys(
                         )):  # no linkbase ref or includes
                val.modelXbrl.error(
                    "SBR.NL.2.2.1.01",
                    _("Taxonomy schema must be a DTS entrypoint OR define linkroles OR arcroles OR link:parts OR context fragments OR abstract items OR tuples OR non-abstract elements OR types OR enumerations OR dimensions OR domains OR hypercubes"
                      ),
                    modelObject=modelDocument)
        if definesConcepts ^ any(  # xor so either concepts and no label LB or no concepts and has label LB
            (refDoc.type == ModelDocument.Type.LINKBASE
             and XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link,
                                    "labelLink") is not None) for refDoc in
                modelDocument.referencesDocument.keys()):  # no label linkbase
            val.modelXbrl.error(
                "SBR.NL.2.2.1.02",
                _("A schema that defines concepts MUST have a linked 2.1 label linkbase"
                  ),
                modelObject=modelDocument)
        if (
                definesNonabstractItems or definesTuples
        ) and not any(  # was xor but changed to and not per RH 1/11/12
            (refDoc.type == ModelDocument.Type.LINKBASE and
             (XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link,
                                 "referenceLink") is not None
              or XmlUtil.descendant(
                  refDoc.xmlRootElement, XbrlConst.link, "label",
                  "{http://www.w3.org/1999/xlink}role",
                  "http://www.xbrl.org/2003/role/documentation") is not None))
                for refDoc in modelDocument.referencesDocument.keys()):
            val.modelXbrl.error(
                "SBR.NL.2.2.1.03",
                _("A schema that defines non-abstract items MUST have a linked (2.1) reference linkbase AND/OR a label linkbase with @xlink:role=documentation"
                  ),
                modelObject=modelDocument)

    elif modelDocument.type == ModelDocument.Type.LINKBASE:
        pass
    visited.remove(modelDocument)
Beispiel #11
0
def checkFilingDTS(val, modelDocument, visited):
    global targetNamespaceDatePattern, efmFilenamePattern, roleTypePattern, arcroleTypePattern, \
            arcroleDefinitionPattern, namePattern, linkroleDefinitionBalanceIncomeSheet, \
            namespacesConflictPattern
    if targetNamespaceDatePattern is None:
        targetNamespaceDatePattern = re.compile(r"/([12][0-9]{3})-([01][0-9])-([0-3][0-9])|"
                                            r"/([12][0-9]{3})([01][0-9])([0-3][0-9])|")
        efmFilenamePattern = re.compile(r"^[a-z0-9][a-zA-Z0-9_\.\-]*(\.xsd|\.xml)$")
        roleTypePattern = re.compile(r"^.*/role/[^/\s]+$")
        arcroleTypePattern = re.compile(r"^.*/arcrole/[^/\s]+$")
        arcroleDefinitionPattern = re.compile(r"^.*[^\\s]+.*$")  # at least one non-whitespace character
        namePattern = re.compile("[][()*+?\\\\/^{}|@#%^=~`\"';:,<>&$\u00a3\u20ac]") # u20ac=Euro, u00a3=pound sterling 
        linkroleDefinitionBalanceIncomeSheet = re.compile(r"[^-]+-\s+Statement\s+-\s+.*(income|balance|financial\W+position)",
                                                          re.IGNORECASE)
        namespacesConflictPattern = re.compile(r"http://(xbrl\.us|fasb\.org|xbrl\.sec\.gov)/(dei|us-types|us-roles|rr)/([0-9]{4}-[0-9]{2}-[0-9]{2})$")
        
    visited.append(modelDocument)
    for referencedDocument, modelDocumentReference in modelDocument.referencesDocument.items():
        #6.07.01 no includes
        if modelDocumentReference.referenceType == "include":
            val.modelXbrl.error("SBR.NL.2.2.0.18",
                _("Taxonomy schema %(schema)s includes %(include)s, only import is allowed"),
                modelObject=modelDocumentReference.referringModelObject,
                    schema=os.path.basename(modelDocument.uri), 
                    include=os.path.basename(referencedDocument.uri))
        if referencedDocument not in visited:
            checkFilingDTS(val, referencedDocument, visited)
            
    if val.disclosureSystem.standardTaxonomiesDict is None:
        pass

    if (modelDocument.type == ModelDocument.Type.SCHEMA and 
        modelDocument.targetNamespace not in val.disclosureSystem.baseTaxonomyNamespaces and
        modelDocument.uri.startswith(val.modelXbrl.uriDir)):
        
        # check schema contents types
        definesLinkroles = False
        definesArcroles = False
        definesLinkParts = False
        definesAbstractItems = False
        definesNonabstractItems = False
        definesConcepts = False
        definesTuples = False
        definesPresentationTuples = False
        definesSpecificationTuples = False
        definesTypes = False
        definesEnumerations = False
        definesDimensions = False
        definesDomains = False
        definesHypercubes = False
                
        genrlSpeclRelSet = val.modelXbrl.relationshipSet(XbrlConst.generalSpecial)
        for modelConcept in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/2001/XMLSchema}element"):
            if isinstance(modelConcept,ModelConcept):
                # 6.7.16 name not duplicated in standard taxonomies
                name = modelConcept.get("name")
                if name is None: 
                    name = ""
                    if modelConcept.get("ref") is not None:
                        continue    # don't validate ref's here
                for c in val.modelXbrl.nameConcepts.get(name, []):
                    if c.modelDocument != modelDocument:
                        if not (genrlSpeclRelSet.isRelated(modelConcept, "child", c) or genrlSpeclRelSet.isRelated(c, "child", modelConcept)):
                            val.modelXbrl.error("SBR.NL.2.2.2.02",
                                _("Concept %(concept)s is also defined in standard taxonomy schema %(standardSchema)s without a general-special relationship"),
                                modelObject=c, concept=modelConcept.qname, standardSchema=os.path.basename(c.modelDocument.uri))
                ''' removed RH 2011-12-23 corresponding set up of table in ValidateFiling
                if val.validateSBRNL and name in val.nameWordsTable:
                    if not any( any( genrlSpeclRelSet.isRelated(c, "child", modelConcept)
                                     for c in val.modelXbrl.nameConcepts.get(partialWordName, []))
                                for partialWordName in val.nameWordsTable[name]):
                        val.modelXbrl.error("SBR.NL.2.3.2.01",
                            _("Concept %(specialName)s is appears to be missing a general-special relationship to %(generalNames)s"),
                            modelObject=c, specialName=modelConcept.qname, generalNames=', or to '.join(val.nameWordsTable[name]))
                '''

                if modelConcept.isTuple:
                    if modelConcept.substitutionGroupQname.localName == "presentationTuple" and modelConcept.substitutionGroupQname.namespaceURI.endswith("/basis/sbr/xbrl/xbrl-syntax-extension"): # namespace may change each year
                        definesPresentationTuples = True
                    elif modelConcept.substitutionGroupQname.localName == "specificationTuple" and modelConcept.substitutionGroupQname.namespaceURI.endswith("/basis/sbr/xbrl/xbrl-syntax-extension"): # namespace may change each year
                        definesSpecificationTuples = True
                    else:
                        definesTuples = True
                    definesConcepts = True
                    if modelConcept.isAbstract:
                        val.modelXbrl.error("SBR.NL.2.2.2.03",
                            _("Concept %(concept)s is an abstract tuple"),
                            modelObject=modelConcept, concept=modelConcept.qname)
                    if tupleCycle(val,modelConcept):
                        val.modelXbrl.error("SBR.NL.2.2.2.07",
                            _("Tuple %(concept)s has a tuple cycle"),
                            modelObject=modelConcept, concept=modelConcept.qname)
                    if modelConcept.get("nillable") != "false" and modelConcept.isRoot:
                        val.modelXbrl.error("SBR.NL.2.2.2.17", #don't want default, just what was really there
                            _("Tuple %(concept)s must have nillable='false'"),
                            modelObject=modelConcept, concept=modelConcept.qname)
                elif modelConcept.isItem:
                    definesConcepts = True
                if modelConcept.abstract == "true":
                    if modelConcept.isRoot:
                        if modelConcept.get("nillable") != "false": #don't want default, just what was really there
                            val.modelXbrl.error("SBR.NL.2.2.2.16",
                                _("Abstract root concept %(concept)s must have nillable='false'"),
                            modelObject=modelConcept, concept=modelConcept.qname)
                        if modelConcept.typeQname != XbrlConst.qnXbrliStringItemType:
                            val.modelXbrl.error("SBR.NL.2.2.2.21",
                                _("Abstract root concept %(concept)s must have type='xbrli:stringItemType'"),
                            modelObject=modelConcept, concept=modelConcept.qname)
                    if modelConcept.balance:
                        val.modelXbrl.error("SBR.NL.2.2.2.22",
                            _("Abstract concept %(concept)s must not have a balance attribute"),
                            modelObject=modelConcept, concept=modelConcept.qname)
                    if modelConcept.isHypercubeItem:
                        definesHypercubes = True
                    elif modelConcept.isDimensionItem:
                        definesDimensions = True
                    elif modelConcept.substitutionGroupQname and modelConcept.substitutionGroupQname.localName in ("domainItem","domainMemberItem"):
                        definesDomains = True
                    elif modelConcept.isItem:
                        definesAbstractItems = True
                else:   # not abstract
                    if modelConcept.isItem:
                        definesNonabstractItems = True
                        if not (modelConcept.label(preferredLabel=XbrlConst.documentationLabel,fallbackToQname=False,lang="nl") or
                                val.modelXbrl.relationshipSet(XbrlConst.conceptReference).fromModelObject(c) or
                                modelConcept.genLabel(role=XbrlConst.genDocumentationLabel,lang="nl") or
                                val.modelXbrl.relationshipSet(XbrlConst.elementReference).fromModelObject(c)):
                            val.modelXbrl.error("SBR.NL.2.2.2.28",
                                _("Concept %(concept)s must have a documentation label or reference"),
                                modelObject=modelConcept, concept=modelConcept.qname)
                if modelConcept.balance and not modelConcept.instanceOfType(XbrlConst.qnXbrliMonetaryItemType):
                    val.modelXbrl.error("SBR.NL.2.2.2.24",
                        _("Non-monetary concept %(concept)s must not have a balance attribute"),
                        modelObject=modelConcept, concept=modelConcept.qname)
                if modelConcept.isLinkPart:
                    definesLinkParts = True
                    val.modelXbrl.error("SBR.NL.2.2.5.01",
                        _("Link:part concept %(concept)s is not allowed"),
                        modelObject=modelConcept, concept=modelConcept.qname)
                    if not modelConcept.genLabel(fallbackToQname=False,lang="nl"):
                        val.modelXbrl.error("SBR.NL.2.2.5.02",
                            _("Link part definition %(concept)s must have a generic label in language 'nl'"),
                            modelObject=modelConcept, concept=modelConcept.qname)

        # 6.7.9 role types authority
        for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}roleType"):
            if isinstance(e,ModelObject):
                roleURI = e.get("roleURI")
                # 6.7.10 only one role type declaration in DTS
                modelRoleTypes = val.modelXbrl.roleTypes.get(roleURI)
                if modelRoleTypes is not None:
                    modelRoleType = modelRoleTypes[0]
                    definition = modelRoleType.definitionNotStripped
                    usedOns = modelRoleType.usedOns
                    if usedOns & XbrlConst.standardExtLinkQnames or XbrlConst.qnGenLink in usedOns:
                        definesLinkroles = True
                        if not e.genLabel():
                            val.modelXbrl.error("SBR.NL.2.2.3.03",
                                _("Link RoleType %(roleType)s missing a generic standard label"),
                                modelObject=e, roleType=roleURI)
                        nlLabel = e.genLabel(lang="nl")
                        if definition != nlLabel:
                            val.modelXbrl.error("SBR.NL.2.2.3.04",
                                _("Link RoleType %(roleType)s definition does not match NL standard generic label, \ndefinition: %(definition)s \nNL label: %(label)s"),
                                modelObject=e, roleType=roleURI, definition=definition, label=nlLabel)
                    if definition and (definition[0].isspace() or definition[-1].isspace()):
                        val.modelXbrl.error("SBR.NL.2.2.3.07",
                            _('Link RoleType %(roleType)s definition has leading or trailing spaces: "%(definition)s"'),
                            modelObject=e, roleType=roleURI, definition=definition)

        # 6.7.13 arcrole types authority
        for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}arcroleType"):
            if isinstance(e,ModelObject):
                arcroleURI = e.get("arcroleURI")
                definesArcroles = True
                val.modelXbrl.error("SBR.NL.2.2.4.01",
                    _("Arcrole type definition is not allowed: %(arcroleURI)s"),
                    modelObject=e, arcroleURI=arcroleURI)
                    
        for appinfoElt in modelDocument.xmlRootElement.iter(tag="{http://www.w3.org/2001/XMLSchema}appinfo"):
            for nonLinkElt in appinfoElt.iterdescendants():
                if isinstance(nonLinkElt, ModelObject) and nonLinkElt.namespaceURI != XbrlConst.link:
                    val.modelXbrl.error("SBR.NL.2.2.11.05",
                        _("Appinfo contains disallowed non-link element %(element)s"),
                        modelObject=nonLinkElt, element=nonLinkElt.qname)

        for cplxTypeElt in modelDocument.xmlRootElement.iter(tag="{http://www.w3.org/2001/XMLSchema}complexType"):
            choiceElt = cplxTypeElt.find("{http://www.w3.org/2001/XMLSchema}choice")
            if choiceElt is not None:
                val.modelXbrl.error("SBR.NL.2.2.11.09",
                    _("ComplexType contains disallowed xs:choice element"),
                    modelObject=choiceElt)
                
        for cplxContentElt in modelDocument.xmlRootElement.iter(tag="{http://www.w3.org/2001/XMLSchema}complexContent"):
            if XmlUtil.descendantAttr(cplxContentElt, "http://www.w3.org/2001/XMLSchema", ("extension","restriction"), "base") != "sbr:placeholder":
                val.modelXbrl.error("SBR.NL.2.2.11.10",
                    _("ComplexContent is disallowed"),
                    modelObject=cplxContentElt)

        for typeEltTag in ("{http://www.w3.org/2001/XMLSchema}complexType",
                            "{http://www.w3.org/2001/XMLSchema}simpleType"):
            for typeElt in modelDocument.xmlRootElement.iter(tag=typeEltTag):
                definesTypes = True
                name = typeElt.get("name")
                if name:
                    if not name[0].islower() or not name.isalnum():
                        val.modelXbrl.error("SBR.NL.3.2.8.09",
                            _("Type name attribute must be lower camelcase: %(name)s."),
                            modelObject=typeElt, name=name)
        
        for enumElt in modelDocument.xmlRootElement.iter(tag="{http://www.w3.org/2001/XMLSchema}enumeration"):
            definesEnumerations = True
            if any(not valueElt.genLabel(lang="nl")
                   for valueElt in enumElt.iter(tag="{http://www.w3.org/2001/XMLSchema}value")):
                val.modelXbrl.error("SBR.NL.2.2.7.05",
                    _("Enumeration element has value(s) without generic label."),
                    modelObject=enumElt)

        if (definesLinkroles + definesArcroles + definesLinkParts +
            definesAbstractItems + definesNonabstractItems + 
            definesTuples + definesPresentationTuples + definesSpecificationTuples + definesTypes +
            definesEnumerations + definesDimensions + definesDomains + 
            definesHypercubes) != 1:
            schemaContents = []
            if definesLinkroles: schemaContents.append(_("linkroles"))
            if definesArcroles: schemaContents.append(_("arcroles"))
            if definesLinkParts: schemaContents.append(_("link parts"))
            if definesAbstractItems: schemaContents.append(_("abstract items"))
            if definesNonabstractItems: schemaContents.append(_("nonabstract items"))
            if definesTuples: schemaContents.append(_("tuples"))
            if definesPresentationTuples: schemaContents.append(_("sbrPresentationTuples"))
            if definesSpecificationTuples: schemaContents.append(_("sbrSpecificationTuples"))
            if definesTypes: schemaContents.append(_("types"))
            if definesEnumerations: schemaContents.append(_("enumerations"))
            if definesDimensions: schemaContents.append(_("dimensions"))
            if definesDomains: schemaContents.append(_("domains"))
            if definesHypercubes: schemaContents.append(_("hypercubes"))
            if schemaContents:
                if not ((definesTuples or definesPresentationTuples or definesSpecificationTuples) and
                        not (definesLinkroles or definesArcroles or definesLinkParts or definesAbstractItems or
                             definesTypes or definesDimensions or definesDomains or definesHypercubes)):
                    val.modelXbrl.error("SBR.NL.2.2.1.01",
                        _("Taxonomy schema may only define one of these: %(contents)s"),
                        modelObject=modelDocument, contents=', '.join(schemaContents))
            elif not any(refDoc.inDTS and refDoc.targetNamespace not in val.disclosureSystem.baseTaxonomyNamespaces
                         for refDoc in modelDocument.referencesDocument.keys()): # no linkbase ref or includes
                val.modelXbrl.error("SBR.NL.2.2.1.01",
                    _("Taxonomy schema must be a DTS entrypoint OR define linkroles OR arcroles OR link:parts OR context fragments OR abstract items OR tuples OR non-abstract elements OR types OR enumerations OR dimensions OR domains OR hypercubes"),
                    modelObject=modelDocument)
        if definesConcepts ^ any(  # xor so either concepts and no label LB or no concepts and has label LB
                   (refDoc.type == ModelDocument.Type.LINKBASE and
                    XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link, "labelLink") is not None)
                   for refDoc in modelDocument.referencesDocument.keys()): # no label linkbase
            val.modelXbrl.error("SBR.NL.2.2.1.02",
                _("A schema that defines concepts MUST have a linked 2.1 label linkbase"),
                modelObject=modelDocument)
        if (definesNonabstractItems or definesTuples) and not any(  # was xor but changed to and not per RH 1/11/12
                   (refDoc.type == ModelDocument.Type.LINKBASE and
                   (XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link, "referenceLink") is not None or
                    XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link, "label", "{http://www.w3.org/1999/xlink}role", "http://www.xbrl.org/2003/role/documentation" ) is not None))
                    for refDoc in modelDocument.referencesDocument.keys()):
            val.modelXbrl.error("SBR.NL.2.2.1.03",
                _("A schema that defines non-abstract items MUST have a linked (2.1) reference linkbase AND/OR a label linkbase with @xlink:role=documentation"),
                modelObject=modelDocument)

    elif modelDocument.type == ModelDocument.Type.LINKBASE:
        pass
    visited.remove(modelDocument)
Beispiel #12
0
    def validate(self, modelXbrl, parameters=None):
        self.parameters = parameters
        self.precisionPattern = re.compile("^([0-9]+|INF)$")
        self.decimalsPattern = re.compile("^(-?[0-9]+|INF)$")
        self.isoCurrencyPattern = re.compile(r"^[A-Z]{3}$")
        self.modelXbrl = modelXbrl
        self.validateDisclosureSystem = modelXbrl.modelManager.validateDisclosureSystem
        self.disclosureSystem = modelXbrl.modelManager.disclosureSystem
        self.validateEFM = self.validateDisclosureSystem and self.disclosureSystem.EFM
        self.validateGFM = self.validateDisclosureSystem and self.disclosureSystem.GFM
        self.validateEFMorGFM = self.validateDisclosureSystem and self.disclosureSystem.EFMorGFM
        self.validateHMRC = self.validateDisclosureSystem and self.disclosureSystem.HMRC
        self.validateSBRNL = self.validateDisclosureSystem and self.disclosureSystem.SBRNL
        self.validateXmlLang = self.validateDisclosureSystem and self.disclosureSystem.xmlLangPattern
        self.validateCalcLB = modelXbrl.modelManager.validateCalcLB
        self.validateInferDecimals = modelXbrl.modelManager.validateInferDecimals
        
        # xlink validation
        modelXbrl.profileStat(None)
        modelXbrl.modelManager.showStatus(_("validating links"))
        modelLinks = set()
        self.remoteResourceLocElements = set()
        self.genericArcArcroles = set()
        for baseSetExtLinks in modelXbrl.baseSets.values():
            for baseSetExtLink in baseSetExtLinks:
                modelLinks.add(baseSetExtLink)    # ext links are unique (no dups)
        for modelLink in modelLinks:
            fromToArcs = {}
            locLabels = {}
            resourceLabels = {}
            resourceArcTos = []
            for arcElt in modelLink.iterchildren():
                if isinstance(arcElt,ModelObject):
                    xlinkType = arcElt.get("{http://www.w3.org/1999/xlink}type")
                    # locator must have an href
                    if xlinkType == "locator":
                        if arcElt.get("{http://www.w3.org/1999/xlink}href") is None:
                            modelXbrl.error("xlink:locatorHref",
                                _("Xlink locator %(xlinkLabel)s missing href in extended link %(linkrole)s"),
                                modelObject=arcElt,
                                linkrole=modelLink.role, 
                                xlinkLabel=arcElt.get("{http://www.w3.org/1999/xlink}label")) 
                        locLabels[arcElt.get("{http://www.w3.org/1999/xlink}label")] = arcElt
                    elif xlinkType == "resource":
                        resourceLabels[arcElt.get("{http://www.w3.org/1999/xlink}label")] = arcElt
                    # can be no duplicated arcs between same from and to
                    elif xlinkType == "arc":
                        fromLabel = arcElt.get("{http://www.w3.org/1999/xlink}from")
                        toLabel = arcElt.get("{http://www.w3.org/1999/xlink}to")
                        fromTo = (fromLabel,toLabel)
                        if fromTo in fromToArcs:
                            modelXbrl.error("xlink:dupArcs",
                                _("Duplicate xlink arcs  in extended link %(linkrole)s from %(xlinkLabelFrom)s to %(xlinkLabelTo)s"),
                                modelObject=arcElt,
                                linkrole=modelLink.role, 
                                xlinkLabelFrom=fromLabel, xlinkLabelTo=toLabel)
                        else:
                            fromToArcs[fromTo] = arcElt
                        if arcElt.namespaceURI == XbrlConst.link:
                            if arcElt.localName in arcNamesTo21Resource: #("labelArc","referenceArc"):
                                resourceArcTos.append((toLabel, arcElt.get("use"), arcElt))
                        elif self.isGenericArc(arcElt):
                            arcrole = arcElt.get("{http://www.w3.org/1999/xlink}arcrole")
                            self.genericArcArcroles.add(arcrole)
                            if arcrole in (XbrlConst.elementLabel, XbrlConst.elementReference):
                                resourceArcTos.append((toLabel, arcrole, arcElt))
                    # values of type (not needed for validating parsers)
                    if xlinkType not in xlinkTypeValues: # ("", "simple", "extended", "locator", "arc", "resource", "title", "none"):
                        modelXbrl.error("xlink:type",
                            _("Xlink type %(xlinkType)s invalid in extended link %(linkrole)s"),
                            modelObject=arcElt, linkrole=modelLink.role, xlinkType=xlinkType)
                    # values of actuate (not needed for validating parsers)
                    xlinkActuate = arcElt.get("{http://www.w3.org/1999/xlink}actuate")
                    if xlinkActuate not in xlinkActuateValues: # ("", "onLoad", "onRequest", "other", "none"):
                        modelXbrl.error("xlink:actuate",
                            _("Actuate %(xlinkActuate)s invalid in extended link %(linkrole)s"),
                            modelObject=arcElt, linkrole=modelLink.role, xlinkActuate=xlinkActuate)
                    # values of show (not needed for validating parsers)
                    xlinkShow = arcElt.get("{http://www.w3.org/1999/xlink}show")
                    if xlinkShow not in xlinkShowValues: # ("", "new", "replace", "embed", "other", "none"):
                        modelXbrl.error("xlink:show",
                            _("Show %(xlinkShow)s invalid in extended link %(linkrole)s"),
                            modelObject=arcElt, linkrole=modelLink.role, xlinkShow=xlinkShow)
            # check from, to of arcs have a resource or loc
            for fromTo, arcElt in fromToArcs.items():
                fromLabel, toLabel = fromTo
                for name, value, sect in (("from", fromLabel, "3.5.3.9.2"),("to",toLabel, "3.5.3.9.3")):
                    if value not in locLabels and value not in resourceLabels:
                        modelXbrl.error("xbrl.{0}:arcResource".format(sect),
                            _("Arc in extended link %(linkrole)s from %(xlinkLabelFrom)s to %(xlinkLabelTo)s attribute '%(attribute)s' has no matching loc or resource label"),
                            modelObject=arcElt, 
                            linkrole=modelLink.role, xlinkLabelFrom=fromLabel, xlinkLabelTo=toLabel, 
                            attribute=name)
                if arcElt.localName == "footnoteArc" and arcElt.namespaceURI == XbrlConst.link and \
                   arcElt.get("{http://www.w3.org/1999/xlink}arcrole") == XbrlConst.factFootnote:
                    if fromLabel not in locLabels:
                        modelXbrl.error("xbrl.4.11.1.3.1:factFootnoteArcFrom",
                            _("Footnote arc in extended link %(linkrole)s from %(xlinkLabelFrom)s to %(xlinkLabelTo)s \"from\" is not a loc"),
                            modelObject=arcElt, 
                            linkrole=modelLink.role, xlinkLabelFrom=fromLabel, xlinkLabelTo=toLabel)
                    if toLabel not in resourceLabels or qname(resourceLabels[toLabel]) != XbrlConst.qnLinkFootnote:
                        modelXbrl.error("xbrl.4.11.1.3.1:factFootnoteArcTo",
                            _("Footnote arc in extended link %(linkrole)s from %(xlinkLabelFrom)s to %(xlinkLabelTo)s \"to\" is not a footnote resource"),
                            modelObject=arcElt, 
                            linkrole=modelLink.role, xlinkLabelFrom=fromLabel, xlinkLabelTo=toLabel)
            # check unprohibited label arcs to remote locs
            for resourceArcTo in resourceArcTos:
                resourceArcToLabel, resourceArcUse, arcElt = resourceArcTo
                if resourceArcToLabel in locLabels:
                    toLabel = locLabels[resourceArcToLabel]
                    if resourceArcUse == "prohibited":
                        self.remoteResourceLocElements.add(toLabel)
                    else:
                        modelXbrl.error("xbrl.5.2.2.3:labelArcRemoteResource",
                            _("Unprohibited labelArc in extended link %(linkrole)s has illegal remote resource loc labeled %(xlinkLabel)s href %(xlinkHref)s"),
                            modelObject=arcElt, 
                            linkrole=modelLink.role, 
                            xlinkLabel=resourceArcToLabel,
                            xlinkHref=toLabel.get("{http://www.w3.org/1999/xlink}href"))
                elif resourceArcToLabel in resourceLabels:
                    toResource = resourceLabels[resourceArcToLabel]
                    if resourceArcUse == XbrlConst.elementLabel:
                        if not self.isGenericLabel(toResource):
                            modelXbrl.error("xbrlle.2.1.1:genericLabelTarget",
                                _("Generic label arc in extended link %(linkrole)s to %(xlinkLabel)s must target a generic label"),
                                modelObject=arcElt, 
                                linkrole=modelLink.role, 
                                xlinkLabel=resourceArcToLabel)
                    elif resourceArcUse == XbrlConst.elementReference:
                        if not self.isGenericReference(toResource):
                            modelXbrl.error("xbrlre.2.1.1:genericReferenceTarget",
                                _("Generic reference arc in extended link %(linkrole)s to %(xlinkLabel)s must target a generic reference"),
                                modelObject=arcElt, 
                                linkrole=modelLink.role, 
                                xlinkLabel=resourceArcToLabel)
            resourceArcTos = None # dereference arcs
        modelXbrl.profileStat(_("validateLinks"))

        modelXbrl.dimensionDefaultConcepts = {}
        modelXbrl.qnameDimensionDefaults = {}
        modelXbrl.qnameDimensionContextElement = {}
        # check base set cycles, dimensions
        modelXbrl.modelManager.showStatus(_("validating relationship sets"))
        for baseSetKey in modelXbrl.baseSets.keys():
            arcrole, ELR, linkqname, arcqname = baseSetKey
            if arcrole.startswith("XBRL-") or ELR is None or \
                linkqname is None or arcqname is None:
                continue
            elif arcrole in XbrlConst.standardArcroleCyclesAllowed:
                # TODO: table should be in this module, where it is used
                cyclesAllowed, specSect = XbrlConst.standardArcroleCyclesAllowed[arcrole]
            elif arcrole in self.modelXbrl.arcroleTypes and len(self.modelXbrl.arcroleTypes[arcrole]) > 0:
                cyclesAllowed = self.modelXbrl.arcroleTypes[arcrole][0].cyclesAllowed
                if arcrole in self.genericArcArcroles:
                    specSect = "xbrlgene:violatedCyclesConstraint"
                else:
                    specSect = "xbrl.5.1.4.3:cycles"
            else:
                cyclesAllowed = "any"
                specSect = None
            if cyclesAllowed != "any" or arcrole in (XbrlConst.summationItem,) \
                                      or arcrole in self.genericArcArcroles  \
                                      or arcrole.startswith(XbrlConst.formulaStartsWith):
                relsSet = modelXbrl.relationshipSet(arcrole,ELR,linkqname,arcqname)
            if cyclesAllowed != "any" and \
                   (XbrlConst.isStandardExtLinkQname(linkqname) and XbrlConst.isStandardArcQname(arcqname)) \
                   or arcrole in self.genericArcArcroles:
                noUndirected = cyclesAllowed == "none"
                fromRelationships = relsSet.fromModelObjects()
                for relFrom, rels in fromRelationships.items():
                    cycleFound = self.fwdCycle(relsSet, rels, noUndirected, {relFrom})
                    if cycleFound is not None:
                        path = str(relFrom.qname) + " " + " - ".join(
                            "{0}:{1} {2}".format(rel.modelDocument.basename, rel.sourceline, rel.toModelObject.qname)
                            for rel in reversed(cycleFound[1:]))
                        modelXbrl.error(specSect,
                            _("Relationships have a %(cycle)s cycle in arcrole %(arcrole)s \nlink role %(linkrole)s \nlink %(linkname)s, \narc %(arcname)s, \npath %(path)s"),
                            modelObject=cycleFound[1:], cycle=cycleFound[0], path=path,
                            arcrole=arcrole, linkrole=ELR, linkname=linkqname, arcname=arcqname), 
                        break
                
            # check calculation arcs for weight issues (note calc arc is an "any" cycles)
            if arcrole == XbrlConst.summationItem:
                for modelRel in relsSet.modelRelationships:
                    weight = modelRel.weight
                    fromConcept = modelRel.fromModelObject
                    toConcept = modelRel.toModelObject
                    if fromConcept is not None and toConcept is not None:
                        if weight == 0:
                            modelXbrl.error("xbrl.5.2.5.2.1:zeroWeight",
                                _("Calculation relationship has zero weight from %(source)s to %(target)s in link role %(linkrole)s"),
                                modelObject=modelRel,
                                source=fromConcept.qname, target=toConcept.qname, linkrole=ELR), 
                        fromBalance = fromConcept.balance
                        toBalance = toConcept.balance
                        if fromBalance and toBalance:
                            if (fromBalance == toBalance and weight < 0) or \
                               (fromBalance != toBalance and weight > 0):
                                modelXbrl.error("xbrl.5.1.1.2:balanceCalcWeightIllegal" +
                                                ("Negative" if weight < 0 else "Positive"),
                                    _("Calculation relationship has illegal weight %(weight)s from %(source)s, %(sourceBalance)s, to %(target)s, %(targetBalance)s, in link role %(linkrole)s (per 5.1.1.2 Table 6)"),
                                    modelObject=modelRel, weight=weight,
                                    source=fromConcept.qname, target=toConcept.qname, linkrole=ELR, 
                                    sourceBalance=fromBalance, targetBalance=toBalance)
                        if not fromConcept.isNumeric or not toConcept.isNumeric:
                            modelXbrl.error("xbrl.5.2.5.2:nonNumericCalc",
                                _("Calculation relationship has illegal concept from %(source)s%(sourceNumericDecorator)s to %(target)s%(targetNumericDecorator)s in link role %(linkrole)s"),
                                modelObject=modelRel,
                                source=fromConcept.qname, target=toConcept.qname, linkrole=ELR, 
                                sourceNumericDecorator="" if fromConcept.isNumeric else _(" (non-numeric)"), 
                                targetNumericDecorator="" if toConcept.isNumeric else _(" (non-numeric)"))
            # check presentation relationships for preferredLabel issues
            elif arcrole == XbrlConst.parentChild:
                for modelRel in relsSet.modelRelationships:
                    preferredLabel = modelRel.preferredLabel
                    toConcept = modelRel.toModelObject
                    if preferredLabel is not None and toConcept is not None and \
                       toConcept.label(preferredLabel=preferredLabel,fallbackToQname=False) is None:
                        modelXbrl.error("xbrl.5.2.4.2.1:preferredLabelMissing",
                            _("Presentation relationship from %(source)s to %(target)s in link role %(linkrole)s missing preferredLabel %(preferredLabel)s"),
                            modelObject=modelRel,
                            source=modelRel.fromModelObject.qname, target=toConcept.qname, linkrole=ELR, 
                            preferredLabel=preferredLabel)
            # check essence-alias relationships
            elif arcrole == XbrlConst.essenceAlias:
                for modelRel in relsSet.modelRelationships:
                    fromConcept = modelRel.fromModelObject
                    toConcept = modelRel.toModelObject
                    if fromConcept is not None and toConcept is not None:
                        if fromConcept.type != toConcept.type or fromConcept.periodType != toConcept.periodType:
                            modelXbrl.error("xbrl.5.2.6.2.2:essenceAliasTypes",
                                _("Essence-alias relationship from %(source)s to %(target)s in link role %(linkrole)s has different types or periodTypes"),
                                modelObject=modelRel,
                                source=fromConcept.qname, target=toConcept.qname, linkrole=ELR)
                        fromBalance = fromConcept.balance
                        toBalance = toConcept.balance
                        if fromBalance and toBalance:
                            if fromBalance and toBalance and fromBalance != toBalance:
                                modelXbrl.error("xbrl.5.2.6.2.2:essenceAliasBalance",
                                    _("Essence-alias relationship from %(source)s to %(target)s in link role %(linkrole)s has different balances")).format(
                                    modelObject=modelRel,
                                    source=fromConcept.qname, target=toConcept.qname, linkrole=ELR)
            elif modelXbrl.hasXDT and arcrole.startswith(XbrlConst.dimStartsWith):
                ValidateXbrlDimensions.checkBaseSet(self, arcrole, ELR, relsSet)             
            elif (modelXbrl.hasFormulae or modelXbrl.hasTableRendering) and arcrole.startswith(XbrlConst.formulaStartsWith):
                ValidateFormula.checkBaseSet(self, arcrole, ELR, relsSet)
        modelXbrl.isDimensionsValidated = True
        modelXbrl.profileStat(_("validateRelationships"))
                            
        # instance checks
        modelXbrl.modelManager.showStatus(_("validating instance"))
        self.footnoteRefs = set()
        if modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE or \
           modelXbrl.modelDocument.type == ModelDocument.Type.INLINEXBRL:
            self.factsWithDeprecatedIxNamespace = []
            self.checkFacts(modelXbrl.facts)
            
            if self.factsWithDeprecatedIxNamespace:
                self.modelXbrl.info("arelle:info",
                    _("%(count)s facts have deprecated transformation namespace %(namespace)s"),
                        modelObject=self.factsWithDeprecatedIxNamespace,
                        count=len(self.factsWithDeprecatedIxNamespace), 
                        namespace=FunctionIxt.deprecatedNamespaceURI)
            del self.factsWithDeprecatedIxNamespace

            
            #instance checks
            for cntx in modelXbrl.contexts.values():
                if cntx.isStartEndPeriod:
                    try:
                        if cntx.endDatetime <= cntx.startDatetime:
                            self.modelXbrl.error("xbrl.4.7.2:periodStartBeforeEnd",
                                _("Context %(contextID)s must have startDate less than endDate"),
                                modelObject=cntx, contextID=cntx.id)
                    except (TypeError, ValueError) as err:
                        self.modelXbrl.error("xbrl.4.7.2:contextDateError",
                            _("Context %(contextID) startDate or endDate: %(error)s"),
                            modelObject=cntx, contextID=cntx.id, error=err)
                elif cntx.isInstantPeriod:
                    try:
                        cntx.instantDatetime #parse field
                    except ValueError as err:
                        self.modelXbrl.error("xbrl.4.7.2:contextDateError",
                            _("Context %(contextID)s instant date: %(error)s"),
                            modelObject=cntx, contextID=cntx.id, error=err)
                self.segmentScenario(cntx.segment, cntx.id, "segment", "4.7.3.2")
                self.segmentScenario(cntx.scenario, cntx.id, "scenario", "4.7.4")
                
            for unit in modelXbrl.units.values():
                mulDivMeasures = unit.measures
                if mulDivMeasures:
                    for measures in mulDivMeasures:
                        for measure in measures:
                            if measure.namespaceURI == XbrlConst.xbrli and not \
                                measure in (XbrlConst.qnXbrliPure, XbrlConst.qnXbrliShares):
                                    self.modelXbrl.error("xbrl.4.8.2:measureElement",
                                        _("Unit %(unitID)s illegal measure: %(measure)s"),
                                        modelObject=unit, unitID=unit.id, measure=measure)
                    for numeratorMeasure in mulDivMeasures[0]:
                        if numeratorMeasure in mulDivMeasures[1]:
                            self.modelXbrl.error("xbrl.4.8.4:measureBothNumDenom",
                                _("Unit %(unitID)s numerator measure: %(measure)s also appears as denominator measure"),
                                modelObject=unit, unitID=unit.id, measure=numeratorMeasure)
            modelXbrl.profileStat(_("validateInstance"))

            if modelXbrl.hasXDT:            
                modelXbrl.modelManager.showStatus(_("validating dimensions"))
                ''' uncomment if using otherFacts in checkFact
                dimCheckableFacts = set(f 
                                        for f in modelXbrl.factsInInstance
                                        if f.concept.isItem and f.context is not None)
                while (dimCheckableFacts): # check one and all of its compatible family members
                    f = dimCheckableFacts.pop()
                    ValidateXbrlDimensions.checkFact(self, f, dimCheckableFacts)
                del dimCheckableFacts
                '''
                self.checkFactDimensions(modelXbrl.facts) # check fact dimensions in document order
                for cntx in modelXbrl.contexts.values():
                    ValidateXbrlDimensions.checkContext(self,cntx)
                modelXbrl.profileStat(_("validateDimensions"))
                    
        # dimensional validity
        #concepts checks
        modelXbrl.modelManager.showStatus(_("validating concepts"))
        for concept in modelXbrl.qnameConcepts.values():
            conceptType = concept.type
            if XbrlConst.isStandardNamespace(concept.qname.namespaceURI) or \
               not concept.modelDocument.inDTS:
                continue
            
            if concept.isTuple:
                # must be global
                if not concept.getparent().localName == "schema":
                    self.modelXbrl.error("xbrl.4.9:tupleGloballyDeclared",
                        _("Tuple %(concept)s must be declared globally"),
                        modelObject=concept, concept=concept.qname)
                if concept.periodType:
                    self.modelXbrl.error("xbrl.4.9:tuplePeriodType",
                        _("Tuple %(concept)s must not have periodType"),
                        modelObject=concept, concept=concept.qname)
                if concept.balance:
                    self.modelXbrl.error("xbrl.4.9:tupleBalance",
                        _("Tuple %(concept)s must not have balance"),
                        modelObject=concept, concept=concept.qname)
                if conceptType is not None:
                    # check attribute declarations
                    for attribute in conceptType.attributes.values():
                        if attribute.qname.namespaceURI in (XbrlConst.xbrli, XbrlConst.link, XbrlConst.xlink, XbrlConst.xl):
                            self.modelXbrl.error("xbrl.4.9:tupleAttribute",
                                _("Tuple %(concept)s must not have attribute in this namespace %(attribute)s"),
                                modelObject=concept, concept=concept.qname, attribute=attribute.qname)
                    # check for mixed="true" or simple content
                    if XmlUtil.descendantAttr(conceptType, XbrlConst.xsd, ("complexType", "complexContent"), "mixed") == "true":
                        self.modelXbrl.error("xbrl.4.9:tupleMixedContent",
                            _("Tuple %(concept)s must not have mixed content"),
                            modelObject=concept, concept=concept.qname)
                    if XmlUtil.descendant(conceptType, XbrlConst.xsd, "simpleContent"):
                        self.modelXbrl.error("xbrl.4.9:tupleSimpleContent",
                            _("Tuple %(concept)s must not have simple content"),
                            modelObject=concept, concept=concept.qname)
                    # child elements must be item or tuple
                    for elementQname in conceptType.elements:
                        childConcept = self.modelXbrl.qnameConcepts.get(elementQname)
                        if childConcept is None:
                            self.modelXbrl.error("xbrl.4.9:tupleElementUndefined",
                                _("Tuple %(concept)s element %(tupleElement)s not defined"),
                                modelObject=concept, concept=str(concept.qname), tupleElement=elementQname)
                        elif not (childConcept.isItem or childConcept.isTuple or # isItem/isTuple do not include item or tuple itself
                                  childConcept.qname == XbrlConst.qnXbrliItem or # subs group includes item as member
                                  childConcept.qname == XbrlConst.qnXbrliTuple):
                            self.modelXbrl.error("xbrl.4.9:tupleElementItemOrTuple",
                                _("Tuple %(concept)s must not have element %(tupleElement)s not an item or tuple"),
                                modelObject=concept, concept=concept.qname, tupleElement=elementQname)
            elif concept.isItem:
                if concept.periodType not in periodTypeValues: #("instant","duration"):
                    self.modelXbrl.error("xbrl.5.1.1.1:itemPeriodType",
                        _("Item %(concept)s must have a valid periodType"),
                        modelObject=concept, concept=concept.qname)
                if concept.isMonetary:
                    if concept.balance not in balanceValues: #(None, "credit","debit"):
                        self.modelXbrl.error("xbrl.5.1.1.2:itemBalance",
                            _("Item %(concept)s must have a valid balance %(balance)s"),
                            modelObject=concept, concept=concept.qname, balance=concept.balance)
                else:
                    if concept.balance:
                        self.modelXbrl.error("xbrl.5.1.1.2:itemBalance",
                            _("Item %(concept)s may not have a balance"),
                            modelObject=concept, concept=concept.qname)
                if concept.baseXbrliType not in baseXbrliTypes:
                    self.modelXbrl.error("xbrl.5.1.1.3:itemType",
                        _("Item %(concept)s type %(itemType)s invalid"),
                        modelObject=concept, concept=concept.qname, itemType=concept.baseXbrliType)
                if modelXbrl.hasXDT:
                    if concept.isHypercubeItem and not concept.abstract == "true":
                        self.modelXbrl.error("xbrldte:HypercubeElementIsNotAbstractError",
                            _("Hypercube item %(concept)s must be abstract"),
                            modelObject=concept, concept=concept.qname)
                    elif concept.isDimensionItem and not concept.abstract == "true":
                        self.modelXbrl.error("xbrldte:DimensionElementIsNotAbstractError",
                            _("Dimension item %(concept)s must be abstract"),
                            modelObject=concept, concept=concept.qname)
            if modelXbrl.hasXDT:
                ValidateXbrlDimensions.checkConcept(self, concept)
        modelXbrl.profileStat(_("validateConcepts"))
            
        modelXbrl.modelManager.showStatus(_("validating DTS"))
        self.DTSreferenceResourceIDs = {}
        ValidateXbrlDTS.checkDTS(self, modelXbrl.modelDocument, [])
        del self.DTSreferenceResourceIDs
        
        global validateUniqueParticleAttribution
        if validateUniqueParticleAttribution is None:
            from arelle.XmlValidateParticles import validateUniqueParticleAttribution
        for modelType in modelXbrl.qnameTypes.values():
            validateUniqueParticleAttribution(modelXbrl, modelType.particlesList, modelType)
        modelXbrl.profileStat(_("validateDTS"))
        
        if self.validateCalcLB:
            modelXbrl.modelManager.showStatus(_("Validating instance calculations"))
            ValidateXbrlCalcs.validate(modelXbrl, inferDecimals=self.validateInferDecimals)
            modelXbrl.profileStat(_("validateCalculations"))
            
        if (modelXbrl.modelManager.validateUtr or
            (self.parameters and self.parameters.get(qname("forceUtrValidation",noPrefixIsNoNamespace=True),(None,"false"))[1] == "true") or
             #(self.validateEFM and 
             #any((concept.namespaceURI in self.disclosureSystem.standardTaxonomiesDict) 
             #    for concept in self.modelXbrl.nameConcepts.get("UTR",())))):
            (self.validateEFM and any(modelDoc.definesUTR for modelDoc in self.modelXbrl.urlDocs.values()))):
            ValidateUtr.validate(modelXbrl)
            modelXbrl.profileStat(_("validateUTR"))
            
        if modelXbrl.hasFormulae or modelXbrl.modelRenderingTables:
            ValidateFormula.validate(self, 
                                     statusMsg=_("compiling formulae and rendering tables") if (modelXbrl.hasFormulae and modelXbrl.modelRenderingTables)
                                     else (_("compiling formulae") if modelXbrl.hasFormulae
                                           else _("compiling rendering tables")))
            
        modelXbrl.modelManager.showStatus(_("ready"), 2000)
Beispiel #13
0
    def validate(self, modelXbrl, parameters=None):
        self.parameters = parameters
        self.NCnamePattern = re.compile(
            "^[_A-Za-z\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]"
            r"[_\-\."
            "\xB7A-Za-z0-9\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\u0300-\u036F\u203F-\u2040]*$"
        )
        self.precisionPattern = re.compile("^([0-9]+|INF)$")
        self.decimalsPattern = re.compile("^(-?[0-9]+|INF)$")
        self.isoCurrencyPattern = re.compile(r"^[A-Z]{3}$")
        self.modelXbrl = modelXbrl
        self.validateDisclosureSystem = modelXbrl.modelManager.validateDisclosureSystem
        self.disclosureSystem = modelXbrl.modelManager.disclosureSystem
        self.validateEFM = self.validateDisclosureSystem and self.disclosureSystem.EFM
        self.validateGFM = self.validateDisclosureSystem and self.disclosureSystem.GFM
        self.validateEFMorGFM = self.validateDisclosureSystem and self.disclosureSystem.EFMorGFM
        self.validateHMRC = self.validateDisclosureSystem and self.disclosureSystem.HMRC
        self.validateSBRNL = self.validateDisclosureSystem and self.disclosureSystem.SBRNL
        self.validateXmlLang = self.validateDisclosureSystem and self.disclosureSystem.xmlLangPattern
        self.validateCalcLB = modelXbrl.modelManager.validateCalcLB
        self.validateInferDecimals = modelXbrl.modelManager.validateInferDecimals

        # xlink validation
        modelXbrl.modelManager.showStatus(_("validating links"))
        modelLinks = set()
        self.remoteResourceLocElements = set()
        self.genericArcArcroles = set()
        for baseSetExtLinks in modelXbrl.baseSets.values():
            for baseSetExtLink in baseSetExtLinks:
                modelLinks.add(
                    baseSetExtLink)  # ext links are unique (no dups)
        for modelLink in modelLinks:
            fromToArcs = {}
            locLabels = {}
            resourceLabels = {}
            resourceArcTos = []
            for arcElt in modelLink.element.childNodes:
                if arcElt.nodeType == 1:
                    xlinkType = arcElt.getAttributeNS(XbrlConst.xlink, "type")
                    # locator must have an href
                    if xlinkType == "locator":
                        if not arcElt.hasAttributeNS(XbrlConst.xlink, "href"):
                            modelXbrl.error(
                                _("Linkbase {0} extended link {1} locator {2} missing href"
                                  ).format(
                                      modelLink.modelDocument.basename,
                                      modelLink.role,
                                      arcElt.hasAttributeNS(
                                          XbrlConst.xlink, "label")), "err",
                                "xlink:locatorHref")
                        locLabels[arcElt.getAttributeNS(
                            XbrlConst.xlink, "label")] = arcElt
                    elif xlinkType == "resource":
                        resourceLabels[arcElt.getAttributeNS(
                            XbrlConst.xlink, "label")] = arcElt
                    # can be no duplicated arcs between same from and to
                    elif xlinkType == "arc":
                        fromLabel = arcElt.getAttributeNS(
                            XbrlConst.xlink, "from")
                        toLabel = arcElt.getAttributeNS(XbrlConst.xlink, "to")
                        fromTo = (fromLabel, toLabel)
                        if fromTo in fromToArcs:
                            modelXbrl.error(
                                _("Linkbase {0} extended link {1} duplicate arcs from {2} to {3}"
                                  ).format(modelLink.modelDocument.basename,
                                           modelLink.role, fromLabel, toLabel),
                                "err", "xlink:dupArcs")
                        else:
                            fromToArcs[fromTo] = arcElt
                        if arcElt.namespaceURI == XbrlConst.link:
                            if arcElt.localName in arcNamesTo21Resource:  #("labelArc","referenceArc"):
                                resourceArcTos.append(
                                    (toLabel, arcElt.getAttribute("use")))
                        elif self.isGenericArc(arcElt):
                            arcrole = arcElt.getAttributeNS(
                                XbrlConst.xlink, "arcrole")
                            self.genericArcArcroles.add(arcrole)
                            if arcrole in (XbrlConst.elementLabel,
                                           XbrlConst.elementReference):
                                resourceArcTos.append((toLabel, arcrole))
                    # values of type (not needed for validating parsers)
                    if xlinkType not in xlinkTypeValues:  # ("", "simple", "extended", "locator", "arc", "resource", "title", "none"):
                        modelXbrl.error(
                            _("Linkbase {0} extended link {1} type {2} invalid"
                              ).format(modelLink.modelDocument.basename,
                                       modelLink.role, xlinkType), "err",
                            "xlink:type")
                    # values of actuate (not needed for validating parsers)
                    xlinkActuate = arcElt.getAttributeNS(
                        XbrlConst.xlink, "actuate")
                    if xlinkActuate not in xlinkActuateValues:  # ("", "onLoad", "onRequest", "other", "none"):
                        modelXbrl.error(
                            _("Linkbase {0} extended link {1} actuate {2} invalid"
                              ).format(modelLink.modelDocument.basename,
                                       modelLink.role, xlinkActuate), "err",
                            "xlink:actuate")
                    # values of show (not needed for validating parsers)
                    xlinkShow = arcElt.getAttributeNS(XbrlConst.xlink, "show")
                    if xlinkShow not in xlinkShowValues:  # ("", "new", "replace", "embed", "other", "none"):
                        modelXbrl.error(
                            _("Linkbase {0} extended link {1} show {2} invalid"
                              ).format(modelLink.modelDocument.basename,
                                       modelLink.role, xlinkShow), "err",
                            "xlink:show")
                    # values of label, from, to (not needed for validating parsers)
                    for name in xlinkLabelAttributes:  # ("label", "from", "to"):
                        value = arcElt.getAttributeNS(XbrlConst.xlink, name)
                        if value != "" and not self.NCnamePattern.match(value):
                            modelXbrl.error(
                                _("Linkbase {0} extended link {1} element {2} {3} '{4}' not an NCname"
                                  ).format(modelLink.modelDocument.basename,
                                           modelLink.role, arcElt.tagName,
                                           name, value), "err",
                                "xlink:{0}".format(name))
            # check from, to of arcs have a resource or loc
            for fromTo, arcElt in fromToArcs.items():
                fromLabel, toLabel in fromTo
                for name, value, sect in (("from", fromLabel, "3.5.3.9.2"),
                                          ("to", toLabel, "3.5.3.9.3")):
                    if value not in locLabels and value not in resourceLabels:
                        modelXbrl.error(
                            _("Arc in linkbase {0} extended link {1} from {2} to {3} attribute \"{4}\" has no matching loc or resource label"
                              ).format(modelLink.modelDocument.basename,
                                       modelLink.role, fromLabel, toLabel,
                                       name), "err",
                            "xbrl.{0}:arcResource".format(sect))
                if arcElt.localName == "footnoteArc" and arcElt.namespaceURI == XbrlConst.link and \
                   arcElt.getAttributeNS(XbrlConst.xlink,"arcrole") == XbrlConst.factFootnote:
                    if fromLabel not in locLabels:
                        modelXbrl.error(
                            _("FootnoteArc in {0} extended link {1} from {2} to {3} \"from\" is not a loc"
                              ).format(modelLink.modelDocument.basename,
                                       modelLink.role, fromLabel, toLabel),
                            "err", "xbrl.4.11.1.3.1:factFootnoteArcFrom")
                    if toLabel not in resourceLabels or qname(
                            resourceLabels[toLabel]
                    ) != XbrlConst.qnLinkFootnote:
                        modelXbrl.error(
                            _("FootnoteArc in {0} extended link {1} from {2} to {3} \"to\" is not a footnote resource"
                              ).format(modelLink.modelDocument.basename,
                                       modelLink.role, fromLabel, toLabel),
                            "err", "xbrl.4.11.1.3.1:factFootnoteArcTo")
            # check unprohibited label arcs to remote locs
            for resourceArcTo in resourceArcTos:
                resourceArcToLabel, resourceArcUse = resourceArcTo
                if resourceArcToLabel in locLabels:
                    toLabel = locLabels[resourceArcToLabel]
                    if resourceArcUse == "prohibited":
                        self.remoteResourceLocElements.add(toLabel)
                    else:
                        modelXbrl.error(
                            _("Unprohibited labelArc in linkbase {0} extended link {1} has illegal remote resource loc labeled {2} href {3}"
                              ).format(
                                  modelLink.modelDocument.basename,
                                  modelLink.role, resourceArcToLabel,
                                  toLabel.getAttributeNS(
                                      XbrlConst.xlink, "href")), "err",
                            "xbrl.5.2.2.3:labelArcRemoteResource")
                elif resourceArcToLabel in resourceLabels:
                    toResource = resourceLabels[resourceArcToLabel]
                    if resourceArcUse == XbrlConst.elementLabel:
                        if not self.isGenericLabel(toResource):
                            modelXbrl.error(
                                _("Generic label arc in linkbase {0} extended link {1} to {2} must target a generic label"
                                  ).format(modelLink.modelDocument.basename,
                                           modelLink.role, resourceArcToLabel),
                                "err", "xbrlle.2.1.1:genericLabelTarget")
                    elif resourceArcUse == XbrlConst.elementReference:
                        if not self.isGenericReference(toResource):
                            modelXbrl.error(
                                _("Generic reference arc in linkbase {0} extended link {1} to {2} must target a generic reference"
                                  ).format(modelLink.modelDocument.basename,
                                           modelLink.role, resourceArcToLabel),
                                "err", "xbrlre.2.1.1:genericReferenceTarget")

        self.dimensionDefaults = {}
        modelXbrl.qnameDimensionDefaults = {}
        modelXbrl.qnameDimensionContextElement = {}
        # check base set cycles, dimensions
        modelXbrl.modelManager.showStatus(_("validating relationship sets"))
        for baseSetKey in modelXbrl.baseSets.keys():
            arcrole, ELR, linkqname, arcqname = baseSetKey
            if arcrole.startswith("XBRL-") or ELR is None or \
                linkqname is None or arcqname is None:
                continue
            elif arcrole in XbrlConst.standardArcroleCyclesAllowed:
                cyclesAllowed, specSect = XbrlConst.standardArcroleCyclesAllowed[
                    arcrole]
            elif arcrole in self.modelXbrl.arcroleTypes and len(
                    self.modelXbrl.arcroleTypes[arcrole]) > 0:
                cyclesAllowed = self.modelXbrl.arcroleTypes[arcrole][
                    0].cyclesAllowed
                if arcrole in self.genericArcArcroles:
                    specSect = "xbrlgene:violatedCyclesConstraint"
                else:
                    specSect = "xbrl.5.1.4.3:cycles"
            else:
                cyclesAllowed = "any"
                specSect = None
            if cyclesAllowed != "any" or arcrole in (XbrlConst.summationItem,) \
                                      or arcrole in self.genericArcArcroles  \
                                      or arcrole.startswith(XbrlConst.formulaStartsWith):
                relsSet = modelXbrl.relationshipSet(arcrole, ELR, linkqname,
                                                    arcqname)
            if cyclesAllowed != "any" and \
                   (XbrlConst.isStandardExtLinkQname(linkqname) and XbrlConst.isStandardArcQname(arcqname)) \
                   or arcrole in self.genericArcArcroles:
                noUndirected = cyclesAllowed == "none"
                fromRelationships = relsSet.fromModelObjects()
                for relFrom, rels in fromRelationships.items():
                    cycleFound = self.fwdCycle(relsSet, rels, noUndirected,
                                               {relFrom})
                    if cycleFound:
                        modelXbrl.error(
                            _("Relationships have a {0} cycle in arcrole {1} link role {2} link {3}, arc {4} starting from {5}"
                              ).format(cycleFound, arcrole, ELR, linkqname,
                                       arcqname, relFrom.qname), "err",
                            "{0}".format(specSect))
                        break

            # check calculation arcs for weight issues (note calc arc is an "any" cycles)
            if arcrole == XbrlConst.summationItem:
                for modelRel in relsSet.modelRelationships:
                    weight = modelRel.weight
                    fromConcept = modelRel.fromModelObject
                    toConcept = modelRel.toModelObject
                    if fromConcept and toConcept:
                        if weight == 0:
                            modelXbrl.error(
                                _("Calculation relationship has zero weight from {0} to {1} in link role {2}"
                                  ).format(fromConcept.qname, toConcept.qname,
                                           ELR), "err",
                                "xbrl.5.2.5.2.1:zeroWeight")
                        fromBalance = fromConcept.balance
                        toBalance = toConcept.balance
                        if fromBalance and toBalance:
                            if (fromBalance == toBalance and weight < 0) or \
                               (fromBalance != toBalance and weight > 0):
                                modelXbrl.error(
                                    _("Calculation relationship has illegal weight {0} from {1}, {2}, to {3}, {4} in link role {5} (per 5.1.1.2 Table 6)"
                                      ).format(weight, fromConcept.qname,
                                               fromBalance, toConcept.qname,
                                               toBalance, ELR), "err",
                                    "xbrl.5.1.1.2:balanceCalcWeight")
                        if not fromConcept.isNumeric or not toConcept.isNumeric:
                            modelXbrl.error(
                                _("Calculation relationship has illegal concept from {0}{1} to {2}{3} in link role {4}"
                                  ).format(
                                      fromConcept.qname,
                                      "" if fromConcept.isNumeric else
                                      " (non-numeric)", toConcept.qname,
                                      "" if fromConcept.isNumeric else
                                      " (non-numeric)", ELR), "err",
                                "xbrl.5.2.5.2:nonNumericCalc")
            # check presentation relationships for preferredLabel issues
            elif arcrole == XbrlConst.parentChild:
                for modelRel in relsSet.modelRelationships:
                    preferredLabel = modelRel.preferredLabel
                    toConcept = modelRel.toModelObject
                    if preferredLabel and toConcept and \
                       toConcept.label(preferredLabel=preferredLabel,fallbackToQname=False) is None:
                        modelXbrl.error(
                            _("Presentation relationship from {0} to {1} in link role {2} missing preferredLabel {3}"
                              ).format(modelRel.fromModelObject.qname,
                                       toConcept.qname, ELR, preferredLabel),
                            "err", "xbrl.5.2.4.2.1:preferredLabelMissing")
            # check essence-alias relationships
            elif arcrole == XbrlConst.essenceAlias:
                for modelRel in relsSet.modelRelationships:
                    fromConcept = modelRel.fromModelObject
                    toConcept = modelRel.toModelObject
                    if fromConcept and toConcept:
                        if fromConcept.type != toConcept.type or fromConcept.periodType != toConcept.periodType:
                            modelXbrl.error(
                                _("Essence-alias relationship from {0} to {1} in link role {2} has different types or periodTypes"
                                  ).format(fromConcept.qname, toConcept.qname,
                                           ELR), "err",
                                "xbrl.5.2.6.2.2:essenceAliasTypes")
                        fromBalance = fromConcept.balance
                        toBalance = toConcept.balance
                        if fromBalance and toBalance:
                            if fromBalance and toBalance and fromBalance != toBalance:
                                modelXbrl.error(
                                    _("Essence-alias relationship from {0} to {1} in link role {2} has different balances"
                                      ).format(fromConcept.qname,
                                               toConcept.qname, ELR), "err",
                                    "xbrl.5.2.6.2.2:essenceAliasBalance")
            elif modelXbrl.hasXDT and arcrole.startswith(
                    XbrlConst.dimStartsWith):
                ValidateXbrlDimensions.checkBaseSet(self, arcrole, ELR,
                                                    relsSet)
            elif modelXbrl.hasFormulae and arcrole.startswith(
                    XbrlConst.formulaStartsWith):
                ValidateFormula.checkBaseSet(self, arcrole, ELR, relsSet)

        # instance checks
        modelXbrl.modelManager.showStatus(_("validating instance"))
        self.footnoteRefs = set()
        if modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE or \
           modelXbrl.modelDocument.type == ModelDocument.Type.INLINEXBRL:
            for f in modelXbrl.facts:
                concept = f.concept
                if concept:
                    if concept.isNumeric:
                        unit = f.unit
                        if f.unitID is None or unit is None:
                            self.modelXbrl.error(
                                _("Fact {0} context {1} is numeric and must have a unit"
                                  ).format(modelXbrl.modelDocument.basename,
                                           f.qname, f.contextID), "err",
                                "xbrl.4.6.2:numericUnit")
                        else:
                            if concept.isMonetary:
                                measures = unit.measures
                                if not measures or len(measures[0]) != 1 or len(measures[1]) != 0 or \
                                    measures[0][0].namespaceURI != XbrlConst.iso4217 or \
                                    not self.isoCurrencyPattern.match(measures[0][0].localName):
                                    self.modelXbrl.error(
                                        _("Fact {0} context {1} must have a monetary unit {2}"
                                          ).format(
                                              modelXbrl.modelDocument.basename,
                                              f.qname, f.contextID, f.unitID),
                                        "err", "xbrl.4.8.2:monetaryFactUnit")
                            elif concept.isShares:
                                measures = unit.measures
                                if not measures or len(measures[0]) != 1 or len(measures[1]) != 0 or \
                                    measures[0][0] != XbrlConst.qnXbrliShares:
                                    self.modelXbrl.error(
                                        _("Fact {0} context {1} must have a xbrli:shares unit {2}"
                                          ).format(f.qname, f.contextID,
                                                   f.unitID), "err",
                                        "xbrl.4.8.2:sharesFactUnit")
                    precision = f.precision
                    hasPrecision = precision is not None
                    if hasPrecision and precision != "INF" and not precision.isdigit(
                    ):
                        self.modelXbrl.error(
                            _("Fact {0} context {1} precision {2} is invalid"
                              ).format(f.qname, f.contextID, precision), "err",
                            "xbrl.4.6.4:precision")
                    decimals = f.decimals
                    hasDecimals = decimals is not None
                    if hasPrecision and not self.precisionPattern.match(
                            precision):
                        self.modelXbrl.error(
                            _("Fact {0} context {1} precision {2} is invalid"
                              ).format(f.qname, f.contextID, precision), "err",
                            "xbrl.4.6.4:precision")
                    if hasPrecision and hasDecimals:
                        self.modelXbrl.error(
                            _("Fact {0} context {1} can not have both precision and decimals"
                              ).format(f.qname, f.contextID), "err",
                            "xbrl.4.6.3:bothPrecisionAndDecimals")
                    if hasDecimals and not self.decimalsPattern.match(
                            decimals):
                        self.modelXbrl.error(
                            _("Fact {0} context {1} decimals {2} is invalid"
                              ).format(f.qname, f.contextID, decimals), "err",
                            "xbrl.4.6.5:decimals")
                    if concept.isItem:
                        context = f.context
                        if context is None:
                            self.modelXbrl.error(
                                _("Item {0} must have a context").format(
                                    f.qname), "err",
                                "xbrl.4.6.1:itemContextRef")
                        else:
                            periodType = concept.periodType
                            if (periodType == "instant" and not context.isInstantPeriod) or \
                               (periodType == "duration" and not (context.isStartEndPeriod or context.isForeverPeriod)):
                                self.modelXbrl.error(
                                    _("Fact {0} context {1} has period type {2} conflict with context"
                                      ).format(f.qname, f.contextID,
                                               periodType), "err",
                                    "xbrl.4.7.2:contextPeriodType")
                            if modelXbrl.hasXDT:
                                ValidateXbrlDimensions.checkFact(self, f)
                        # check precision and decimals
                        if f.xsiNil == "true":
                            if hasPrecision or hasDecimals:
                                self.modelXbrl.error(
                                    _("Fact {0} context {1} can not be nil and have either precision or decimals"
                                      ).format(f.qname, f.contextID), "err",
                                    "xbrl.4.6.3:nilPrecisionDecimals")
                        elif concept.isFraction:
                            if hasPrecision or hasDecimals:
                                self.modelXbrl.error(
                                    _("Fact {0} context {1} is a fraction concept and cannot have either precision or decimals"
                                      ).format(f.qname, f.contextID), "err",
                                    "xbrl.4.6.3:fractionPrecisionDecimals")
                                numerator, denominator = f.fractionValue
                                if not (numerator == "INF"
                                        or numerator.isnumeric()):
                                    self.modelXbrl.error(
                                        _("Fact {0} context {1} is a fraction with invalid numerator {2}"
                                          ).format(f.qname, f.contextID,
                                                   numerator), "err",
                                        "xbrl.5.1.1:fractionPrecisionDecimals")
                                if not denominator.isnumeric() or int(
                                        denominator) == 0:
                                    self.modelXbrl.error(
                                        _("Fact {0} context {1} is a fraction with invalid denominator {2}"
                                          ).format(f.qname, f.contextID,
                                                   denominator), "err",
                                        "xbrl.5.1.1:fractionPrecisionDecimals")
                        else:
                            if modelXbrl.modelDocument.type != ModelDocument.Type.INLINEXBRL:
                                for child in f.element.childNodes:
                                    if child.nodeType == 1:
                                        self.modelXbrl.error(
                                            _("Fact {0} context {1} may not have child elements {2}"
                                              ).format(f.qname, f.contextID,
                                                       child.tagName), "err",
                                            "xbrl.5.1.1:itemMixedContent")
                                        break
                            if concept.isNumeric and not hasPrecision and not hasDecimals:
                                self.modelXbrl.error(
                                    _("Fact {0} context {1} is a numeric concept and must have either precision or decimals"
                                      ).format(f.qname, f.contextID), "err",
                                    "xbrl.4.6.3:missingPrecisionDecimals")
                    elif concept.isTuple:
                        if f.contextID:
                            self.modelXbrl.error(
                                _("Tuple {0} must not have a context").format(
                                    f.qname), "err",
                                "xbrl.4.6.1:tupleContextRef")
                        if hasPrecision or hasDecimals:
                            self.modelXbrl.error(
                                _("Fact {0} is a tuple and cannot have either precision or decimals"
                                  ).format(f.qname), "err",
                                "xbrl.4.6.3:tuplePrecisionDecimals")
                        # custom attributes may be allowed by anyAttribute but not by 2.1
                        for attrQname, attrValue in XbrlUtil.attributes(
                                self.modelXbrl, concept, f.element):
                            if attrQname.namespaceURI in (XbrlConst.xbrli,
                                                          XbrlConst.link,
                                                          XbrlConst.xlink,
                                                          XbrlConst.xl):
                                self.modelXbrl.error(
                                    _("Fact {0} is a tuple and must not have attribute in this namespace {1}"
                                      ).format(f.qname, attrQname), "err",
                                    "xbrl.4.9:tupleAttribute")

                    else:
                        self.modelXbrl.error(
                            _("Fact {0} must be an item or tuple").format(
                                f.qname), "err", "xbrl.4.6:notItemOrTuple")

                from arelle.ModelObject import ModelInlineFact
                if isinstance(f, ModelInlineFact):
                    self.footnoteRefs.update(f.footnoteRefs)

            #instance checks
            for cntx in modelXbrl.contexts.values():
                if cntx.isStartEndPeriod:
                    try:
                        if cntx.endDatetime <= cntx.startDatetime:
                            self.modelXbrl.error(
                                _("Context {0} must have startDate less than endDate"
                                  ).format(cntx.id), "err",
                                "xbrl.4.7.2:periodStartBeforeEnd")
                    except ValueError as err:
                        self.modelXbrl.error(
                            _("Context {0} startDate or endDate: {1}").format(
                                cntx.id, err), "err",
                            "xbrl.4.7.2:contextDateError")
                elif cntx.isInstantPeriod:
                    try:
                        cntx.instantDatetime  #parse field
                    except ValueError as err:
                        self.modelXbrl.error(
                            _("Context {0} instant date: {1}").format(
                                cntx.id, err), "err",
                            "xbrl.4.7.2:contextDateError")
                self.segmentScenario(cntx.segment, cntx.id, "segment",
                                     "4.7.3.2")
                self.segmentScenario(cntx.scenario, cntx.id, "scenario",
                                     "4.7.4")
                if modelXbrl.hasXDT:
                    ValidateXbrlDimensions.checkContext(self, cntx)

            for unit in modelXbrl.units.values():
                mulDivMeasures = unit.measures
                if mulDivMeasures:
                    for measures in mulDivMeasures:
                        for measure in measures:
                            if measure.namespaceURI == XbrlConst.xbrli and not \
                                measure in (XbrlConst.qnXbrliPure, XbrlConst.qnXbrliShares):
                                self.modelXbrl.error(
                                    _("Unit {0} illegal measure: {1}").format(
                                        unit.id, measure), "err",
                                    "xbrl.4.8.2:measureElement")
                    for numeratorMeasure in mulDivMeasures[0]:
                        if numeratorMeasure in mulDivMeasures[1]:
                            self.modelXbrl.error(
                                _("Unit {0} numerator measure: {1} also appears as denominator measure"
                                  ).format(unit.id, numeratorMeasure), "err",
                                "xbrl.4.8.4:measureBothNumDenom")

        #concepts checks
        modelXbrl.modelManager.showStatus(_("validating concepts"))
        for concept in modelXbrl.qnameConcepts.values():
            conceptType = concept.type
            if XbrlConst.isStandardNamespace(concept.namespaceURI) or \
               not concept.modelDocument.inDTS:
                continue

            if concept.isTuple:
                # must be global
                if not concept.element.parentNode.localName == "schema":
                    self.modelXbrl.error(
                        _("Tuple {0} must be declared globally").format(
                            concept.qname), "err",
                        "xbrl.4.9:tupleGloballyDeclared")
                if concept.periodType:
                    self.modelXbrl.error(
                        _("Tuple {0} must not have periodType").format(
                            concept.qname), "err", "xbrl.4.9:tuplePeriodType")
                if concept.balance:
                    self.modelXbrl.error(
                        _("Tuple {0} must not have balance").format(
                            concept.qname), "err", "xbrl.4.9:tupleBalance")
                # check attribute declarations
                for attributeQname in conceptType.attributes:
                    if attributeQname.namespaceURI in (XbrlConst.xbrli,
                                                       XbrlConst.link,
                                                       XbrlConst.xlink,
                                                       XbrlConst.xl):
                        self.modelXbrl.error(
                            _("Tuple {0} must not have attribute in this namespace {1}"
                              ).format(concept.qname, attributeQname), "err",
                            "xbrl.4.9:tupleAttribute")
                # check for mixed="true" or simple content
                if XmlUtil.descendantAttr(conceptType.element, XbrlConst.xsd,
                                          ("complexType", "complexContent"),
                                          "mixed") == "true":
                    self.modelXbrl.error(
                        _("Tuple {0} must not have mixed content").format(
                            concept.qname), "err",
                        "xbrl.4.9:tupleMixedContent")
                if XmlUtil.descendant(conceptType.element, XbrlConst.xsd,
                                      "simpleContent"):
                    self.modelXbrl.error(
                        _("Tuple {0} must not have simple content").format(
                            concept.qname), "err",
                        "xbrl.4.9:tupleSimpleContent")
                # child elements must be item or tuple
                for elementQname in conceptType.elements:
                    childConcept = self.modelXbrl.qnameConcepts.get(
                        elementQname)
                    if childConcept is None:
                        self.modelXbrl.error(
                            _("Tuple {0} element {1} not defined").format(
                                concept.qname, elementQname), "err",
                            "xbrl.4.9:tupleElementUndefined")
                    elif not (
                            childConcept.isItem or childConcept.isTuple
                            or  # isItem/isTuple do not include item or tuple itself
                            childConcept.qname == XbrlConst.qnXbrliItem
                            or  # subs group includes item as member
                            childConcept.qname == XbrlConst.qnXbrliTuple):
                        self.modelXbrl.error(
                            _("Tuple {0} must not have element {1} not an item or tuple"
                              ).format(concept.qname, elementQname), "err",
                            "xbrl.4.9:tupleElementItemOrTuple")
            elif concept.isItem:
                if concept.periodType not in periodTypeValues:  #("instant","duration"):
                    self.modelXbrl.error(
                        _("Item {0} must have a valid periodType").format(
                            concept.qname), "err",
                        "xbrl.5.1.1.1:itemPeriodType")
                if concept.isMonetary:
                    if concept.balance not in balanceValues:  #(None, "credit","debit"):
                        self.modelXbrl.error(
                            _("Item {0} must have a valid balance {1}").format(
                                concept.qname, concept.balance), "err",
                            "xbrl.5.1.1.2:itemBalance")
                else:
                    if concept.balance:
                        self.modelXbrl.error(
                            _("Item {0} may not have a balance").format(
                                concept.qname), "err",
                            "xbrl.5.1.1.2:itemBalance")
                if concept.baseXbrliType not in baseXbrliTypes:
                    self.modelXbrl.error(
                        _("Item {0} type {1} invalid").format(
                            concept.qname, concept.baseXbrliType), "err",
                        "xbrl.5.1.1.3:itemType")
                if modelXbrl.hasXDT:
                    if concept.isHypercubeItem and not concept.abstract == "true":
                        self.modelXbrl.error(
                            _("Hypercube item {0} must be abstract").format(
                                concept.qname), "err",
                            "xbrldte:HypercubeElementIsNotAbstractError")
                    elif concept.isDimensionItem and not concept.abstract == "true":
                        self.modelXbrl.error(
                            _("Dimension item {0} must be abstract").format(
                                concept.qname), "err",
                            "xbrldte:DimensionElementIsNotAbstractError")
            if modelXbrl.hasXDT:
                ValidateXbrlDimensions.checkConcept(self, concept)

        modelXbrl.modelManager.showStatus(_("validating DTS"))
        self.DTSreferenceResourceIDs = {}
        ValidateXbrlDTS.checkDTS(self, modelXbrl.modelDocument, [])
        del self.DTSreferenceResourceIDs

        if self.validateCalcLB:
            modelXbrl.modelManager.showStatus(
                _("Validating instance calculations"))
            ValidateXbrlCalcs.validate(
                modelXbrl, inferPrecision=(not self.validateInferDecimals))

        if modelXbrl.modelManager.validateUtr:
            ValidateUtr.validate(modelXbrl)

        if modelXbrl.hasFormulae:
            ValidateFormula.validate(self)

        modelXbrl.modelManager.showStatus(_("ready"), 2000)
Beispiel #14
0
def checkDTS(val, modelDocument, visited):
    global targetNamespaceDatePattern, roleTypePattern, arcroleTypePattern, arcroleDefinitionPattern
    if targetNamespaceDatePattern is None:
        targetNamespaceDatePattern = re.compile(r"/([12][0-9]{3})-([01][0-9])-([0-3][0-9])|"
                                            r"/([12][0-9]{3})([01][0-9])([0-3][0-9])|")
        roleTypePattern = re.compile(r".*/role/[^/]+")
        arcroleTypePattern = re.compile(r".*/arcrole/[^/]+")
        arcroleDefinitionPattern = re.compile(r"^.*[^\\s]+.*$")  # at least one non-whitespace character
        
    visited.append(modelDocument)
    definesLabelLinkbase = False
    for referencedDocument in modelDocument.referencesDocument.items():
        #6.07.01 no includes
        if referencedDocument[1] == "include":
            val.modelXbrl.error(("EFM.6.07.01", "GFM.1.03.01", "SBR.NL.2.2.0.18"),
                _("Taxonomy schema %(schema)s includes %(include)s, only import is allowed"),
                modelObject=modelDocument,
                    schema=os.path.basename(modelDocument.uri), 
                    include=os.path.basename(referencedDocument[0].uri))
        if referencedDocument[0] not in visited:
            checkDTS(val, referencedDocument[0], visited)
            
    if val.disclosureSystem.standardTaxonomiesDict is None:
        pass
    if (modelDocument.type == ModelDocument.Type.SCHEMA and 
        modelDocument.targetNamespace not in val.disclosureSystem.baseTaxonomyNamespaces and
        modelDocument.uri.startswith(val.modelXbrl.uriDir)):
        
        # check schema contents types
        if val.validateSBRNL:
            definesLinkroles = False
            definesArcroles = False
            definesLinkParts = False
            definesAbstractItems = False
            definesNonabstractItems = False
            definesConcepts = False
            definesTuples = False
            definesPresentationTuples = False
            definesSpecificationTuples = False
            definesTypes = False
            definesEnumerations = False
            definesDimensions = False
            definesDomains = False
            definesHypercubes = False
                
        # 6.7.3 check namespace for standard authority
        targetNamespaceAuthority = UrlUtil.authority(modelDocument.targetNamespace) 
        if targetNamespaceAuthority in val.disclosureSystem.standardAuthorities:
            val.modelXbrl.error(("EFM.6.07.03", "GFM.1.03.03"),
                _("Taxonomy schema %(schema)s namespace %(targetNamespace)s is a disallowed authority"),
                modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace)
            
        # 6.7.4 check namespace format
        if modelDocument.targetNamespace is None:
            match = None
        elif val.validateEFMorGFM:
            targetNamespaceDate = modelDocument.targetNamespace[len(targetNamespaceAuthority):]
            match = targetNamespaceDatePattern.match(targetNamespaceDate)
        else:
            match = None
        if match is not None:
            try:
                if match.lastindex == 3:
                    datetime.date(int(match.group(1)),int(match.group(2)),int(match.group(3)))
                elif match.lastindex == 6:
                    datetime.date(int(match.group(4)),int(match.group(5)),int(match.group(6)))
                else:
                    match = None
            except ValueError:
                match = None
        if match is None:
            val.modelXbrl.error(("EFM.6.07.04", "GFM.1.03.04"),
                _("Taxonomy schema %(schema)s namespace %(targetNamespace)s must have format http://{authority}/{versionDate}"),
                modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace)

        if modelDocument.targetNamespace is not None:
            # 6.7.5 check prefix for _
            prefix = XmlUtil.xmlnsprefix(modelDocument.xmlRootElement,modelDocument.targetNamespace)
            if prefix and "_" in prefix:
                val.modelXbrl.error(("EFM.6.07.07", "GFM.1.03.07"),
                    _("Taxonomy schema %(schema)s namespace %(targetNamespace)s prefix %(prefix)s must not have an '_'"),
                    modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace, prefix=prefix)

            if val.validateSBRNL:
                genrlSpeclRelSet = val.modelXbrl.relationshipSet(XbrlConst.generalSpecial)
            for modelConcept in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/2001/XMLSchema}element"):
                if isinstance(modelConcept,ModelConcept):
                    # 6.7.16 name not duplicated in standard taxonomies
                    name = modelConcept.get("name")
                    if name is None: 
                        name = ""
                        if modelConcept.get("ref") is not None:
                            continue    # don't validate ref's here
                    for c in val.modelXbrl.nameConcepts.get(name, []):
                        if c.modelDocument != modelDocument:
                            if (val.validateEFMorGFM and
                                  not c.modelDocument.uri.startswith(val.modelXbrl.uriDir)):
                                val.modelXbrl.error(("EFM.6.07.16", "GFM.1.03.18"),
                                    _("Concept %(concept)s is also defined in standard taxonomy schema schema %(standardSchema)s"),
                                    modelObject=c, concept=modelConcept.qname, standardSchema=os.path.basename(c.modelDocument.uri))
                            elif val.validateSBRNL:
                                if not (genrlSpeclRelSet.isRelated(modelConcept, "child", c) or genrlSpeclRelSet.isRelated(c, "child", modelConcept)):
                                    val.modelXbrl.error("SBR.NL.2.2.2.02",
                                        _("Concept %(concept)s is also defined in standard taxonomy schema %(standardSchema)s without a general-special relationship"),
                                        modelObject=c, concept=modelConcept.qname, standardSchema=os.path.basename(c.modelDocument.uri))
                    ''' removed RH 2011-12-23 corresponding set up of table in ValidateFiling
                    if val.validateSBRNL and name in val.nameWordsTable:
                        if not any( any( genrlSpeclRelSet.isRelated(c, "child", modelConcept)
                                         for c in val.modelXbrl.nameConcepts.get(partialWordName, []))
                                    for partialWordName in val.nameWordsTable[name]):
                            val.modelXbrl.error("SBR.NL.2.3.2.01",
                                _("Concept %(specialName)s is appears to be missing a general-special relationship to %(generalNames)s"),
                                modelObject=c, specialName=modelConcept.qname, generalNames=', or to '.join(val.nameWordsTable[name]))
                    '''

                    # 6.7.17 id properly formed
                    id = modelConcept.id
                    requiredId = (prefix if prefix is not None else "") + "_" + name
                    if val.validateEFMorGFM and id != requiredId:
                        val.modelXbrl.error(("EFM.6.07.17", "GFM.1.03.19"),
                            _("Concept %(concept)s id %(id)s should be $(requiredId)s"),
                            modelObject=modelConcept, concept=modelConcept.qname, id=id, requiredId=requiredId)
                        
                    # 6.7.18 nillable is true
                    nillable = modelConcept.get("nillable")
                    if nillable != "true":
                        val.modelXbrl.error(("EFM.6.07.18", "GFM.1.03.20"),
                            _("Taxonomy schema %(schema)s element %(concept)s nillable %(nillable)s should be 'true'"),
                            modelObject=modelConcept, schema=os.path.basename(modelDocument.uri),
                            concept=name, nillable=nillable)
        
                    # 6.7.19 not tuple
                    if modelConcept.isTuple:
                        if val.validateEFMorGFM:
                            val.modelXbrl.error(("EFM.6.07.19", "GFM.1.03.21"),
                                _("Concept %(concept)s is a tuple"),
                                modelObject=modelConcept, concept=modelConcept.qname)
                        
                    # 6.7.20 no typed domain ref
                    if modelConcept.isTypedDimension:
                        val.modelXbrl.error(("EFM.6.07.20", "GFM.1.03.22"),
                            _("Concept %(concept)s has typedDomainRef %(typedDomainRef)s"),
                            modelObject=modelConcept, concept=modelConcept.qname,
                            typedDomainRef=modelConcept.typedDomainElement.qname if modelConcept.typedDomainElement is not None else modelConcept.typedDomainRef)
                        
                    # 6.7.21 abstract must be duration
                    isDuration = modelConcept.periodType == "duration"
                    if modelConcept.abstract == "true" and not isDuration:
                        val.modelXbrl.error(("EFM.6.07.21", "GFM.1.03.23"),
                            _("Taxonomy schema %(schema)s element %(concept)s is abstract but period type is not duration"),
                            modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=name)
                        
                    # 6.7.22 abstract must be stringItemType
                    ''' removed SEC EFM v.17, Edgar release 10.4, and GFM 2011-04-08
                    if modelConcept.abstract == "true" and modelConcept.typeQname != XbrlConst. qnXbrliStringItemType:
                        val.modelXbrl.error(("EFM.6.07.22", "GFM.1.03.24"),
                            _("Concept %(concept)s  is abstract but type is not xbrli:stringItemType"),
                            modelObject=modelConcept, concept=modelConcept.qname)
					'''
                    substititutionGroupQname = modelConcept.substitutionGroupQname
                    # 6.7.23 Axis must be subs group dimension
                    if name.endswith("Axis") ^ (substititutionGroupQname == XbrlConst.qnXbrldtDimensionItem):
                        val.modelXbrl.error(("EFM.6.07.23", "GFM.1.03.25"),
                            _("Concept %(concept)s must end in Axis to be in xbrldt:dimensionItem substitution group"),
                            modelObject=modelConcept, concept=modelConcept.qname)

                    # 6.7.24 Table must be subs group hypercube
                    if name.endswith("Table") ^ (substititutionGroupQname == XbrlConst.qnXbrldtHypercubeItem):
                        val.modelXbrl.error(("EFM.6.07.24", "GFM.1.03.26"),
                            _("Concept %(concept)s must end in Table to be in xbrldt:hypercubeItem substitution group"),
                            modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=modelConcept.qname)

                    # 6.7.25 if neither hypercube or dimension, substitution group must be item
                    if substititutionGroupQname not in (None,
                                                        XbrlConst.qnXbrldtDimensionItem, 
                                                        XbrlConst.qnXbrldtHypercubeItem,
                                                        XbrlConst.qnXbrliItem):                           
                        val.modelXbrl.error(("EFM.6.07.25", "GFM.1.03.27"),
                            _("Concept %(concept)s has disallowed substitution group %(substitutionGroup)s"),
                            modelObject=modelConcept, concept=modelConcept.qname,
                            substitutionGroup=modelConcept.substitutionGroupQname)
                        
                    # 6.7.26 Table must be subs group hypercube
                    if name.endswith("LineItems") and modelConcept.abstract != "true":
                        val.modelXbrl.error(("EFM.6.07.26", "GFM.1.03.28"),
                            _("Concept %(concept)s is a LineItems but not abstract"),
                            modelObject=modelConcept, concept=modelConcept.qname)

                    # 6.7.27 type domainMember must end with Domain or Member
                    conceptType = modelConcept.type
                    isDomainItemType = conceptType is not None and conceptType.isDomainItemType
                    endsWithDomainOrMember = name.endswith("Domain") or name.endswith("Member")
                    if isDomainItemType != endsWithDomainOrMember:
                        val.modelXbrl.error(("EFM.6.07.27", "GFM.1.03.29"),
                            _("Concept %(concept)s must end with Domain or Member for type of domainItemType"),
                            modelObject=modelConcept, concept=modelConcept.qname)

                    # 6.7.28 domainItemType must be duration
                    if isDomainItemType and not isDuration:
                        val.modelXbrl.error(("EFM.6.07.28", "GFM.1.03.30"),
                            _("Concept %(concept)s is a domainItemType and must be periodType duration"),
                            modelObject=modelConcept, concept=modelConcept.qname)
                    
                    if val.validateSBRNL:
                        if modelConcept.isTuple:
                            if modelConcept.substitutionGroupQname.localName == "presentationTuple" and modelConcept.substitutionGroupQname.namespaceURI.endswith("/basis/sbr/xbrl/xbrl-syntax-extension"): # namespace may change each year
                                definesPresentationTuples = True
                            elif modelConcept.substitutionGroupQname.localName == "specificationTuple" and modelConcept.substitutionGroupQname.namespaceURI.endswith("/basis/sbr/xbrl/xbrl-syntax-extension"): # namespace may change each year
                                definesSpecificationTuples = True
                            else:
                                definesTuples = True
                            definesConcepts = True
                            if modelConcept.isAbstract:
                                val.modelXbrl.error("SBR.NL.2.2.2.03",
                                    _("Concept %(concept)s is an abstract tuple"),
                                    modelObject=modelConcept, concept=modelConcept.qname)
                            if tupleCycle(val,modelConcept):
                                val.modelXbrl.error("SBR.NL.2.2.2.07",
                                    _("Tuple %(concept)s has a tuple cycle"),
                                    modelObject=modelConcept, concept=modelConcept.qname)
                            if modelConcept.get("nillable") != "false" and modelConcept.isRoot:
                                val.modelXbrl.error("SBR.NL.2.2.2.17", #don't want default, just what was really there
                                    _("Tuple %(concept)s must have nillable='false'"),
                                    modelObject=modelConcept, concept=modelConcept.qname)
                        elif modelConcept.isItem:
                            definesConcepts = True
                        if modelConcept.abstract == "true":
                            if modelConcept.isRoot:
                                if modelConcept.get("nillable") != "false": #don't want default, just what was really there
                                    val.modelXbrl.error("SBR.NL.2.2.2.16",
                                        _("Abstract root concept %(concept)s must have nillable='false'"),
                                    modelObject=modelConcept, concept=modelConcept.qname)
                                if modelConcept.typeQname != XbrlConst.qnXbrliStringItemType:
                                    val.modelXbrl.error("SBR.NL.2.2.2.21",
                                        _("Abstract root concept %(concept)s must have type='xbrli:stringItemType'"),
                                    modelObject=modelConcept, concept=modelConcept.qname)
                            if modelConcept.balance:
                                val.modelXbrl.error("SBR.NL.2.2.2.22",
                                    _("Abstract concept %(concept)s must not have a balance attribute"),
                                    modelObject=modelConcept, concept=modelConcept.qname)
                            if modelConcept.isHypercubeItem:
                                definesHypercubes = True
                            elif modelConcept.isDimensionItem:
                                definesDimensions = True
                            elif substititutionGroupQname and substititutionGroupQname.localName in ("domainItem","domainMemberItem"):
                                definesDomains = True
                            elif modelConcept.isItem:
                                definesAbstractItems = True
                        else:   # not abstract
                            if modelConcept.isItem:
                                definesNonabstractItems = True
                                if not (modelConcept.label(preferredLabel=XbrlConst.documentationLabel,fallbackToQname=False,lang="nl") or
                                        val.modelXbrl.relationshipSet(XbrlConst.conceptReference).fromModelObject(c) or
                                        modelConcept.genLabel(role=XbrlConst.genDocumentationLabel,lang="nl") or
                                        val.modelXbrl.relationshipSet(XbrlConst.elementReference).fromModelObject(c)):
                                    val.modelXbrl.error("SBR.NL.2.2.2.28",
                                        _("Concept %(concept)s must have a documentation label or reference"),
                                        modelObject=modelConcept, concept=modelConcept.qname)
                        if modelConcept.balance and not modelConcept.instanceOfType(XbrlConst.qnXbrliMonetaryItemType):
                            val.modelXbrl.error("SBR.NL.2.2.2.24",
                                _("Non-monetary concept %(concept)s must not have a balance attribute"),
                                modelObject=modelConcept, concept=modelConcept.qname)
                        if modelConcept.isLinkPart:
                            definesLinkParts = True
                            val.modelXbrl.error("SBR.NL.2.2.5.01",
                                _("Link:part concept %(concept)s is not allowed"),
                                modelObject=modelConcept, concept=modelConcept.qname)
                            if not modelConcept.genLabel(fallbackToQname=False,lang="nl"):
                                val.modelXbrl.error("SBR.NL.2.2.5.02",
                                    _("Link part definition %(concept)s must have a generic label in language 'nl'"),
                                    modelObject=modelConcept, concept=modelConcept.qname)
        # 6.7.8 check for embedded linkbase
        for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}linkbase"):
            if isinstance(e,ModelObject):
                val.modelXbrl.error(("EFM.6.07.08", "GFM.1.03.08"),
                    _("Taxonomy schema %(schema)s contains an embedded linkbase"),
                    modelObject=e, schema=os.path.basename(modelDocument.uri))
                break

        requiredUsedOns = {XbrlConst.qnLinkPresentationLink,
                           XbrlConst.qnLinkCalculationLink,
                           XbrlConst.qnLinkDefinitionLink}

        # 6.7.9 role types authority
        for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}roleType"):
            if isinstance(e,ModelObject):
                roleURI = e.get("roleURI")
                if targetNamespaceAuthority != UrlUtil.authority(roleURI):
                    val.modelXbrl.error(("EFM.6.07.09", "GFM.1.03.09"),
                        _("RoleType %(roleType)s does not match authority %(targetNamespaceAuthority)s"),
                        modelObject=e, roleType=roleURI, targetNamespaceAuthority=targetNamespaceAuthority)
                # 6.7.9 end with .../role/lc3 name
                if not roleTypePattern.match(roleURI):
                    val.modelXbrl.warning(("EFM.6.07.09", "GFM.1.03.09"),
                        "RoleType %(roleType)s should end with /role/{LC3name}",
                        modelObject=e, roleType=roleURI)
                    
                # 6.7.10 only one role type declaration in DTS
                modelRoleTypes = val.modelXbrl.roleTypes.get(roleURI)
                if modelRoleTypes is not None:
                    modelRoleType = modelRoleTypes[0]
                    definition = modelRoleType.definitionNotStripped
                    usedOns = modelRoleType.usedOns
                    if len(modelRoleTypes) > 1:
                        val.modelXbrl.error(("EFM.6.07.10", "GFM.1.03.10"),
                            _("RoleType %(roleType)s is defined in multiple taxonomies"),
                            modelObject=e, roleType=roleURI)
                    elif len(modelRoleTypes) == 1:
                        # 6.7.11 used on's for pre, cal, def if any has a used on
                        if not usedOns.isdisjoint(requiredUsedOns) and len(requiredUsedOns - usedOns) > 0:
                            val.modelXbrl.error(("EFM.6.07.11", "GFM.1.03.11"),
                                _("RoleType %(roleType)s missing used on %(usedOn)s"),
                                modelObject=e, roleType=roleURI, usedOn=requiredUsedOns - usedOns)
                            
                        # 6.7.12 definition match pattern
                        if (val.disclosureSystem.roleDefinitionPattern is not None and
                            (definition is None or not val.disclosureSystem.roleDefinitionPattern.match(definition))):
                            val.modelXbrl.error(("EFM.6.07.12", "GFM.1.03.12-14"),
                                _("RoleType %(roleType)s definition \"%(definition)s\" must match {Sortcode} - {Type} - {Title}"),
                                modelObject=e, roleType=roleURI, definition=definition)
                        
                    if val.validateSBRNL:
                        if usedOns & XbrlConst.standardExtLinkQnames or XbrlConst.qnGenLink in usedOns:
                            definesLinkroles = True
                            if not e.genLabel():
                                val.modelXbrl.error("SBR.NL.2.2.3.03",
                                    _("Link RoleType %(roleType)s missing a generic standard label"),
                                    modelObject=e, roleType=roleURI)
                            nlLabel = e.genLabel(lang="nl")
                            if definition != nlLabel:
                                val.modelXbrl.error("SBR.NL.2.2.3.04",
                                    _("Link RoleType %(roleType)s definition does not match NL standard generic label, \ndefinition: %(definition)s \nNL label: %(label)s"),
                                    modelObject=e, roleType=roleURI, definition=definition, label=nlLabel)
                        if definition and (definition[0].isspace() or definition[-1].isspace()):
                            val.modelXbrl.error("SBR.NL.2.2.3.07",
                                _('Link RoleType %(roleType)s definition has leading or trailing spaces: "%(definition)s"'),
                                modelObject=e, roleType=roleURI, definition=definition)

        # 6.7.13 arcrole types authority
        for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}arcroleType"):
            if isinstance(e,ModelObject):
                arcroleURI = e.get("arcroleURI")
                if targetNamespaceAuthority != UrlUtil.authority(arcroleURI):
                    val.modelXbrl.error(("EFM.6.07.13", "GFM.1.03.15"),
                        _("ArcroleType %(arcroleType)s does not match authority %(targetNamespaceAuthority)s"),
                        modelObject=e, arcroleType=arcroleURI, targetNamespaceAuthority=targetNamespaceAuthority)
                # 6.7.13 end with .../arcrole/lc3 name
                if not arcroleTypePattern.match(arcroleURI):
                    val.modelXbrl.warning(("EFM.6.07.13", "GFM.1.03.15"),
                        _("ArcroleType %(arcroleType)s should end with /arcrole/{LC3name}"),
                        modelObject=e, arcroleType=arcroleURI)
                    
                # 6.7.14 only one arcrole type declaration in DTS
                modelRoleTypes = val.modelXbrl.arcroleTypes[arcroleURI]
                if len(modelRoleTypes) > 1:
                    val.modelXbrl.error(("EFM.6.07.14", "GFM.1.03.16"),
                        _("ArcroleType %(arcroleType)s is defined in multiple taxonomies"),
                        modelObject=e, arcroleType=arcroleURI)
                    
                # 6.7.15 definition match pattern
                definition = modelRoleTypes[0].definition
                if definition is None or not arcroleDefinitionPattern.match(definition):
                    val.modelXbrl.error(("EFM.6.07.15", "GFM.1.03.17"),
                        _("ArcroleType %(arcroleType)s definition must be non-empty"),
                        modelObject=e, arcroleType=arcroleURI)
    
                if val.validateSBRNL:
                    definesArcroles = True
                    val.modelXbrl.error("SBR.NL.2.2.4.01",
                        _("Arcrole type definition is not allowed: %(arcroleURI)s"),
                        modelObject=e, arcroleURI=arcroleURI)
                    
        if val.validateSBRNL:
            for appinfoElt in modelDocument.xmlRootElement.iter(tag="{http://www.w3.org/2001/XMLSchema}appinfo"):
                for nonLinkElt in appinfoElt.iterdescendants():
                    if isinstance(nonLinkElt, ModelObject) and nonLinkElt.namespaceURI != XbrlConst.link:
                        val.modelXbrl.error("SBR.NL.2.2.11.05",
                            _("Appinfo contains disallowed non-link element %(element)s"),
                            modelObject=nonLinkElt, element=nonLinkElt.qname)

            for cplxTypeElt in modelDocument.xmlRootElement.iter(tag="{http://www.w3.org/2001/XMLSchema}complexType"):
                choiceElt = cplxTypeElt.find("{http://www.w3.org/2001/XMLSchema}choice")
                if choiceElt is not None:
                    val.modelXbrl.error("SBR.NL.2.2.11.09",
                        _("ComplexType contains disallowed xs:choice element"),
                        modelObject=choiceElt)
                    
            for cplxContentElt in modelDocument.xmlRootElement.iter(tag="{http://www.w3.org/2001/XMLSchema}complexContent"):
                if XmlUtil.descendantAttr(cplxContentElt, "http://www.w3.org/2001/XMLSchema", ("extension","restriction"), "base") != "sbr:placeholder":
                    val.modelXbrl.error("SBR.NL.2.2.11.10",
                        _("ComplexContent is disallowed"),
                        modelObject=cplxContentElt)

            definesTypes = (modelDocument.xmlRootElement.find("{http://www.w3.org/2001/XMLSchema}complexType") is not None or
                            modelDocument.xmlRootElement.find("{http://www.w3.org/2001/XMLSchema}simpleType") is not None)
            if (definesLinkroles + definesArcroles + definesLinkParts +
                definesAbstractItems + definesNonabstractItems + 
                definesTuples + definesPresentationTuples + definesSpecificationTuples + definesTypes +
                definesEnumerations + definesDimensions + definesDomains + 
                definesHypercubes) != 1:
                schemaContents = []
                if definesLinkroles: schemaContents.append(_("linkroles"))
                if definesArcroles: schemaContents.append(_("arcroles"))
                if definesLinkParts: schemaContents.append(_("link parts"))
                if definesAbstractItems: schemaContents.append(_("abstract items"))
                if definesNonabstractItems: schemaContents.append(_("nonabstract items"))
                if definesTuples: schemaContents.append(_("tuples"))
                if definesPresentationTuples: schemaContents.append(_("sbrPresentationTuples"))
                if definesSpecificationTuples: schemaContents.append(_("sbrSpecificationTuples"))
                if definesTypes: schemaContents.append(_("types"))
                if definesEnumerations: schemaContents.append(_("enumerations"))
                if definesDimensions: schemaContents.append(_("dimensions"))
                if definesDomains: schemaContents.append(_("domains"))
                if definesHypercubes: schemaContents.append(_("hypercubes"))
                if schemaContents:
                    val.modelXbrl.error("SBR.NL.2.2.1.01",
                        _("Taxonomy schema may only define one of these: %(contents)s"),
                        modelObject=modelDocument, contents=', '.join(schemaContents))
                elif not any(refDoc.inDTS and refDoc.targetNamespace not in val.disclosureSystem.baseTaxonomyNamespaces
                             for refDoc in modelDocument.referencesDocument.keys()): # no linkbase ref or includes
                    val.modelXbrl.error("SBR.NL.2.2.1.01",
                        _("Taxonomy schema must be a DTS entrypoint OR define linkroles OR arcroles OR link:parts OR context fragments OR abstract items OR tuples OR non-abstract elements OR types OR enumerations OR dimensions OR domains OR hypercubes"),
                        modelObject=modelDocument)
            if definesConcepts ^ any(  # xor so either concepts and no label LB or no concepts and has label LB
                       (refDoc.type == ModelDocument.Type.LINKBASE and
                        XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link, "labelLink") is not None)
                       for refDoc in modelDocument.referencesDocument.keys()): # no label linkbase
                val.modelXbrl.error("SBR.NL.2.2.1.02",
                    _("A schema that defines concepts MUST have a linked 2.1 label linkbase"),
                    modelObject=modelDocument)
            if (definesNonabstractItems or definesTuples) and not any(  # was xor but changed to and not per RH 1/11/12
                       (refDoc.type == ModelDocument.Type.LINKBASE and
                       (XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link, "referenceLink") is not None or
                        XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link, "label", "{http://www.w3.org/1999/xlink}role", "http://www.xbrl.org/2003/role/documentation" ) is not None))
                        for refDoc in modelDocument.referencesDocument.keys()):
                val.modelXbrl.error("SBR.NL.2.2.1.03",
                    _("A schema that defines non-abstract items MUST have a linked (2.1) reference linkbase AND/OR a label linkbase with @xlink:role=documentation"),
                    modelObject=modelDocument)

    visited.remove(modelDocument)
Beispiel #15
0
def checkDTS(val, modelDocument, visited):
    global targetNamespaceDatePattern, efmFilenamePattern, roleTypePattern, arcroleTypePattern, \
            arcroleDefinitionPattern, namePattern, linkroleDefinitionBalanceIncomeSheet, \
            namespacesConflictPattern
    if targetNamespaceDatePattern is None:
        targetNamespaceDatePattern = re.compile(r"/([12][0-9]{3})-([01][0-9])-([0-3][0-9])|"
                                            r"/([12][0-9]{3})([01][0-9])([0-3][0-9])|")
        efmFilenamePattern = re.compile(r"^[a-z0-9][a-zA-Z0-9_\.\-]*(\.xsd|\.xml)$")
        roleTypePattern = re.compile(r"^.*/role/[^/\s]+$")
        arcroleTypePattern = re.compile(r"^.*/arcrole/[^/\s]+$")
        arcroleDefinitionPattern = re.compile(r"^.*[^\\s]+.*$")  # at least one non-whitespace character
        namePattern = re.compile("[][()*+?\\\\/^{}|@#%^=~`\"';:,<>&$\u00a3\u20ac]") # u20ac=Euro, u00a3=pound sterling 
        linkroleDefinitionBalanceIncomeSheet = re.compile(r"[^-]+-\s+Statement\s+-\s+.*(income|balance|financial\W+position)",
                                                          re.IGNORECASE)
        namespacesConflictPattern = re.compile(r"http://(xbrl\.us|fasb\.org|xbrl\.sec\.gov)/(dei|us-types|us-roles|rr)/([0-9]{4}-[0-9]{2}-[0-9]{2})$")
    nonDomainItemNameProblemPattern = re.compile(
        r"({0})|(FirstQuarter|SecondQuarter|ThirdQuarter|FourthQuarter|[1-4]Qtr|Qtr[1-4]|ytd|YTD|HalfYear)(?:$|[A-Z\W])"
        .format(re.sub(r"\W", "", (val.entityRegistrantName or "").title())))
    
        
    visited.append(modelDocument)
    definesLabelLinkbase = False
    for referencedDocument, modelDocumentReference in modelDocument.referencesDocument.items():
        #6.07.01 no includes
        if modelDocumentReference.referenceType == "include":
            val.modelXbrl.error(("EFM.6.07.01", "GFM.1.03.01", "SBR.NL.2.2.0.18"),
                _("Taxonomy schema %(schema)s includes %(include)s, only import is allowed"),
                modelObject=modelDocumentReference.referringModelObject,
                    schema=os.path.basename(modelDocument.uri), 
                    include=os.path.basename(referencedDocument.uri))
        if referencedDocument not in visited:
            checkDTS(val, referencedDocument, visited)
            
    if val.disclosureSystem.standardTaxonomiesDict is None:
        pass

    if val.validateEFM: 
        if modelDocument.uri in val.disclosureSystem.standardTaxonomiesDict:
            if modelDocument.targetNamespace:
                # check for duplicates of us-types, dei, and rr taxonomies
                match = namespacesConflictPattern.match(modelDocument.targetNamespace)
                if match is not None:
                    val.standardNamespaceConflicts[match.group(2)].add(modelDocument)
        else:
            if len(modelDocument.basename) > 32:
                val.modelXbrl.error("EFM.5.01.01.tooManyCharacters",
                    _("Document file name %(filename)s must not exceed 32 characters."),
                    modelObject=modelDocument, filename=modelDocument.basename)
            if not efmFilenamePattern.match(modelDocument.basename):
                val.modelXbrl.error("EFM.5.01.01",
                    _("Document file name %(filename)s must start with a-z or 0-9, contain upper or lower case letters, ., -, _, and end with .xsd or .xml."),
                    modelObject=modelDocument, filename=modelDocument.basename)
    
    if (modelDocument.type == ModelDocument.Type.SCHEMA and 
        modelDocument.targetNamespace not in val.disclosureSystem.baseTaxonomyNamespaces and
        modelDocument.uri.startswith(val.modelXbrl.uriDir)):
        
        # check schema contents types
        if val.validateSBRNL:
            definesLinkroles = False
            definesArcroles = False
            definesLinkParts = False
            definesAbstractItems = False
            definesNonabstractItems = False
            definesConcepts = False
            definesTuples = False
            definesPresentationTuples = False
            definesSpecificationTuples = False
            definesTypes = False
            definesEnumerations = False
            definesDimensions = False
            definesDomains = False
            definesHypercubes = False
                
        # 6.7.3 check namespace for standard authority
        targetNamespaceAuthority = UrlUtil.authority(modelDocument.targetNamespace) 
        if targetNamespaceAuthority in val.disclosureSystem.standardAuthorities:
            val.modelXbrl.error(("EFM.6.07.03", "GFM.1.03.03"),
                _("Taxonomy schema %(schema)s namespace %(targetNamespace)s is a disallowed authority"),
                modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace, 
                targetNamespaceAuthority=UrlUtil.authority(modelDocument.targetNamespace, includeScheme=False))
            
        # 6.7.4 check namespace format
        if modelDocument.targetNamespace is None or not modelDocument.targetNamespace.startswith("http://"):
            match = None
        elif val.validateEFMorGFM:
            targetNamespaceDate = modelDocument.targetNamespace[len(targetNamespaceAuthority):]
            match = targetNamespaceDatePattern.match(targetNamespaceDate)
        else:
            match = None
        if match is not None:
            try:
                if match.lastindex == 3:
                    date = datetime.date(int(match.group(1)),int(match.group(2)),int(match.group(3)))
                elif match.lastindex == 6:
                    date = datetime.date(int(match.group(4)),int(match.group(5)),int(match.group(6)))
                else:
                    match = None
            except ValueError:
                match = None
        if match is None:
            val.modelXbrl.error(("EFM.6.07.04", "GFM.1.03.04"),
                _("Taxonomy schema %(schema)s namespace %(targetNamespace)s must have format http://{authority}/{versionDate}"),
                modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace)
        elif val.fileNameDate and date > val.fileNameDate:
            val.modelXbrl.info(("EFM.6.07.06", "GFM.1.03.06"),
                _("Warning: Taxonomy schema %(schema)s namespace %(targetNamespace)s has date later than document name date %(docNameDate)s"),
                modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace,
                docNameDate=val.fileNameDate)

        if modelDocument.targetNamespace is not None:
            # 6.7.5 check prefix for _
            authority = UrlUtil.authority(modelDocument.targetNamespace)
            if not re.match(r"(http://|https://|ftp://|urn:)\w+",authority):
                val.modelXbrl.error(("EFM.6.07.05", "GFM.1.03.05"),
                    _("Taxonomy schema %(schema)s namespace %(targetNamespace)s must be a valid URL with a valid authority for the namespace."),
                    modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace)
            prefix = XmlUtil.xmlnsprefix(modelDocument.xmlRootElement,modelDocument.targetNamespace)
            if not prefix:
                val.modelXbrl.error(("EFM.6.07.07", "GFM.1.03.07"),
                    _("Taxonomy schema %(schema)s namespace %(targetNamespace)s missing prefix for the namespace."),
                    modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace)
            elif "_" in prefix:
                val.modelXbrl.error(("EFM.6.07.07", "GFM.1.03.07"),
                    _("Taxonomy schema %(schema)s namespace %(targetNamespace)s prefix %(prefix)s must not have an '_'"),
                    modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace, prefix=prefix)

            if val.validateSBRNL:
                genrlSpeclRelSet = val.modelXbrl.relationshipSet(XbrlConst.generalSpecial)
            for modelConcept in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/2001/XMLSchema}element"):
                if isinstance(modelConcept,ModelConcept):
                    # 6.7.16 name not duplicated in standard taxonomies
                    name = modelConcept.get("name")
                    if name is None: 
                        name = ""
                        if modelConcept.get("ref") is not None:
                            continue    # don't validate ref's here
                    for c in val.modelXbrl.nameConcepts.get(name, []):
                        if c.modelDocument != modelDocument:
                            if (val.validateEFMorGFM and
                                  not c.modelDocument.uri.startswith(val.modelXbrl.uriDir)):
                                val.modelXbrl.error(("EFM.6.07.16", "GFM.1.03.18"),
                                    _("Concept %(concept)s is also defined in standard taxonomy schema schema %(standardSchema)s"),
                                    modelObject=(modelConcept,c), concept=modelConcept.qname, standardSchema=os.path.basename(c.modelDocument.uri), standardConcept=c.qname)
                            elif val.validateSBRNL:
                                if not (genrlSpeclRelSet.isRelated(modelConcept, "child", c) or genrlSpeclRelSet.isRelated(c, "child", modelConcept)):
                                    val.modelXbrl.error("SBR.NL.2.2.2.02",
                                        _("Concept %(concept)s is also defined in standard taxonomy schema %(standardSchema)s without a general-special relationship"),
                                        modelObject=c, concept=modelConcept.qname, standardSchema=os.path.basename(c.modelDocument.uri))
                    ''' removed RH 2011-12-23 corresponding set up of table in ValidateFiling
                    if val.validateSBRNL and name in val.nameWordsTable:
                        if not any( any( genrlSpeclRelSet.isRelated(c, "child", modelConcept)
                                         for c in val.modelXbrl.nameConcepts.get(partialWordName, []))
                                    for partialWordName in val.nameWordsTable[name]):
                            val.modelXbrl.error("SBR.NL.2.3.2.01",
                                _("Concept %(specialName)s is appears to be missing a general-special relationship to %(generalNames)s"),
                                modelObject=c, specialName=modelConcept.qname, generalNames=', or to '.join(val.nameWordsTable[name]))
                    '''

                    # 6.7.17 id properly formed
                    id = modelConcept.id
                    requiredId = (prefix if prefix is not None else "") + "_" + name
                    if val.validateEFMorGFM and id != requiredId:
                        val.modelXbrl.error(("EFM.6.07.17", "GFM.1.03.19"),
                            _("Concept %(concept)s id %(id)s should be $(requiredId)s"),
                            modelObject=modelConcept, concept=modelConcept.qname, id=id, requiredId=requiredId)
                        
                    # 6.7.18 nillable is true
                    nillable = modelConcept.get("nillable")
                    if nillable != "true":
                        val.modelXbrl.error(("EFM.6.07.18", "GFM.1.03.20"),
                            _("Taxonomy schema %(schema)s element %(concept)s nillable %(nillable)s should be 'true'"),
                            modelObject=modelConcept, schema=os.path.basename(modelDocument.uri),
                            concept=name, nillable=nillable)
        
                    # 6.7.19 not tuple
                    if modelConcept.isTuple:
                        if val.validateEFMorGFM:
                            val.modelXbrl.error(("EFM.6.07.19", "GFM.1.03.21"),
                                _("Concept %(concept)s is a tuple"),
                                modelObject=modelConcept, concept=modelConcept.qname)
                        
                    # 6.7.20 no typed domain ref
                    if modelConcept.isTypedDimension:
                        val.modelXbrl.error(("EFM.6.07.20", "GFM.1.03.22"),
                            _("Concept %(concept)s has typedDomainRef %(typedDomainRef)s"),
                            modelObject=modelConcept, concept=modelConcept.qname,
                            typedDomainRef=modelConcept.typedDomainElement.qname if modelConcept.typedDomainElement is not None else modelConcept.typedDomainRef)
                        
                    # 6.7.21 abstract must be duration
                    isDuration = modelConcept.periodType == "duration"
                    if modelConcept.abstract == "true" and not isDuration:
                        val.modelXbrl.error(("EFM.6.07.21", "GFM.1.03.23"),
                            _("Taxonomy schema %(schema)s element %(concept)s is abstract but period type is not duration"),
                            modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=modelConcept.qname)
                        
                    # 6.7.22 abstract must be stringItemType
                    ''' removed SEC EFM v.17, Edgar release 10.4, and GFM 2011-04-08
                    if modelConcept.abstract == "true" and modelConcept.typeQname != XbrlConst. qnXbrliStringItemType:
                        val.modelXbrl.error(("EFM.6.07.22", "GFM.1.03.24"),
                            _("Concept %(concept)s  is abstract but type is not xbrli:stringItemType"),
                            modelObject=modelConcept, concept=modelConcept.qname)
					'''
                    substititutionGroupQname = modelConcept.substitutionGroupQname
                    # 6.7.23 Axis must be subs group dimension
                    if name.endswith("Axis") ^ (substititutionGroupQname == XbrlConst.qnXbrldtDimensionItem):
                        val.modelXbrl.error(("EFM.6.07.23", "GFM.1.03.25"),
                            _("Concept %(concept)s must end in Axis to be in xbrldt:dimensionItem substitution group"),
                            modelObject=modelConcept, concept=modelConcept.qname)

                    # 6.7.24 Table must be subs group hypercube
                    if name.endswith("Table") ^ (substititutionGroupQname == XbrlConst.qnXbrldtHypercubeItem):
                        val.modelXbrl.error(("EFM.6.07.24", "GFM.1.03.26"),
                            _("Concept %(concept)s must end in Table to be in xbrldt:hypercubeItem substitution group"),
                            modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=modelConcept.qname)

                    # 6.7.25 if neither hypercube or dimension, substitution group must be item
                    if substititutionGroupQname not in (None,
                                                        XbrlConst.qnXbrldtDimensionItem, 
                                                        XbrlConst.qnXbrldtHypercubeItem,
                                                        XbrlConst.qnXbrliItem):                           
                        val.modelXbrl.error(("EFM.6.07.25", "GFM.1.03.27"),
                            _("Concept %(concept)s has disallowed substitution group %(substitutionGroup)s"),
                            modelObject=modelConcept, concept=modelConcept.qname,
                            substitutionGroup=modelConcept.substitutionGroupQname)
                        
                    # 6.7.26 Table must be subs group hypercube
                    if name.endswith("LineItems") and modelConcept.abstract != "true":
                        val.modelXbrl.error(("EFM.6.07.26", "GFM.1.03.28"),
                            _("Concept %(concept)s is a LineItems but not abstract"),
                            modelObject=modelConcept, concept=modelConcept.qname)

                    # 6.7.27 type domainMember must end with Domain or Member
                    conceptType = modelConcept.type
                    isDomainItemType = conceptType is not None and conceptType.isDomainItemType
                    endsWithDomainOrMember = name.endswith("Domain") or name.endswith("Member")
                    if isDomainItemType != endsWithDomainOrMember:
                        val.modelXbrl.error(("EFM.6.07.27", "GFM.1.03.29"),
                            _("Concept %(concept)s must end with Domain or Member for type of domainItemType"),
                            modelObject=modelConcept, concept=modelConcept.qname)

                    # 6.7.28 domainItemType must be duration
                    if isDomainItemType and not isDuration:
                        val.modelXbrl.error(("EFM.6.07.28", "GFM.1.03.30"),
                            _("Concept %(concept)s is a domainItemType and must be periodType duration"),
                            modelObject=modelConcept, concept=modelConcept.qname)
                        
                    # 6.8.5 semantic check, check LC3 name
                    if not name[0].isupper():
                        val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.05.firstLetter", "GFM.2.03.05.firstLetter"),
                            _("Concept %(concept)s name must start with a capital letter"),
                            modelObject=modelConcept, concept=modelConcept.qname)
                    if namePattern.search(name):
                        val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.05.disallowedCharacter", "GFM.2.03.05.disallowedCharacter"),
                            _("Concept %(concept)s has disallowed name character"),
                            modelObject=modelConcept, concept=modelConcept.qname)
                    if len(name) > 200:
                        val.modelXbrl.log("ERROR-SEMANTIC", "EFM.6.08.05.nameLength",
                            _("Concept %(concept)s name length %(namelength)s exceeds 200 characters"),
                            modelObject=modelConcept, concept=modelConcept.qname, namelength=len(name))
                        
                    if val.validateEFM:
                        label = modelConcept.label(lang="en-US", fallbackToQname=False)
                        if label:
                            # allow Joe's Bar, N.A.  to be JoesBarNA -- remove ', allow A. as not article "a"
                            lc3name = ''.join(re.sub(r"['.-]", "", (w[0] or w[2] or w[3] or w[4])).title()
                                              for w in re.findall(r"((\w+')+\w+)|(A[.-])|([.-]A(?=\W|$))|(\w+)", label) # EFM implies this should allow - and . re.findall(r"[\w\-\.]+", label)
                                              if w[4].lower() not in ("the", "a", "an"))
                            if not(name == lc3name or 
                                   (name and lc3name and lc3name[0].isdigit() and name[1:] == lc3name and (name[0].isalpha() or name[0] == '_'))):
                                val.modelXbrl.log("WARNING-SEMANTIC", "EFM.6.08.05.LC3",
                                    _("Concept %(concept)s should match expected LC3 composition %(lc3name)s"),
                                    modelObject=modelConcept, concept=modelConcept.qname, lc3name=lc3name)
                                
                    if conceptType is not None:
                        # 6.8.6 semantic check
                        if not isDomainItemType and conceptType.qname != XbrlConst.qnXbrliDurationItemType:
                            nameProblems = nonDomainItemNameProblemPattern.findall(name)
                            if any(any(t) for t in nameProblems):  # list of tuples with possibly nonempty strings
                                val.modelXbrl.log("WARNING-SEMANTIC", ("EFM.6.08.06", "GFM.2.03.06"),
                                    _("Concept %(concept)s should not contain company or period information, found: %(matches)s"),
                                    modelObject=modelConcept, concept=modelConcept.qname, 
                                    matches=", ".join(''.join(t) for t in nameProblems))
                        
                        if conceptType.qname == XbrlConst.qnXbrliMonetaryItemType:
                            if not modelConcept.balance:
                                # 6.8.11 may not appear on a income or balance statement
                                if any(linkroleDefinitionBalanceIncomeSheet.match(roleType.definition)
                                       for rel in val.modelXbrl.relationshipSet(XbrlConst.parentChild).toModelObject(modelConcept)
                                       for roleType in val.modelXbrl.roleTypes.get(rel.linkrole,())):
                                    val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.11", "GFM.2.03.11"),
                                        _("Concept %(concept)s must have a balance because it appears in a statement of income or balance sheet"),
                                        modelObject=modelConcept, concept=modelConcept.qname)
                                # 6.11.5 semantic check, must have a documentation label
                                stdLabel = modelConcept.label(lang="en-US", fallbackToQname=False)
                                defLabel = modelConcept.label(preferredLabel=XbrlConst.documentationLabel, lang="en-US", fallbackToQname=False)
                                if not defLabel or ( # want different words than std label
                                    stdLabel and re.findall(r"\w+", stdLabel) == re.findall(r"\w+", defLabel)):
                                    val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.11.05", "GFM.2.04.04"),
                                        _("Concept %(concept)s is monetary without a balance and must have a documentation label that disambiguates its sign"),
                                        modelObject=modelConcept, concept=modelConcept.qname)
                        
                        # 6.8.16 semantic check
                        if conceptType.qname == XbrlConst.qnXbrliDateItemType and modelConcept.periodType != "duration":
                            val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.16", "GFM.2.03.16"),
                                _("Concept %(concept)s of type xbrli:dateItemType must have periodType duration"),
                                modelObject=modelConcept, concept=modelConcept.qname)
                        
                        # 6.8.17 semantic check
                        if conceptType.qname == XbrlConst.qnXbrliStringItemType and modelConcept.periodType != "duration":
                            val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.17", "GFM.2.03.17"),
                                _("Concept %(concept)s of type xbrli:stringItemType must have periodType duration"),
                                modelObject=modelConcept, concept=modelConcept.qname)
                        
                    if val.validateSBRNL:
                        if modelConcept.isTuple:
                            if modelConcept.substitutionGroupQname.localName == "presentationTuple" and modelConcept.substitutionGroupQname.namespaceURI.endswith("/basis/sbr/xbrl/xbrl-syntax-extension"): # namespace may change each year
                                definesPresentationTuples = True
                            elif modelConcept.substitutionGroupQname.localName == "specificationTuple" and modelConcept.substitutionGroupQname.namespaceURI.endswith("/basis/sbr/xbrl/xbrl-syntax-extension"): # namespace may change each year
                                definesSpecificationTuples = True
                            else:
                                definesTuples = True
                            definesConcepts = True
                            if modelConcept.isAbstract:
                                val.modelXbrl.error("SBR.NL.2.2.2.03",
                                    _("Concept %(concept)s is an abstract tuple"),
                                    modelObject=modelConcept, concept=modelConcept.qname)
                            if tupleCycle(val,modelConcept):
                                val.modelXbrl.error("SBR.NL.2.2.2.07",
                                    _("Tuple %(concept)s has a tuple cycle"),
                                    modelObject=modelConcept, concept=modelConcept.qname)
                            if modelConcept.get("nillable") != "false" and modelConcept.isRoot:
                                val.modelXbrl.error("SBR.NL.2.2.2.17", #don't want default, just what was really there
                                    _("Tuple %(concept)s must have nillable='false'"),
                                    modelObject=modelConcept, concept=modelConcept.qname)
                        elif modelConcept.isItem:
                            definesConcepts = True
                        if modelConcept.abstract == "true":
                            if modelConcept.isRoot:
                                if modelConcept.get("nillable") != "false": #don't want default, just what was really there
                                    val.modelXbrl.error("SBR.NL.2.2.2.16",
                                        _("Abstract root concept %(concept)s must have nillable='false'"),
                                    modelObject=modelConcept, concept=modelConcept.qname)
                                if modelConcept.typeQname != XbrlConst.qnXbrliStringItemType:
                                    val.modelXbrl.error("SBR.NL.2.2.2.21",
                                        _("Abstract root concept %(concept)s must have type='xbrli:stringItemType'"),
                                    modelObject=modelConcept, concept=modelConcept.qname)
                            if modelConcept.balance:
                                val.modelXbrl.error("SBR.NL.2.2.2.22",
                                    _("Abstract concept %(concept)s must not have a balance attribute"),
                                    modelObject=modelConcept, concept=modelConcept.qname)
                            if modelConcept.isHypercubeItem:
                                definesHypercubes = True
                            elif modelConcept.isDimensionItem:
                                definesDimensions = True
                            elif substititutionGroupQname and substititutionGroupQname.localName in ("domainItem","domainMemberItem"):
                                definesDomains = True
                            elif modelConcept.isItem:
                                definesAbstractItems = True
                        else:   # not abstract
                            if modelConcept.isItem:
                                definesNonabstractItems = True
                                if not (modelConcept.label(preferredLabel=XbrlConst.documentationLabel,fallbackToQname=False,lang="nl") or
                                        val.modelXbrl.relationshipSet(XbrlConst.conceptReference).fromModelObject(c) or
                                        modelConcept.genLabel(role=XbrlConst.genDocumentationLabel,lang="nl") or
                                        val.modelXbrl.relationshipSet(XbrlConst.elementReference).fromModelObject(c)):
                                    val.modelXbrl.error("SBR.NL.2.2.2.28",
                                        _("Concept %(concept)s must have a documentation label or reference"),
                                        modelObject=modelConcept, concept=modelConcept.qname)
                        if modelConcept.balance and not modelConcept.instanceOfType(XbrlConst.qnXbrliMonetaryItemType):
                            val.modelXbrl.error("SBR.NL.2.2.2.24",
                                _("Non-monetary concept %(concept)s must not have a balance attribute"),
                                modelObject=modelConcept, concept=modelConcept.qname)
                        if modelConcept.isLinkPart:
                            definesLinkParts = True
                            val.modelXbrl.error("SBR.NL.2.2.5.01",
                                _("Link:part concept %(concept)s is not allowed"),
                                modelObject=modelConcept, concept=modelConcept.qname)
                            if not modelConcept.genLabel(fallbackToQname=False,lang="nl"):
                                val.modelXbrl.error("SBR.NL.2.2.5.02",
                                    _("Link part definition %(concept)s must have a generic label in language 'nl'"),
                                    modelObject=modelConcept, concept=modelConcept.qname)
        # 6.7.8 check for embedded linkbase
        for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}linkbase"):
            if isinstance(e,ModelObject):
                val.modelXbrl.error(("EFM.6.07.08", "GFM.1.03.08"),
                    _("Taxonomy schema %(schema)s contains an embedded linkbase"),
                    modelObject=e, schema=modelDocument.basename)
                break

        requiredUsedOns = {XbrlConst.qnLinkPresentationLink,
                           XbrlConst.qnLinkCalculationLink,
                           XbrlConst.qnLinkDefinitionLink}
        
        standardUsedOns = {XbrlConst.qnLinkLabel, XbrlConst.qnLinkReference, 
                           XbrlConst.qnLinkDefinitionArc, XbrlConst.qnLinkCalculationArc, XbrlConst.qnLinkPresentationArc, 
                           XbrlConst.qnLinkLabelArc, XbrlConst.qnLinkReferenceArc, 
                           # per WH, private footnote arc and footnore resource roles are not allowed
                           XbrlConst.qnLinkFootnoteArc, XbrlConst.qnLinkFootnote,
                           }

        # 6.7.9 role types authority
        for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}roleType"):
            if isinstance(e,ModelObject):
                roleURI = e.get("roleURI")
                if targetNamespaceAuthority != UrlUtil.authority(roleURI):
                    val.modelXbrl.error(("EFM.6.07.09", "GFM.1.03.09"),
                        _("RoleType %(roleType)s does not match authority %(targetNamespaceAuthority)s"),
                        modelObject=e, roleType=roleURI, targetNamespaceAuthority=targetNamespaceAuthority, targetNamespace=modelDocument.targetNamespace)
                # 6.7.9 end with .../role/lc3 name
                if not roleTypePattern.match(roleURI):
                    val.modelXbrl.warning(("EFM.6.07.09.roleEnding", "GFM.1.03.09"),
                        "RoleType %(roleType)s should end with /role/{LC3name}",
                        modelObject=e, roleType=roleURI)
                    
                # 6.7.10 only one role type declaration in DTS
                modelRoleTypes = val.modelXbrl.roleTypes.get(roleURI)
                if modelRoleTypes is not None:
                    modelRoleType = modelRoleTypes[0]
                    definition = modelRoleType.definitionNotStripped
                    usedOns = modelRoleType.usedOns
                    if len(modelRoleTypes) == 1:
                        # 6.7.11 used on's for pre, cal, def if any has a used on
                        if not usedOns.isdisjoint(requiredUsedOns) and len(requiredUsedOns - usedOns) > 0:
                            val.modelXbrl.error(("EFM.6.07.11", "GFM.1.03.11"),
                                _("RoleType %(roleType)s missing used on %(usedOn)s"),
                                modelObject=e, roleType=roleURI, usedOn=requiredUsedOns - usedOns)
                            
                        # 6.7.12 definition match pattern
                        if (val.disclosureSystem.roleDefinitionPattern is not None and
                            (definition is None or not val.disclosureSystem.roleDefinitionPattern.match(definition))):
                            val.modelXbrl.error(("EFM.6.07.12", "GFM.1.03.12-14"),
                                _("RoleType %(roleType)s definition \"%(definition)s\" must match {Sortcode} - {Type} - {Title}"),
                                modelObject=e, roleType=roleURI, definition=(definition or ""))

                    if usedOns & standardUsedOns: # semantics check
                        val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.03", "GFM.2.03.03"),
                            _("RoleType %(roleuri)s is defined using role types already defined by standard roles for: %(qnames)s"),
                            modelObject=e, roleuri=roleURI, qnames=', '.join(str(qn) for qn in usedOns & standardUsedOns))

                    if val.validateSBRNL:
                        if usedOns & XbrlConst.standardExtLinkQnames or XbrlConst.qnGenLink in usedOns:
                            definesLinkroles = True
                            if not e.genLabel():
                                val.modelXbrl.error("SBR.NL.2.2.3.03",
                                    _("Link RoleType %(roleType)s missing a generic standard label"),
                                    modelObject=e, roleType=roleURI)
                            nlLabel = e.genLabel(lang="nl")
                            if definition != nlLabel:
                                val.modelXbrl.error("SBR.NL.2.2.3.04",
                                    _("Link RoleType %(roleType)s definition does not match NL standard generic label, \ndefinition: %(definition)s \nNL label: %(label)s"),
                                    modelObject=e, roleType=roleURI, definition=definition, label=nlLabel)
                        if definition and (definition[0].isspace() or definition[-1].isspace()):
                            val.modelXbrl.error("SBR.NL.2.2.3.07",
                                _('Link RoleType %(roleType)s definition has leading or trailing spaces: "%(definition)s"'),
                                modelObject=e, roleType=roleURI, definition=definition)

        # 6.7.13 arcrole types authority
        for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}arcroleType"):
            if isinstance(e,ModelObject):
                arcroleURI = e.get("arcroleURI")
                if targetNamespaceAuthority != UrlUtil.authority(arcroleURI):
                    val.modelXbrl.error(("EFM.6.07.13", "GFM.1.03.15"),
                        _("ArcroleType %(arcroleType)s does not match authority %(targetNamespaceAuthority)s"),
                        modelObject=e, arcroleType=arcroleURI, targetNamespaceAuthority=targetNamespaceAuthority, targetNamespace=modelDocument.targetNamespace)
                # 6.7.13 end with .../arcrole/lc3 name
                if not arcroleTypePattern.match(arcroleURI):
                    val.modelXbrl.warning(("EFM.6.07.13.arcroleEnding", "GFM.1.03.15"),
                        _("ArcroleType %(arcroleType)s should end with /arcrole/{LC3name}"),
                        modelObject=e, arcroleType=arcroleURI)
                    
                # 6.7.15 definition match pattern
                modelRoleTypes = val.modelXbrl.arcroleTypes[arcroleURI]
                definition = modelRoleTypes[0].definition
                if definition is None or not arcroleDefinitionPattern.match(definition):
                    val.modelXbrl.error(("EFM.6.07.15", "GFM.1.03.17"),
                        _("ArcroleType %(arcroleType)s definition must be non-empty"),
                        modelObject=e, arcroleType=arcroleURI)
    
                # semantic checks
                usedOns = modelRoleTypes[0].usedOns
                if usedOns & standardUsedOns: # semantics check
                    val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.03", "GFM.2.03.03"),
                        _("ArcroleType %(arcroleuri)s is defined using role types already defined by standard arcroles for: %(qnames)s"),
                        modelObject=e, arcroleuri=arcroleURI, qnames=', '.join(str(qn) for qn in usedOns & standardUsedOns))

                if val.validateSBRNL:
                    definesArcroles = True
                    val.modelXbrl.error("SBR.NL.2.2.4.01",
                        _("Arcrole type definition is not allowed: %(arcroleURI)s"),
                        modelObject=e, arcroleURI=arcroleURI)
                    
        if val.validateSBRNL:
            for appinfoElt in modelDocument.xmlRootElement.iter(tag="{http://www.w3.org/2001/XMLSchema}appinfo"):
                for nonLinkElt in appinfoElt.iterdescendants():
                    if isinstance(nonLinkElt, ModelObject) and nonLinkElt.namespaceURI != XbrlConst.link:
                        val.modelXbrl.error("SBR.NL.2.2.11.05",
                            _("Appinfo contains disallowed non-link element %(element)s"),
                            modelObject=nonLinkElt, element=nonLinkElt.qname)

            for cplxTypeElt in modelDocument.xmlRootElement.iter(tag="{http://www.w3.org/2001/XMLSchema}complexType"):
                choiceElt = cplxTypeElt.find("{http://www.w3.org/2001/XMLSchema}choice")
                if choiceElt is not None:
                    val.modelXbrl.error("SBR.NL.2.2.11.09",
                        _("ComplexType contains disallowed xs:choice element"),
                        modelObject=choiceElt)
                    
            for cplxContentElt in modelDocument.xmlRootElement.iter(tag="{http://www.w3.org/2001/XMLSchema}complexContent"):
                if XmlUtil.descendantAttr(cplxContentElt, "http://www.w3.org/2001/XMLSchema", ("extension","restriction"), "base") != "sbr:placeholder":
                    val.modelXbrl.error("SBR.NL.2.2.11.10",
                        _("ComplexContent is disallowed"),
                        modelObject=cplxContentElt)

            definesTypes = (modelDocument.xmlRootElement.find("{http://www.w3.org/2001/XMLSchema}complexType") is not None or
                            modelDocument.xmlRootElement.find("{http://www.w3.org/2001/XMLSchema}simpleType") is not None)
            
            for enumElt in modelDocument.xmlRootElement.iter(tag="{http://www.w3.org/2001/XMLSchema}enumeration"):
                definesEnumerations = True
                if any(not valueElt.genLabel(lang="nl")
                       for valueElt in enumElt.iter(tag="{http://www.w3.org/2001/XMLSchema}value")):
                    val.modelXbrl.error("SBR.NL.2.2.7.05",
                        _("Enumeration element has value(s) without generic label."),
                        modelObject=enumElt)

            if (definesLinkroles + definesArcroles + definesLinkParts +
                definesAbstractItems + definesNonabstractItems + 
                definesTuples + definesPresentationTuples + definesSpecificationTuples + definesTypes +
                definesEnumerations + definesDimensions + definesDomains + 
                definesHypercubes) != 1:
                schemaContents = []
                if definesLinkroles: schemaContents.append(_("linkroles"))
                if definesArcroles: schemaContents.append(_("arcroles"))
                if definesLinkParts: schemaContents.append(_("link parts"))
                if definesAbstractItems: schemaContents.append(_("abstract items"))
                if definesNonabstractItems: schemaContents.append(_("nonabstract items"))
                if definesTuples: schemaContents.append(_("tuples"))
                if definesPresentationTuples: schemaContents.append(_("sbrPresentationTuples"))
                if definesSpecificationTuples: schemaContents.append(_("sbrSpecificationTuples"))
                if definesTypes: schemaContents.append(_("types"))
                if definesEnumerations: schemaContents.append(_("enumerations"))
                if definesDimensions: schemaContents.append(_("dimensions"))
                if definesDomains: schemaContents.append(_("domains"))
                if definesHypercubes: schemaContents.append(_("hypercubes"))
                if schemaContents:
                    if not ((definesTuples or definesPresentationTuples or definesSpecificationTuples) and
                            not (definesLinkroles or definesArcroles or definesLinkParts or definesAbstractItems or
                                 definesTypes or definesDimensions or definesDomains or definesHypercubes)):
                        val.modelXbrl.error("SBR.NL.2.2.1.01",
                            _("Taxonomy schema may only define one of these: %(contents)s"),
                            modelObject=modelDocument, contents=', '.join(schemaContents))
                elif not any(refDoc.inDTS and refDoc.targetNamespace not in val.disclosureSystem.baseTaxonomyNamespaces
                             for refDoc in modelDocument.referencesDocument.keys()): # no linkbase ref or includes
                    val.modelXbrl.error("SBR.NL.2.2.1.01",
                        _("Taxonomy schema must be a DTS entrypoint OR define linkroles OR arcroles OR link:parts OR context fragments OR abstract items OR tuples OR non-abstract elements OR types OR enumerations OR dimensions OR domains OR hypercubes"),
                        modelObject=modelDocument)
            if definesConcepts ^ any(  # xor so either concepts and no label LB or no concepts and has label LB
                       (refDoc.type == ModelDocument.Type.LINKBASE and
                        XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link, "labelLink") is not None)
                       for refDoc in modelDocument.referencesDocument.keys()): # no label linkbase
                val.modelXbrl.error("SBR.NL.2.2.1.02",
                    _("A schema that defines concepts MUST have a linked 2.1 label linkbase"),
                    modelObject=modelDocument)
            if (definesNonabstractItems or definesTuples) and not any(  # was xor but changed to and not per RH 1/11/12
                       (refDoc.type == ModelDocument.Type.LINKBASE and
                       (XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link, "referenceLink") is not None or
                        XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link, "label", "{http://www.w3.org/1999/xlink}role", "http://www.xbrl.org/2003/role/documentation" ) is not None))
                        for refDoc in modelDocument.referencesDocument.keys()):
                val.modelXbrl.error("SBR.NL.2.2.1.03",
                    _("A schema that defines non-abstract items MUST have a linked (2.1) reference linkbase AND/OR a label linkbase with @xlink:role=documentation"),
                    modelObject=modelDocument)

        #6.3.3 filename check
        m = re.match(r"^\w+-([12][0-9]{3}[01][0-9][0-3][0-9]).xsd$", modelDocument.basename)
        if m:
            try: # check date value
                datetime.datetime.strptime(m.group(1),"%Y%m%d").date()
                # date and format are ok, check "should" part of 6.3.3
                if val.fileNameBasePart:
                    expectedFilename = "{0}-{1}.xsd".format(val.fileNameBasePart, val.fileNameDatePart)
                    if modelDocument.basename != expectedFilename:
                        val.modelXbrl.log("WARNING-SEMANTIC", ("EFM.6.03.03.matchInstance", "GFM.1.01.01.matchInstance"),
                            _('Schema file name warning: %(filename)s, should match %(expectedFilename)s'),
                            modelObject=modelDocument, filename=modelDocument.basename, expectedFilename=expectedFilename)
            except ValueError:
                val.modelXbrl.error(("EFM.6.03.03", "GFM.1.01.01"),
                    _('Invalid schema file base name part (date) in "{base}-{yyyymmdd}.xsd": %(filename)s'),
                    modelObject=modelDocument, filename=modelDocument.basename)
        else:
            val.modelXbrl.error(("EFM.6.03.03", "GFM.1.01.01"),
                _('Invalid schema file name, must match "{base}-{yyyymmdd}.xsd": %(filename)s'),
                modelObject=modelDocument, filename=modelDocument.basename)

    elif modelDocument.type == ModelDocument.Type.LINKBASE:
        # if it is part of the submission (in same directory) check name
        if modelDocument.filepath.startswith(val.modelXbrl.modelDocument.filepathdir):
            #6.3.3 filename check
            extLinkElt = XmlUtil.descendant(modelDocument.xmlRootElement, XbrlConst.link, "*", "{http://www.w3.org/1999/xlink}type", "extended")
            if extLinkElt is None:# no ext link element
                val.modelXbrl.error(("EFM.6.03.03.noLinkElement", "GFM.1.01.01.noLinkElement"),
                    _('Invalid linkbase file name: %(filename)s, has no extended link element, cannot determine link type.'),
                    modelObject=modelDocument, filename=modelDocument.basename)
            elif extLinkElt.localName not in extLinkEltFileNameEnding:
                val.modelXbrl.error("EFM.6.03.02",
                    _('Invalid linkbase link element %(linkElement)s in %(filename)s'),
                    modelObject=modelDocument, linkElement=extLinkElt.localName, filename=modelDocument.basename)
            else:
                m = re.match(r"^\w+-([12][0-9]{3}[01][0-9][0-3][0-9])(_[a-z]{3}).xml$", modelDocument.basename)
                expectedSuffix = extLinkEltFileNameEnding[extLinkElt.localName]
                if m and m.group(2) == expectedSuffix:
                    try: # check date value
                        datetime.datetime.strptime(m.group(1),"%Y%m%d").date()
                        # date and format are ok, check "should" part of 6.3.3
                        if val.fileNameBasePart:
                            expectedFilename = "{0}-{1}{2}.xml".format(val.fileNameBasePart, val.fileNameDatePart, expectedSuffix)
                            if modelDocument.basename != expectedFilename:
                                val.modelXbrl.log("WARNING-SEMANTIC", ("EFM.6.03.03.matchInstance", "GFM.1.01.01.matchInstance"),
                                    _('Linkbase name warning: %(filename)s should match %(expectedFilename)s'),
                                    modelObject=modelDocument, filename=modelDocument.basename, expectedFilename=expectedFilename)
                    except ValueError:
                        val.modelXbrl.error(("EFM.6.03.03", "GFM.1.01.01"),
                            _('Invalid linkbase base file name part (date) in "{base}-{yyyymmdd}_{suffix}.xml": %(filename)s'),
                            modelObject=modelDocument, filename=modelDocument.basename)
                else:
                    val.modelXbrl.error(("EFM.6.03.03", "GFM.1.01.01"),
                        _('Invalid linkbase name, must match "{base}-{yyyymmdd}%(expectedSuffix)s.xml": %(filename)s'),
                        modelObject=modelDocument, filename=modelDocument.basename, expectedSuffix=expectedSuffix)
    visited.remove(modelDocument)