def testcaseDiscover(self, testcaseElement): isTransformTestcase = testcaseElement.namespaceURI == "http://xbrl.org/2011/conformance-rendering/transforms" if XmlUtil.xmlnsprefix(testcaseElement, XbrlConst.cfcn) or isTransformTestcase: self.type = Type.REGISTRYTESTCASE self.outpath = self.xmlRootElement.get("outpath") self.testcaseVariations = [] priorTransformName = None for modelVariation in XmlUtil.descendants(testcaseElement, testcaseElement.namespaceURI, "variation"): self.testcaseVariations.append(modelVariation) if isTransformTestcase and modelVariation.getparent().get( "name") is not None: transformName = modelVariation.getparent().get("name") if transformName != priorTransformName: priorTransformName = transformName variationNumber = 1 modelVariation._name = "{0} v-{1:02}".format( priorTransformName, variationNumber) variationNumber += 1 if len(self.testcaseVariations) == 0: # may be a inline test case if XbrlConst.ixbrl in testcaseElement.values(): self.testcaseVariations.append(testcaseElement)
def testcaseDiscover(self, testcaseElement): if XmlUtil.xmlnsprefix(testcaseElement, XbrlConst.cfcn): self.type = Type.REGISTRYTESTCASE self.testcaseVariations = [ModelObject.createTestcaseVariation(self, variationElement) for variationElement in testcaseElement.getElementsByTagName("variation")] if len(self.testcaseVariations) == 0: # may be a inline test case for i in range(len(testcaseElement.attributes)): if testcaseElement.attributes.item(i).value == XbrlConst.ixbrl: modelVariation = ModelObject.createTestcaseVariation(self, testcaseElement) self.testcaseVariations.append(modelVariation) break
def qname(self): try: return self._xsdQname except AttributeError: name = self.name if self.name: if self.parentQname == XbrlConst.qnXsdSchema or self.isQualifiedForm: prefix = XmlUtil.xmlnsprefix(self.modelDocument.xmlRootElement,self.modelDocument.targetNamespace) self._xsdQname = ModelValue.QName(prefix, self.modelDocument.targetNamespace, name) else: self._xsdQname = ModelValue.QName(None, None, name) else: self._xsdQname = None return self._xsdQname
def parse(modelFormula): if sys.version[0] >= '3': # python 3 requires modified parser to allow release of global objects when closing DTS from arelle.pyparsing.pyparsing_py3 import ParseException, ParseSyntaxException else: from pyparsing import ParseException, ParseSyntaxException modelXbrl = modelFormula.modelXbrl global formulaFile, lineno, xmlns, logMessage logMessage = modelXbrl.log formulaFile = modelFormula.modelDocument.uri baseName = modelFormula.modelDocument.basename lineno = modelFormula.sourceline sourceString = modelFormula.select xmlns = modelFormula.nsmap # cdr doesn't declare xsd namespace URI prefixes for ns, nsDocs in modelXbrl.namespaceDocs.items(): for nsDoc in nsDocs: nsDocPrefix = XmlUtil.xmlnsprefix(nsDoc.xmlRootElement, ns) if nsDocPrefix and nsDocPrefix not in xmlns: xmlns[nsDocPrefix] = ns xmlns[nsDocPrefix.upper()] = ns # cdr also has upper case prefixes intermixed break try: modelFormula.prog = cdrProg.parseString( sourceString, parseAll=True ) successful = True except (ParseException, ParseSyntaxException) as err: from arelle.XPathParser import exceptionErrorIndication logMessage("ERROR", "cdrFormula:syntaxError", _("Parse error: \n%(error)s"), formulaFile=formulaFile, sourceFileLines=((baseName, lineno),), error=exceptionErrorIndication(err)) successful = False except (ValueError) as err: logMessage("ERROR", "cdrFormula:valueError", _("Parsing terminated due to error: \n%(error)s"), formulaFile=formulaFile, sourceFileLines=((baseName, lineno),), error=err) successful = False except Exception as err: logMessage("ERROR", "cdrFormula:parserException", _("Parsing of terminated due to error: \n%(error)s"), formulaFile=formulaFile, sourceFileLines=((baseName, lineno),), error=err, exc_info=True) successful = False return successful
def testcaseDiscover(self, testcaseElement): if XmlUtil.xmlnsprefix(testcaseElement, XbrlConst.cfcn): self.type = Type.REGISTRYTESTCASE self.testcaseVariations = [ ModelObject.createTestcaseVariation(self, variationElement) for variationElement in testcaseElement.getElementsByTagName( "variation") ] if len(self.testcaseVariations) == 0: # may be a inline test case for i in range(len(testcaseElement.attributes)): if testcaseElement.attributes.item(i).value == XbrlConst.ixbrl: modelVariation = ModelObject.createTestcaseVariation( self, testcaseElement) self.testcaseVariations.append(modelVariation) break
def qname(self): try: return self._xsdQname except AttributeError: name = self.name if self.name: if self.parentQname == XbrlConst.qnXsdSchema or self.isQualifiedForm: prefix = XmlUtil.xmlnsprefix( self.modelDocument.xmlRootElement, self.modelDocument.targetNamespace) self._xsdQname = ModelValue.QName( prefix, self.modelDocument.targetNamespace, name) else: self._xsdQname = ModelValue.QName(None, None, name) else: self._xsdQname = None return self._xsdQname
def testcaseDiscover(self, testcaseElement): isTransformTestcase = testcaseElement.namespaceURI == "http://xbrl.org/2011/conformance-rendering/transforms" if XmlUtil.xmlnsprefix(testcaseElement, XbrlConst.cfcn) or isTransformTestcase: self.type = Type.REGISTRYTESTCASE self.testcaseVariations = [] priorTransformName = None for modelVariation in XmlUtil.descendants(testcaseElement, testcaseElement.namespaceURI, "variation"): self.testcaseVariations.append(modelVariation) if isTransformTestcase and modelVariation.getparent().get("name") is not None: transformName = modelVariation.getparent().get("name") if transformName != priorTransformName: priorTransformName = transformName variationNumber = 1 modelVariation._name = "{0} v-{1:02}".format(priorTransformName, variationNumber) variationNumber += 1 if len(self.testcaseVariations) == 0: # may be a inline test case if XbrlConst.ixbrl in testcaseElement.values(): self.testcaseVariations.append(testcaseElement)
def saveLoadableExcel(dts, excelFile): from arelle import ModelDocument, XmlUtil from openpyxl import Workbook, cell from openpyxl.styles import Font, PatternFill, Border, Alignment, Color, fills, Side from openpyxl.worksheet.dimensions import ColumnDimension workbook = Workbook(encoding="utf-8") # remove pre-existing worksheets while len(workbook.worksheets)>0: workbook.remove_sheet(workbook.worksheets[0]) conceptsWs = workbook.create_sheet(title="Concepts") dtsWs = workbook.create_sheet(title="DTS") # identify type of taxonomy conceptsWsHeaders = None cellFontArgs = None for doc in dts.urlDocs.values(): if doc.type == ModelDocument.Type.SCHEMA and doc.inDTS: for i in range(len(headersStyles)): if re.match(headersStyles[i][0], doc.targetNamespace): cellFontArgs = headersStyles[i][1] # use as arguments to Font() conceptsWsHeaders = headersStyles[i][2] break if conceptsWsHeaders is None: dts.info("error:saveLoadableExcel", _("Referenced taxonomy style not identified, assuming general pattern."), modelObject=dts) cellFontArgs = headersStyles[-1][1] # use as arguments to Font() conceptsWsHeaders = headersStyles[-1][2] hdrCellFont = Font(**cellFontArgs) hdrCellFill = PatternFill(patternType=fills.FILL_SOLID, fgColor=Color("00FFBF5F")) # Excel's light orange fill color = 00FF990 cellFont = Font(**cellFontArgs) def writeCell(ws,row,col,value,fontBold=False,borders=True,indent=0,hAlign=None,vAlign=None,hdr=False): cell = ws.cell(row=row,column=col) cell.value = value if hdr: cell.font = hdrCellFont cell.fill = hdrCellFill if not hAlign: hAlign = "center" if not vAlign: vAlign = "center" else: cell.font = cellFont if not hAlign: hAlign = "left" if not vAlign: vAlign = "top" if borders: cell.border = Border(top=Side(border_style="thin"), left=Side(border_style="thin"), right=Side(border_style="thin"), bottom=Side(border_style="thin")) cell.alignment = Alignment(horizontal=hAlign, vertical=vAlign, wrap_text=True, indent=indent) # sheet 1 col widths for i, hdr in enumerate(conceptsWsHeaders): colLetter = cell.get_column_letter(i+1) conceptsWs.column_dimensions[colLetter] = ColumnDimension(conceptsWs, customWidth=True) conceptsWs.column_dimensions[colLetter].width = headerWidths.get(hdr[1], 40) # sheet 2 headers for i, hdr in enumerate(dtsWsHeaders): colLetter = cell.get_column_letter(i+1) dtsWs.column_dimensions[colLetter] = ColumnDimension(conceptsWs, customWidth=True) dtsWs.column_dimensions[colLetter].width = hdr[1] writeCell(dtsWs, 1, i+1, hdr[0], hdr=True) # referenced taxonomies conceptsRow = 1 dtsRow = 3 # identify extension schema extensionSchemaDoc = None if dts.modelDocument.type == ModelDocument.Type.SCHEMA: extensionSchemaDoc = dts.modelDocument elif dts.modelDocument.type == ModelDocument.Type.INSTANCE: for doc, docReference in dts.modelDocument.referencesDocument.items(): if docReference.referenceType == "href": extensionSchemaDoc = doc break if extensionSchemaDoc is None: dts.info("error:saveLoadableExcel", _("Unable to identify extension taxonomy."), modelObject=dts) return for doc, docReference in extensionSchemaDoc.referencesDocument.items(): if docReference.referenceType == "import" and doc.targetNamespace != XbrlConst.xbrli: writeCell(dtsWs, dtsRow, 1, "import") writeCell(dtsWs, dtsRow, 2, "schema") writeCell(dtsWs, dtsRow, 3, XmlUtil.xmlnsprefix(doc.xmlRootElement, doc.targetNamespace)) writeCell(dtsWs, dtsRow, 4, doc.uri) writeCell(dtsWs, dtsRow, 5, doc.targetNamespace) dtsRow += 1 dtsRow += 1 doc = extensionSchemaDoc writeCell(dtsWs, dtsRow, 1, "extension") writeCell(dtsWs, dtsRow, 2, "schema") writeCell(dtsWs, dtsRow, 3, XmlUtil.xmlnsprefix(doc.xmlRootElement, doc.targetNamespace)) writeCell(dtsWs, dtsRow, 4, os.path.basename(doc.uri)) writeCell(dtsWs, dtsRow, 5, doc.targetNamespace) dtsRow += 1 for doc, docReference in extensionSchemaDoc.referencesDocument.items(): if docReference.referenceType == "href" and doc.type == ModelDocument.Type.LINKBASE: linkbaseType = "" role = docReference.referringModelObject.get("{http://www.w3.org/1999/xlink}role") or "" if role.startswith("http://www.xbrl.org/2003/role/") and role.endswith("LinkbaseRef"): linkbaseType = os.path.basename(role)[0:-11] writeCell(dtsWs, dtsRow, 1, "extension") writeCell(dtsWs, dtsRow, 2, "linkbase") writeCell(dtsWs, dtsRow, 3, linkbaseType) writeCell(dtsWs, dtsRow, 4, os.path.basename(doc.uri)) writeCell(dtsWs, dtsRow, 5, "") dtsRow += 1 dtsRow += 1 # extended link roles defined in this document for roleURI, roleTypes in sorted(dts.roleTypes.items(), # sort on definition if any else URI key=lambda item: (item[1][0].definition or item[0])): for roleType in roleTypes: if roleType.modelDocument == extensionSchemaDoc: writeCell(dtsWs, dtsRow, 1, "extension") writeCell(dtsWs, dtsRow, 2, "role") writeCell(dtsWs, dtsRow, 3, "") writeCell(dtsWs, dtsRow, 4, roleType.definition) writeCell(dtsWs, dtsRow, 5, roleURI) dtsRow += 1 # tree walk recursive function def treeWalk(row, depth, concept, preferredLabel, arcrole, preRelSet, visited): if concept is not None: # calc parents calcRelSet = dts.relationshipSet(XbrlConst.summationItem, preRelSet.linkrole) calcRel = None for modelRel in calcRelSet.toModelObject(concept): calcRel = modelRel break for i, hdr in enumerate(conceptsWsHeaders): colType = hdr[1] value = "" if colType == "name": value = str(concept.name) elif colType == "prefix" and concept.qname is not None: value = concept.qname.prefix elif colType == "type" and concept.type is not None: value = str(concept.type.qname) elif colType == "substitutionGroup": value = str(concept.substitutionGroupQname) elif colType == "abstract": value = "true" if concept.isAbstract else "false" elif colType == "nillable": if concept.isNillable: value = "true" elif colType == "periodType": value = concept.periodType elif colType == "balance": value = concept.balance elif colType == "label": role = hdr[2] lang = hdr[3] if role == XbrlConst.standardLabel: if "indented" in hdr: roleUri = preferredLabel elif "overridePreferred" in hdr: if preferredLabel and preferredLabel != XbrlConst.standardLabel: roleUri = role else: roleUri = "**no value**" # skip putting a value in this column else: roleUri = role else: roleUri = role if roleUri != "**no value**": value = concept.label(roleUri, linkroleHint=preRelSet.linkrole, lang=lang, fallbackToQname=(role == XbrlConst.standardLabel)) elif colType == "preferredLabel" and preferredLabel: if preferredLabel.startswith("http://www.xbrl.org/2003/role/"): value = os.path.basename(preferredLabel) else: value = preferredLabel elif colType == "calculationParent" and calcRel is not None: calcParent = calcRel.fromModelObject if calcParent is not None: value = str(calcParent.qname) elif colType == "calculationWeight" and calcRel is not None: value = calcRel.weight elif colType == "depth": value = depth if "indented" in hdr: indent = min(depth, MAXINDENT) else: indent = 0 writeCell(conceptsWs, row, i+1, value, indent=indent) row += 1 if concept not in visited: visited.add(concept) for modelRel in preRelSet.fromModelObject(concept): if modelRel.toModelObject is not None: row = treeWalk(row, depth + 1, modelRel.toModelObject, modelRel.preferredLabel, arcrole, preRelSet, visited) visited.remove(concept) return row # use presentation relationships for conceptsWs arcrole = XbrlConst.parentChild # sort URIs by definition linkroleUris = [] relationshipSet = dts.relationshipSet(arcrole) if relationshipSet: for linkroleUri in relationshipSet.linkRoleUris: modelRoleTypes = dts.roleTypes.get(linkroleUri) if modelRoleTypes: roledefinition = (modelRoleTypes[0].genLabel(strip=True) or modelRoleTypes[0].definition or linkroleUri) else: roledefinition = linkroleUri linkroleUris.append((roledefinition, linkroleUri)) linkroleUris.sort() # for each URI in definition order for roledefinition, linkroleUri in linkroleUris: # write linkrole writeCell(conceptsWs, conceptsRow, 1, (roledefinition or linkroleUri), borders=False) # ELR has no boarders, just font specified conceptsRow += 1 # write header row for i, hdr in enumerate(conceptsWsHeaders): writeCell(conceptsWs, conceptsRow, i+1, hdr[0], hdr=True) conceptsRow += 1 # elr relationships for tree walk linkRelationshipSet = dts.relationshipSet(arcrole, linkroleUri) for rootConcept in linkRelationshipSet.rootConcepts: conceptsRow = treeWalk(conceptsRow, 0, rootConcept, None, arcrole, linkRelationshipSet, set()) conceptsRow += 1 # double space rows between tables else: # write header row for i, hdr in enumerate(conceptsWsHeaders): writeCell(conceptsWs, conceptsRow, i, hdr[0], hdr=True) conceptsRow += 1 # get lang lang = None for i, hdr in enumerate(conceptsWsHeaders): colType = hdr[1] if colType == "label": lang = hdr[3] if colType == "label": role = hdr[2] lang = hdr[3] lbls = defaultdict(list) for concept in set(dts.qnameConcepts.values()): # may be twice if unqualified, with and without namespace lbls[concept.label(role,lang=lang)].append(concept.objectId()) srtLbls = sorted(lbls.keys()) excludedNamespaces = XbrlConst.ixbrlAll.union( (XbrlConst.xbrli, XbrlConst.link, XbrlConst.xlink, XbrlConst.xl, XbrlConst.xbrldt, XbrlConst.xhtml)) for label in srtLbls: for objectId in lbls[label]: concept = dts.modelObject(objectId) if concept.modelDocument.targetNamespace not in excludedNamespaces: for i, hdr in enumerate(conceptsWsHeaders): colType = hdr[1] value = "" if colType == "name": value = str(concept.qname.localName) elif colType == "prefix": value = concept.qname.prefix elif colType == "type": value = str(concept.type.qname) elif colType == "substitutionGroup": value = str(concept.substitutionGroupQname) elif colType == "abstract": value = "true" if concept.isAbstract else "false" elif colType == "periodType": value = concept.periodType elif colType == "balance": value = concept.balance elif colType == "label": role = hdr[2] lang = hdr[3] value = concept.label(role, lang=lang) elif colType == "depth": value = 0 if "indented" in hdr: indent = min(0, MAXINDENT) else: indent = 0 writeCell(conceptsWs, conceptsRow, i, value, indent=indent) conceptsRow += 1 try: workbook.save(excelFile) dts.info("info:saveLoadableExcel", _("Saved Excel file: %(excelFile)s"), excelFile=os.path.basename(excelFile), modelXbrl=dts) except Exception as ex: dts.error("exception:saveLoadableExcel", _("File saving exception: %(error)s"), error=ex, modelXbrl=dts)
def checkDTSdocument(val, modelDocument): modelXbrl = val.modelXbrl if modelDocument.type in (ModelDocument.Type.SCHEMA, ModelDocument.Type.LINKBASE): isSchema = modelDocument.type == ModelDocument.Type.SCHEMA docinfo = modelDocument.xmlDocument.docinfo if docinfo and docinfo.xml_version != "1.0": modelXbrl.error("SBR.NL.2.2.0.02" if isSchema else "SBR.NL.2.3.0.02", _('%(docType)s xml version must be "1.0" but is "%(xmlVersion)s"'), modelObject=modelDocument, docType=modelDocument.gettype().title(), xmlVersion=docinfo.xml_version) if modelDocument.documentEncoding.lower() != "utf-8": modelXbrl.error("SBR.NL.2.2.0.03" if isSchema else "SBR.NL.2.3.0.03", _('%(docType)s encoding must be "utf-8" but is "%(xmlEncoding)s"'), modelObject=modelDocument, docType=modelDocument.gettype().title(), xmlEncoding=modelDocument.documentEncoding) lookingForPrecedingComment = True for commentNode in modelDocument.xmlRootElement.itersiblings(preceding=True): if isinstance(commentNode, etree._Comment): if lookingForPrecedingComment: lookingForPrecedingComment = False else: modelXbrl.error("SBR.NL.2.2.0.05" if isSchema else "SBR.NL.2.3.0.05", _('%(docType)s must have only one comment node before schema element'), modelObject=modelDocument, docType=modelDocument.gettype().title()) if lookingForPrecedingComment: modelXbrl.error("SBR.NL.2.2.0.04" if isSchema else "SBR.NL.2.3.0.04", _('%(docType)s must have comment node only on line 2'), modelObject=modelDocument, docType=modelDocument.gettype().title()) # check namespaces are used for prefix, ns in modelDocument.xmlRootElement.nsmap.items(): if ((prefix not in val.valUsedPrefixes) and (modelDocument.type != ModelDocument.Type.SCHEMA or ns != modelDocument.targetNamespace)): modelXbrl.error("SBR.NL.2.2.0.11" if modelDocument.type == ModelDocument.Type.SCHEMA else "SBR.NL.2.3.0.08", _('%(docType)s namespace declaration "%(declaration)s" is not used'), modelObject=modelDocument, docType=modelDocument.gettype().title(), declaration=("xmlns" + (":" + prefix if prefix else "") + "=" + ns)) if isSchema and val.annotationsCount > 1: modelXbrl.error("SBR.NL.2.2.0.22", _('Schema has %(annotationsCount)s xs:annotation elements, only 1 allowed'), modelObject=modelDocument, annotationsCount=val.annotationsCount) if modelDocument.type == ModelDocument.Type.LINKBASE: if not val.containsRelationship: modelXbrl.error("SBR.NL.2.3.0.12", "Linkbase has no relationships", modelObject=modelDocument) # check file name suffixes extLinkElt = XmlUtil.descendant(modelDocument.xmlRootElement, XbrlConst.link, "*", "{http://www.w3.org/1999/xlink}type", "extended") if extLinkElt is not None: expectedSuffix = None if extLinkElt.localName == "labelLink": anyLabel = XmlUtil.descendant(modelDocument.xmlRootElement, XbrlConst.link, "label", "{http://www.w3.org/XML/1998/namespace}lang", "*") if anyLabel is not None: xmlLang = anyLabel.get("{http://www.w3.org/XML/1998/namespace}lang") expectedSuffix = "-lab-{0}.xml".format(xmlLang) else: expectedSuffix = {"referenceLink": "-ref.xml", "presentationLink": "-pre.xml", "definitionLink": "-def.xml"}.get(extLinkElt.localName, None) if expectedSuffix: if not modelDocument.uri.endswith(expectedSuffix): modelXbrl.error("SBR.NL.3.2.1.09", "Linkbase filename expected to end with %(expectedSuffix)s: %(filename)s", modelObject=modelDocument, expectedSuffix=expectedSuffix, filename=modelDocument.uri) elif extLinkElt.qname == XbrlConst.qnGenLink: anyLabel = XmlUtil.descendant(modelDocument.xmlRootElement, XbrlConst.link, "label", "{http://www.w3.org/XML/1998/namespace}lang", "*") if anyLabel is not None: xmlLang = anyLabel.get("{http://www.w3.org/XML/1998/namespace}lang") expectedSuffix = "-generic-lab-{0}.xml".format(xmlLang) elif XmlUtil.descendant(modelDocument.xmlRootElement, XbrlConst.link, "reference") is not None: expectedSuffix = "-generic-ref.xml" if expectedSuffix and not modelDocument.uri.endswith(expectedSuffix): modelXbrl.error("SBR.NL.3.2.1.10", "Generic linkbase filename expected to end with %(expectedSuffix)s: %(filename)s", modelObject=modelDocument, expectedSuffix=expectedSuffix, filename=modelDocument.uri) # label checks for qnLabel in (XbrlConst.qnLinkLabel, XbrlConst.qnGenLabel): for modelLabel in modelDocument.xmlRootElement.iterdescendants(tag=qnLabel.clarkNotation): if isinstance(modelLabel, ModelResource): if not modelLabel.text or not modelLabel.text[:1].isupper(): modelXbrl.error("SBR.NL.3.2.7.05", _("Labels MUST have a capital first letter, label %(label)s: %(text)s"), modelObject=modelLabel, label=modelLabel.xlinkLabel, text=modelLabel.text[:64]) if modelLabel.role in (XbrlConst.standardLabel, XbrlConst.genStandardLabel): if len(modelLabel.text) > 255: modelXbrl.error("SBR.NL.3.2.7.06", _("Labels with the 'standard' role MUST NOT exceed 255 characters, label %(label)s: %(text)s"), modelObject=modelLabel, label=modelLabel.xlinkLabel, text=modelLabel.text[:64]) if modelLabel.role in (XbrlConst.standardLabel, XbrlConst.genStandardLabel): if len(modelLabel.text) > 255: modelXbrl.error("SBR.NL.3.2.7.06", _("Labels with the 'standard' role MUST NOT exceed 255 characters, label %(label)s: %(text)s"), modelObject=modelLabel, label=modelLabel.xlinkLabel, text=modelLabel.text[:64]) for modelResource in modelDocument.modelObjects: # locator checks if isinstance(modelResource, ModelLocator): hrefModelObject = modelResource.dereference() if isinstance(hrefModelObject, ModelObject): expectedLocLabel = hrefModelObject.id + "_loc" if modelResource.xlinkLabel != expectedLocLabel: modelXbrl.error("SBR.NL.3.2.11.01", _("Locator @xlink:label names MUST be concatenated from: @id from the XML node, underscore, 'loc', expected %(expectedLocLabel)s, found %(foundLocLabel)s"), modelObject=modelResource, expectedLocLabel=expectedLocLabel, foundLocLabel=modelResource.xlinkLabel) # xlinkLabel checks if isinstance(modelResource, ModelResource): if re.match(r"[^a-zA-Z0-9_-]", modelResource.xlinkLabel): modelXbrl.error("SBR.NL.3.2.11.03", _("@xlink:label names MUST use a-zA-Z0-9_- characters only: %(xlinkLabel)s"), modelObject=modelResource, xlinkLabel=modelResource.xlinkLabel) elif modelDocument.targetNamespace: # SCHEMA with targetNamespace # check for unused imports for referencedDocument in modelDocument.referencesDocument.keys(): if (referencedDocument.type == ModelDocument.Type.SCHEMA and referencedDocument.targetNamespace not in {XbrlConst.xbrli, XbrlConst.link} and referencedDocument.targetNamespace not in val.referencedNamespaces): modelXbrl.error("SBR.NL.2.2.0.15", _("A schema import schemas of which no content is being addressed: %(importedFile)s"), modelObject=modelDocument, importedFile=referencedDocument.basename) if modelDocument.targetNamespace != modelDocument.targetNamespace.lower(): modelXbrl.error("SBR.NL.3.2.3.02", _("Namespace URI's MUST be lower case: %(namespaceURI)s"), modelObject=modelDocument, namespaceURI=modelDocument.targetNamespace) if len(modelDocument.targetNamespace) > 255: modelXbrl.error("SBR.NL.3.2.3.03", _("Namespace URI's MUST NOT be longer than 255 characters: %(namespaceURI)s"), modelObject=modelDocument, namespaceURI=modelDocument.targetNamespace) if re.match(r"[^a-z0-9_/-]", modelDocument.targetNamespace): modelXbrl.error("SBR.NL.3.2.3.04", _("Namespace URI's MUST use only signs from a-z0-9_-/: %(namespaceURI)s"), modelObject=modelDocument, namespaceURI=modelDocument.targetNamespace) if not modelDocument.targetNamespace.startswith('http://www.nltaxonomie.nl'): modelXbrl.error("SBR.NL.3.2.3.05", _("Namespace URI's MUST start with 'http://www.nltaxonomie.nl': %(namespaceURI)s"), modelObject=modelDocument, namespaceURI=modelDocument.targetNamespace) namespacePrefix = XmlUtil.xmlnsprefix(modelDocument.xmlRootElement, modelDocument.targetNamespace) if not namespacePrefix: modelXbrl.error("SBR.NL.3.2.4.01", _("TargetNamespaces MUST have a prefix: %(namespaceURI)s"), modelObject=modelDocument, namespaceURI=modelDocument.targetNamespace) elif namespacePrefix in val.prefixNamespace: modelXbrl.error("SBR.NL.3.2.4.02", _("Namespace prefix MUST be unique within the NT but prefix '%(prefix)s' is used by both %(namespaceURI)s and %(namespaceURI2)s."), modelObject=modelDocument, prefix=namespacePrefix, namespaceURI=modelDocument.targetNamespace, namespaceURI2=val.prefixNamespace[namespacePrefix]) else: val.prefixNamespace[namespacePrefix] = modelDocument.targetNamespace val.namespacePrefix[modelDocument.targetNamespace] = namespacePrefix if namespacePrefix in {"xsi", "xsd", "xs", "xbrli", "link", "xlink", "xbrldt", "xbrldi", "gen", "xl"}: modelXbrl.error("SBR.NL.3.2.4.03", _("Namespace prefix '%(prefix)s' reserved by organizations for international specifications is used %(namespaceURI)s."), modelObject=modelDocument, prefix=namespacePrefix, namespaceURI=modelDocument.targetNamespace) if len(namespacePrefix) > 20: modelXbrl.warning("SBR.NL.3.2.4.06", _("Namespace prefix '%(prefix)s' SHOULD not exceed 20 characters %(namespaceURI)s."), modelObject=modelDocument, prefix=namespacePrefix, namespaceURI=modelDocument.targetNamespace) # check every non-targetnamespace prefix against its schema requiredLinkrole = None # only set for extension taxonomies # check folder names if modelDocument.filepathdir.startswith(modelXbrl.uriDir): partnerPrefix = None pathDir = modelDocument.filepathdir[len(modelXbrl.uriDir) + 1:].replace("\\", "/") lastPathSegment = None for pathSegment in pathDir.split("/"): if pathSegment.lower() != pathSegment: modelXbrl.error("SBR.NL.3.2.1.02", _("Folder names must be in lower case: %(folder)s"), modelObject=modelDocument, folder=pathSegment) if len(pathSegment) >= 15 : modelXbrl.error("SBR.NL.3.2.1.03", _("Folder names must be less than 15 characters: %(folder)s"), modelObject=modelDocument, folder=pathSegment) if pathSegment in ("bd", "kvk", "cbs"): partnerPrefix = pathSegment + '-' lastPathSegment = pathSegment if modelDocument.basename.lower() != modelDocument.basename: modelXbrl.error("SBR.NL.3.2.1.05", _("File names must be in lower case: %(file)s"), modelObject=modelDocument, file=modelDocument.basename) if partnerPrefix and not modelDocument.basename.startswith(partnerPrefix): modelXbrl.error("SBR.NL.3.2.1.14", "NT Partner DTS files MUST start with %(partnerPrefix)s consistently: %(filename)s", modelObject=modelDocument, partnerPrefix=partnerPrefix, filename=modelDocument.uri) if modelDocument.type == ModelDocument.Type.SCHEMA: if modelDocument.targetNamespace: nsParts = modelDocument.targetNamespace.split("/") # [0] = https, [1] = // [2] = nl.taxonomie [3] = year requiredNamespace = "http://www.nltaxonomie.nl/" + nsParts[3] + "/" + pathDir + "/" + modelDocument.basename[:-4] requiredLinkrole = "http://www.nltaxonomie.nl/" + nsParts[3] + "/" + pathDir + "/" if modelDocument == modelXbrl.modelDocument: # entry point requiredNamespace += '-' + nsParts[3] if not modelDocument.targetNamespace.startswith(requiredNamespace): modelXbrl.error("SBR.NL.3.2.3.06", _("Namespace URI's MUST be constructed like %(requiredNamespace)s: %(namespaceURI)s"), modelObject=modelDocument, requiredNamespace=requiredNamespace, namespaceURI=modelDocument.targetNamespace) else: requiredLinkrole = '' # concept checks for modelConcept in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/2001/XMLSchema}element"): if isinstance(modelConcept, ModelConcept): # 6.7.16 name not duplicated in standard taxonomies name = modelConcept.get("name") if name: substititutionGroupQname = modelConcept.substitutionGroupQname if substititutionGroupQname: if name.endswith("Member") ^ (substititutionGroupQname.localName == "domainMemberItem" and substititutionGroupQname.namespaceURI.endswith("/xbrl/xbrl-syntax-extension")): modelXbrl.error("SBR.NL.3.2.5.11", _("Concept %(concept)s must end in Member to be in sbr:domainMemberItem substitution group"), modelObject=modelConcept, concept=modelConcept.qname) if name.endswith("Domain") ^ (substititutionGroupQname.localName == "domainItem" and substititutionGroupQname.namespaceURI.endswith("/xbrl/xbrl-syntax-extension")): modelXbrl.error("SBR.NL.3.2.5.12", _("Concept %(concept)s must end in Domain to be in sbr:domainItem substitution group"), modelObject=modelConcept, concept=modelConcept.qname) if name.endswith("TypedAxis") ^ (substititutionGroupQname == XbrlConst.qnXbrldtDimensionItem and modelConcept.isTypedDimension): modelXbrl.error("SBR.NL.3.2.5.14", _("Concept %(concept)s must end in TypedAxis to be in xbrldt:dimensionItem substitution group if they represent a typed dimension"), modelObject=modelConcept, concept=modelConcept.qname) if (name.endswith("Axis") and not name.endswith("TypedAxis")) ^ (substititutionGroupQname == XbrlConst.qnXbrldtDimensionItem and modelConcept.isExplicitDimension): modelXbrl.error("SBR.NL.3.2.5.13", _("Concept %(concept)s must end in Axis to be in xbrldt:dimensionItem substitution group if they represent an explicit dimension"), modelObject=modelConcept, concept=modelConcept.qname) if name.endswith("Table") ^ (substititutionGroupQname == XbrlConst.qnXbrldtHypercubeItem): modelXbrl.error("SBR.NL.3.2.5.15", _("Concept %(concept)s must end in Table to be in xbrldt:hypercubeItem substitution group"), modelObject=modelConcept, concept=modelConcept.qname) if name.endswith("Title") ^ (substititutionGroupQname.localName == "presentationItem" and substititutionGroupQname.namespaceURI.endswith("/xbrl/xbrl-syntax-extension")): modelXbrl.error("SBR.NL.3.2.5.16", _("Concept %(concept)s must end in Title to be in sbr:presentationItem substitution group"), modelObject=modelConcept, concept=modelConcept.qname) if len(name) > 200: modelXbrl.error("SBR.NL.3.2.12.02" if modelConcept.isLinkPart else "SBR.NL.3.2.5.21" if (modelConcept.isItem or modelConcept.isTuple) else "SBR.NL.3.2.14.01", _("Concept %(concept)s name length %(namelength)s exceeds 200 characters"), modelObject=modelConcept, concept=modelConcept.qname, namelength=len(name)) # type checks for typeType in ("simpleType", "complexType"): for modelType in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/2001/XMLSchema}" + typeType): if isinstance(modelType, ModelType): name = modelType.get("name") if name is None: name = "" if modelType.get("ref") is not None: continue # don't validate ref's here if len(name) > 200: modelXbrl.error("SBR.NL.3.2.5.21", _("Type %(type)s name length %(namelength)s exceeds 200 characters"), modelObject=modelType, type=modelType.qname, namelength=len(name)) if modelType.qnameDerivedFrom and modelType.qnameDerivedFrom.namespaceURI != XbrlConst.xbrli: modelXbrl.error("SBR.NL.3.2.8.01", _("Custom datatypes MUST be a restriction from XII defined datatypes: %(type)s"), modelObject=modelType, type=modelType.qname) if re.match(r"[^a-zA-Z0-9_-]", name): modelXbrl.error("SBR.NL.3.2.8.02", _("Datatype names MUST use characters a-zA-Z0-9_- only: %(type)s"), modelObject=modelDocument, type=modelType.qname) if modelType.facets and "enumeration" in modelType.facets: if not modelType.qnameDerivedFrom == XbrlConst.qnXbrliStringItemType: modelXbrl.error("SBR.NL.3.2.13.01", _("Enumerations MUST use a restriction on xbrli:stringItemType: %(type)s"), modelObject=modelDocument, type=modelType.qname) if lastPathSegment == "entrypoints": if not modelDocument.xmlRootElement.id: modelXbrl.error("SBR.NL.2.2.0.23", _("xs:schema/@id MUST be present in schema files in the reports/{NT partner}/entrypoints/ folder"), modelObject=modelDocument) # check for idObject conflicts for id, modelObject in modelDocument.idObjects.items(): if id in val.idObjects: modelXbrl.error("SBR.NL.3.2.6.01", _("ID %(id)s must be unique in the DTS but is present on two elements."), modelObject=(modelObject, val.idObjects[id]), id=id) else: val.idObjects[id] = modelObject for roleURI, modelRoleTypes in modelXbrl.roleTypes.items(): if not roleURI.startswith("http://www.xbrl.org"): usedOns = set.union(*[modelRoleType.usedOns for modelRoleType in modelRoleTypes]) # check roletypes for linkroles (only) if usedOns & {XbrlConst.qnLinkPresentationLink, XbrlConst.qnLinkCalculationLink, XbrlConst.qnLinkDefinitionLink, XbrlConst.qnLinkLabel, XbrlConst.qnLinkReference, XbrlConst.qnLinkFootnote}: if len(modelRoleTypes) > 1: modelXbrl.error("SBR.NL.3.2.9.01", _("Linkrole URI's MUST be unique in the NT: %(linkrole)s"), modelObject=modelRoleTypes, linkrole=roleURI) if roleURI.lower() != roleURI: modelXbrl.error("SBR.NL.3.2.9.02", _("Linkrole URI's MUST be in lowercase: %(linkrole)s"), modelObject=modelRoleTypes, linkrole=roleURI) if re.match(r"[^a-z0-9_/-]", roleURI): modelXbrl.error("SBR.NL.3.2.9.03", _("Linkrole URI's MUST use characters a-z0-9_-/ only: %(linkrole)s"), modelObject=modelRoleTypes, linkrole=roleURI) if len(roleURI) > 255: modelXbrl.error("SBR.NL.3.2.9.04", _("Linkrole URI's MUST NOT be longer than 255 characters, length is %(len)s: %(linkrole)s"), modelObject=modelRoleTypes, len=len(roleURI), linkrole=roleURI) ''' removed per RH 2013-03-13 e-mail if not roleURI.startswith('http://www.nltaxonomie.nl'): modelXbrl.error("SBR.NL.3.2.9.05", _("Linkrole URI's MUST start with 'http://www.nltaxonomie.nl': %(linkrole)s"), modelObject=modelRoleTypes, linkrole=roleURI) if (requiredLinkrole and not roleURI.startswith(requiredLinkrole) and re.match(r".*(domain$|axis$|table$|lineitem$)", roleURI)): modelXbrl.error("SBR.NL.3.2.9.06", _("Linkrole URI's MUST have the following construct: http://www.nltaxonomie.nl / {folder path} / {functional name} - {domain or axis or table or lineitem}: %(linkrole)s"), modelObject=modelRoleTypes, linkrole=roleURI) ''' for modelRoleType in modelRoleTypes: if len(modelRoleType.id) > 255: modelXbrl.error("SBR.NL.3.2.10.02", _("Linkrole @id MUST NOT exceed 255 characters, length is %(length)s: %(linkroleID)s"), modelObject=modelRoleType, length=len(modelRoleType.id), linkroleID=modelRoleType.id) partnerPrefix = modelRoleTypes[0].modelDocument.basename.split('-') if partnerPrefix: # first element before dash is prefix urnPartnerLinkroleStart = "urn:{0}:linkrole:".format(partnerPrefix[0]) if not roleURI.startswith(urnPartnerLinkroleStart): modelXbrl.error("SBR.NL.3.2.9.10", _("Linkrole MUST start with urn:{NT partner code}:linkrole:, \nexpecting: %(expectedStart)s..., \nfound: %(linkrole)s"), modelObject=modelRoleType, expectedStart=urnPartnerLinkroleStart, linkrole=roleURI)
def checkDTS(val, modelDocument, visited): global targetNamespaceDatePattern, roleTypePattern, arcroleTypePattern, arcroleDefinitionPattern if targetNamespaceDatePattern is None: targetNamespaceDatePattern = re.compile(r"/([12][0-9]{3})-([01][0-9])-([0-3][0-9])|" r"/([12][0-9]{3})([01][0-9])([0-3][0-9])|") roleTypePattern = re.compile(r".*/role/[^/]+") arcroleTypePattern = re.compile(r".*/arcrole/[^/]+") arcroleDefinitionPattern = re.compile(r"^.*[^\\s]+.*$") # at least one non-whitespace character visited.append(modelDocument) definesLabelLinkbase = False for referencedDocument in modelDocument.referencesDocument.items(): #6.07.01 no includes if referencedDocument[1] == "include": val.modelXbrl.error(("EFM.6.07.01", "GFM.1.03.01", "SBR.NL.2.2.0.18"), _("Taxonomy schema %(schema)s includes %(include)s, only import is allowed"), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), include=os.path.basename(referencedDocument[0].uri)) if referencedDocument[0] not in visited: checkDTS(val, referencedDocument[0], visited) if val.disclosureSystem.standardTaxonomiesDict is None: pass if (modelDocument.type == ModelDocument.Type.SCHEMA and modelDocument.targetNamespace not in val.disclosureSystem.baseTaxonomyNamespaces and modelDocument.uri.startswith(val.modelXbrl.uriDir)): # check schema contents types if val.validateSBRNL: definesLinkroles = False definesArcroles = False definesLinkParts = False definesAbstractItems = False definesNonabstractItems = False definesConcepts = False definesTuples = False definesPresentationTuples = False definesSpecificationTuples = False definesTypes = False definesEnumerations = False definesDimensions = False definesDomains = False definesHypercubes = False # 6.7.3 check namespace for standard authority targetNamespaceAuthority = UrlUtil.authority(modelDocument.targetNamespace) if targetNamespaceAuthority in val.disclosureSystem.standardAuthorities: val.modelXbrl.error(("EFM.6.07.03", "GFM.1.03.03"), _("Taxonomy schema %(schema)s namespace %(targetNamespace)s is a disallowed authority"), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace) # 6.7.4 check namespace format if modelDocument.targetNamespace is None: match = None elif val.validateEFMorGFM: targetNamespaceDate = modelDocument.targetNamespace[len(targetNamespaceAuthority):] match = targetNamespaceDatePattern.match(targetNamespaceDate) else: match = None if match is not None: try: if match.lastindex == 3: datetime.date(int(match.group(1)),int(match.group(2)),int(match.group(3))) elif match.lastindex == 6: datetime.date(int(match.group(4)),int(match.group(5)),int(match.group(6))) else: match = None except ValueError: match = None if match is None: val.modelXbrl.error(("EFM.6.07.04", "GFM.1.03.04"), _("Taxonomy schema %(schema)s namespace %(targetNamespace)s must have format http://{authority}/{versionDate}"), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace) if modelDocument.targetNamespace is not None: # 6.7.5 check prefix for _ prefix = XmlUtil.xmlnsprefix(modelDocument.xmlRootElement,modelDocument.targetNamespace) if prefix and "_" in prefix: val.modelXbrl.error(("EFM.6.07.07", "GFM.1.03.07"), _("Taxonomy schema %(schema)s namespace %(targetNamespace)s prefix %(prefix)s must not have an '_'"), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace, prefix=prefix) if val.validateSBRNL: genrlSpeclRelSet = val.modelXbrl.relationshipSet(XbrlConst.generalSpecial) for modelConcept in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/2001/XMLSchema}element"): if isinstance(modelConcept,ModelConcept): # 6.7.16 name not duplicated in standard taxonomies name = modelConcept.get("name") if name is None: name = "" if modelConcept.get("ref") is not None: continue # don't validate ref's here for c in val.modelXbrl.nameConcepts.get(name, []): if c.modelDocument != modelDocument: if (val.validateEFMorGFM and not c.modelDocument.uri.startswith(val.modelXbrl.uriDir)): val.modelXbrl.error(("EFM.6.07.16", "GFM.1.03.18"), _("Concept %(concept)s is also defined in standard taxonomy schema schema %(standardSchema)s"), modelObject=c, concept=modelConcept.qname, standardSchema=os.path.basename(c.modelDocument.uri)) elif val.validateSBRNL: if not (genrlSpeclRelSet.isRelated(modelConcept, "child", c) or genrlSpeclRelSet.isRelated(c, "child", modelConcept)): val.modelXbrl.error("SBR.NL.2.2.2.02", _("Concept %(concept)s is also defined in standard taxonomy schema %(standardSchema)s without a general-special relationship"), modelObject=c, concept=modelConcept.qname, standardSchema=os.path.basename(c.modelDocument.uri)) ''' removed RH 2011-12-23 corresponding set up of table in ValidateFiling if val.validateSBRNL and name in val.nameWordsTable: if not any( any( genrlSpeclRelSet.isRelated(c, "child", modelConcept) for c in val.modelXbrl.nameConcepts.get(partialWordName, [])) for partialWordName in val.nameWordsTable[name]): val.modelXbrl.error("SBR.NL.2.3.2.01", _("Concept %(specialName)s is appears to be missing a general-special relationship to %(generalNames)s"), modelObject=c, specialName=modelConcept.qname, generalNames=', or to '.join(val.nameWordsTable[name])) ''' # 6.7.17 id properly formed id = modelConcept.id requiredId = (prefix if prefix is not None else "") + "_" + name if val.validateEFMorGFM and id != requiredId: val.modelXbrl.error(("EFM.6.07.17", "GFM.1.03.19"), _("Concept %(concept)s id %(id)s should be $(requiredId)s"), modelObject=modelConcept, concept=modelConcept.qname, id=id, requiredId=requiredId) # 6.7.18 nillable is true nillable = modelConcept.get("nillable") if nillable != "true": val.modelXbrl.error(("EFM.6.07.18", "GFM.1.03.20"), _("Taxonomy schema %(schema)s element %(concept)s nillable %(nillable)s should be 'true'"), modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=name, nillable=nillable) # 6.7.19 not tuple if modelConcept.isTuple: if val.validateEFMorGFM: val.modelXbrl.error(("EFM.6.07.19", "GFM.1.03.21"), _("Concept %(concept)s is a tuple"), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.20 no typed domain ref if modelConcept.isTypedDimension: val.modelXbrl.error(("EFM.6.07.20", "GFM.1.03.22"), _("Concept %(concept)s has typedDomainRef %(typedDomainRef)s"), modelObject=modelConcept, concept=modelConcept.qname, typedDomainRef=modelConcept.typedDomainElement.qname if modelConcept.typedDomainElement is not None else modelConcept.typedDomainRef) # 6.7.21 abstract must be duration isDuration = modelConcept.periodType == "duration" if modelConcept.abstract == "true" and not isDuration: val.modelXbrl.error(("EFM.6.07.21", "GFM.1.03.23"), _("Taxonomy schema %(schema)s element %(concept)s is abstract but period type is not duration"), modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=name) # 6.7.22 abstract must be stringItemType ''' removed SEC EFM v.17, Edgar release 10.4, and GFM 2011-04-08 if modelConcept.abstract == "true" and modelConcept.typeQname != XbrlConst. qnXbrliStringItemType: val.modelXbrl.error(("EFM.6.07.22", "GFM.1.03.24"), _("Concept %(concept)s is abstract but type is not xbrli:stringItemType"), modelObject=modelConcept, concept=modelConcept.qname) ''' substititutionGroupQname = modelConcept.substitutionGroupQname # 6.7.23 Axis must be subs group dimension if name.endswith("Axis") ^ (substititutionGroupQname == XbrlConst.qnXbrldtDimensionItem): val.modelXbrl.error(("EFM.6.07.23", "GFM.1.03.25"), _("Concept %(concept)s must end in Axis to be in xbrldt:dimensionItem substitution group"), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.24 Table must be subs group hypercube if name.endswith("Table") ^ (substititutionGroupQname == XbrlConst.qnXbrldtHypercubeItem): val.modelXbrl.error(("EFM.6.07.24", "GFM.1.03.26"), _("Concept %(concept)s must end in Table to be in xbrldt:hypercubeItem substitution group"), modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=modelConcept.qname) # 6.7.25 if neither hypercube or dimension, substitution group must be item if substititutionGroupQname not in (None, XbrlConst.qnXbrldtDimensionItem, XbrlConst.qnXbrldtHypercubeItem, XbrlConst.qnXbrliItem): val.modelXbrl.error(("EFM.6.07.25", "GFM.1.03.27"), _("Concept %(concept)s has disallowed substitution group %(substitutionGroup)s"), modelObject=modelConcept, concept=modelConcept.qname, substitutionGroup=modelConcept.substitutionGroupQname) # 6.7.26 Table must be subs group hypercube if name.endswith("LineItems") and modelConcept.abstract != "true": val.modelXbrl.error(("EFM.6.07.26", "GFM.1.03.28"), _("Concept %(concept)s is a LineItems but not abstract"), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.27 type domainMember must end with Domain or Member conceptType = modelConcept.type isDomainItemType = conceptType is not None and conceptType.isDomainItemType endsWithDomainOrMember = name.endswith("Domain") or name.endswith("Member") if isDomainItemType != endsWithDomainOrMember: val.modelXbrl.error(("EFM.6.07.27", "GFM.1.03.29"), _("Concept %(concept)s must end with Domain or Member for type of domainItemType"), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.28 domainItemType must be duration if isDomainItemType and not isDuration: val.modelXbrl.error(("EFM.6.07.28", "GFM.1.03.30"), _("Concept %(concept)s is a domainItemType and must be periodType duration"), modelObject=modelConcept, concept=modelConcept.qname) if val.validateSBRNL: if modelConcept.isTuple: if modelConcept.substitutionGroupQname.localName == "presentationTuple" and modelConcept.substitutionGroupQname.namespaceURI.endswith("/basis/sbr/xbrl/xbrl-syntax-extension"): # namespace may change each year definesPresentationTuples = True elif modelConcept.substitutionGroupQname.localName == "specificationTuple" and modelConcept.substitutionGroupQname.namespaceURI.endswith("/basis/sbr/xbrl/xbrl-syntax-extension"): # namespace may change each year definesSpecificationTuples = True else: definesTuples = True definesConcepts = True if modelConcept.isAbstract: val.modelXbrl.error("SBR.NL.2.2.2.03", _("Concept %(concept)s is an abstract tuple"), modelObject=modelConcept, concept=modelConcept.qname) if tupleCycle(val,modelConcept): val.modelXbrl.error("SBR.NL.2.2.2.07", _("Tuple %(concept)s has a tuple cycle"), modelObject=modelConcept, concept=modelConcept.qname) if modelConcept.get("nillable") != "false" and modelConcept.isRoot: val.modelXbrl.error("SBR.NL.2.2.2.17", #don't want default, just what was really there _("Tuple %(concept)s must have nillable='false'"), modelObject=modelConcept, concept=modelConcept.qname) elif modelConcept.isItem: definesConcepts = True if modelConcept.abstract == "true": if modelConcept.isRoot: if modelConcept.get("nillable") != "false": #don't want default, just what was really there val.modelXbrl.error("SBR.NL.2.2.2.16", _("Abstract root concept %(concept)s must have nillable='false'"), modelObject=modelConcept, concept=modelConcept.qname) if modelConcept.typeQname != XbrlConst.qnXbrliStringItemType: val.modelXbrl.error("SBR.NL.2.2.2.21", _("Abstract root concept %(concept)s must have type='xbrli:stringItemType'"), modelObject=modelConcept, concept=modelConcept.qname) if modelConcept.balance: val.modelXbrl.error("SBR.NL.2.2.2.22", _("Abstract concept %(concept)s must not have a balance attribute"), modelObject=modelConcept, concept=modelConcept.qname) if modelConcept.isHypercubeItem: definesHypercubes = True elif modelConcept.isDimensionItem: definesDimensions = True elif substititutionGroupQname and substititutionGroupQname.localName in ("domainItem","domainMemberItem"): definesDomains = True elif modelConcept.isItem: definesAbstractItems = True else: # not abstract if modelConcept.isItem: definesNonabstractItems = True if not (modelConcept.label(preferredLabel=XbrlConst.documentationLabel,fallbackToQname=False,lang="nl") or val.modelXbrl.relationshipSet(XbrlConst.conceptReference).fromModelObject(c) or modelConcept.genLabel(role=XbrlConst.genDocumentationLabel,lang="nl") or val.modelXbrl.relationshipSet(XbrlConst.elementReference).fromModelObject(c)): val.modelXbrl.error("SBR.NL.2.2.2.28", _("Concept %(concept)s must have a documentation label or reference"), modelObject=modelConcept, concept=modelConcept.qname) if modelConcept.balance and not modelConcept.instanceOfType(XbrlConst.qnXbrliMonetaryItemType): val.modelXbrl.error("SBR.NL.2.2.2.24", _("Non-monetary concept %(concept)s must not have a balance attribute"), modelObject=modelConcept, concept=modelConcept.qname) if modelConcept.isLinkPart: definesLinkParts = True val.modelXbrl.error("SBR.NL.2.2.5.01", _("Link:part concept %(concept)s is not allowed"), modelObject=modelConcept, concept=modelConcept.qname) if not modelConcept.genLabel(fallbackToQname=False,lang="nl"): val.modelXbrl.error("SBR.NL.2.2.5.02", _("Link part definition %(concept)s must have a generic label in language 'nl'"), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.8 check for embedded linkbase for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}linkbase"): if isinstance(e,ModelObject): val.modelXbrl.error(("EFM.6.07.08", "GFM.1.03.08"), _("Taxonomy schema %(schema)s contains an embedded linkbase"), modelObject=e, schema=os.path.basename(modelDocument.uri)) break requiredUsedOns = {XbrlConst.qnLinkPresentationLink, XbrlConst.qnLinkCalculationLink, XbrlConst.qnLinkDefinitionLink} # 6.7.9 role types authority for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}roleType"): if isinstance(e,ModelObject): roleURI = e.get("roleURI") if targetNamespaceAuthority != UrlUtil.authority(roleURI): val.modelXbrl.error(("EFM.6.07.09", "GFM.1.03.09"), _("RoleType %(roleType)s does not match authority %(targetNamespaceAuthority)s"), modelObject=e, roleType=roleURI, targetNamespaceAuthority=targetNamespaceAuthority) # 6.7.9 end with .../role/lc3 name if not roleTypePattern.match(roleURI): val.modelXbrl.warning(("EFM.6.07.09", "GFM.1.03.09"), "RoleType %(roleType)s should end with /role/{LC3name}", modelObject=e, roleType=roleURI) # 6.7.10 only one role type declaration in DTS modelRoleTypes = val.modelXbrl.roleTypes.get(roleURI) if modelRoleTypes is not None: modelRoleType = modelRoleTypes[0] definition = modelRoleType.definitionNotStripped usedOns = modelRoleType.usedOns if len(modelRoleTypes) > 1: val.modelXbrl.error(("EFM.6.07.10", "GFM.1.03.10"), _("RoleType %(roleType)s is defined in multiple taxonomies"), modelObject=e, roleType=roleURI) elif len(modelRoleTypes) == 1: # 6.7.11 used on's for pre, cal, def if any has a used on if not usedOns.isdisjoint(requiredUsedOns) and len(requiredUsedOns - usedOns) > 0: val.modelXbrl.error(("EFM.6.07.11", "GFM.1.03.11"), _("RoleType %(roleType)s missing used on %(usedOn)s"), modelObject=e, roleType=roleURI, usedOn=requiredUsedOns - usedOns) # 6.7.12 definition match pattern if (val.disclosureSystem.roleDefinitionPattern is not None and (definition is None or not val.disclosureSystem.roleDefinitionPattern.match(definition))): val.modelXbrl.error(("EFM.6.07.12", "GFM.1.03.12-14"), _("RoleType %(roleType)s definition \"%(definition)s\" must match {Sortcode} - {Type} - {Title}"), modelObject=e, roleType=roleURI, definition=definition) if val.validateSBRNL: if usedOns & XbrlConst.standardExtLinkQnames or XbrlConst.qnGenLink in usedOns: definesLinkroles = True if not e.genLabel(): val.modelXbrl.error("SBR.NL.2.2.3.03", _("Link RoleType %(roleType)s missing a generic standard label"), modelObject=e, roleType=roleURI) nlLabel = e.genLabel(lang="nl") if definition != nlLabel: val.modelXbrl.error("SBR.NL.2.2.3.04", _("Link RoleType %(roleType)s definition does not match NL standard generic label, \ndefinition: %(definition)s \nNL label: %(label)s"), modelObject=e, roleType=roleURI, definition=definition, label=nlLabel) if definition and (definition[0].isspace() or definition[-1].isspace()): val.modelXbrl.error("SBR.NL.2.2.3.07", _('Link RoleType %(roleType)s definition has leading or trailing spaces: "%(definition)s"'), modelObject=e, roleType=roleURI, definition=definition) # 6.7.13 arcrole types authority for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}arcroleType"): if isinstance(e,ModelObject): arcroleURI = e.get("arcroleURI") if targetNamespaceAuthority != UrlUtil.authority(arcroleURI): val.modelXbrl.error(("EFM.6.07.13", "GFM.1.03.15"), _("ArcroleType %(arcroleType)s does not match authority %(targetNamespaceAuthority)s"), modelObject=e, arcroleType=arcroleURI, targetNamespaceAuthority=targetNamespaceAuthority) # 6.7.13 end with .../arcrole/lc3 name if not arcroleTypePattern.match(arcroleURI): val.modelXbrl.warning(("EFM.6.07.13", "GFM.1.03.15"), _("ArcroleType %(arcroleType)s should end with /arcrole/{LC3name}"), modelObject=e, arcroleType=arcroleURI) # 6.7.14 only one arcrole type declaration in DTS modelRoleTypes = val.modelXbrl.arcroleTypes[arcroleURI] if len(modelRoleTypes) > 1: val.modelXbrl.error(("EFM.6.07.14", "GFM.1.03.16"), _("ArcroleType %(arcroleType)s is defined in multiple taxonomies"), modelObject=e, arcroleType=arcroleURI) # 6.7.15 definition match pattern definition = modelRoleTypes[0].definition if definition is None or not arcroleDefinitionPattern.match(definition): val.modelXbrl.error(("EFM.6.07.15", "GFM.1.03.17"), _("ArcroleType %(arcroleType)s definition must be non-empty"), modelObject=e, arcroleType=arcroleURI) if val.validateSBRNL: definesArcroles = True val.modelXbrl.error("SBR.NL.2.2.4.01", _("Arcrole type definition is not allowed: %(arcroleURI)s"), modelObject=e, arcroleURI=arcroleURI) if val.validateSBRNL: for appinfoElt in modelDocument.xmlRootElement.iter(tag="{http://www.w3.org/2001/XMLSchema}appinfo"): for nonLinkElt in appinfoElt.iterdescendants(): if isinstance(nonLinkElt, ModelObject) and nonLinkElt.namespaceURI != XbrlConst.link: val.modelXbrl.error("SBR.NL.2.2.11.05", _("Appinfo contains disallowed non-link element %(element)s"), modelObject=nonLinkElt, element=nonLinkElt.qname) for cplxTypeElt in modelDocument.xmlRootElement.iter(tag="{http://www.w3.org/2001/XMLSchema}complexType"): choiceElt = cplxTypeElt.find("{http://www.w3.org/2001/XMLSchema}choice") if choiceElt is not None: val.modelXbrl.error("SBR.NL.2.2.11.09", _("ComplexType contains disallowed xs:choice element"), modelObject=choiceElt) for cplxContentElt in modelDocument.xmlRootElement.iter(tag="{http://www.w3.org/2001/XMLSchema}complexContent"): if XmlUtil.descendantAttr(cplxContentElt, "http://www.w3.org/2001/XMLSchema", ("extension","restriction"), "base") != "sbr:placeholder": val.modelXbrl.error("SBR.NL.2.2.11.10", _("ComplexContent is disallowed"), modelObject=cplxContentElt) definesTypes = (modelDocument.xmlRootElement.find("{http://www.w3.org/2001/XMLSchema}complexType") is not None or modelDocument.xmlRootElement.find("{http://www.w3.org/2001/XMLSchema}simpleType") is not None) if (definesLinkroles + definesArcroles + definesLinkParts + definesAbstractItems + definesNonabstractItems + definesTuples + definesPresentationTuples + definesSpecificationTuples + definesTypes + definesEnumerations + definesDimensions + definesDomains + definesHypercubes) != 1: schemaContents = [] if definesLinkroles: schemaContents.append(_("linkroles")) if definesArcroles: schemaContents.append(_("arcroles")) if definesLinkParts: schemaContents.append(_("link parts")) if definesAbstractItems: schemaContents.append(_("abstract items")) if definesNonabstractItems: schemaContents.append(_("nonabstract items")) if definesTuples: schemaContents.append(_("tuples")) if definesPresentationTuples: schemaContents.append(_("sbrPresentationTuples")) if definesSpecificationTuples: schemaContents.append(_("sbrSpecificationTuples")) if definesTypes: schemaContents.append(_("types")) if definesEnumerations: schemaContents.append(_("enumerations")) if definesDimensions: schemaContents.append(_("dimensions")) if definesDomains: schemaContents.append(_("domains")) if definesHypercubes: schemaContents.append(_("hypercubes")) if schemaContents: val.modelXbrl.error("SBR.NL.2.2.1.01", _("Taxonomy schema may only define one of these: %(contents)s"), modelObject=modelDocument, contents=', '.join(schemaContents)) elif not any(refDoc.inDTS and refDoc.targetNamespace not in val.disclosureSystem.baseTaxonomyNamespaces for refDoc in modelDocument.referencesDocument.keys()): # no linkbase ref or includes val.modelXbrl.error("SBR.NL.2.2.1.01", _("Taxonomy schema must be a DTS entrypoint OR define linkroles OR arcroles OR link:parts OR context fragments OR abstract items OR tuples OR non-abstract elements OR types OR enumerations OR dimensions OR domains OR hypercubes"), modelObject=modelDocument) if definesConcepts ^ any( # xor so either concepts and no label LB or no concepts and has label LB (refDoc.type == ModelDocument.Type.LINKBASE and XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link, "labelLink") is not None) for refDoc in modelDocument.referencesDocument.keys()): # no label linkbase val.modelXbrl.error("SBR.NL.2.2.1.02", _("A schema that defines concepts MUST have a linked 2.1 label linkbase"), modelObject=modelDocument) if (definesNonabstractItems or definesTuples) and not any( # was xor but changed to and not per RH 1/11/12 (refDoc.type == ModelDocument.Type.LINKBASE and (XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link, "referenceLink") is not None or XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link, "label", "{http://www.w3.org/1999/xlink}role", "http://www.xbrl.org/2003/role/documentation" ) is not None)) for refDoc in modelDocument.referencesDocument.keys()): val.modelXbrl.error("SBR.NL.2.2.1.03", _("A schema that defines non-abstract items MUST have a linked (2.1) reference linkbase AND/OR a label linkbase with @xlink:role=documentation"), modelObject=modelDocument) visited.remove(modelDocument)
def final(val): if not (val.validateEBA or val.validateEIOPA): return modelXbrl = val.modelXbrl modelDocument = modelXbrl.modelDocument _statusMsg = _("validating {0} filing rules").format(val.disclosureSystem.name) modelXbrl.profileActivity() val.showStatus(_statusMsg) if modelDocument.type == ModelDocument.Type.INSTANCE and (val.validateEBA or val.validateEIOPA): if not modelDocument.uri.endswith(".xbrl"): modelXbrl.warning("EBA.1.1", _('XBRL instance documents SHOULD use the extension ".xbrl" but it is "%(extension)s"'), modelObject=modelDocument, xmlEncoding=os.path.splitext(modelDocument.basename)[1]) if modelDocument.documentEncoding.lower() not in ("utf-8", "utf-8-sig"): modelXbrl.error("EBA.1.4", _('XBRL instance documents MUST use "UTF-8" encoding but is "%(xmlEncoding)s"'), modelObject=modelDocument, xmlEncoding=modelDocument.documentEncoding) schemaRefElts = [] schemaRefFileNames = [] for doc, docRef in modelDocument.referencesDocument.items(): if docRef.referenceType == "href": if docRef.referringModelObject.localName == "schemaRef": schemaRefElts.append(docRef.referringModelObject) schemaRefFileNames.append(doc.basename) if not UrlUtil.isAbsolute(doc.uri): modelXbrl.error("EBA.2.2", _('The link:schemaRef element in submitted instances MUST resolve to the full published entry point URL: %(url)s.'), modelObject=docRef.referringModelObject, url=doc.uri) elif docRef.referringModelObject.localName == "linkbaseRef": modelXbrl.error("EBA.2.3", _('The link:linkbaseRef element is not allowed: %(fileName)s.'), modelObject=docRef.referringModelObject, fileName=doc.basename) if len(schemaRefFileNames) > 1: modelXbrl.error("EBA.1.5", _('XBRL instance documents MUST reference only one entry point schema but %(numEntryPoints)s were found: %(entryPointNames)s'), modelObject=modelDocument, numEntryPoints=len(schemaRefFileNames), entryPointNames=', '.join(sorted(schemaRefFileNames))) ### check entry point names appropriate for filing indicator (DPM DB?) if len(schemaRefElts) != 1: modelXbrl.error("EBA.2.3", _('Any reported XBRL instance document MUST contain only one xbrli:xbrl/link:schemaRef node, but %(entryPointCount)s.'), modelObject=schemaRefElts, entryPointCount=len(schemaRefElts)) filingIndicators = {} for fIndicator in modelXbrl.factsByQname[qnFilingIndicator]: _value = (fIndicator.xValue or fIndicator.value) # use validated xValue if DTS else value for skipDTS if _value in filingIndicators: modelXbrl.error("EBA.1.6.1", _('Multiple filing indicators facts for indicator %(filingIndicator)s.'), modelObject=(fIndicator, filingIndicators[_value]), filingIndicator=_value) filingIndicators[_value] = fIndicator if not filingIndicators: modelXbrl.error("EBA.1.6", _('Missing filing indicators. Reported XBRL instances MUST include appropriate filing indicator elements'), modelObject=modelDocument) numFilingIndicatorTuples = len(modelXbrl.factsByQname[qnFIndicators]) if numFilingIndicatorTuples > 1 and not getattr(modelXbrl, "isStreamingMode", False): modelXbrl.info("EBA.1.6.2", _('Multiple filing indicators tuples when not in streaming mode (info).'), modelObject=modelXbrl.factsByQname[qnFIndicators]) # note EBA 2.1 is in ModelDocument.py cntxIDs = set() cntxEntities = set() cntxDates = defaultdict(list) timelessDatePattern = re.compile(r"\s*([0-9]{4})-([0-9]{2})-([0-9]{2})\s*$") for cntx in modelXbrl.contexts.values(): cntxIDs.add(cntx.id) cntxEntities.add(cntx.entityIdentifier) dateElts = XmlUtil.descendants(cntx, XbrlConst.xbrli, ("startDate","endDate","instant")) if any(not timelessDatePattern.match(e.textValue) for e in dateElts): modelXbrl.error("EBA.2.10", _('Period dates must be whole dates without time or timezone: %(dates)s.'), modelObject=cntx, dates=", ".join(e.text for e in dateElts)) if cntx.isForeverPeriod: modelXbrl.error("EBA.2.11", _('Forever context period is not allowed.'), modelObject=cntx) elif cntx.isStartEndPeriod: modelXbrl.error("EBA.2.13", _('Start-End (flow) context period is not allowed.'), modelObject=cntx) elif cntx.isInstantPeriod: cntxDates[cntx.instantDatetime].append(cntx) if XmlUtil.hasChild(cntx, XbrlConst.xbrli, "segment"): modelXbrl.error("EBA.2.14", _("The segment element not allowed in context Id: %(context)s"), modelObject=cntx, context=cntx.contextID) for scenElt in XmlUtil.descendants(cntx, XbrlConst.xbrli, "scenario"): childTags = ", ".join([child.prefixedName for child in scenElt.iterchildren() if isinstance(child,ModelObject) and child.tag != "{http://xbrl.org/2006/xbrldi}explicitMember" and child.tag != "{http://xbrl.org/2006/xbrldi}typedMember"]) if len(childTags) > 0: modelXbrl.error("EBA.2.15", _("Scenario of context Id %(context)s has disallowed content: %(content)s"), modelObject=cntx, context=cntx.id, content=childTags) if len(cntxDates) > 1: modelXbrl.error("EBA.2.13", _('Contexts must have the same date: %(dates)s.'), modelObject=[_cntx for _cntxs in cntxDates.values() for _cntx in _cntxs], dates=', '.join(XmlUtil.dateunionValue(_dt, subtractOneDay=True) for _dt in cntxDates.keys())) unusedCntxIDs = cntxIDs - {fact.contextID for fact in modelXbrl.factsInInstance if fact.contextID} # skip tuples if unusedCntxIDs: modelXbrl.warning("EBA.2.7", _('Unused xbrli:context nodes SHOULD NOT be present in the instance: %(unusedContextIDs)s.'), modelObject=[modelXbrl.contexts[unusedCntxID] for unusedCntxID in unusedCntxIDs], unusedContextIDs=", ".join(sorted(unusedCntxIDs))) if len(cntxEntities) > 1: modelXbrl.warning("EBA.2.9", _('All entity identifiers and schemes must be the same, %(count)s found: %(entities)s.'), modelObject=modelDocument, count=len(cntxEntities), entities=", ".join(sorted(str(cntxEntity) for cntxEntity in cntxEntities))) otherFacts = {} # (contextHash, unitHash, xmlLangHash) : fact nilFacts = [] stringFactsWithoutXmlLang = [] nonMonetaryNonPureFacts = [] unitIDsUsed = set() currenciesUsed = {} for qname, facts in modelXbrl.factsByQname.items(): for f in facts: if modelXbrl.skipDTS: c = f.qname.localName[0] isNumeric = c in ('m', 'p', 'i') isMonetary = c == 'm' isInteger = c == 'i' isPercent = c == 'p' isString = c == 's' else: concept = f.concept if concept is not None: isNumeric = concept.isNumeric isMonetary = concept.isMonetary isInteger = concept.baseXbrliType in integerItemTypes isPercent = concept.typeQname == qnPercentItemType isString = concept.baseXbrliType in ("stringItemType", "normalizedStringItemType") else: isNumeric = isString = False # error situation k = (f.getparent().objectIndex, f.qname, f.context.contextDimAwareHash if f.context is not None else None, f.unit.hash if f.unit is not None else None, hash(f.xmlLang)) if f.qname == qnFIndicators and val.validateEIOPA: pass elif k not in otherFacts: otherFacts[k] = {f} else: matches = [o for o in otherFacts[k] if (f.getparent().objectIndex == o.getparent().objectIndex and f.qname == o.qname and f.context.isEqualTo(o.context) if f.context is not None and o.context is not None else True) and (f.unit.isEqualTo(o.unit) if f.unit is not None and o.unit is not None else True) and (f.xmlLang == o.xmlLang)] if matches: contexts = [f.contextID] + [o.contextID for o in matches] modelXbrl.error("EBA.2.16", _('Facts are duplicates %(fact)s contexts %(contexts)s.'), modelObject=[f] + matches, fact=f.qname, contexts=', '.join(contexts)) else: otherFacts[k].add(f) if isNumeric: if f.precision: modelXbrl.error("EBA.2.17", _("Numeric fact %(fact)s of context %(contextID)s has a precision attribute '%(precision)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, precision=f.precision) if f.decimals and f.decimals != "INF": try: dec = int(f.decimals) if isMonetary: if dec < -3: modelXbrl.error("EBA.2.17", _("Monetary fact %(fact)s of context %(contextID)s has a decimal attribute < -3: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) elif isInteger: if dec != 0: modelXbrl.error("EBA.2.17", _("Integer fact %(fact)s of context %(contextID)s has a decimal attribute \u2260 0: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) elif isPercent: if dec < 4: modelXbrl.error("EBA.2.17", _("Percent fact %(fact)s of context %(contextID)s has a decimal attribute < 4: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) except ValueError: pass # should have been reported as a schema error by loader '''' (not intended by EBA 2.18, paste here is from EFM) if not f.isNil and getattr(f,"xValid", 0) == 4: try: insignificance = insignificantDigits(f.xValue, decimals=f.decimals) if insignificance: # if not None, returns (truncatedDigits, insiginficantDigits) modelXbrl.error(("EFM.6.05.37", "GFM.1.02.26"), _("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s has nonzero digits in insignificant portion %(insignificantDigits)s."), modelObject=f1, fact=f1.qname, contextID=f1.contextID, decimals=f1.decimals, value=f1.xValue, truncatedDigits=insignificance[0], insignificantDigits=insignificance[1]) except (ValueError,TypeError): modelXbrl.error(("EBA.2.18"), _("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s causes Value Error exception."), modelObject=f1, fact=f1.qname, contextID=f1.contextID, decimals=f1.decimals, value=f1.value) ''' unit = f.unit if unit is not None: if isMonetary: if unit.measures[0]: currenciesUsed[unit.measures[0][0]] = unit elif not unit.isSingleMeasure or unit.measures[0][0] != XbrlConst.qnXbrliPure: nonMonetaryNonPureFacts.append(f) elif isString: if not f.xmlLang: stringFactsWithoutXmlLang.append(f) if f.unitID is not None: unitIDsUsed.add(f.unitID) if f.isNil: nilFacts.append(f) if nilFacts: modelXbrl.error("EBA.2.19", _('Nil facts MUST NOT be present in the instance: %(nilFacts)s.'), modelObject=nilFacts, nilFacts=", ".join(str(f.qname) for f in nilFacts)) if stringFactsWithoutXmlLang: modelXbrl.error("EBA.2.20", _("String facts need to report xml:lang: '%(langLessFacts)s'"), modelObject=stringFactsWithoutXmlLang, langLEssFacts=", ".join(str(f.qname) for f in stringFactsWithoutXmlLang)) if nonMonetaryNonPureFacts: modelXbrl.error("EBA.3.2", _("Non monetary (numeric) facts MUST use the pure unit: '%(langLessFacts)s'"), modelObject=nonMonetaryNonPureFacts, langLessFacts=", ".join(str(f.qname) for f in nonMonetaryNonPureFacts)) unusedUnitIDs = modelXbrl.units.keys() - unitIDsUsed if unusedUnitIDs: modelXbrl.warning("EBA.2.21", _('Unused xbrli:unit nodes SHOULD NOT be present in the instance: %(unusedUnitIDs)s.'), modelObject=[modelXbrl.units[unusedUnitID] for unusedUnitID in unusedUnitIDs], unusedUnitIDs=", ".join(sorted(unusedUnitIDs))) unitHashes = {} for unit in modelXbrl.units.values(): h = hash(unit) if h in unitHashes and unit.isEqualTo(unitHashes[h]): modelXbrl.warning("EBA.2.32", _("Duplicate units SHOULD NOT be reported, units %(unit1)s and %(unit2)s have same measures.'"), modelObject=(unit, unitHashes[h]), unit1=unit.id, unit2=unitHashes[h].id) else: unitHashes[h] = unit if len(currenciesUsed) > 1: modelXbrl.error("EBA.3.1", _("There MUST be only one currency but %(numCurrencies)s were found: %(currencies)s.'"), modelObject=currenciesUsed.values(), numCurrencies=len(currenciesUsed), currencies=", ".join(str(c) for c in currenciesUsed.keys())) namespacePrefixesUsed = defaultdict(set) prefixesUnused = set(modelDocument.xmlRootElement.keys()).copy() for elt in modelDocument.xmlRootElement.iter(): if isinstance(elt, ModelObject): # skip comments and processing instructions namespacePrefixesUsed[elt.qname.namespaceURI].add(elt.qname.prefix) prefixesUnused.discard(elt.qname.prefix) if prefixesUnused: modelXbrl.warning("EBA.3.4", _("There SHOULD be no unused prefixes but these were declared: %(unusedPrefixes)s.'"), modelObject=modelDocument, unusedPrefixes=', '.join(sorted(prefixesUnused))) for ns, prefixes in namespacePrefixesUsed.items(): nsDocs = modelXbrl.namespaceDocs.get(ns) if nsDocs: for nsDoc in nsDocs: nsDocPrefix = XmlUtil.xmlnsprefix(nsDoc.xmlRootElement, ns) if any(prefix != nsDocPrefix for prefix in prefixes if prefix is not None): modelXbrl.warning("EBA.3.5", _("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s"), modelObject=modelDocument, namespace=ns, declaredPrefix=nsDocPrefix, foundPrefixes=', '.join(sorted(prefixes - {None}))) modelXbrl.profileActivity(_statusMsg, minTimeToShow=0.0) val.showStatus(None) del val.prefixNamespace, val.namespacePrefix, val.idObjects, val.typedDomainElements
def final(val): if not (val.validateEBA or val.validateEIOPA): return modelXbrl = val.modelXbrl modelDocument = modelXbrl.modelDocument _statusMsg = _("validating {0} filing rules").format(val.disclosureSystem.name) modelXbrl.profileActivity() modelXbrl.modelManager.showStatus(_statusMsg) if modelDocument.type == ModelDocument.Type.INSTANCE and (val.validateEBA or val.validateEIOPA): if not modelDocument.uri.endswith(".xbrl"): modelXbrl.warning("EBA.1.1", _('XBRL instance documents SHOULD use the extension ".xbrl" but it is "%(extension)s"'), modelObject=modelDocument, extension=os.path.splitext(modelDocument.basename)[1]) modelXbrl.error("EIOPA.S.1.1.a", _('XBRL instance documents MUST use the extension ".xbrl" but it is "%(extension)s"'), modelObject=modelDocument, extension=os.path.splitext(modelDocument.basename)[1]) if val.isEIOPA_2_0_1: _encodings = ("UTF-8", "utf-8-sig") else: _encodings = ("utf-8", "UTF-8", "utf-8-sig") if modelDocument.documentEncoding not in _encodings: modelXbrl.error(("EBA.1.4", "EIOPA.1.4"), _('XBRL instance documents MUST use "UTF-8" encoding but is "%(xmlEncoding)s"'), modelObject=modelDocument, xmlEncoding=modelDocument.documentEncoding) schemaRefElts = [] schemaRefFileNames = [] for doc, docRef in modelDocument.referencesDocument.items(): if docRef.referenceType == "href": if docRef.referringModelObject.localName == "schemaRef": schemaRefElts.append(docRef.referringModelObject) schemaRefFileNames.append(doc.basename) if not UrlUtil.isAbsolute(doc.uri): modelXbrl.error(("EBA.2.2", "EIOPA.S.1.5.a" if val.isEIOPAfullVersion else "EIOPA.S.1.5.b"), _('The link:schemaRef element in submitted instances MUST resolve to the full published entry point URL: %(url)s.'), modelObject=docRef.referringModelObject, url=doc.uri, messageCodes=("EBA.2.2", "EIOPA.S.1.5.a","EIOPA.S.1.5.b")) elif docRef.referringModelObject.localName == "linkbaseRef": modelXbrl.error(("EBA.2.3","EIOPA.S.1.5.a"), _('The link:linkbaseRef element is not allowed: %(fileName)s.'), modelObject=docRef.referringModelObject, fileName=doc.basename) _numSchemaRefs = len(XmlUtil.children(modelDocument.xmlRootElement, XbrlConst.link, "schemaRef")) if _numSchemaRefs > 1: modelXbrl.error(("EIOPA.S.1.5.a", "EBA.1.5"), _('XBRL instance documents MUST reference only one entry point schema but %(numEntryPoints)s were found: %(entryPointNames)s'), modelObject=modelDocument, numEntryPoints=_numSchemaRefs, entryPointNames=', '.join(sorted(schemaRefFileNames))) ### check entry point names appropriate for filing indicator (DPM DB?) if len(schemaRefElts) != 1: modelXbrl.error("EBA.2.3", _('Any reported XBRL instance document MUST contain only one xbrli:xbrl/link:schemaRef node, but %(entryPointCount)s.'), modelObject=schemaRefElts, entryPointCount=len(schemaRefElts)) # non-streaming EBA checks if not getattr(modelXbrl, "isStreamingMode", False): val.qnReportedCurrency = None if val.isEIOPA_2_0_1 and qnMetReportingCurrency in modelXbrl.factsByQname: for _multiCurrencyFact in modelXbrl.factsByQname[qnMetReportingCurrency]: # multi-currency fact val.qnReportedCurrency = _multiCurrencyFact.xValue break validateFacts(val, modelXbrl.facts) # check sum of fact md5s (otherwise checked in streaming process) xbrlFactsCheckVersion = None expectedSumOfFactMd5s = None for pi in modelDocument.xmlRootElement.getchildren(): if isinstance(pi, etree._ProcessingInstruction) and pi.target == "xbrl-facts-check": _match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']", pi.text) if _match: _matchGroups = _match.groups() if len(_matchGroups) == 2: if _matchGroups[0] == "version": xbrlFactsCheckVersion = _matchGroups[1] elif _matchGroups[0] == "sum-of-fact-md5s": try: expectedSumOfFactMd5s = Md5Sum(_matchGroups[1]) except ValueError: modelXbrl.error("EIOPA:xbrlFactsCheckError", _("Invalid sum-of-md5s %(sumOfMd5)s"), modelObject=modelXbrl, sumOfMd5=_matchGroups[1]) if xbrlFactsCheckVersion and expectedSumOfFactMd5s: sumOfFactMd5s = Md5Sum() for f in modelXbrl.factsInInstance: sumOfFactMd5s += f.md5sum if sumOfFactMd5s != expectedSumOfFactMd5s: modelXbrl.warning("EIOPA:xbrlFactsCheckWarning", _("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"), modelObject=modelXbrl, expectedMd5=expectedSumOfFactMd5s, actualMd5Sum=sumOfFactMd5s) else: modelXbrl.info("info", _("Successful XBRL facts sum of md5s."), modelObject=modelXbrl) if any(badError in modelXbrl.errors for badError in ("EBA.2.1", "EIOPA.2.1", "EIOPA.S.1.5.a/EIOPA.S.1.5.b")): pass # skip checking filingIndicators if bad errors elif not val.filingIndicators: modelXbrl.error(("EBA.1.6", "EIOPA.1.6.a"), _('Missing filing indicators. Reported XBRL instances MUST include appropriate (positive) filing indicator elements'), modelObject=modelDocument) elif all(filed == False for filed in val.filingIndicators.values()): modelXbrl.error(("EBA.1.6", "EIOPA.1.6.a"), _('All filing indicators are filed="false". Reported XBRL instances MUST include appropriate (positive) filing indicator elements'), modelObject=modelDocument) if val.numFilingIndicatorTuples > 1: modelXbrl.warning(("EBA.1.6.2", "EIOPA.1.6.2"), _('Multiple filing indicators tuples when not in streaming mode (info).'), modelObject=modelXbrl.factsByQname[qnFIndicators]) if len(val.cntxDates) > 1: modelXbrl.error(("EBA.2.13","EIOPA.2.13"), _('Contexts must have the same date: %(dates)s.'), # when streaming values are no longer available, but without streaming they can be logged modelObject=set(_cntx for _cntxs in val.cntxDates.values() for _cntx in _cntxs), dates=', '.join(XmlUtil.dateunionValue(_dt, subtractOneDay=True) for _dt in val.cntxDates.keys())) if val.unusedCntxIDs: if val.isEIOPA_2_0_1: modelXbrl.error("EIOPA.2.7", _('Unused xbrli:context nodes MUST NOT be present in the instance: %(unusedContextIDs)s.'), modelObject=[modelXbrl.contexts[unusedCntxID] for unusedCntxID in val.unusedCntxIDs if unusedCntxID in modelXbrl.contexts], unusedContextIDs=", ".join(sorted(val.unusedCntxIDs))) else: modelXbrl.warning(("EBA.2.7", "EIOPA.2.7"), _('Unused xbrli:context nodes SHOULD NOT be present in the instance: %(unusedContextIDs)s.'), modelObject=[modelXbrl.contexts[unusedCntxID] for unusedCntxID in val.unusedCntxIDs if unusedCntxID in modelXbrl.contexts], unusedContextIDs=", ".join(sorted(val.unusedCntxIDs))) if len(val.cntxEntities) > 1: modelXbrl.error(("EBA.2.9", "EIOPA.2.9"), _('All entity identifiers and schemes MUST be the same, %(count)s found: %(entities)s.'), modelObject=modelDocument, count=len(val.cntxEntities), entities=", ".join(sorted(str(cntxEntity) for cntxEntity in val.cntxEntities))) for _scheme, _LEI in val.cntxEntities: if (_scheme in ("http://standards.iso.org/iso/17442", "http://standard.iso.org/iso/17442", "LEI") or (not val.isEIOPAfullVersion and _scheme == "PRE-LEI")): result = LeiUtil.checkLei(_LEI) if result == LeiUtil.LEI_INVALID_LEXICAL: modelXbrl.error("EIOPA.S.2.8.c", _("Context has lexically invalid LEI %(lei)s."), modelObject=modelDocument, lei=_LEI) elif result == LeiUtil.LEI_INVALID_CHECKSUM: modelXbrl.error("EIOPA.S.2.8.c", _("Context has LEI checksum error in %(lei)s."), modelObject=modelDocument, lei=_LEI) if _scheme == "http://standard.iso.org/iso/17442": modelXbrl.warning("EIOPA.S.2.8.c", _("Warning, context has entity scheme %(scheme)s should be plural: http://standards.iso.org/iso/17442."), modelObject=modelDocument, scheme=_scheme) elif _scheme == "SC": pass # anything is ok for Specific Code else: modelXbrl.error("EIOPA.S.2.8.c", _("Context has unrecognized entity scheme %(scheme)s."), modelObject=modelDocument, scheme=_scheme) if val.unusedUnitIDs: if val.isEIOPA_2_0_1: modelXbrl.error("EIOPA.2.22", _('Unused xbrli:unit nodes MUST NOT be present in the instance: %(unusedUnitIDs)s.'), modelObject=[modelXbrl.units[unusedUnitID] for unusedUnitID in val.unusedUnitIDs if unusedUnitID in modelXbrl.units], unusedUnitIDs=", ".join(sorted(val.unusedUnitIDs))) else: modelXbrl.warning(("EBA.2.22", "EIOPA.2.22"), _('Unused xbrli:unit nodes SHOULD NOT be present in the instance: %(unusedUnitIDs)s.'), modelObject=[modelXbrl.units[unusedUnitID] for unusedUnitID in val.unusedUnitIDs if unusedUnitID in modelXbrl.units], unusedUnitIDs=", ".join(sorted(val.unusedUnitIDs))) if len(val.currenciesUsed) > 1: modelXbrl.error(("EBA.3.1","EIOPA.3.1"), _("There MUST be only one currency but %(numCurrencies)s were found: %(currencies)s.'"), modelObject=val.currenciesUsed.values(), numCurrencies=len(val.currenciesUsed), currencies=", ".join(str(c) for c in val.currenciesUsed.keys())) elif val.isEIOPA_2_0_1 and any(_measure.localName != val.reportingCurrency for _measure in val.currenciesUsed.keys()): modelXbrl.error("EIOPA.3.1", _("There MUST be only one currency but reporting currency %(reportingCurrency)s differs from unit currencies: %(currencies)s.'"), modelObject=val.currenciesUsed.values(), reportingCurrency=val.reportingCurrency, currencies=", ".join(str(c) for c in val.currenciesUsed.keys())) if val.prefixesUnused: modelXbrl.warning(("EBA.3.4", "EIOPA.3.4"), _("There SHOULD be no unused prefixes but these were declared: %(unusedPrefixes)s.'"), modelObject=modelDocument, unusedPrefixes=', '.join(sorted(val.prefixesUnused))) for ns, prefixes in val.namespacePrefixesUsed.items(): nsDocs = modelXbrl.namespaceDocs.get(ns) if nsDocs: for nsDoc in nsDocs: nsDocPrefix = XmlUtil.xmlnsprefix(nsDoc.xmlRootElement, ns) if any(prefix != nsDocPrefix for prefix in prefixes if prefix is not None): modelXbrl.warning(("EBA.3.5", "EIOPA.3.5"), _("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s"), modelObject=modelDocument, namespace=ns, declaredPrefix=nsDocPrefix, foundPrefixes=', '.join(sorted(prefixes - {None}))) elif ns in CANONICAL_PREFIXES and any(prefix != CANONICAL_PREFIXES[ns] for prefix in prefixes if prefix is not None): modelXbrl.warning(("EBA.3.5", "EIOPA.3.5"), _("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s"), modelObject=modelDocument, namespace=ns, declaredPrefix=CANONICAL_PREFIXES[ns], foundPrefixes=', '.join(sorted(prefixes - {None}))) modelXbrl.profileActivity(_statusMsg, minTimeToShow=0.0) modelXbrl.modelManager.showStatus(None) del val.prefixNamespace, val.namespacePrefix, val.idObjects, val.typedDomainElements del val.utrValidator, val.firstFact, val.footnotesRelationshipSet
def checkDTSdocument(val, modelDocument): modelXbrl = val.modelXbrl if modelDocument.type == ModelDocument.Type.INSTANCE: if not modelDocument.uri.endswith(".xbrl"): modelXbrl.warning("EBA.1.1", _('XBRL instance documents SHOULD use the extension ".xbrl" encoding but it is "%(extension)s"'), modelObject=modelDocument, xmlEncoding=os.path.splitext(modelDocument.basename)[1]) if modelDocument.documentEncoding.lower() not in ("utf-8", "utf-8-sig"): modelXbrl.error("EBA.1.4", _('XBRL instance documents MUST use "UTF-8" encoding but is "%(xmlEncoding)s"'), modelObject=modelDocument, xmlEncoding=modelDocument.documentEncoding) schemaRefElts = [] schemaRefFileNames = [] for doc, docRef in modelDocument.referencesDocument.items(): if docRef.referenceType == "href": if docRef.referringModelObject.localName == "schemaRef": schemaRefElts.append(docRef.referringModelObject) schemaRefFileNames.append(doc.basename) if not UrlUtil.isAbsolute(doc.uri): modelXbrl.error("EBA.2.2", _('The link:schemaRef element in submitted instances MUST resolve to the full published entry point URL: %(url)s.'), modelObject=docRef.referringModelObject, url=doc.uri) elif docRef.referringModelObject.localName == "linkbaseRef": modelXbrl.error("EBA.2.3", _('The link:linkbaseRef element is not allowed: %(fileName)s.'), modelObject=docRef.referringModelObject, fileName=doc.basename) if len(schemaRefFileNames) > 1: modelXbrl.error("EBA.1.5", _('XBRL instance documents MUST reference only one entry point schema but %(numEntryPoints)s were found: %(entryPointNames)s'), modelObject=modelDocument, numEntryPoints=len(schemaRefFileNames), entryPointNames=', '.join(sorted(schemaRefFileNames))) ### check entry point names appropriate for filing indicator (DPM DB?) if len(schemaRefElts) != 1: modelXbrl.error("EBA.2.3", _('Any reported XBRL instance document MUST contain only one xbrli:xbrl/link:schemaRef node, but %(entryPointCount)s.'), modelObject=schemaRefElts, entryPointCount=len(schemaRefElts)) filingIndicators = {} for fIndicator in modelXbrl.factsByQname[qnFilingIndicator]: if fIndicator.xValue in filingIndicators: modelXbrl.error("EBA.1.6.1", _('Multiple filing indicators facts for indicator %(filingIndicator)s.'), modelObject=(fIndicator, filingIndicators[filingIndicators]), filingIndicator=fIndicator.xValue) filingIndicators[fIndicator.xValue] = fIndicator if not filingIndicators: modelXbrl.error("EBA.1.6", _('Missing filing indicators. Reported XBRL instances MUST include appropriate filing indicator elements'), modelObject=modelDocument) numFilingIndicatorTuples = len(modelXbrl.factsByQname[qnFilingIndicators]) if numFilingIndicatorTuples > 1 and not getattr(modelXbrl, "isStreamingMode", False): modelXbrl.info("EBA.1.6.2", _('Multiple filing indicators tuples when not in streaming mode (info).'), modelObject=modelXbrl.factsByQname[qnFilingIndicators]) # note EBA 2.1 is in ModelDocument.py cntxIDs = set() cntxEntities = set() timelessDatePattern = re.compile(r"\s*([0-9]{4})-([0-9]{2})-([0-9]{2})\s*$") for cntx in modelXbrl.contexts.values(): cntxIDs.add(cntx.id) cntxEntities.add(cntx.entityIdentifier) dateElts = XmlUtil.descendants(cntx, XbrlConst.xbrli, ("startDate","endDate","instant")) if any(not timelessDatePattern.match(e.textValue) for e in dateElts): modelXbrl.error("EBA.2.10", _('Period dates must be whole dates without time or timezone: %(dates)s.'), modelObject=cntx, dates=", ".join(e.text for e in dateElts)) if cntx.isForeverPeriod: modelXbrl.error("EBA.2.11", _('Forever context period is not allowed.'), modelObject=cntx) if XmlUtil.hasChild(cntx, XbrlConst.xbrli, "segment"): modelXbrl.error("EBA.2.14", _("The segment element not allowed in context Id: %(context)s"), modelObject=cntx, context=cntx.contextID) for scenElt in XmlUtil.descendants(cntx, XbrlConst.xbrli, "scenario"): childTags = ", ".join([child.prefixedName for child in scenElt.iterchildren() if isinstance(child,ModelObject) and child.tag != "{http://xbrl.org/2006/xbrldi}explicitMember" and child.tag != "{http://xbrl.org/2006/xbrldi}typedMember"]) if len(childTags) > 0: modelXbrl.error("EBA.2.15", _("Scenario of context Id %(context)s has disallowed content: %(content)s"), modelObject=cntx, context=cntx.id, content=childTags) unusedCntxIDs = cntxIDs - {fact.contextID for fact in modelXbrl.facts} if unusedCntxIDs: modelXbrl.warning("EBA.2.7", _('Unused xbrli:context nodes SHOULD NOT be present in the instance: %(unusedContextIDs)s.'), modelObject=[modelXbrl.contexts[unusedCntxID] for unusedCntxID in unusedCntxIDs], unusedContextIDs=", ".join(sorted(unusedCntxIDs))) if len(cntxEntities) > 1: modelXbrl.warning("EBA.2.9", _('All entity identifiers and schemes must be the same, %(count)s found: %(entities)s.'), modelObject=modelDocument, count=len(cntxEntities), entities=", ".join(sorted(str(cntxEntity) for cntxEntity in cntxEntities))) otherFacts = {} # (contextHash, unitHash, xmlLangHash) : fact nilFacts = [] unitIDsUsed = set() currenciesUsed = {} for qname, facts in modelXbrl.factsByQname.items(): for f in facts: concept = f.concept k = (f.context.contextDimAwareHash if f.context is not None else None, f.unit.hash if f.unit is not None else None, hash(f.xmlLang)) if k not in otherFacts: otherFacts[k] = {f} else: matches = [o for o in otherFacts[k] if (f.context.isEqualTo(o.context) if f.context is not None and o.context is not None else True) and (f.unit.isEqualTo(o.unit) if f.unit is not None and o.unit is not None else True) and (f.xmlLang == o.xmlLang)] if matches: modelXbrl.error("EBA.2.16", _('Facts are duplicates %(fact)s and %(otherFacts)s.'), modelObject=[f] + matches, fact=f.qname, otherFacts=', '.join(str(f.qname) for f in matches)) else: otherFacts[k].add(f) if concept is not None and concept.isNumeric: if f.precision: modelXbrl.error("EBA.2.17", _("Numeric fact %(fact)s of context %(contextID)s has a precision attribute '%(precision)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, precision=f.precision) '''' (not intended by EBA 2.18) if f.decimals and f.decimals != "INF" and not f.isNil and getattr(f,"xValid", 0) == 4: try: insignificance = insignificantDigits(f.xValue, decimals=f.decimals) if insignificance: # if not None, returns (truncatedDigits, insiginficantDigits) modelXbrl.error(("EFM.6.05.37", "GFM.1.02.26"), _("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s has nonzero digits in insignificant portion %(insignificantDigits)s."), modelObject=f1, fact=f1.qname, contextID=f1.contextID, decimals=f1.decimals, value=f1.xValue, truncatedDigits=insignificance[0], insignificantDigits=insignificance[1]) except (ValueError,TypeError): modelXbrl.error(("EFM.6.05.37", "GFM.1.02.26"), _("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s causes Value Error exception."), modelObject=f1, fact=f1.qname, contextID=f1.contextID, decimals=f1.decimals, value=f1.value) ''' unit = f.unit if concept.isMonetary and unit is not None and unit.measures[0]: currenciesUsed[unit.measures[0][0]] = unit if f.unitID is not None: unitIDsUsed.add(f.unitID) if f.isNil: nilFacts.append(f) if nilFacts: modelXbrl.warning("EBA.2.19", _('Nil facts SHOULD NOT be present in the instance: %(nilFacts)s.'), modelObject=nilFacts, nilFacts=", ".join(str(f.qname) for f in nilFacts)) unusedUnitIDs = modelXbrl.units.keys() - unitIDsUsed if unusedUnitIDs: modelXbrl.warning("EBA.2.21", _('Unused xbrli:unit nodes SHOULD NOT be present in the instance: %(unusedUnitIDs)s.'), modelObject=[modelXbrl.units[unusedUnitID] for unusedUnitID in unusedUnitIDs], unusedUnitIDs=", ".join(sorted(unusedUnitIDs))) unitHashes = {} for unit in modelXbrl.units.values(): h = hash(unit) if h in unitHashes and unit.isEqualTo(unitHashes[h]): modelXbrl.error("EBA.2.32", _("Units %(unit1)s and %(unit2)s have same measures.'"), modelObject=(unit, unitHashes[h]), unit1=unit.id, unit2=unitHashes[h].id) else: unitHashes[h] = unit if len(currenciesUsed) > 1: modelXbrl.error("EBA.3.1", _("There MUST be only one currency but %(numCurrencies)s were found: %(currencies)s.'"), modelObject=currenciesUsed.values(), numCurrencies=len(currenciesUsed), currencies=", ".join(str(c) for c in currenciesUsed.keys())) namespacePrefixesUsed = defaultdict(set) prefixesUnused = set(modelDocument.xmlRootElement.keys()).copy() for elt in modelDocument.xmlRootElement.iter(): if isinstance(elt, ModelObject): # skip comments and processing instructions namespacePrefixesUsed[elt.qname.namespaceURI].add(elt.qname.prefix) prefixesUnused.discard(elt.qname.prefix) if prefixesUnused: modelXbrl.warning("EBA.3.4", _("There SHOULD be no unused prefixes but these were declared: %(unusedPrefixes)s.'"), modelObject=modelDocument, unusedPrefixes=', '.join(sorted(prefixesUnused))) for ns, prefixes in namespacePrefixesUsed.items(): nsDocs = modelXbrl.namespaceDocs.get(ns) if nsDocs: for nsDoc in nsDocs: nsDocPrefix = XmlUtil.xmlnsprefix(nsDoc.xmlRootElement, ns) if any(prefix != nsDocPrefix for prefix in prefixes if prefix is not None): modelXbrl.warning("EBA.3.5", _("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s"), modelObject=modelDocument, namespace=ns, declaredPrefix=nsDocPrefix, foundPrefixes=', '.join(sorted(prefixes - {None})))
def checkDTS(val, modelDocument, visited): global targetNamespaceDatePattern, efmFilenamePattern, roleTypePattern, arcroleTypePattern, \ arcroleDefinitionPattern, namePattern, linkroleDefinitionBalanceIncomeSheet, \ namespacesConflictPattern if targetNamespaceDatePattern is None: targetNamespaceDatePattern = re.compile(r"/([12][0-9]{3})-([01][0-9])-([0-3][0-9])|" r"/([12][0-9]{3})([01][0-9])([0-3][0-9])|") efmFilenamePattern = re.compile(r"^[a-z0-9][a-zA-Z0-9_\.\-]*(\.xsd|\.xml)$") roleTypePattern = re.compile(r"^.*/role/[^/\s]+$") arcroleTypePattern = re.compile(r"^.*/arcrole/[^/\s]+$") arcroleDefinitionPattern = re.compile(r"^.*[^\\s]+.*$") # at least one non-whitespace character namePattern = re.compile("[][()*+?\\\\/^{}|@#%^=~`\"';:,<>&$\u00a3\u20ac]") # u20ac=Euro, u00a3=pound sterling linkroleDefinitionBalanceIncomeSheet = re.compile(r"[^-]+-\s+Statement\s+-\s+.*(income|balance|financial\W+position)", re.IGNORECASE) namespacesConflictPattern = re.compile(r"http://(xbrl\.us|fasb\.org|xbrl\.sec\.gov)/(dei|us-types|us-roles|rr)/([0-9]{4}-[0-9]{2}-[0-9]{2})$") nonDomainItemNameProblemPattern = re.compile( r"({0})|(FirstQuarter|SecondQuarter|ThirdQuarter|FourthQuarter|[1-4]Qtr|Qtr[1-4]|ytd|YTD|HalfYear)(?:$|[A-Z\W])" .format(re.sub(r"\W", "", (val.entityRegistrantName or "").title()))) visited.append(modelDocument) definesLabelLinkbase = False for referencedDocument, modelDocumentReference in modelDocument.referencesDocument.items(): #6.07.01 no includes if modelDocumentReference.referenceType == "include": val.modelXbrl.error(("EFM.6.07.01", "GFM.1.03.01", "SBR.NL.2.2.0.18"), _("Taxonomy schema %(schema)s includes %(include)s, only import is allowed"), modelObject=modelDocumentReference.referringModelObject, schema=os.path.basename(modelDocument.uri), include=os.path.basename(referencedDocument.uri)) if referencedDocument not in visited: checkDTS(val, referencedDocument, visited) if val.disclosureSystem.standardTaxonomiesDict is None: pass if val.validateEFM: if modelDocument.uri in val.disclosureSystem.standardTaxonomiesDict: if modelDocument.targetNamespace: # check for duplicates of us-types, dei, and rr taxonomies match = namespacesConflictPattern.match(modelDocument.targetNamespace) if match is not None: val.standardNamespaceConflicts[match.group(2)].add(modelDocument) else: if len(modelDocument.basename) > 32: val.modelXbrl.error("EFM.5.01.01.tooManyCharacters", _("Document file name %(filename)s must not exceed 32 characters."), modelObject=modelDocument, filename=modelDocument.basename) if not efmFilenamePattern.match(modelDocument.basename): val.modelXbrl.error("EFM.5.01.01", _("Document file name %(filename)s must start with a-z or 0-9, contain upper or lower case letters, ., -, _, and end with .xsd or .xml."), modelObject=modelDocument, filename=modelDocument.basename) if (modelDocument.type == ModelDocument.Type.SCHEMA and modelDocument.targetNamespace not in val.disclosureSystem.baseTaxonomyNamespaces and modelDocument.uri.startswith(val.modelXbrl.uriDir)): # check schema contents types if val.validateSBRNL: definesLinkroles = False definesArcroles = False definesLinkParts = False definesAbstractItems = False definesNonabstractItems = False definesConcepts = False definesTuples = False definesPresentationTuples = False definesSpecificationTuples = False definesTypes = False definesEnumerations = False definesDimensions = False definesDomains = False definesHypercubes = False # 6.7.3 check namespace for standard authority targetNamespaceAuthority = UrlUtil.authority(modelDocument.targetNamespace) if targetNamespaceAuthority in val.disclosureSystem.standardAuthorities: val.modelXbrl.error(("EFM.6.07.03", "GFM.1.03.03"), _("Taxonomy schema %(schema)s namespace %(targetNamespace)s is a disallowed authority"), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace, targetNamespaceAuthority=UrlUtil.authority(modelDocument.targetNamespace, includeScheme=False)) # 6.7.4 check namespace format if modelDocument.targetNamespace is None or not modelDocument.targetNamespace.startswith("http://"): match = None elif val.validateEFMorGFM: targetNamespaceDate = modelDocument.targetNamespace[len(targetNamespaceAuthority):] match = targetNamespaceDatePattern.match(targetNamespaceDate) else: match = None if match is not None: try: if match.lastindex == 3: date = datetime.date(int(match.group(1)),int(match.group(2)),int(match.group(3))) elif match.lastindex == 6: date = datetime.date(int(match.group(4)),int(match.group(5)),int(match.group(6))) else: match = None except ValueError: match = None if match is None: val.modelXbrl.error(("EFM.6.07.04", "GFM.1.03.04"), _("Taxonomy schema %(schema)s namespace %(targetNamespace)s must have format http://{authority}/{versionDate}"), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace) elif val.fileNameDate and date > val.fileNameDate: val.modelXbrl.info(("EFM.6.07.06", "GFM.1.03.06"), _("Warning: Taxonomy schema %(schema)s namespace %(targetNamespace)s has date later than document name date %(docNameDate)s"), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace, docNameDate=val.fileNameDate) if modelDocument.targetNamespace is not None: # 6.7.5 check prefix for _ authority = UrlUtil.authority(modelDocument.targetNamespace) if not re.match(r"(http://|https://|ftp://|urn:)\w+",authority): val.modelXbrl.error(("EFM.6.07.05", "GFM.1.03.05"), _("Taxonomy schema %(schema)s namespace %(targetNamespace)s must be a valid URL with a valid authority for the namespace."), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace) prefix = XmlUtil.xmlnsprefix(modelDocument.xmlRootElement,modelDocument.targetNamespace) if not prefix: val.modelXbrl.error(("EFM.6.07.07", "GFM.1.03.07"), _("Taxonomy schema %(schema)s namespace %(targetNamespace)s missing prefix for the namespace."), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace) elif "_" in prefix: val.modelXbrl.error(("EFM.6.07.07", "GFM.1.03.07"), _("Taxonomy schema %(schema)s namespace %(targetNamespace)s prefix %(prefix)s must not have an '_'"), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace, prefix=prefix) if val.validateSBRNL: genrlSpeclRelSet = val.modelXbrl.relationshipSet(XbrlConst.generalSpecial) for modelConcept in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/2001/XMLSchema}element"): if isinstance(modelConcept,ModelConcept): # 6.7.16 name not duplicated in standard taxonomies name = modelConcept.get("name") if name is None: name = "" if modelConcept.get("ref") is not None: continue # don't validate ref's here for c in val.modelXbrl.nameConcepts.get(name, []): if c.modelDocument != modelDocument: if (val.validateEFMorGFM and not c.modelDocument.uri.startswith(val.modelXbrl.uriDir)): val.modelXbrl.error(("EFM.6.07.16", "GFM.1.03.18"), _("Concept %(concept)s is also defined in standard taxonomy schema schema %(standardSchema)s"), modelObject=(modelConcept,c), concept=modelConcept.qname, standardSchema=os.path.basename(c.modelDocument.uri), standardConcept=c.qname) elif val.validateSBRNL: if not (genrlSpeclRelSet.isRelated(modelConcept, "child", c) or genrlSpeclRelSet.isRelated(c, "child", modelConcept)): val.modelXbrl.error("SBR.NL.2.2.2.02", _("Concept %(concept)s is also defined in standard taxonomy schema %(standardSchema)s without a general-special relationship"), modelObject=c, concept=modelConcept.qname, standardSchema=os.path.basename(c.modelDocument.uri)) ''' removed RH 2011-12-23 corresponding set up of table in ValidateFiling if val.validateSBRNL and name in val.nameWordsTable: if not any( any( genrlSpeclRelSet.isRelated(c, "child", modelConcept) for c in val.modelXbrl.nameConcepts.get(partialWordName, [])) for partialWordName in val.nameWordsTable[name]): val.modelXbrl.error("SBR.NL.2.3.2.01", _("Concept %(specialName)s is appears to be missing a general-special relationship to %(generalNames)s"), modelObject=c, specialName=modelConcept.qname, generalNames=', or to '.join(val.nameWordsTable[name])) ''' # 6.7.17 id properly formed id = modelConcept.id requiredId = (prefix if prefix is not None else "") + "_" + name if val.validateEFMorGFM and id != requiredId: val.modelXbrl.error(("EFM.6.07.17", "GFM.1.03.19"), _("Concept %(concept)s id %(id)s should be $(requiredId)s"), modelObject=modelConcept, concept=modelConcept.qname, id=id, requiredId=requiredId) # 6.7.18 nillable is true nillable = modelConcept.get("nillable") if nillable != "true": val.modelXbrl.error(("EFM.6.07.18", "GFM.1.03.20"), _("Taxonomy schema %(schema)s element %(concept)s nillable %(nillable)s should be 'true'"), modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=name, nillable=nillable) # 6.7.19 not tuple if modelConcept.isTuple: if val.validateEFMorGFM: val.modelXbrl.error(("EFM.6.07.19", "GFM.1.03.21"), _("Concept %(concept)s is a tuple"), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.20 no typed domain ref if modelConcept.isTypedDimension: val.modelXbrl.error(("EFM.6.07.20", "GFM.1.03.22"), _("Concept %(concept)s has typedDomainRef %(typedDomainRef)s"), modelObject=modelConcept, concept=modelConcept.qname, typedDomainRef=modelConcept.typedDomainElement.qname if modelConcept.typedDomainElement is not None else modelConcept.typedDomainRef) # 6.7.21 abstract must be duration isDuration = modelConcept.periodType == "duration" if modelConcept.abstract == "true" and not isDuration: val.modelXbrl.error(("EFM.6.07.21", "GFM.1.03.23"), _("Taxonomy schema %(schema)s element %(concept)s is abstract but period type is not duration"), modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=modelConcept.qname) # 6.7.22 abstract must be stringItemType ''' removed SEC EFM v.17, Edgar release 10.4, and GFM 2011-04-08 if modelConcept.abstract == "true" and modelConcept.typeQname != XbrlConst. qnXbrliStringItemType: val.modelXbrl.error(("EFM.6.07.22", "GFM.1.03.24"), _("Concept %(concept)s is abstract but type is not xbrli:stringItemType"), modelObject=modelConcept, concept=modelConcept.qname) ''' substititutionGroupQname = modelConcept.substitutionGroupQname # 6.7.23 Axis must be subs group dimension if name.endswith("Axis") ^ (substititutionGroupQname == XbrlConst.qnXbrldtDimensionItem): val.modelXbrl.error(("EFM.6.07.23", "GFM.1.03.25"), _("Concept %(concept)s must end in Axis to be in xbrldt:dimensionItem substitution group"), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.24 Table must be subs group hypercube if name.endswith("Table") ^ (substititutionGroupQname == XbrlConst.qnXbrldtHypercubeItem): val.modelXbrl.error(("EFM.6.07.24", "GFM.1.03.26"), _("Concept %(concept)s must end in Table to be in xbrldt:hypercubeItem substitution group"), modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=modelConcept.qname) # 6.7.25 if neither hypercube or dimension, substitution group must be item if substititutionGroupQname not in (None, XbrlConst.qnXbrldtDimensionItem, XbrlConst.qnXbrldtHypercubeItem, XbrlConst.qnXbrliItem): val.modelXbrl.error(("EFM.6.07.25", "GFM.1.03.27"), _("Concept %(concept)s has disallowed substitution group %(substitutionGroup)s"), modelObject=modelConcept, concept=modelConcept.qname, substitutionGroup=modelConcept.substitutionGroupQname) # 6.7.26 Table must be subs group hypercube if name.endswith("LineItems") and modelConcept.abstract != "true": val.modelXbrl.error(("EFM.6.07.26", "GFM.1.03.28"), _("Concept %(concept)s is a LineItems but not abstract"), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.27 type domainMember must end with Domain or Member conceptType = modelConcept.type isDomainItemType = conceptType is not None and conceptType.isDomainItemType endsWithDomainOrMember = name.endswith("Domain") or name.endswith("Member") if isDomainItemType != endsWithDomainOrMember: val.modelXbrl.error(("EFM.6.07.27", "GFM.1.03.29"), _("Concept %(concept)s must end with Domain or Member for type of domainItemType"), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.28 domainItemType must be duration if isDomainItemType and not isDuration: val.modelXbrl.error(("EFM.6.07.28", "GFM.1.03.30"), _("Concept %(concept)s is a domainItemType and must be periodType duration"), modelObject=modelConcept, concept=modelConcept.qname) # 6.8.5 semantic check, check LC3 name if not name[0].isupper(): val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.05.firstLetter", "GFM.2.03.05.firstLetter"), _("Concept %(concept)s name must start with a capital letter"), modelObject=modelConcept, concept=modelConcept.qname) if namePattern.search(name): val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.05.disallowedCharacter", "GFM.2.03.05.disallowedCharacter"), _("Concept %(concept)s has disallowed name character"), modelObject=modelConcept, concept=modelConcept.qname) if len(name) > 200: val.modelXbrl.log("ERROR-SEMANTIC", "EFM.6.08.05.nameLength", _("Concept %(concept)s name length %(namelength)s exceeds 200 characters"), modelObject=modelConcept, concept=modelConcept.qname, namelength=len(name)) if val.validateEFM: label = modelConcept.label(lang="en-US", fallbackToQname=False) if label: # allow Joe's Bar, N.A. to be JoesBarNA -- remove ', allow A. as not article "a" lc3name = ''.join(re.sub(r"['.-]", "", (w[0] or w[2] or w[3] or w[4])).title() for w in re.findall(r"((\w+')+\w+)|(A[.-])|([.-]A(?=\W|$))|(\w+)", label) # EFM implies this should allow - and . re.findall(r"[\w\-\.]+", label) if w[4].lower() not in ("the", "a", "an")) if not(name == lc3name or (name and lc3name and lc3name[0].isdigit() and name[1:] == lc3name and (name[0].isalpha() or name[0] == '_'))): val.modelXbrl.log("WARNING-SEMANTIC", "EFM.6.08.05.LC3", _("Concept %(concept)s should match expected LC3 composition %(lc3name)s"), modelObject=modelConcept, concept=modelConcept.qname, lc3name=lc3name) if conceptType is not None: # 6.8.6 semantic check if not isDomainItemType and conceptType.qname != XbrlConst.qnXbrliDurationItemType: nameProblems = nonDomainItemNameProblemPattern.findall(name) if any(any(t) for t in nameProblems): # list of tuples with possibly nonempty strings val.modelXbrl.log("WARNING-SEMANTIC", ("EFM.6.08.06", "GFM.2.03.06"), _("Concept %(concept)s should not contain company or period information, found: %(matches)s"), modelObject=modelConcept, concept=modelConcept.qname, matches=", ".join(''.join(t) for t in nameProblems)) if conceptType.qname == XbrlConst.qnXbrliMonetaryItemType: if not modelConcept.balance: # 6.8.11 may not appear on a income or balance statement if any(linkroleDefinitionBalanceIncomeSheet.match(roleType.definition) for rel in val.modelXbrl.relationshipSet(XbrlConst.parentChild).toModelObject(modelConcept) for roleType in val.modelXbrl.roleTypes.get(rel.linkrole,())): val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.11", "GFM.2.03.11"), _("Concept %(concept)s must have a balance because it appears in a statement of income or balance sheet"), modelObject=modelConcept, concept=modelConcept.qname) # 6.11.5 semantic check, must have a documentation label stdLabel = modelConcept.label(lang="en-US", fallbackToQname=False) defLabel = modelConcept.label(preferredLabel=XbrlConst.documentationLabel, lang="en-US", fallbackToQname=False) if not defLabel or ( # want different words than std label stdLabel and re.findall(r"\w+", stdLabel) == re.findall(r"\w+", defLabel)): val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.11.05", "GFM.2.04.04"), _("Concept %(concept)s is monetary without a balance and must have a documentation label that disambiguates its sign"), modelObject=modelConcept, concept=modelConcept.qname) # 6.8.16 semantic check if conceptType.qname == XbrlConst.qnXbrliDateItemType and modelConcept.periodType != "duration": val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.16", "GFM.2.03.16"), _("Concept %(concept)s of type xbrli:dateItemType must have periodType duration"), modelObject=modelConcept, concept=modelConcept.qname) # 6.8.17 semantic check if conceptType.qname == XbrlConst.qnXbrliStringItemType and modelConcept.periodType != "duration": val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.17", "GFM.2.03.17"), _("Concept %(concept)s of type xbrli:stringItemType must have periodType duration"), modelObject=modelConcept, concept=modelConcept.qname) if val.validateSBRNL: if modelConcept.isTuple: if modelConcept.substitutionGroupQname.localName == "presentationTuple" and modelConcept.substitutionGroupQname.namespaceURI.endswith("/basis/sbr/xbrl/xbrl-syntax-extension"): # namespace may change each year definesPresentationTuples = True elif modelConcept.substitutionGroupQname.localName == "specificationTuple" and modelConcept.substitutionGroupQname.namespaceURI.endswith("/basis/sbr/xbrl/xbrl-syntax-extension"): # namespace may change each year definesSpecificationTuples = True else: definesTuples = True definesConcepts = True if modelConcept.isAbstract: val.modelXbrl.error("SBR.NL.2.2.2.03", _("Concept %(concept)s is an abstract tuple"), modelObject=modelConcept, concept=modelConcept.qname) if tupleCycle(val,modelConcept): val.modelXbrl.error("SBR.NL.2.2.2.07", _("Tuple %(concept)s has a tuple cycle"), modelObject=modelConcept, concept=modelConcept.qname) if modelConcept.get("nillable") != "false" and modelConcept.isRoot: val.modelXbrl.error("SBR.NL.2.2.2.17", #don't want default, just what was really there _("Tuple %(concept)s must have nillable='false'"), modelObject=modelConcept, concept=modelConcept.qname) elif modelConcept.isItem: definesConcepts = True if modelConcept.abstract == "true": if modelConcept.isRoot: if modelConcept.get("nillable") != "false": #don't want default, just what was really there val.modelXbrl.error("SBR.NL.2.2.2.16", _("Abstract root concept %(concept)s must have nillable='false'"), modelObject=modelConcept, concept=modelConcept.qname) if modelConcept.typeQname != XbrlConst.qnXbrliStringItemType: val.modelXbrl.error("SBR.NL.2.2.2.21", _("Abstract root concept %(concept)s must have type='xbrli:stringItemType'"), modelObject=modelConcept, concept=modelConcept.qname) if modelConcept.balance: val.modelXbrl.error("SBR.NL.2.2.2.22", _("Abstract concept %(concept)s must not have a balance attribute"), modelObject=modelConcept, concept=modelConcept.qname) if modelConcept.isHypercubeItem: definesHypercubes = True elif modelConcept.isDimensionItem: definesDimensions = True elif substititutionGroupQname and substititutionGroupQname.localName in ("domainItem","domainMemberItem"): definesDomains = True elif modelConcept.isItem: definesAbstractItems = True else: # not abstract if modelConcept.isItem: definesNonabstractItems = True if not (modelConcept.label(preferredLabel=XbrlConst.documentationLabel,fallbackToQname=False,lang="nl") or val.modelXbrl.relationshipSet(XbrlConst.conceptReference).fromModelObject(c) or modelConcept.genLabel(role=XbrlConst.genDocumentationLabel,lang="nl") or val.modelXbrl.relationshipSet(XbrlConst.elementReference).fromModelObject(c)): val.modelXbrl.error("SBR.NL.2.2.2.28", _("Concept %(concept)s must have a documentation label or reference"), modelObject=modelConcept, concept=modelConcept.qname) if modelConcept.balance and not modelConcept.instanceOfType(XbrlConst.qnXbrliMonetaryItemType): val.modelXbrl.error("SBR.NL.2.2.2.24", _("Non-monetary concept %(concept)s must not have a balance attribute"), modelObject=modelConcept, concept=modelConcept.qname) if modelConcept.isLinkPart: definesLinkParts = True val.modelXbrl.error("SBR.NL.2.2.5.01", _("Link:part concept %(concept)s is not allowed"), modelObject=modelConcept, concept=modelConcept.qname) if not modelConcept.genLabel(fallbackToQname=False,lang="nl"): val.modelXbrl.error("SBR.NL.2.2.5.02", _("Link part definition %(concept)s must have a generic label in language 'nl'"), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.8 check for embedded linkbase for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}linkbase"): if isinstance(e,ModelObject): val.modelXbrl.error(("EFM.6.07.08", "GFM.1.03.08"), _("Taxonomy schema %(schema)s contains an embedded linkbase"), modelObject=e, schema=modelDocument.basename) break requiredUsedOns = {XbrlConst.qnLinkPresentationLink, XbrlConst.qnLinkCalculationLink, XbrlConst.qnLinkDefinitionLink} standardUsedOns = {XbrlConst.qnLinkLabel, XbrlConst.qnLinkReference, XbrlConst.qnLinkDefinitionArc, XbrlConst.qnLinkCalculationArc, XbrlConst.qnLinkPresentationArc, XbrlConst.qnLinkLabelArc, XbrlConst.qnLinkReferenceArc, # per WH, private footnote arc and footnore resource roles are not allowed XbrlConst.qnLinkFootnoteArc, XbrlConst.qnLinkFootnote, } # 6.7.9 role types authority for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}roleType"): if isinstance(e,ModelObject): roleURI = e.get("roleURI") if targetNamespaceAuthority != UrlUtil.authority(roleURI): val.modelXbrl.error(("EFM.6.07.09", "GFM.1.03.09"), _("RoleType %(roleType)s does not match authority %(targetNamespaceAuthority)s"), modelObject=e, roleType=roleURI, targetNamespaceAuthority=targetNamespaceAuthority, targetNamespace=modelDocument.targetNamespace) # 6.7.9 end with .../role/lc3 name if not roleTypePattern.match(roleURI): val.modelXbrl.warning(("EFM.6.07.09.roleEnding", "GFM.1.03.09"), "RoleType %(roleType)s should end with /role/{LC3name}", modelObject=e, roleType=roleURI) # 6.7.10 only one role type declaration in DTS modelRoleTypes = val.modelXbrl.roleTypes.get(roleURI) if modelRoleTypes is not None: modelRoleType = modelRoleTypes[0] definition = modelRoleType.definitionNotStripped usedOns = modelRoleType.usedOns if len(modelRoleTypes) == 1: # 6.7.11 used on's for pre, cal, def if any has a used on if not usedOns.isdisjoint(requiredUsedOns) and len(requiredUsedOns - usedOns) > 0: val.modelXbrl.error(("EFM.6.07.11", "GFM.1.03.11"), _("RoleType %(roleType)s missing used on %(usedOn)s"), modelObject=e, roleType=roleURI, usedOn=requiredUsedOns - usedOns) # 6.7.12 definition match pattern if (val.disclosureSystem.roleDefinitionPattern is not None and (definition is None or not val.disclosureSystem.roleDefinitionPattern.match(definition))): val.modelXbrl.error(("EFM.6.07.12", "GFM.1.03.12-14"), _("RoleType %(roleType)s definition \"%(definition)s\" must match {Sortcode} - {Type} - {Title}"), modelObject=e, roleType=roleURI, definition=(definition or "")) if usedOns & standardUsedOns: # semantics check val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.03", "GFM.2.03.03"), _("RoleType %(roleuri)s is defined using role types already defined by standard roles for: %(qnames)s"), modelObject=e, roleuri=roleURI, qnames=', '.join(str(qn) for qn in usedOns & standardUsedOns)) if val.validateSBRNL: if usedOns & XbrlConst.standardExtLinkQnames or XbrlConst.qnGenLink in usedOns: definesLinkroles = True if not e.genLabel(): val.modelXbrl.error("SBR.NL.2.2.3.03", _("Link RoleType %(roleType)s missing a generic standard label"), modelObject=e, roleType=roleURI) nlLabel = e.genLabel(lang="nl") if definition != nlLabel: val.modelXbrl.error("SBR.NL.2.2.3.04", _("Link RoleType %(roleType)s definition does not match NL standard generic label, \ndefinition: %(definition)s \nNL label: %(label)s"), modelObject=e, roleType=roleURI, definition=definition, label=nlLabel) if definition and (definition[0].isspace() or definition[-1].isspace()): val.modelXbrl.error("SBR.NL.2.2.3.07", _('Link RoleType %(roleType)s definition has leading or trailing spaces: "%(definition)s"'), modelObject=e, roleType=roleURI, definition=definition) # 6.7.13 arcrole types authority for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}arcroleType"): if isinstance(e,ModelObject): arcroleURI = e.get("arcroleURI") if targetNamespaceAuthority != UrlUtil.authority(arcroleURI): val.modelXbrl.error(("EFM.6.07.13", "GFM.1.03.15"), _("ArcroleType %(arcroleType)s does not match authority %(targetNamespaceAuthority)s"), modelObject=e, arcroleType=arcroleURI, targetNamespaceAuthority=targetNamespaceAuthority, targetNamespace=modelDocument.targetNamespace) # 6.7.13 end with .../arcrole/lc3 name if not arcroleTypePattern.match(arcroleURI): val.modelXbrl.warning(("EFM.6.07.13.arcroleEnding", "GFM.1.03.15"), _("ArcroleType %(arcroleType)s should end with /arcrole/{LC3name}"), modelObject=e, arcroleType=arcroleURI) # 6.7.15 definition match pattern modelRoleTypes = val.modelXbrl.arcroleTypes[arcroleURI] definition = modelRoleTypes[0].definition if definition is None or not arcroleDefinitionPattern.match(definition): val.modelXbrl.error(("EFM.6.07.15", "GFM.1.03.17"), _("ArcroleType %(arcroleType)s definition must be non-empty"), modelObject=e, arcroleType=arcroleURI) # semantic checks usedOns = modelRoleTypes[0].usedOns if usedOns & standardUsedOns: # semantics check val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.03", "GFM.2.03.03"), _("ArcroleType %(arcroleuri)s is defined using role types already defined by standard arcroles for: %(qnames)s"), modelObject=e, arcroleuri=arcroleURI, qnames=', '.join(str(qn) for qn in usedOns & standardUsedOns)) if val.validateSBRNL: definesArcroles = True val.modelXbrl.error("SBR.NL.2.2.4.01", _("Arcrole type definition is not allowed: %(arcroleURI)s"), modelObject=e, arcroleURI=arcroleURI) if val.validateSBRNL: for appinfoElt in modelDocument.xmlRootElement.iter(tag="{http://www.w3.org/2001/XMLSchema}appinfo"): for nonLinkElt in appinfoElt.iterdescendants(): if isinstance(nonLinkElt, ModelObject) and nonLinkElt.namespaceURI != XbrlConst.link: val.modelXbrl.error("SBR.NL.2.2.11.05", _("Appinfo contains disallowed non-link element %(element)s"), modelObject=nonLinkElt, element=nonLinkElt.qname) for cplxTypeElt in modelDocument.xmlRootElement.iter(tag="{http://www.w3.org/2001/XMLSchema}complexType"): choiceElt = cplxTypeElt.find("{http://www.w3.org/2001/XMLSchema}choice") if choiceElt is not None: val.modelXbrl.error("SBR.NL.2.2.11.09", _("ComplexType contains disallowed xs:choice element"), modelObject=choiceElt) for cplxContentElt in modelDocument.xmlRootElement.iter(tag="{http://www.w3.org/2001/XMLSchema}complexContent"): if XmlUtil.descendantAttr(cplxContentElt, "http://www.w3.org/2001/XMLSchema", ("extension","restriction"), "base") != "sbr:placeholder": val.modelXbrl.error("SBR.NL.2.2.11.10", _("ComplexContent is disallowed"), modelObject=cplxContentElt) definesTypes = (modelDocument.xmlRootElement.find("{http://www.w3.org/2001/XMLSchema}complexType") is not None or modelDocument.xmlRootElement.find("{http://www.w3.org/2001/XMLSchema}simpleType") is not None) for enumElt in modelDocument.xmlRootElement.iter(tag="{http://www.w3.org/2001/XMLSchema}enumeration"): definesEnumerations = True if any(not valueElt.genLabel(lang="nl") for valueElt in enumElt.iter(tag="{http://www.w3.org/2001/XMLSchema}value")): val.modelXbrl.error("SBR.NL.2.2.7.05", _("Enumeration element has value(s) without generic label."), modelObject=enumElt) if (definesLinkroles + definesArcroles + definesLinkParts + definesAbstractItems + definesNonabstractItems + definesTuples + definesPresentationTuples + definesSpecificationTuples + definesTypes + definesEnumerations + definesDimensions + definesDomains + definesHypercubes) != 1: schemaContents = [] if definesLinkroles: schemaContents.append(_("linkroles")) if definesArcroles: schemaContents.append(_("arcroles")) if definesLinkParts: schemaContents.append(_("link parts")) if definesAbstractItems: schemaContents.append(_("abstract items")) if definesNonabstractItems: schemaContents.append(_("nonabstract items")) if definesTuples: schemaContents.append(_("tuples")) if definesPresentationTuples: schemaContents.append(_("sbrPresentationTuples")) if definesSpecificationTuples: schemaContents.append(_("sbrSpecificationTuples")) if definesTypes: schemaContents.append(_("types")) if definesEnumerations: schemaContents.append(_("enumerations")) if definesDimensions: schemaContents.append(_("dimensions")) if definesDomains: schemaContents.append(_("domains")) if definesHypercubes: schemaContents.append(_("hypercubes")) if schemaContents: if not ((definesTuples or definesPresentationTuples or definesSpecificationTuples) and not (definesLinkroles or definesArcroles or definesLinkParts or definesAbstractItems or definesTypes or definesDimensions or definesDomains or definesHypercubes)): val.modelXbrl.error("SBR.NL.2.2.1.01", _("Taxonomy schema may only define one of these: %(contents)s"), modelObject=modelDocument, contents=', '.join(schemaContents)) elif not any(refDoc.inDTS and refDoc.targetNamespace not in val.disclosureSystem.baseTaxonomyNamespaces for refDoc in modelDocument.referencesDocument.keys()): # no linkbase ref or includes val.modelXbrl.error("SBR.NL.2.2.1.01", _("Taxonomy schema must be a DTS entrypoint OR define linkroles OR arcroles OR link:parts OR context fragments OR abstract items OR tuples OR non-abstract elements OR types OR enumerations OR dimensions OR domains OR hypercubes"), modelObject=modelDocument) if definesConcepts ^ any( # xor so either concepts and no label LB or no concepts and has label LB (refDoc.type == ModelDocument.Type.LINKBASE and XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link, "labelLink") is not None) for refDoc in modelDocument.referencesDocument.keys()): # no label linkbase val.modelXbrl.error("SBR.NL.2.2.1.02", _("A schema that defines concepts MUST have a linked 2.1 label linkbase"), modelObject=modelDocument) if (definesNonabstractItems or definesTuples) and not any( # was xor but changed to and not per RH 1/11/12 (refDoc.type == ModelDocument.Type.LINKBASE and (XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link, "referenceLink") is not None or XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link, "label", "{http://www.w3.org/1999/xlink}role", "http://www.xbrl.org/2003/role/documentation" ) is not None)) for refDoc in modelDocument.referencesDocument.keys()): val.modelXbrl.error("SBR.NL.2.2.1.03", _("A schema that defines non-abstract items MUST have a linked (2.1) reference linkbase AND/OR a label linkbase with @xlink:role=documentation"), modelObject=modelDocument) #6.3.3 filename check m = re.match(r"^\w+-([12][0-9]{3}[01][0-9][0-3][0-9]).xsd$", modelDocument.basename) if m: try: # check date value datetime.datetime.strptime(m.group(1),"%Y%m%d").date() # date and format are ok, check "should" part of 6.3.3 if val.fileNameBasePart: expectedFilename = "{0}-{1}.xsd".format(val.fileNameBasePart, val.fileNameDatePart) if modelDocument.basename != expectedFilename: val.modelXbrl.log("WARNING-SEMANTIC", ("EFM.6.03.03.matchInstance", "GFM.1.01.01.matchInstance"), _('Schema file name warning: %(filename)s, should match %(expectedFilename)s'), modelObject=modelDocument, filename=modelDocument.basename, expectedFilename=expectedFilename) except ValueError: val.modelXbrl.error(("EFM.6.03.03", "GFM.1.01.01"), _('Invalid schema file base name part (date) in "{base}-{yyyymmdd}.xsd": %(filename)s'), modelObject=modelDocument, filename=modelDocument.basename) else: val.modelXbrl.error(("EFM.6.03.03", "GFM.1.01.01"), _('Invalid schema file name, must match "{base}-{yyyymmdd}.xsd": %(filename)s'), modelObject=modelDocument, filename=modelDocument.basename) elif modelDocument.type == ModelDocument.Type.LINKBASE: # if it is part of the submission (in same directory) check name if modelDocument.filepath.startswith(val.modelXbrl.modelDocument.filepathdir): #6.3.3 filename check extLinkElt = XmlUtil.descendant(modelDocument.xmlRootElement, XbrlConst.link, "*", "{http://www.w3.org/1999/xlink}type", "extended") if extLinkElt is None:# no ext link element val.modelXbrl.error(("EFM.6.03.03.noLinkElement", "GFM.1.01.01.noLinkElement"), _('Invalid linkbase file name: %(filename)s, has no extended link element, cannot determine link type.'), modelObject=modelDocument, filename=modelDocument.basename) elif extLinkElt.localName not in extLinkEltFileNameEnding: val.modelXbrl.error("EFM.6.03.02", _('Invalid linkbase link element %(linkElement)s in %(filename)s'), modelObject=modelDocument, linkElement=extLinkElt.localName, filename=modelDocument.basename) else: m = re.match(r"^\w+-([12][0-9]{3}[01][0-9][0-3][0-9])(_[a-z]{3}).xml$", modelDocument.basename) expectedSuffix = extLinkEltFileNameEnding[extLinkElt.localName] if m and m.group(2) == expectedSuffix: try: # check date value datetime.datetime.strptime(m.group(1),"%Y%m%d").date() # date and format are ok, check "should" part of 6.3.3 if val.fileNameBasePart: expectedFilename = "{0}-{1}{2}.xml".format(val.fileNameBasePart, val.fileNameDatePart, expectedSuffix) if modelDocument.basename != expectedFilename: val.modelXbrl.log("WARNING-SEMANTIC", ("EFM.6.03.03.matchInstance", "GFM.1.01.01.matchInstance"), _('Linkbase name warning: %(filename)s should match %(expectedFilename)s'), modelObject=modelDocument, filename=modelDocument.basename, expectedFilename=expectedFilename) except ValueError: val.modelXbrl.error(("EFM.6.03.03", "GFM.1.01.01"), _('Invalid linkbase base file name part (date) in "{base}-{yyyymmdd}_{suffix}.xml": %(filename)s'), modelObject=modelDocument, filename=modelDocument.basename) else: val.modelXbrl.error(("EFM.6.03.03", "GFM.1.01.01"), _('Invalid linkbase name, must match "{base}-{yyyymmdd}%(expectedSuffix)s.xml": %(filename)s'), modelObject=modelDocument, filename=modelDocument.basename, expectedSuffix=expectedSuffix) visited.remove(modelDocument)
def saveLoadableExcel(dts, excelFile): from arelle import ModelDocument, XmlUtil from openpyxl import Workbook, cell from openpyxl.styles import Font, PatternFill, Border, Alignment, Color, fills, Side from openpyxl.worksheet.dimensions import ColumnDimension workbook = Workbook(encoding="utf-8") # remove pre-existing worksheets while len(workbook.worksheets) > 0: workbook.remove_sheet(workbook.worksheets[0]) conceptsWs = workbook.create_sheet(title="Concepts") dtsWs = workbook.create_sheet(title="DTS") # identify type of taxonomy conceptsWsHeaders = None cellFontArgs = None for doc in dts.urlDocs.values(): if doc.type == ModelDocument.Type.SCHEMA and doc.inDTS: for i in range(len(headersStyles)): if re.match(headersStyles[i][0], doc.targetNamespace): cellFontArgs = headersStyles[i][ 1] # use as arguments to Font() conceptsWsHeaders = headersStyles[i][2] break if conceptsWsHeaders is None: dts.info( "error:saveLoadableExcel", _("Referenced taxonomy style not identified, assuming general pattern." ), modelObject=dts) cellFontArgs = headersStyles[-1][1] # use as arguments to Font() conceptsWsHeaders = headersStyles[-1][2] hdrCellFont = Font(**cellFontArgs) hdrCellFill = PatternFill( patternType=fills.FILL_SOLID, fgColor=Color("00FFBF5F")) # Excel's light orange fill color = 00FF990 cellFont = Font(**cellFontArgs) def writeCell(ws, row, col, value, fontBold=False, borders=True, indent=0, hAlign=None, vAlign=None, hdr=False): cell = ws.cell(row=row, column=col) cell.value = value if hdr: cell.font = hdrCellFont cell.fill = hdrCellFill if not hAlign: hAlign = "center" if not vAlign: vAlign = "center" else: cell.font = cellFont if not hAlign: hAlign = "left" if not vAlign: vAlign = "top" if borders: cell.border = Border(top=Side(border_style="thin"), left=Side(border_style="thin"), right=Side(border_style="thin"), bottom=Side(border_style="thin")) cell.alignment = Alignment(horizontal=hAlign, vertical=vAlign, wrap_text=True, indent=indent) # sheet 1 col widths for i, hdr in enumerate(conceptsWsHeaders): colLetter = cell.get_column_letter(i + 1) conceptsWs.column_dimensions[colLetter] = ColumnDimension( conceptsWs, customWidth=True) conceptsWs.column_dimensions[colLetter].width = headerWidths.get( hdr[1], 40) # sheet 2 headers for i, hdr in enumerate(dtsWsHeaders): colLetter = cell.get_column_letter(i + 1) dtsWs.column_dimensions[colLetter] = ColumnDimension(conceptsWs, customWidth=True) dtsWs.column_dimensions[colLetter].width = hdr[1] writeCell(dtsWs, 1, i + 1, hdr[0], hdr=True) # referenced taxonomies conceptsRow = 1 dtsRow = 3 # identify extension schema extensionSchemaDoc = None if dts.modelDocument.type == ModelDocument.Type.SCHEMA: extensionSchemaDoc = dts.modelDocument elif dts.modelDocument.type == ModelDocument.Type.INSTANCE: for doc, docReference in dts.modelDocument.referencesDocument.items(): if docReference.referenceType == "href": extensionSchemaDoc = doc break if extensionSchemaDoc is None: dts.info("error:saveLoadableExcel", _("Unable to identify extension taxonomy."), modelObject=dts) return for doc, docReference in extensionSchemaDoc.referencesDocument.items(): if docReference.referenceType == "import" and doc.targetNamespace != XbrlConst.xbrli: writeCell(dtsWs, dtsRow, 1, "import") writeCell(dtsWs, dtsRow, 2, "schema") writeCell( dtsWs, dtsRow, 3, XmlUtil.xmlnsprefix(doc.xmlRootElement, doc.targetNamespace)) writeCell(dtsWs, dtsRow, 4, doc.uri) writeCell(dtsWs, dtsRow, 5, doc.targetNamespace) dtsRow += 1 dtsRow += 1 doc = extensionSchemaDoc writeCell(dtsWs, dtsRow, 1, "extension") writeCell(dtsWs, dtsRow, 2, "schema") writeCell(dtsWs, dtsRow, 3, XmlUtil.xmlnsprefix(doc.xmlRootElement, doc.targetNamespace)) writeCell(dtsWs, dtsRow, 4, os.path.basename(doc.uri)) writeCell(dtsWs, dtsRow, 5, doc.targetNamespace) dtsRow += 1 for doc, docReference in extensionSchemaDoc.referencesDocument.items(): if docReference.referenceType == "href" and doc.type == ModelDocument.Type.LINKBASE: linkbaseType = "" role = docReference.referringModelObject.get( "{http://www.w3.org/1999/xlink}role") or "" if role.startswith("http://www.xbrl.org/2003/role/" ) and role.endswith("LinkbaseRef"): linkbaseType = os.path.basename(role)[0:-11] writeCell(dtsWs, dtsRow, 1, "extension") writeCell(dtsWs, dtsRow, 2, "linkbase") writeCell(dtsWs, dtsRow, 3, linkbaseType) writeCell(dtsWs, dtsRow, 4, os.path.basename(doc.uri)) writeCell(dtsWs, dtsRow, 5, "") dtsRow += 1 dtsRow += 1 # extended link roles defined in this document for roleURI, roleTypes in sorted( dts.roleTypes.items(), # sort on definition if any else URI key=lambda item: (item[1][0].definition or item[0])): for roleType in roleTypes: if roleType.modelDocument == extensionSchemaDoc: writeCell(dtsWs, dtsRow, 1, "extension") writeCell(dtsWs, dtsRow, 2, "role") writeCell(dtsWs, dtsRow, 3, "") writeCell(dtsWs, dtsRow, 4, roleType.definition) writeCell(dtsWs, dtsRow, 5, roleURI) dtsRow += 1 # tree walk recursive function def treeWalk(row, depth, concept, preferredLabel, arcrole, preRelSet, visited): if concept is not None: # calc parents calcRelSet = dts.relationshipSet(XbrlConst.summationItem, preRelSet.linkrole) calcRel = None for modelRel in calcRelSet.toModelObject(concept): calcRel = modelRel break for i, hdr in enumerate(conceptsWsHeaders): colType = hdr[1] value = "" if colType == "name": value = str(concept.name) elif colType == "prefix" and concept.qname is not None: value = concept.qname.prefix elif colType == "type" and concept.type is not None: value = str(concept.type.qname) elif colType == "substitutionGroup": value = str(concept.substitutionGroupQname) elif colType == "abstract": value = "true" if concept.isAbstract else "false" elif colType == "nillable": if concept.isNillable: value = "true" elif colType == "periodType": value = concept.periodType elif colType == "balance": value = concept.balance elif colType == "label": role = hdr[2] lang = hdr[3] if role == XbrlConst.standardLabel: if "indented" in hdr: roleUri = preferredLabel elif "overridePreferred" in hdr: if preferredLabel and preferredLabel != XbrlConst.standardLabel: roleUri = role else: roleUri = "**no value**" # skip putting a value in this column else: roleUri = role else: roleUri = role if roleUri != "**no value**": value = concept.label( roleUri, linkroleHint=preRelSet.linkrole, lang=lang, fallbackToQname=(role == XbrlConst.standardLabel)) elif colType == "preferredLabel" and preferredLabel: if preferredLabel.startswith( "http://www.xbrl.org/2003/role/"): value = os.path.basename(preferredLabel) else: value = preferredLabel elif colType == "calculationParent" and calcRel is not None: calcParent = calcRel.fromModelObject if calcParent is not None: value = str(calcParent.qname) elif colType == "calculationWeight" and calcRel is not None: value = calcRel.weight elif colType == "depth": value = depth if "indented" in hdr: indent = min(depth, MAXINDENT) else: indent = 0 writeCell(conceptsWs, row, i + 1, value, indent=indent) row += 1 if concept not in visited: visited.add(concept) for modelRel in preRelSet.fromModelObject(concept): if modelRel.toModelObject is not None: row = treeWalk(row, depth + 1, modelRel.toModelObject, modelRel.preferredLabel, arcrole, preRelSet, visited) visited.remove(concept) return row # use presentation relationships for conceptsWs arcrole = XbrlConst.parentChild # sort URIs by definition linkroleUris = [] relationshipSet = dts.relationshipSet(arcrole) if relationshipSet: for linkroleUri in relationshipSet.linkRoleUris: modelRoleTypes = dts.roleTypes.get(linkroleUri) if modelRoleTypes: roledefinition = (modelRoleTypes[0].genLabel(strip=True) or modelRoleTypes[0].definition or linkroleUri) else: roledefinition = linkroleUri linkroleUris.append((roledefinition, linkroleUri)) linkroleUris.sort() # for each URI in definition order for roledefinition, linkroleUri in linkroleUris: # write linkrole writeCell( conceptsWs, conceptsRow, 1, (roledefinition or linkroleUri), borders=False) # ELR has no boarders, just font specified conceptsRow += 1 # write header row for i, hdr in enumerate(conceptsWsHeaders): writeCell(conceptsWs, conceptsRow, i + 1, hdr[0], hdr=True) conceptsRow += 1 # elr relationships for tree walk linkRelationshipSet = dts.relationshipSet(arcrole, linkroleUri) for rootConcept in linkRelationshipSet.rootConcepts: conceptsRow = treeWalk(conceptsRow, 0, rootConcept, None, arcrole, linkRelationshipSet, set()) conceptsRow += 1 # double space rows between tables else: # write header row for i, hdr in enumerate(conceptsWsHeaders): writeCell(conceptsWs, conceptsRow, i, hdr[0], hdr=True) conceptsRow += 1 # get lang lang = None for i, hdr in enumerate(conceptsWsHeaders): colType = hdr[1] if colType == "label": lang = hdr[3] if colType == "label": role = hdr[2] lang = hdr[3] lbls = defaultdict(list) for concept in set(dts.qnameConcepts.values( )): # may be twice if unqualified, with and without namespace lbls[concept.label(role, lang=lang)].append(concept.objectId()) srtLbls = sorted(lbls.keys()) excludedNamespaces = XbrlConst.ixbrlAll.union( (XbrlConst.xbrli, XbrlConst.link, XbrlConst.xlink, XbrlConst.xl, XbrlConst.xbrldt, XbrlConst.xhtml)) for label in srtLbls: for objectId in lbls[label]: concept = dts.modelObject(objectId) if concept.modelDocument.targetNamespace not in excludedNamespaces: for i, hdr in enumerate(conceptsWsHeaders): colType = hdr[1] value = "" if colType == "name": value = str(concept.qname.localName) elif colType == "prefix": value = concept.qname.prefix elif colType == "type": value = str(concept.type.qname) elif colType == "substitutionGroup": value = str(concept.substitutionGroupQname) elif colType == "abstract": value = "true" if concept.isAbstract else "false" elif colType == "periodType": value = concept.periodType elif colType == "balance": value = concept.balance elif colType == "label": role = hdr[2] lang = hdr[3] value = concept.label(role, lang=lang) elif colType == "depth": value = 0 if "indented" in hdr: indent = min(0, MAXINDENT) else: indent = 0 writeCell(conceptsWs, conceptsRow, i, value, indent=indent) conceptsRow += 1 try: workbook.save(excelFile) dts.info("info:saveLoadableExcel", _("Saved Excel file: %(excelFile)s"), excelFile=os.path.basename(excelFile), modelXbrl=dts) except Exception as ex: dts.error("exception:saveLoadableExcel", _("File saving exception: %(error)s"), error=ex, modelXbrl=dts)
def saveLoadableExcel(dts, excelFile): from arelle import ModelDocument, XmlUtil, xlwt workbook = xlwt.Workbook(encoding="utf-8") sheet1 = workbook.add_sheet("Sheet1") sheet2 = workbook.add_sheet("Sheet2") # identify type of taxonomy sheet1Headers = None cellFont = None for doc in dts.urlDocs.values(): if doc.type == ModelDocument.Type.SCHEMA and doc.inDTS: for i in range(len(headersStyles)): if re.match(headersStyles[i][0], doc.targetNamespace): cellFont = headersStyles[i][1] sheet1Headers = headersStyles[i][2] break if sheet1Headers is None: dts.info( "error:saveLoadableExcel", _("Referenced taxonomy style not identified, assuming general pattern." ), modelObject=dts) cellFont = headersStyles[-1][1] sheet1Headers = headersStyles[-1][2] hdrCellFmt = xlwt.easyxf( cellFont + "align: wrap on, vert center, horiz center; " "pattern: pattern solid_fill, fore_color light_orange; " "border: top thin, right thin, bottom thin, left thin; ") # sheet 1 col widths for i, hdr in enumerate(sheet1Headers): sheet1.col(i).width = 256 * headerWidths.get(hdr[1], 40) # sheet 2 headers for i, hdr in enumerate(sheet2Headers): sheet2.col(i).width = 256 * hdr[1] sheet2.write(0, i, hdr[0], hdrCellFmt) # referenced taxonomies sheet1row = 0 sheet2row = 2 cellFmt = xlwt.easyxf( cellFont + "border: top thin, right thin, bottom thin, left thin; " "align: wrap on, vert top, horiz left;") cellFmtIndented = dict( (i, xlwt.easyxf( cellFont + "border: top thin, right thin, bottom thin, left thin; " "align: wrap on, vert top, horiz left, indent {0};".format(i))) for i in range(16)) # identify extension schema extensionSchemaDoc = None if dts.modelDocument.type == ModelDocument.Type.SCHEMA: extensionSchemaDoc = dts.modelDocument elif dts.modelDocument.type == ModelDocument.Type.INSTANCE: for doc, docReference in dts.modelDocument.referencesDocument.items(): if docReference.referenceType == "href": extensionSchemaDoc = doc break if extensionSchemaDoc is None: dts.info("error:saveLoadableExcel", _("Unable to identify extension taxonomy."), modelObject=dts) return for doc, docReference in extensionSchemaDoc.referencesDocument.items(): if docReference.referenceType == "import" and doc.targetNamespace != XbrlConst.xbrli: sheet2.write(sheet2row, 0, "import", cellFmt) sheet2.write(sheet2row, 1, "schema", cellFmt) sheet2.write( sheet2row, 2, XmlUtil.xmlnsprefix(doc.xmlRootElement, doc.targetNamespace), cellFmt) sheet2.write(sheet2row, 3, doc.uri, cellFmt) sheet2.write(sheet2row, 4, doc.targetNamespace, cellFmt) sheet2row += 1 sheet2row += 1 doc = extensionSchemaDoc sheet2.write(sheet2row, 0, "extension", cellFmt) sheet2.write(sheet2row, 1, "schema", cellFmt) sheet2.write(sheet2row, 2, XmlUtil.xmlnsprefix(doc.xmlRootElement, doc.targetNamespace), cellFmt) sheet2.write(sheet2row, 3, os.path.basename(doc.uri), cellFmt) sheet2.write(sheet2row, 4, doc.targetNamespace, cellFmt) sheet2row += 1 for doc, docReference in extensionSchemaDoc.referencesDocument.items(): if docReference.referenceType == "href" and doc.type == ModelDocument.Type.LINKBASE: linkbaseType = "" role = docReference.referringModelObject.get( "{http://www.w3.org/1999/xlink}role") or "" if role.startswith("http://www.xbrl.org/2003/role/" ) and role.endswith("LinkbaseRef"): linkbaseType = os.path.basename(role)[0:-11] sheet2.write(sheet2row, 0, "extension", cellFmt) sheet2.write(sheet2row, 1, "linkbase", cellFmt) sheet2.write(sheet2row, 2, linkbaseType, cellFmt) sheet2.write(sheet2row, 3, os.path.basename(doc.uri), cellFmt) sheet2.write(sheet2row, 4, "", cellFmt) sheet2row += 1 sheet2row += 1 # extended link roles defined in this document for roleURI, roleTypes in sorted( dts.roleTypes.items(), # sort on definition if any else URI key=lambda item: (item[1][0].definition or item[0])): for roleType in roleTypes: if roleType.modelDocument == extensionSchemaDoc: sheet2.write(sheet2row, 0, "extension", cellFmt) sheet2.write(sheet2row, 1, "role", cellFmt) sheet2.write(sheet2row, 2, "", cellFmt) sheet2.write(sheet2row, 3, roleType.definition, cellFmt) sheet2.write(sheet2row, 4, roleURI, cellFmt) sheet2row += 1 # tree walk recursive function def treeWalk(row, depth, concept, preferredLabel, arcrole, preRelSet, visited): if concept is not None: # calc parents calcRelSet = dts.relationshipSet(XbrlConst.summationItem, preRelSet.linkrole) calcRel = None for modelRel in calcRelSet.toModelObject(concept): calcRel = modelRel break for i, hdr in enumerate(sheet1Headers): colType = hdr[1] value = "" if colType == "name": value = str(concept.name) elif colType == "prefix" and concept.qname is not None: value = concept.qname.prefix elif colType == "type" and concept.type is not None: value = str(concept.type.qname) elif colType == "substitutionGroup": value = str(concept.substitutionGroupQname) elif colType == "abstract": value = "true" if concept.isAbstract else "false" elif colType == "periodType": value = concept.periodType elif colType == "balance": value = concept.balance elif colType == "label": role = hdr[2] lang = hdr[3] value = concept.label(preferredLabel if role == XbrlConst.standardLabel else role, linkroleHint=preRelSet.linkrole, lang=lang) elif colType == "preferredLabel" and preferredLabel: if preferredLabel.startswith( "http://www.xbrl.org/2003/role/"): value = os.path.basename(preferredLabel) else: value = preferredLabel elif colType == "calculationParent" and calcRel is not None: calcParent = calcRel.fromModelObject if calcParent is not None: value = str(calcParent.qname) elif colType == "calculationWeight" and calcRel is not None: value = calcRel.weight elif colType == "depth": value = depth if "indented" in hdr: style = cellFmtIndented[min(depth, len(cellFmtIndented) - 1)] else: style = cellFmt sheet1.write(row, i, value, style) row += 1 if concept not in visited: visited.add(concept) for modelRel in preRelSet.fromModelObject(concept): if modelRel.toModelObject is not None: row = treeWalk(row, depth + 1, modelRel.toModelObject, modelRel.preferredLabel, arcrole, preRelSet, visited) visited.remove(concept) return row # use presentation relationships for Sheet1 arcrole = XbrlConst.parentChild # sort URIs by definition linkroleUris = [] relationshipSet = dts.relationshipSet(arcrole) if relationshipSet: for linkroleUri in relationshipSet.linkRoleUris: modelRoleTypes = dts.roleTypes.get(linkroleUri) if modelRoleTypes: roledefinition = (modelRoleTypes[0].genLabel(strip=True) or modelRoleTypes[0].definition or linkroleUri) else: roledefinition = linkroleUri linkroleUris.append((roledefinition, linkroleUri)) linkroleUris.sort() # for each URI in definition order for roledefinition, linkroleUri in linkroleUris: # write linkrole sheet1.write( sheet1row, 0, (roledefinition or linkroleUri), xlwt.easyxf( cellFont)) # ELR has no boarders, just font specified sheet1row += 1 # write header row for i, hdr in enumerate(sheet1Headers): sheet1.write(sheet1row, i, hdr[0], hdrCellFmt) sheet1row += 1 # elr relationships for tree walk linkRelationshipSet = dts.relationshipSet(arcrole, linkroleUri) for rootConcept in linkRelationshipSet.rootConcepts: sheet1row = treeWalk(sheet1row, 0, rootConcept, None, arcrole, linkRelationshipSet, set()) sheet1row += 1 # double space rows between tables else: # write header row for i, hdr in enumerate(sheet1Headers): sheet1.write(sheet1row, i, hdr[0], hdrCellFmt) sheet1row += 1 # get lang lang = None for i, hdr in enumerate(sheet1Headers): colType = hdr[1] if colType == "label": lang = hdr[3] if colType == "label": role = hdr[2] lang = hdr[3] lbls = defaultdict(list) for concept in set(dts.qnameConcepts.values( )): # may be twice if unqualified, with and without namespace lbls[concept.label(role, lang=lang)].append(concept.objectId()) srtLbls = sorted(lbls.keys()) excludedNamespaces = XbrlConst.ixbrlAll.union( (XbrlConst.xbrli, XbrlConst.link, XbrlConst.xlink, XbrlConst.xl, XbrlConst.xbrldt, XbrlConst.xhtml)) for label in srtLbls: for objectId in lbls[label]: concept = dts.modelObject(objectId) if concept.modelDocument.targetNamespace not in excludedNamespaces: for i, hdr in enumerate(sheet1Headers): colType = hdr[1] value = "" if colType == "name": value = str(concept.qname.localName) elif colType == "prefix": value = concept.qname.prefix elif colType == "type": value = str(concept.type.qname) elif colType == "substitutionGroup": value = str(concept.substitutionGroupQname) elif colType == "abstract": value = "true" if concept.isAbstract else "false" elif colType == "periodType": value = concept.periodType elif colType == "balance": value = concept.balance elif colType == "label": role = hdr[2] lang = hdr[3] value = concept.label(role, lang=lang) elif colType == "depth": value = 0 if "indented" in hdr: style = cellFmtIndented[min( 0, len(cellFmtIndented) - 1)] else: style = cellFmt sheet1.write(sheet1row, i, value, style) sheet1row += 1 try: workbook.save(excelFile) dts.info("info:saveLoadableExcel", _("Saved Excel file: %(excelFile)s"), excelFile=os.path.basename(excelFile), modelXbrl=dts) except Exception as ex: dts.error("exception:saveLoadableExcel", _("File saving exception: %(error)s"), error=ex, modelXbrl=dts)
def final(val): if not (val.validateEBA or val.validateEIOPA): return modelXbrl = val.modelXbrl modelDocument = modelXbrl.modelDocument _statusMsg = _("validating {0} filing rules").format( val.disclosureSystem.name) modelXbrl.profileActivity() modelXbrl.modelManager.showStatus(_statusMsg) if modelDocument.type == ModelDocument.Type.INSTANCE and ( val.validateEBA or val.validateEIOPA): if not modelDocument.uri.endswith(".xbrl"): modelXbrl.warning( "EBA.1.1", _('XBRL instance documents SHOULD use the extension ".xbrl" but it is "%(extension)s"' ), modelObject=modelDocument, extension=os.path.splitext(modelDocument.basename)[1]) if modelDocument.documentEncoding.lower() not in ("utf-8", "utf-8-sig"): modelXbrl.error( "EBA.1.4", _('XBRL instance documents MUST use "UTF-8" encoding but is "%(xmlEncoding)s"' ), modelObject=modelDocument, xmlEncoding=modelDocument.documentEncoding) schemaRefElts = [] schemaRefFileNames = [] for doc, docRef in modelDocument.referencesDocument.items(): if docRef.referenceType == "href": if docRef.referringModelObject.localName == "schemaRef": schemaRefElts.append(docRef.referringModelObject) schemaRefFileNames.append(doc.basename) if not UrlUtil.isAbsolute(doc.uri): modelXbrl.error( "EBA.2.2", _('The link:schemaRef element in submitted instances MUST resolve to the full published entry point URL: %(url)s.' ), modelObject=docRef.referringModelObject, url=doc.uri) elif docRef.referringModelObject.localName == "linkbaseRef": modelXbrl.error( "EBA.2.3", _('The link:linkbaseRef element is not allowed: %(fileName)s.' ), modelObject=docRef.referringModelObject, fileName=doc.basename) if len(schemaRefFileNames) > 1: modelXbrl.error( "EBA.1.5", _('XBRL instance documents MUST reference only one entry point schema but %(numEntryPoints)s were found: %(entryPointNames)s' ), modelObject=modelDocument, numEntryPoints=len(schemaRefFileNames), entryPointNames=', '.join(sorted(schemaRefFileNames))) ### check entry point names appropriate for filing indicator (DPM DB?) if len(schemaRefElts) != 1: modelXbrl.error( "EBA.2.3", _('Any reported XBRL instance document MUST contain only one xbrli:xbrl/link:schemaRef node, but %(entryPointCount)s.' ), modelObject=schemaRefElts, entryPointCount=len(schemaRefElts)) # non-streaming EBA checks if not getattr(modelXbrl, "isStreamingMode", False): validateFacts(val, modelXbrl.facts) # check sum of fact md5s (otherwise checked in streaming process) xbrlFactsCheckVersion = None expectedSumOfFactMd5s = None for pi in modelDocument.xmlRootElement.getchildren(): if isinstance(pi, etree._ProcessingInstruction ) and pi.target == "xbrl-facts-check": _match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']", pi.text) if _match: _matchGroups = _match.groups() if len(_matchGroups) == 2: if _matchGroups[0] == "version": xbrlFactsCheckVersion = _matchGroups[1] elif _matchGroups[0] == "sum-of-fact-md5s": try: expectedSumOfFactMd5s = Md5Sum( _matchGroups[1]) except ValueError: modelXbrl.error( "EIOPA:xbrlFactsCheckError", _("Invalid sum-of-md5s %(sumOfMd5)s"), modelObject=modelXbrl, sumOfMd5=_matchGroups[1]) if xbrlFactsCheckVersion and expectedSumOfFactMd5s: sumOfFactMd5s = Md5Sum() for f in modelXbrl.factsInInstance: sumOfFactMd5s += f.md5sum if sumOfFactMd5s != expectedSumOfFactMd5s: modelXbrl.warning( "EIOPA:xbrlFactsCheckWarning", _("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s" ), modelObject=modelXbrl, expectedMd5=expectedSumOfFactMd5s, actualMd5Sum=sumOfFactMd5s) else: modelXbrl.info("info", _("Successful XBRL facts sum of md5s."), modelObject=modelXbrl) if not val.filingIndicators: modelXbrl.error( "EBA.1.6", _('Missing filing indicators. Reported XBRL instances MUST include appropriate filing indicator elements' ), modelObject=modelDocument) if val.numFilingIndicatorTuples > 1: modelXbrl.info( "EBA.1.6.2", _('Multiple filing indicators tuples when not in streaming mode (info).' ), modelObject=modelXbrl.factsByQname[qnFIndicators]) if len(val.cntxDates) > 1: modelXbrl.error( "EBA.2.13", _('Contexts must have the same date: %(dates)s.'), # when streaming values are no longer available, but without streaming they can be logged modelObject=set(_cntx for _cntxs in val.cntxDates.values() for _cntx in _cntxs), dates=', '.join( XmlUtil.dateunionValue(_dt, subtractOneDay=True) for _dt in val.cntxDates.keys())) if val.unusedCntxIDs: modelXbrl.warning( "EBA.2.7", _('Unused xbrli:context nodes SHOULD NOT be present in the instance: %(unusedContextIDs)s.' ), modelObject=[ modelXbrl.contexts[unusedCntxID] for unusedCntxID in val.unusedCntxIDs if unusedCntxID in modelXbrl.contexts ], unusedContextIDs=", ".join(sorted(val.unusedCntxIDs))) if len(val.cntxEntities) > 1: modelXbrl.warning( "EBA.2.9", _('All entity identifiers and schemes must be the same, %(count)s found: %(entities)s.' ), modelObject=modelDocument, count=len(val.cntxEntities), entities=", ".join( sorted(str(cntxEntity) for cntxEntity in val.cntxEntities))) if val.unusedUnitIDs: modelXbrl.warning( "EBA.2.21", _('Unused xbrli:unit nodes SHOULD NOT be present in the instance: %(unusedUnitIDs)s.' ), modelObject=[ modelXbrl.units[unusedUnitID] for unusedUnitID in val.unusedUnitIDs if unusedUnitID in modelXbrl.units ], unusedUnitIDs=", ".join(sorted(val.unusedUnitIDs))) if len(val.currenciesUsed) > 1: modelXbrl.error( "EBA.3.1", _("There MUST be only one currency but %(numCurrencies)s were found: %(currencies)s.'" ), modelObject=val.currenciesUsed.values(), numCurrencies=len(val.currenciesUsed), currencies=", ".join( str(c) for c in val.currenciesUsed.keys())) if val.prefixesUnused: modelXbrl.warning( "EBA.3.4", _("There SHOULD be no unused prefixes but these were declared: %(unusedPrefixes)s.'" ), modelObject=modelDocument, unusedPrefixes=', '.join(sorted(val.prefixesUnused))) for ns, prefixes in val.namespacePrefixesUsed.items(): nsDocs = modelXbrl.namespaceDocs.get(ns) if nsDocs: for nsDoc in nsDocs: nsDocPrefix = XmlUtil.xmlnsprefix(nsDoc.xmlRootElement, ns) if any(prefix != nsDocPrefix for prefix in prefixes if prefix is not None): modelXbrl.warning( "EBA.3.5", _("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s" ), modelObject=modelDocument, namespace=ns, declaredPrefix=nsDocPrefix, foundPrefixes=', '.join(sorted(prefixes - {None}))) modelXbrl.profileActivity(_statusMsg, minTimeToShow=0.0) modelXbrl.modelManager.showStatus(None) del val.prefixNamespace, val.namespacePrefix, val.idObjects, val.typedDomainElements del val.utrValidator
def checkDTS(val, modelDocument, visited): visited.append(modelDocument) definesLabelLinkbase = False for referencedDocument in modelDocument.referencesDocument.items(): #6.07.01 no includes if referencedDocument[1] == "include": val.modelXbrl.error( _("Taxonomy schema {0} includes {1}, only import is allowed").format( os.path.basename(modelDocument.uri), os.path.basename(referencedDocument[0].uri)), "err", "EFM.6.07.01", "GFM.1.03.01", "SBR.NL.2.2.0.18") if referencedDocument[0] not in visited: checkDTS(val, referencedDocument[0], visited) if val.disclosureSystem.standardTaxonomiesDict is None: pass if (modelDocument.type == ModelDocument.Type.SCHEMA and modelDocument.targetNamespace not in val.disclosureSystem.baseTaxonomyNamespaces and modelDocument.uri.startswith(val.modelXbrl.uriDir)): # check schema contents types if val.validateSBRNL: definesConcepts = False definesLinkroles = False definesArcroles = False definesLinkParts = False definesAbstractItems = False definesNonabstractItems = False definesTuples = False definesEnumerations = False definesDimensions = False definesDomains = False definesHypercubes = False # 6.7.3 check namespace for standard authority targetNamespaceAuthority = UrlUtil.authority(modelDocument.targetNamespace) if targetNamespaceAuthority in val.disclosureSystem.standardAuthorities: val.modelXbrl.error( _("Taxonomy schema {0} namespace {1} is a disallowed authority").format( os.path.basename(modelDocument.uri), modelDocument.targetNamespace), "err", "EFM.6.07.03", "GFM.1.03.03") # 6.7.4 check namespace format if modelDocument.targetNamespace is None: match = None elif val.validateEFMorGFM: targetNamespaceDate = modelDocument.targetNamespace[len(targetNamespaceAuthority):] match = val.targetNamespaceDatePattern.match(targetNamespaceDate) else: match = None if match is not None: try: if match.lastindex == 3: datetime.date(int(match.group(1)),int(match.group(2)),int(match.group(3))) elif match.lastindex == 6: datetime.date(int(match.group(4)),int(match.group(5)),int(match.group(6))) else: match = None except ValueError: match = None if match is None: val.modelXbrl.error( _("Taxonomy schema {0} namespace {1} must have format http://{2}authority{3}/{2}versionDate{3}").format( os.path.basename(modelDocument.uri), modelDocument.targetNamespace, "{", "}"), "err", "EFM.6.07.04", "GFM.1.03.04") if modelDocument.targetNamespace is not None: # 6.7.5 check prefix for _ prefix = XmlUtil.xmlnsprefix(modelDocument.xmlRootElement,modelDocument.targetNamespace) if prefix and "_" in prefix: val.modelXbrl.error( _("Taxonomy schema {0} namespace {1} prefix {2} must not have an '_'").format( os.path.basename(modelDocument.uri), modelDocument.targetNamespace, prefix), "err", "EFM.6.07.07", "GFM.1.03.07") for eltName in modelDocument.xmlRootElement.getElementsByTagNameNS(XbrlConst.xsd, "element"): # 6.7.16 name not duplicated in standard taxonomies name = eltName.getAttribute("name") concepts = val.modelXbrl.nameConcepts.get(name) modelConcept = None if concepts is not None: for c in concepts: if c.modelDocument == modelDocument: modelConcept = c elif (val.validateEFMorGFM and not c.modelDocument.uri.startswith(val.modelXbrl.uriDir)): val.modelXbrl.error( _("Taxonomy schema {0} element {1} is also defined in standard taxonomy schema {2}").format( os.path.basename(modelDocument.uri), name, os.path.basename(c.modelDocument.uri)), "err", "EFM.6.07.16", "GFM.1.03.18") if val.validateSBRNL: for c in concepts: if c.modelDocument != modelDocument: relSet = val.modelXbrl.relationshipSet(XbrlConst.generalSpecial) if not (relSet.isRelated(modelConcept, "child", c) or relSet.isRelated(modelConcept, "child", c)): val.modelXbrl.error( _("Taxonomy schema {0} element {1} is also defined in standard taxonomy schema {2} without a general-special relationship").format( os.path.basename(modelDocument.uri), name, os.path.basename(c.modelDocument.uri)), "err", "SBR.NL.2.2.2.02") # 6.7.17 id properly formed id = eltName.getAttribute("id") requiredId = str(prefix) + "_" + name if val.validateEFMorGFM and id != requiredId: val.modelXbrl.error( _("Taxonomy schema {0} element {1} id {2} should be {3}").format( os.path.basename(modelDocument.uri), name, id, requiredId), "err", "EFM.6.07.17", "GFM.1.03.19") # 6.7.18 nillable is true nillable = eltName.getAttribute("nillable") if nillable != "true": val.modelXbrl.error( _("Taxonomy schema {0} element {1} nillable {2} should be 'true'").format( os.path.basename(modelDocument.uri), name, nillable), "err", "EFM.6.07.18", "GFM.1.03.20") if modelConcept is not None: # 6.7.19 not tuple if modelConcept.isTuple: if val.validateEFMorGFM: val.modelXbrl.error( _("Taxonomy schema {0} element {1} is a tuple").format( os.path.basename(modelDocument.uri), name), "err", "EFM.6.07.19", "GFM.1.03.21") # 6.7.20 no typed domain ref if modelConcept.typedDomainRefQname is not None: val.modelXbrl.error( _("Taxonomy schema {0} element {1} has typedDomainRef {2}").format( os.path.basename(modelDocument.uri), name, modelConcept.typedDomainRefQname), "err", "EFM.6.07.20", "GFM.1.03.22") # 6.7.21 abstract must be duration isDuration = modelConcept.periodType == "duration" if modelConcept.abstract == "true" and not isDuration: val.modelXbrl.error( _("Taxonomy schema {0} element {1} is abstract but period type is not duration").format( os.path.basename(modelDocument.uri), name), "err", "EFM.6.07.21", "GFM.1.03.23") # 6.7.22 abstract must be stringItemType ''' removed SEC EFM v.17, Edgar release 10.4, and GFM 2011-04-08 if modelConcept.abstract == "true" and modelConcept.typeQname != XbrlConst. qnXbrliStringItemType: val.modelXbrl.error( _("Taxonomy schema {0} element {1} is abstract but type is not xbrli:stringItemType").format( os.path.basename(modelDocument.uri), name), "err", "EFM.6.07.22", "GFM.1.03.24") ''' substititutionGroupQname = modelConcept.substitutionGroupQname # 6.7.23 Axis must be subs group dimension if name.endswith("Axis") ^ (substititutionGroupQname == XbrlConst.qnXbrldtDimensionItem): val.modelXbrl.error( _("Taxonomy schema {0} element {1} must end in Axis to be in dimensionItem substitution group").format( os.path.basename(modelDocument.uri), name), "err", "EFM.6.07.23", "GFM.1.03.25") # 6.7.24 Table must be subs group hypercube if name.endswith("Table") ^ (substititutionGroupQname == XbrlConst.qnXbrldtHypercubeItem): val.modelXbrl.error( _("Taxonomy schema {0} element {1} is an Axis but not in hypercubeItem substitution group").format( os.path.basename(modelDocument.uri), name), "err", "EFM.6.07.24", "GFM.1.03.26") # 6.7.25 if neither hypercube or dimension, substitution group must be item if substititutionGroupQname not in (None, XbrlConst.qnXbrldtDimensionItem, XbrlConst.qnXbrldtHypercubeItem, XbrlConst.qnXbrliItem): val.modelXbrl.error( _("Taxonomy schema {0} element {1} has disallowed substitution group {2}").format( os.path.basename(modelDocument.uri), name, modelConcept.substitutionGroupQname), "err", "EFM.6.07.25", "GFM.1.03.27") # 6.7.26 Table must be subs group hypercube if name.endswith("LineItems") and modelConcept.abstract != "true": val.modelXbrl.error( _("Taxonomy schema {0} element {1} is a LineItems but not abstract").format( os.path.basename(modelDocument.uri), name), "err", "EFM.6.07.26", "GFM.1.03.28") # 6.7.27 type domainMember must end with Domain or Member conceptType = modelConcept.type isDomainItemType = conceptType is not None and conceptType.isDomainItemType endsWithDomainOrMember = name.endswith("Domain") or name.endswith("Member") if isDomainItemType != endsWithDomainOrMember: val.modelXbrl.error( _("Taxonomy schema {0} element {1} must end with Domain or Member for type of domainItemType").format( os.path.basename(modelDocument.uri), name), "err", "EFM.6.07.27", "GFM.1.03.29") # 6.7.28 domainItemType must be duration if isDomainItemType and not isDuration: val.modelXbrl.error( _("Taxonomy schema {0} element {1} is a domainItemType and must be periodType duration").format( os.path.basename(modelDocument.uri), name), "err", "EFM.6.07.28", "GFM.1.03.30") if val.validateSBRNL: definesConcepts = True if modelConcept.isTuple: definesTuples = True if modelConcept.abstract == "true": val.modelXbrl.error( _("Taxonomy schema {0} element {1} is an abstract tuple").format( os.path.basename(modelDocument.uri), modelConcept.qname), "err", "SBR.NL.2.2.2.03") if tupleCycle(val,modelConcept): val.modelXbrl.error( _("Taxonomy schema {0} tuple {1} has a tuple cycle").format( os.path.basename(modelDocument.uri), modelConcept.qname), "err", "SBR.NL.2.2.2.07") if modelConcept.nillable != "false" and modelConcept.isRoot: val.modelXbrl.error( _("Taxonomy schema {0} tuple {1} must have nillable='false'").format( os.path.basename(modelDocument.uri), modelConcept.qname), "err", "SBR.NL.2.2.2.17") if modelConcept.abstract == "true": if modelConcept.isRoot: if modelConcept.nillable != "false": val.modelXbrl.error( _("Taxonomy schema {0} abstract root concept {1} must have nillable='false'").format( os.path.basename(modelDocument.uri), modelConcept.qname), "err", "SBR.NL.2.2.2.16") if modelConcept.typeQname != XbrlConst.qnXbrliStringItemType: val.modelXbrl.error( _("Taxonomy schema {0} abstract root concept {1} must have type='xbrli:stringItemType'").format( os.path.basename(modelDocument.uri), modelConcept.qname), "err", "SBR.NL.2.2.2.21") else: # not root if modelConcept.isItem: val.modelXbrl.error( _("Taxonomy schema {0} abstract item {1} must not be a child of a tuple").format( os.path.basename(modelDocument.uri), modelConcept.qname), "err", "SBR.NL.2.2.2.31") if modelConcept.balance: val.modelXbrl.error( _("Taxonomy schema {0} abstract concept {1} must not have a balance attribute").format( os.path.basename(modelDocument.uri), modelConcept.qname), "err", "SBR.NL.2.2.2.22") if modelConcept.isTuple: val.modelXbrl.error( _("Taxonomy schema {0} tuple {1} must not be abstract").format( os.path.basename(modelDocument.uri), modelConcept.qname), "err", "SBR.NL.2.2.2.31") if modelConcept.isHypercubeItem: definesHypercubes = True elif modelConcept.isDimensionItem: definesDimensions = True elif substititutionGroupQname.localName == "domainItem": definesDomains = True elif modelConcept.isItem: definesAbstractItems = True else: # not abstract if not (modelConcept.label(preferredLabel=XbrlConst.documentationLabel,fallbackToQname=False,lang="nl") or val.modelXbrl.relationshipSet(XbrlConst.conceptReference).fromModelObject(c)): val.modelXbrl.error( _("Taxonomy schema {0} {1} must have a documentation label or reference").format( os.path.basename(modelDocument.uri), modelConcept.qname), "err", "SBR.NL.2.2.2.28") if modelConcept.balance and not modelConcept.instanceOfType(XbrlConst.qnXbrliMonetaryItemType): val.modelXbrl.error( _("Taxonomy schema {0} non-monetary concept {1} must not have a balance attribute").format( os.path.basename(modelDocument.uri), modelConcept.qname), "err", "SBR.NL.2.2.2.24") if not modelConcept.label(fallbackToQname=False,lang="nl"): val.modelXbrl.error( _("Taxonomy schema {0} {1} must have a standard label in language 'nl'").format( os.path.basename(modelDocument.uri), modelConcept.qname), "err", "SBR.NL.2.2.2.26") if not modelConcept.isRoot: # tuple child if modelConcept.element.hasAttribute("maxOccurs") and modelConcept.element.getAttribute("maxOccurs") != "1": val.modelXbrl.error( _("Taxonomy schema {0} tuple concept {1} must have maxOccurs='1'").format( os.path.basename(modelDocument.uri), modelConcept.qname), "err", "SBR.NL.2.2.2.30") if modelConcept.isLinkPart: definesLinkParts = True val.modelXbrl.error( _("Taxonomy schema {0} link:part concept {1} is not allowed").format( os.path.basename(modelDocument.uri), modelConcept.qname), "err", "SBR.NL.2.2.5.01") if modelConcept.isTypedDimension: domainElt = modelConcept.typedDomainElement if domainElt and domainElt.element and domainElt.element.localName == "complexType": val.modelXbrl.error( _("Taxonomy schema {0} typed dimension {1} domain element {2} has disallowed complex content").format( os.path.basename(modelDocument.uri), modelConcept.qname, domainElt.qname), "err", "SBR.NL.2.2.8.02") # 6.7.8 check for embedded linkbase for e in modelDocument.xmlRootElement.getElementsByTagNameNS(XbrlConst.link, "linkbase"): val.modelXbrl.error( _("Taxonomy schema {0} contains an embedded linkbase").format( os.path.basename(modelDocument.uri)), "err", "EFM.6.07.08", "GFM.1.03.08") break requiredUsedOns = {XbrlConst.qnLinkPresentationLink, XbrlConst.qnLinkCalculationLink, XbrlConst.qnLinkDefinitionLink} # 6.7.9 role types authority for e in modelDocument.xmlRootElement.getElementsByTagNameNS(XbrlConst.link, "roleType"): roleURI = e.getAttribute("roleURI") if targetNamespaceAuthority != UrlUtil.authority(roleURI): val.modelXbrl.error( _("Taxonomy schema {0} roleType {1} does not match authority {2}").format( os.path.basename(modelDocument.uri), roleURI, targetNamespaceAuthority), "err", "EFM.6.07.09", "GFM.1.03.09") # 6.7.9 end with .../role/lc3 name if not val.roleTypePattern.match(roleURI): val.modelXbrl.error( _("Taxonomy schema {0} roleType {1} should end with /role/{2}LC3name{3}").format( os.path.basename(modelDocument.uri), roleURI, '{', '}'), "wrn", "EFM.6.07.09", "GFM.1.03.09") # 6.7.10 only one role type declaration in DTS modelRoleTypes = val.modelXbrl.roleTypes.get(roleURI) if modelRoleTypes is not None: if len(modelRoleTypes) > 1: val.modelXbrl.error( _("Taxonomy schema {0} roleType {1} is defined in multiple taxonomies").format( os.path.basename(modelDocument.uri), roleURI), "err", "EFM.6.07.10", "GFM.1.03.10") elif len(modelRoleTypes) == 1: # 6.7.11 used on's for pre, cal, def if any has a used on usedOns = modelRoleTypes[0].usedOns if not usedOns.isdisjoint(requiredUsedOns) and len(requiredUsedOns - usedOns) > 0: val.modelXbrl.error( _("Taxonomy schema {0} roleType {1} missing used on {2}").format( os.path.basename(modelDocument.uri), roleURI, requiredUsedOns - usedOns), "err", "EFM.6.07.11", "GFM.1.03.11") # 6.7.12 definition match pattern definition = modelRoleTypes[0].definitionNotStripped if definition is None or not val.disclosureSystem.roleDefinitionPattern.match(definition): val.modelXbrl.error( _("Taxonomy schema {0} roleType {1} definition \"{2}\" must match {3}Sortcode{4} - {3}Type{4} - {3}Title{4}").format( os.path.basename(modelDocument.uri), roleURI, definition, '{', '}'), "err", "EFM.6.07.12", "GFM.1.03.12-14") if val.validateSBRNL and (usedOns & XbrlConst.standardExtLinkQnames): definesLinkroles = True # 6.7.13 arcrole types authority for e in modelDocument.xmlRootElement.getElementsByTagNameNS(XbrlConst.link, "arcroleType"): arcroleURI = e.getAttribute("arcroleURI") if targetNamespaceAuthority != UrlUtil.authority(arcroleURI): val.modelXbrl.error( _("Taxonomy schema {0} arcroleType {1} does not match authority {2}").format( os.path.basename(modelDocument.uri), arcroleURI, targetNamespaceAuthority), "err", "EFM.6.07.13", "GFM.1.03.15") # 6.7.13 end with .../arcrole/lc3 name if not val.arcroleTypePattern.match(arcroleURI): val.modelXbrl.error( _("Taxonomy schema {0} arcroleType {1} should end with /arcrole/{2}LC3name{3}").format( os.path.basename(modelDocument.uri), arcroleURI, '{', '}'), "wrn", "EFM.6.07.13", "GFM.1.03.15") # 6.7.14 only one arcrole type declaration in DTS modelRoleTypes = val.modelXbrl.arcroleTypes[arcroleURI] if len(modelRoleTypes) > 1: val.modelXbrl.error( _("Taxonomy schema {0} arcroleType {1} is defined in multiple taxonomies").format( os.path.basename(modelDocument.uri), arcroleURI), "err", "EFM.6.07.14", "GFM.1.03.16") # 6.7.15 definition match pattern definition = modelRoleTypes[0].definition if definition is None or not val.arcroleDefinitionPattern.match(definition): val.modelXbrl.error( _("Taxonomy schema {0} arcroleType {1} definition must be non-empty").format( os.path.basename(modelDocument.uri), arcroleURI, '{', '}'), "err", "EFM.6.07.15", "GFM.1.03.17") if val.validateSBRNL: definesArcroles = True if val.validateSBRNL and (definesLinkroles + definesArcroles + definesLinkParts + definesAbstractItems + definesNonabstractItems + definesTuples + definesEnumerations + definesDimensions + definesDomains + definesHypercubes) > 1: schemaContents = [] if definesLinkroles: schemaContents.append(_("linkroles")) if definesArcroles: schemaContents.append(_("arcroles")) if definesLinkParts: schemaContents.append(_("link parts")) if definesAbstractItems: schemaContents.append(_("abstract items")) if definesNonabstractItems: schemaContents.append(_("nonabstract items")) if definesTuples: schemaContents.append(_("tuples")) if definesEnumerations: schemaContents.append(_("enumerations")) if definesDimensions: schemaContents.append(_("dimensions")) if definesDomains: schemaContents.append(_("domains")) if definesHypercubes: schemaContents.append(_("hypercubes")) val.modelXbrl.error( _("Taxonomy schema {0} may only define one of these: {1}").format( os.path.basename(modelDocument.uri), ', '.join(schemaContents)), "err", "SBR.NL.2.2.1.01") visited.remove(modelDocument)
def checkHierarchyConstraints(elt): constraints = ixHierarchyConstraints.get(elt.localName) if constraints: for _rel, names in constraints: reqt = _rel[0] rel = _rel[1:] if reqt in ('&', '^', '1'): nameFilter = ('*',) else: nameFilter = names if nameFilter == ('*',): namespaceFilter = namespacePrefix = '*' elif len(nameFilter) == 1 and "}" in nameFilter[0] and nameFilter[0][0] == "{": namespaceFilter, _sep, nameFilter = nameFilter[0][1:].partition("}") namespacePrefix = XmlUtil.xmlnsprefix(elt,namespaceFilter) else: namespaceFilter = elt.namespaceURI namespacePrefix = elt.prefix relations = {"ancestor": XmlUtil.ancestor, "parent": XmlUtil.parent, "child-choice": XmlUtil.children, "child-sequence": XmlUtil.children, "child-or-text": XmlUtil.children, "descendant": XmlUtil.descendants}[rel]( elt, namespaceFilter, nameFilter) if rel in ("ancestor", "parent"): if relations is None: relations = [] else: relations = [relations] if rel == "child-or-text": relations += XmlUtil.innerTextNodes(elt, ixExclude=True, ixEscape=False, ixContinuation=False, ixResolveUris=False) issue = '' if reqt in ('^',): if not any(r.localName in names and r.namespaceURI == elt.namespaceURI for r in relations): issue = " and is missing one of " + ', '.join(names) if reqt in ('1',) and not elt.isNil: if sum(r.localName in names and r.namespaceURI == elt.namespaceURI for r in relations) != 1: issue = " and must have exactly one of " + ', '.join(names) if reqt in ('&', '^'): disallowed = [str(r.elementQname) for r in relations if not (r.tag in names or (r.localName in names and r.namespaceURI == elt.namespaceURI))] if disallowed: issue += " and may not have " + ", ".join(disallowed) elif rel == "child-sequence": sequencePosition = 0 for i, r in enumerate(relations): rPos = names.index(str(r.localName)) if rPos < sequencePosition: issue += " and is out of sequence: " + str(r.elementQname) else: sequencePosition = rPos if reqt == '?' and len(relations) > 1: issue = " may only have 0 or 1 but {0} present ".format(len(relations)) if reqt == '+' and len(relations) == 0: issue = " must have at least 1 but none present " disallowedChildText = bool(reqt == '&' and rel in ("child-sequence", "child-choice") and elt.textValue.strip()) if ((reqt == '+' and not relations) or (reqt == '-' and relations) or (issue) or disallowedChildText): code = "{}:{}".format(ixSect[elt.namespaceURI].get(elt.localName,"other")["constraint"], { 'ancestor': "ancestorNode", 'parent': "parentNode", 'child-choice': "childNodes", 'child-sequence': "childNodes", 'child-or-text': "childNodesOrText", 'descendant': "descendantNodes"}[rel] + { '+': "Required", '-': "Disallowed", '&': "Allowed", '^': "Specified", '1': "Specified"}.get(reqt, "Specified")) msg = _("Inline XBRL ix:{0} {1} {2} {3} {4} element{5}").format( elt.localName, {'+': "must", '-': "may not", '&': "may only", '?': "may", '+': "must", '^': "must", '1': "must"}[reqt], {'ancestor': "be nested in", 'parent': "have parent", 'child-choice': "have child", 'child-sequence': "have child", 'child-or-text': "have child or text,", 'descendant': "have as descendant"}[rel], '' if rel == 'child-or-text' else ', '.join(str(r.elementQname) for r in relations) if names == ('*',) and relations else ", ".join("{}:{}".format(namespacePrefix, n) for n in names), issue, " and no child text (\"{}\")".format(elt.textValue.strip()[:32]) if disallowedChildText else "") modelXbrl.error(code, msg, modelObject=[elt] + relations, requirement=reqt, messageCodes=("ix{ver.sect}:ancestorNode{Required|Disallowed}", "ix{ver.sect}:childNodesOrTextRequired", "ix{ver.sect}:childNodes{Required|Disallowed|Allowed}", "ix{ver.sect}:descendantNodesDisallowed", "ix{ver.sect}:parentNodeRequired")) # other static element checks (that don't require a complete object model, context, units, etc if elt.localName == "nonFraction": childElts = XmlUtil.children(elt, '*', '*') hasText = (elt.text or "") or any((childElt.tail or "") for childElt in childElts) if elt.isNil: ancestorNonFractions = XmlUtil.ancestors(elt, _ixNS, elt.localName) if ancestorNonFractions: modelXbrl.error(ixMsgCode("nonFractionAncestors", elt), _("Fact %(fact)s is a nil nonFraction and MUST not have an ancestor ix:nonFraction"), modelObject=[elt] + ancestorNonFractions, fact=elt.qname) if childElts or hasText: modelXbrl.error(ixMsgCode("nonFractionTextAndElementChildren", elt), _("Fact %(fact)s is a nil nonFraction and MUST not have an child elements or text"), modelObject=[elt] + childElts, fact=elt.qname) elt.setInvalid() # prevent further validation or cascading errors else: if ((childElts and (len(childElts) != 1 or childElts[0].namespaceURI != _ixNS or childElts[0].localName != "nonFraction")) or (childElts and hasText)): modelXbrl.error(ixMsgCode("nonFractionTextAndElementChildren", elt), _("Fact %(fact)s is a non-nil nonFraction and MUST have exactly one ix:nonFraction child element or text."), modelObject=[elt] + childElts, fact=elt.qname) elt.setInvalid() if elt.localName == "fraction": if elt.isNil: ancestorFractions = XmlUtil.ancestors(elt, _ixNS, elt.localName) if ancestorFractions: modelXbrl.error(ixMsgCode("fractionAncestors", elt), _("Fact %(fact)s is a nil fraction and MUST not have an ancestor ix:fraction"), modelObject=[elt] + ancestorFractions, fact=elt.qname) else: nonFrChildren = [e for e in XmlUtil.children(elt, _ixNS, '*') if e.localName not in ("fraction", "numerator", "denominator")] if nonFrChildren: modelXbrl.error(ixMsgCode("fractionElementChildren", elt), _("Fact %(fact)s is a non-nil fraction and not have any child elements except ix:fraction, ix:numerator and ix:denominator: %(children)s"), modelObject=[elt] + nonFrChildren, fact=elt.qname, children=", ".join(e.localName for e in nonFrChildren)) for ancestorFraction in XmlUtil.ancestors(elt, XbrlConst.ixbrl11, "fraction"): # only ix 1.1 if normalizeSpace(elt.get("unitRef")) != normalizeSpace(ancestorFraction.get("unitRef")): modelXbrl.error(ixMsgCode("fractionNestedUnitRef", elt), _("Fact %(fact)s fraction and ancestor fractions must have matching unitRefs: %(unitRef)s, %(unitRef2)s"), modelObject=[elt] + nonFrChildren, fact=elt.qname, unitRef=elt.get("unitRef"), unitRef2=ancestorFraction.get("unitRef")) if elt.localName in ("nonFraction", "numerator", "denominator", "nonNumeric"): fmt = elt.format if fmt: if fmt in _customTransforms: pass elif fmt.namespaceURI not in FunctionIxt.ixtNamespaceFunctions: modelXbrl.error(ixMsgCode("invalidTransformation", elt, sect="validation"), _("Fact %(fact)s has unrecognized transformation namespace %(namespace)s"), modelObject=elt, fact=elt.qname, transform=fmt, namespace=fmt.namespaceURI) elt.setInvalid() elif fmt.localName not in FunctionIxt.ixtNamespaceFunctions[fmt.namespaceURI]: modelXbrl.error(ixMsgCode("invalidTransformation", elt, sect="validation"), _("Fact %(fact)s has unrecognized transformation name %(name)s"), modelObject=elt, fact=elt.qname, transform=fmt, name=fmt.localName) elt.setInvalid()
def final(val): if not (val.validateEBA or val.validateEIOPA): return modelXbrl = val.modelXbrl modelDocument = modelXbrl.modelDocument _statusMsg = _("validating {0} filing rules").format(val.disclosureSystem.name) modelXbrl.profileActivity() modelXbrl.modelManager.showStatus(_statusMsg) if modelDocument.type == ModelDocument.Type.INSTANCE and (val.validateEBA or val.validateEIOPA): if not modelDocument.uri.endswith(".xbrl"): modelXbrl.warning("EBA.1.1", _('XBRL instance documents SHOULD use the extension ".xbrl" but it is "%(extension)s"'), modelObject=modelDocument, extension=os.path.splitext(modelDocument.basename)[1]) modelXbrl.error("EIOPA.S.1.1.a", _('XBRL instance documents MUST use the extension ".xbrl" but it is "%(extension)s"'), modelObject=modelDocument, extension=os.path.splitext(modelDocument.basename)[1]) if val.isEIOPA_2_0_1: _encodings = ("UTF-8", "utf-8-sig") else: _encodings = ("utf-8", "UTF-8", "utf-8-sig") if modelDocument.documentEncoding not in _encodings: modelXbrl.error(("EBA.1.4", "EIOPA.1.4"), _('XBRL instance documents MUST use "UTF-8" encoding but is "%(xmlEncoding)s"'), modelObject=modelDocument, xmlEncoding=modelDocument.documentEncoding) schemaRefElts = [] schemaRefFileNames = [] for doc, docRef in modelDocument.referencesDocument.items(): if docRef.referenceType == "href": if docRef.referringModelObject.localName == "schemaRef": schemaRefElts.append(docRef.referringModelObject) schemaRefFileNames.append(doc.basename) if not UrlUtil.isAbsolute(doc.uri): modelXbrl.error(("EBA.2.2", "EIOPA.S.1.5.a" if val.isEIOPAfullVersion else "EIOPA.S.1.5.b"), _('The link:schemaRef element in submitted instances MUST resolve to the full published entry point URL: %(url)s.'), modelObject=docRef.referringModelObject, url=doc.uri, messageCodes=("EBA.2.2", "EIOPA.S.1.5.a","EIOPA.S.1.5.b")) elif docRef.referringModelObject.localName == "linkbaseRef": modelXbrl.error(("EBA.2.3","EIOPA.S.1.5.a"), _('The link:linkbaseRef element is not allowed: %(fileName)s.'), modelObject=docRef.referringModelObject, fileName=doc.basename) _numSchemaRefs = len(XmlUtil.children(modelDocument.xmlRootElement, XbrlConst.link, "schemaRef")) if _numSchemaRefs > 1: modelXbrl.error(("EIOPA.S.1.5.a", "EBA.1.5"), _('XBRL instance documents MUST reference only one entry point schema but %(numEntryPoints)s were found: %(entryPointNames)s'), modelObject=modelDocument, numEntryPoints=_numSchemaRefs, entryPointNames=', '.join(sorted(schemaRefFileNames))) ### check entry point names appropriate for filing indicator (DPM DB?) if len(schemaRefElts) != 1: modelXbrl.error("EBA.2.3", _('Any reported XBRL instance document MUST contain only one xbrli:xbrl/link:schemaRef node, but %(entryPointCount)s.'), modelObject=schemaRefElts, entryPointCount=len(schemaRefElts)) # non-streaming EBA checks if not getattr(modelXbrl, "isStreamingMode", False): val.qnReportedCurrency = None if val.isEIOPA_2_0_1 and qnMetReportingCurrency in modelXbrl.factsByQname: for _multiCurrencyFact in modelXbrl.factsByQname[qnMetReportingCurrency]: # multi-currency fact val.qnReportedCurrency = _multiCurrencyFact.xValue break validateFacts(val, modelXbrl.facts) # check sum of fact md5s (otherwise checked in streaming process) xbrlFactsCheckVersion = None expectedSumOfFactMd5s = None for pi in modelDocument.xmlRootElement.getchildren(): if isinstance(pi, etree._ProcessingInstruction) and pi.target == "xbrl-facts-check": _match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']", pi.text) if _match: _matchGroups = _match.groups() if len(_matchGroups) == 2: if _matchGroups[0] == "version": xbrlFactsCheckVersion = _matchGroups[1] elif _matchGroups[0] == "sum-of-fact-md5s": try: expectedSumOfFactMd5s = Md5Sum(_matchGroups[1]) except ValueError: modelXbrl.error("EIOPA:xbrlFactsCheckError", _("Invalid sum-of-md5s %(sumOfMd5)s"), modelObject=modelXbrl, sumOfMd5=_matchGroups[1]) if xbrlFactsCheckVersion and expectedSumOfFactMd5s: sumOfFactMd5s = Md5Sum() for f in modelXbrl.factsInInstance: sumOfFactMd5s += f.md5sum if sumOfFactMd5s != expectedSumOfFactMd5s: modelXbrl.warning("EIOPA:xbrlFactsCheckWarning", _("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"), modelObject=modelXbrl, expectedMd5=expectedSumOfFactMd5s, actualMd5Sum=sumOfFactMd5s) else: modelXbrl.info("info", _("Successful XBRL facts sum of md5s."), modelObject=modelXbrl) if any(badError in modelXbrl.errors for badError in ("EBA.2.1", "EIOPA.2.1", "EIOPA.S.1.5.a/EIOPA.S.1.5.b")): pass # skip checking filingIndicators if bad errors elif not val.filingIndicators: modelXbrl.error(("EBA.1.6", "EIOPA.1.6.a"), _('Missing filing indicators. Reported XBRL instances MUST include appropriate (positive) filing indicator elements'), modelObject=modelDocument) elif all(filed == False for filed in val.filingIndicators.values()): modelXbrl.error(("EBA.1.6", "EIOPA.1.6.a"), _('All filing indicators are filed="false". Reported XBRL instances MUST include appropriate (positive) filing indicator elements'), modelObject=modelDocument) if val.numFilingIndicatorTuples > 1: modelXbrl.warning(("EBA.1.6.2", "EIOPA.1.6.2"), _('Multiple filing indicators tuples when not in streaming mode (info).'), modelObject=modelXbrl.factsByQname[qnFIndicators]) if len(val.cntxDates) > 1: modelXbrl.error(("EBA.2.13","EIOPA.2.13"), _('Contexts must have the same date: %(dates)s.'), # when streaming values are no longer available, but without streaming they can be logged modelObject=set(_cntx for _cntxs in val.cntxDates.values() for _cntx in _cntxs), dates=', '.join(XmlUtil.dateunionValue(_dt, subtractOneDay=True) for _dt in val.cntxDates.keys())) if val.unusedCntxIDs: if val.isEIOPA_2_0_1: modelXbrl.error("EIOPA.2.7", _('Unused xbrli:context nodes MUST NOT be present in the instance: %(unusedContextIDs)s.'), modelObject=[modelXbrl.contexts[unusedCntxID] for unusedCntxID in val.unusedCntxIDs if unusedCntxID in modelXbrl.contexts], unusedContextIDs=", ".join(sorted(val.unusedCntxIDs))) else: modelXbrl.warning(("EBA.2.7", "EIOPA.2.7"), _('Unused xbrli:context nodes SHOULD NOT be present in the instance: %(unusedContextIDs)s.'), modelObject=[modelXbrl.contexts[unusedCntxID] for unusedCntxID in val.unusedCntxIDs if unusedCntxID in modelXbrl.contexts], unusedContextIDs=", ".join(sorted(val.unusedCntxIDs))) if len(val.cntxEntities) > 1: modelXbrl.error(("EBA.2.9", "EIOPA.2.9"), _('All entity identifiers and schemes MUST be the same, %(count)s found: %(entities)s.'), modelObject=modelDocument, count=len(val.cntxEntities), entities=", ".join(sorted(str(cntxEntity) for cntxEntity in val.cntxEntities))) for _scheme, _LEI in val.cntxEntities: if (_scheme in ("http://standards.iso.org/iso/17442", "http://standard.iso.org/iso/17442", "LEI") or (not val.isEIOPAfullVersion and _scheme == "PRE-LEI")): if _scheme == "http://standard.iso.org/iso/17442": modelXbrl.warning(("EBA.3.6", "EIOPA.S.2.8.c"), _("Warning, context has entity scheme %(scheme)s should be plural: http://standards.iso.org/iso/17442."), modelObject=modelDocument, scheme=_scheme) result = LeiUtil.checkLei(_LEI) if result == LeiUtil.LEI_INVALID_LEXICAL: modelXbrl.error("EIOPA.S.2.8.c", _("Context has lexically invalid LEI %(lei)s."), modelObject=modelDocument, lei=_LEI) elif result == LeiUtil.LEI_INVALID_CHECKSUM: modelXbrl.error("EIOPA.S.2.8.c", _("Context has LEI checksum error in %(lei)s."), modelObject=modelDocument, lei=_LEI) elif _scheme == "SC": pass # anything is ok for Specific Code else: modelXbrl.error("EIOPA.S.2.8.c", _("Context has unrecognized entity scheme %(scheme)s."), modelObject=modelDocument, scheme=_scheme) if val.unusedUnitIDs: if val.isEIOPA_2_0_1: modelXbrl.error("EIOPA.2.22", _('Unused xbrli:unit nodes MUST NOT be present in the instance: %(unusedUnitIDs)s.'), modelObject=[modelXbrl.units[unusedUnitID] for unusedUnitID in val.unusedUnitIDs if unusedUnitID in modelXbrl.units], unusedUnitIDs=", ".join(sorted(val.unusedUnitIDs))) else: modelXbrl.warning(("EBA.2.22", "EIOPA.2.22"), _('Unused xbrli:unit nodes SHOULD NOT be present in the instance: %(unusedUnitIDs)s.'), modelObject=[modelXbrl.units[unusedUnitID] for unusedUnitID in val.unusedUnitIDs if unusedUnitID in modelXbrl.units], unusedUnitIDs=", ".join(sorted(val.unusedUnitIDs))) if len(val.currenciesUsed) > 1: modelXbrl.error(("EBA.3.1","EIOPA.3.1"), _("There MUST be only one currency but %(numCurrencies)s were found: %(currencies)s.'"), modelObject=val.currenciesUsed.values(), numCurrencies=len(val.currenciesUsed), currencies=", ".join(str(c) for c in val.currenciesUsed.keys())) elif val.isEIOPA_2_0_1 and any(_measure.localName != val.reportingCurrency for _measure in val.currenciesUsed.keys()): modelXbrl.error("EIOPA.3.1", _("There MUST be only one currency but reporting currency %(reportingCurrency)s differs from unit currencies: %(currencies)s.'"), modelObject=val.currenciesUsed.values(), reportingCurrency=val.reportingCurrency, currencies=", ".join(str(c) for c in val.currenciesUsed.keys())) if val.prefixesUnused: modelXbrl.warning(("EBA.3.4", "EIOPA.3.4"), _("There SHOULD be no unused prefixes but these were declared: %(unusedPrefixes)s.'"), modelObject=modelDocument, unusedPrefixes=', '.join(sorted(val.prefixesUnused))) for ns, prefixes in val.namespacePrefixesUsed.items(): nsDocs = modelXbrl.namespaceDocs.get(ns) if nsDocs: for nsDoc in nsDocs: nsDocPrefix = XmlUtil.xmlnsprefix(nsDoc.xmlRootElement, ns) if any(prefix != nsDocPrefix for prefix in prefixes if prefix is not None): modelXbrl.warning(("EBA.3.5", "EIOPA.3.5"), _("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s"), modelObject=modelDocument, namespace=ns, declaredPrefix=nsDocPrefix, foundPrefixes=', '.join(sorted(prefixes - {None}))) elif ns in CANONICAL_PREFIXES and any(prefix != CANONICAL_PREFIXES[ns] for prefix in prefixes if prefix is not None): modelXbrl.warning(("EBA.3.5", "EIOPA.3.5"), _("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s"), modelObject=modelDocument, namespace=ns, declaredPrefix=CANONICAL_PREFIXES[ns], foundPrefixes=', '.join(sorted(prefixes - {None}))) modelXbrl.profileActivity(_statusMsg, minTimeToShow=0.0) modelXbrl.modelManager.showStatus(None) del val.prefixNamespace, val.namespacePrefix, val.idObjects, val.typedDomainElements del val.utrValidator, val.firstFact, val.footnotesRelationshipSet
def checkFilingDTS(val, modelDocument, isEFM, isGFM, visited): global targetNamespaceDatePattern, efmFilenamePattern, htmlFileNamePattern, roleTypePattern, arcroleTypePattern, \ arcroleDefinitionPattern, namePattern, linkroleDefinitionBalanceIncomeSheet, \ namespacesConflictPattern if targetNamespaceDatePattern is None: targetNamespaceDatePattern = re.compile( r"/([12][0-9]{3})-([01][0-9])-([0-3][0-9])|" r"/([12][0-9]{3})([01][0-9])([0-3][0-9])|") efmFilenamePattern = re.compile( r"^[a-z0-9][a-zA-Z0-9_\.\-]*(\.xsd|\.xml|\.htm)$") htmlFileNamePattern = re.compile( r"^[a-zA-Z0-9][._a-zA-Z0-9-]*(\.htm)$") roleTypePattern = re.compile(r"^.*/role/[^/\s]+$") arcroleTypePattern = re.compile(r"^.*/arcrole/[^/\s]+$") arcroleDefinitionPattern = re.compile( r"^.*[^\\s]+.*$") # at least one non-whitespace character namePattern = re.compile( "[][()*+?\\\\/^{}|@#%^=~`\"';:,<>&$\u00a3\u20ac]" ) # u20ac=Euro, u00a3=pound sterling linkroleDefinitionBalanceIncomeSheet = re.compile( r"[^-]+-\s+Statement\s+-\s+.*(income|balance|financial\W+position)", re.IGNORECASE) namespacesConflictPattern = re.compile( r"http://(xbrl\.us|fasb\.org|xbrl\.sec\.gov)/(dei|us-types|us-roles|rr)/([0-9]{4}-[0-9]{2}-[0-9]{2})$" ) nonDomainItemNameProblemPattern = re.compile( r"({0})|(FirstQuarter|SecondQuarter|ThirdQuarter|FourthQuarter|[1-4]Qtr|Qtr[1-4]|ytd|YTD|HalfYear)(?:$|[A-Z\W])" .format(re.sub(r"\W", "", (val.entityRegistrantName or "").title()))) visited.append(modelDocument) for referencedDocument, modelDocumentReference in modelDocument.referencesDocument.items( ): #6.07.01 no includes if modelDocumentReference.referenceType == "include": val.modelXbrl.error( ("EFM.6.07.01", "GFM.1.03.01"), _("Taxonomy schema %(schema)s includes %(include)s, only import is allowed" ), modelObject=modelDocumentReference.referringModelObject, schema=os.path.basename(modelDocument.uri), include=os.path.basename(referencedDocument.uri)) if referencedDocument not in visited and referencedDocument.inDTS: # ignore EdgarRenderer added non-DTS documents checkFilingDTS(val, referencedDocument, isEFM, isGFM, visited) if val.disclosureSystem.standardTaxonomiesDict is None: pass if isEFM: if modelDocument.uri in val.disclosureSystem.standardTaxonomiesDict: if modelDocument.targetNamespace: # check for duplicates of us-types, dei, and rr taxonomies match = namespacesConflictPattern.match( modelDocument.targetNamespace) if match is not None: val.standardNamespaceConflicts[match.group(2)].add( modelDocument) else: if len(modelDocument.basename) > 32: val.modelXbrl.error( "EFM.5.01.01.tooManyCharacters", _("Document file name %(filename)s must not exceed 32 characters." ), modelObject=modelDocument, filename=modelDocument.basename) if modelDocument.type == ModelDocument.Type.INLINEXBRL: if not htmlFileNamePattern.match(modelDocument.basename): val.modelXbrl.error( "EFM.5.01.01", _("Document file name %(filename)s must start with a-z or 0-9, contain upper or lower case letters, ., -, _, and end with .htm." ), modelObject=modelDocument, filename=modelDocument.basename) elif not efmFilenamePattern.match(modelDocument.basename): val.modelXbrl.error( "EFM.5.01.01", _("Document file name %(filename)s must start with a-z or 0-9, contain upper or lower case letters, ., -, _, and end with .xsd or .xml." ), modelObject=modelDocument, filename=modelDocument.basename) if (modelDocument.type == ModelDocument.Type.SCHEMA and modelDocument.targetNamespace not in val.disclosureSystem.baseTaxonomyNamespaces and modelDocument.uri.startswith(val.modelXbrl.uriDir)): val.hasExtensionSchema = True # check schema contents types # 6.7.3 check namespace for standard authority targetNamespaceAuthority = UrlUtil.authority( modelDocument.targetNamespace) if targetNamespaceAuthority in val.disclosureSystem.standardAuthorities: val.modelXbrl.error( ("EFM.6.07.03", "GFM.1.03.03"), _("Taxonomy schema %(schema)s namespace %(targetNamespace)s is a disallowed authority" ), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace, targetNamespaceAuthority=UrlUtil.authority( modelDocument.targetNamespace, includeScheme=False)) # 6.7.4 check namespace format if modelDocument.targetNamespace is None or not modelDocument.targetNamespace.startswith( "http://"): match = None else: targetNamespaceDate = modelDocument.targetNamespace[ len(targetNamespaceAuthority):] match = targetNamespaceDatePattern.match(targetNamespaceDate) if match is not None: try: if match.lastindex == 3: date = datetime.date(int(match.group(1)), int(match.group(2)), int(match.group(3))) elif match.lastindex == 6: date = datetime.date(int(match.group(4)), int(match.group(5)), int(match.group(6))) else: match = None except ValueError: match = None if match is None: val.modelXbrl.error( ("EFM.6.07.04", "GFM.1.03.04"), _("Taxonomy schema %(schema)s namespace %(targetNamespace)s must have format http://{authority}/{versionDate}" ), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace) elif val.fileNameDate and date > val.fileNameDate: val.modelXbrl.info( ("EFM.6.07.06", "GFM.1.03.06"), _("Warning: Taxonomy schema %(schema)s namespace %(targetNamespace)s has date later than document name date %(docNameDate)s" ), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace, docNameDate=val.fileNameDate) if modelDocument.targetNamespace is not None: # 6.7.5 check prefix for _ authority = UrlUtil.authority(modelDocument.targetNamespace) if not re.match(r"(http://|https://|ftp://|urn:)\w+", authority): val.modelXbrl.error( ("EFM.6.07.05", "GFM.1.03.05"), _("Taxonomy schema %(schema)s namespace %(targetNamespace)s must be a valid URL with a valid authority for the namespace." ), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace) prefix = XmlUtil.xmlnsprefix(modelDocument.xmlRootElement, modelDocument.targetNamespace) if not prefix: val.modelXbrl.error( ("EFM.6.07.07", "GFM.1.03.07"), _("Taxonomy schema %(schema)s namespace %(targetNamespace)s missing prefix for the namespace." ), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace) elif "_" in prefix: val.modelXbrl.error( ("EFM.6.07.07", "GFM.1.03.07"), _("Taxonomy schema %(schema)s namespace %(targetNamespace)s prefix %(prefix)s must not have an '_'" ), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace, prefix=prefix) for modelConcept in modelDocument.xmlRootElement.iterdescendants( tag="{http://www.w3.org/2001/XMLSchema}element"): if isinstance(modelConcept, ModelConcept): # 6.7.16 name not duplicated in standard taxonomies name = modelConcept.get("name") if name is None: name = "" if modelConcept.get("ref") is not None: continue # don't validate ref's here for c in val.modelXbrl.nameConcepts.get(name, []): if c.modelDocument != modelDocument: if not c.modelDocument.uri.startswith( val.modelXbrl.uriDir): val.modelXbrl.error( ("EFM.6.07.16", "GFM.1.03.18"), _("Concept %(concept)s is also defined in standard taxonomy schema schema %(standardSchema)s" ), modelObject=(modelConcept, c), concept=modelConcept.qname, standardSchema=os.path.basename( c.modelDocument.uri), standardConcept=c.qname) # 6.7.17 id properly formed _id = modelConcept.id requiredId = (prefix if prefix is not None else "") + "_" + name if _id != requiredId: val.modelXbrl.error( ("EFM.6.07.17", "GFM.1.03.19"), _("Concept %(concept)s id %(id)s should be %(requiredId)s" ), modelObject=modelConcept, concept=modelConcept.qname, id=_id, requiredId=requiredId) # 6.7.18 nillable is true nillable = modelConcept.get("nillable") if nillable != "true" and modelConcept.isItem: val.modelXbrl.error( ("EFM.6.07.18", "GFM.1.03.20"), _("Taxonomy schema %(schema)s element %(concept)s nillable %(nillable)s should be 'true'" ), modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=name, nillable=nillable) # 6.7.19 not tuple if modelConcept.isTuple: val.modelXbrl.error( ("EFM.6.07.19", "GFM.1.03.21"), _("Concept %(concept)s is a tuple"), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.20 no typed domain ref if modelConcept.isTypedDimension: val.modelXbrl.error( ("EFM.6.07.20", "GFM.1.03.22"), _("Concept %(concept)s has typedDomainRef %(typedDomainRef)s" ), modelObject=modelConcept, concept=modelConcept.qname, typedDomainRef=modelConcept.typedDomainElement. qname if modelConcept.typedDomainElement is not None else modelConcept.typedDomainRef) # 6.7.21 abstract must be duration isDuration = modelConcept.periodType == "duration" if modelConcept.isAbstract and not isDuration: val.modelXbrl.error( ("EFM.6.07.21", "GFM.1.03.23"), _("Taxonomy schema %(schema)s element %(concept)s is abstract but period type is not duration" ), modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=modelConcept.qname) # 6.7.22 abstract must be stringItemType ''' removed SEC EFM v.17, Edgar release 10.4, and GFM 2011-04-08 if modelConcept.abstract == "true" and modelConcept.typeQname != XbrlConst. qnXbrliStringItemType: val.modelXbrl.error(("EFM.6.07.22", "GFM.1.03.24"), _("Concept %(concept)s is abstract but type is not xbrli:stringItemType"), modelObject=modelConcept, concept=modelConcept.qname) ''' substitutionGroupQname = modelConcept.substitutionGroupQname # 6.7.23 Axis must be subs group dimension if name.endswith("Axis") ^ ( substitutionGroupQname == XbrlConst.qnXbrldtDimensionItem): val.modelXbrl.error( ("EFM.6.07.23", "GFM.1.03.25"), _("Concept %(concept)s must end in Axis to be in xbrldt:dimensionItem substitution group" ), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.24 Table must be subs group hypercube if name.endswith("Table") ^ ( substitutionGroupQname == XbrlConst.qnXbrldtHypercubeItem): val.modelXbrl.error( ("EFM.6.07.24", "GFM.1.03.26"), _("Concept %(concept)s must end in Table to be in xbrldt:hypercubeItem substitution group" ), modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=modelConcept.qname) # 6.7.25 if neither hypercube or dimension, substitution group must be item if substitutionGroupQname not in ( None, XbrlConst.qnXbrldtDimensionItem, XbrlConst.qnXbrldtHypercubeItem, XbrlConst.qnXbrliItem): val.modelXbrl.error( ("EFM.6.07.25", "GFM.1.03.27"), _("Concept %(concept)s has disallowed substitution group %(substitutionGroup)s" ), modelObject=modelConcept, concept=modelConcept.qname, substitutionGroup=modelConcept. substitutionGroupQname) # 6.7.26 Table must be subs group hypercube if name.endswith( "LineItems") and modelConcept.abstract != "true": val.modelXbrl.error( ("EFM.6.07.26", "GFM.1.03.28"), _("Concept %(concept)s is a LineItems but not abstract" ), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.27 type domainMember must end with Domain or Member conceptType = modelConcept.type isDomainItemType = conceptType is not None and conceptType.isDomainItemType endsWithDomainOrMember = name.endswith( "Domain") or name.endswith("Member") if isDomainItemType != endsWithDomainOrMember: val.modelXbrl.error( ("EFM.6.07.27", "GFM.1.03.29"), _("Concept %(concept)s must end with Domain or Member for type of domainItemType" ), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.28 domainItemType must be duration if isDomainItemType and not isDuration: val.modelXbrl.error( ("EFM.6.07.28", "GFM.1.03.30"), _("Concept %(concept)s is a domainItemType and must be periodType duration" ), modelObject=modelConcept, concept=modelConcept.qname) #6.7.31 (version 27) fractions if modelConcept.isFraction: val.modelXbrl.error( "EFM.6.07.31", _("Concept %(concept)s is a fraction"), modelObject=modelConcept, concept=modelConcept.qname) #6.7.32 (version 27) instant non numeric if modelConcept.isItem and (not modelConcept.isNumeric and not isDuration and not modelConcept.isAbstract and not isDomainItemType): val.modelXbrl.error( "EFM.6.07.32", _("Taxonomy schema %(schema)s element %(concept)s is non-numeric but period type is not duration" ), modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=modelConcept.qname) # 6.8.5 semantic check, check LC3 name if name: if not name[0].isupper(): val.modelXbrl.log( "ERROR-SEMANTIC", ("EFM.6.08.05.firstLetter", "GFM.2.03.05.firstLetter"), _("Concept %(concept)s name must start with a capital letter" ), modelObject=modelConcept, concept=modelConcept.qname) if namePattern.search(name): val.modelXbrl.log( "ERROR-SEMANTIC", ("EFM.6.08.05.disallowedCharacter", "GFM.2.03.05.disallowedCharacter"), _("Concept %(concept)s has disallowed name character" ), modelObject=modelConcept, concept=modelConcept.qname) if len(name) > 200: val.modelXbrl.log( "ERROR-SEMANTIC", "EFM.6.08.05.nameLength", _("Concept %(concept)s name length %(namelength)s exceeds 200 characters" ), modelObject=modelConcept, concept=modelConcept.qname, namelength=len(name)) if isEFM: label = modelConcept.label(lang="en-US", fallbackToQname=False) if label: # allow Joe's Bar, N.A. to be JoesBarNA -- remove ', allow A. as not article "a" lc3name = ''.join( re.sub(r"['.-]", "", ( w[0] or w[2] or w[3] or w[4])).title() for w in re.findall( r"((\w+')+\w+)|(A[.-])|([.-]A(?=\W|$))|(\w+)", label ) # EFM implies this should allow - and . re.findall(r"[\w\-\.]+", label) if w[4].lower() not in ("the", "a", "an")) if not (name == lc3name or (name and lc3name and lc3name[0].isdigit() and name[1:] == lc3name and (name[0].isalpha() or name[0] == '_'))): val.modelXbrl.log( "WARNING-SEMANTIC", "EFM.6.08.05.LC3", _("Concept %(concept)s should match expected LC3 composition %(lc3name)s" ), modelObject=modelConcept, concept=modelConcept.qname, lc3name=lc3name) if conceptType is not None: # 6.8.6 semantic check if not isDomainItemType and conceptType.qname != XbrlConst.qnXbrliDurationItemType: nameProblems = nonDomainItemNameProblemPattern.findall( name) if any( any(t) for t in nameProblems ): # list of tuples with possibly nonempty strings val.modelXbrl.log( "WARNING-SEMANTIC", ("EFM.6.08.06", "GFM.2.03.06"), _("Concept %(concept)s should not contain company or period information, found: %(matches)s" ), modelObject=modelConcept, concept=modelConcept.qname, matches=", ".join(''.join(t) for t in nameProblems)) if conceptType.qname == XbrlConst.qnXbrliMonetaryItemType: if not modelConcept.balance: # 6.8.11 may not appear on a income or balance statement if any( linkroleDefinitionBalanceIncomeSheet. match(roleType.definition) for rel in val.modelXbrl. relationshipSet(XbrlConst.parentChild). toModelObject(modelConcept) for roleType in val.modelXbrl. roleTypes.get(rel.linkrole, ())): val.modelXbrl.log( "ERROR-SEMANTIC", ("EFM.6.08.11", "GFM.2.03.11"), _("Concept %(concept)s must have a balance because it appears in a statement of income or balance sheet" ), modelObject=modelConcept, concept=modelConcept.qname) # 6.11.5 semantic check, must have a documentation label stdLabel = modelConcept.label( lang="en-US", fallbackToQname=False) defLabel = modelConcept.label( preferredLabel=XbrlConst. documentationLabel, lang="en-US", fallbackToQname=False) if not defLabel or ( # want different words than std label stdLabel and re.findall(r"\w+", stdLabel) == re.findall(r"\w+", defLabel)): val.modelXbrl.log( "ERROR-SEMANTIC", ("EFM.6.11.05", "GFM.2.04.04"), _("Concept %(concept)s is monetary without a balance and must have a documentation label that disambiguates its sign" ), modelObject=modelConcept, concept=modelConcept.qname) # 6.8.16 semantic check if conceptType.qname == XbrlConst.qnXbrliDateItemType and modelConcept.periodType != "duration": val.modelXbrl.log( "ERROR-SEMANTIC", ("EFM.6.08.16", "GFM.2.03.16"), _("Concept %(concept)s of type xbrli:dateItemType must have periodType duration" ), modelObject=modelConcept, concept=modelConcept.qname) # 6.8.17 semantic check if conceptType.qname == XbrlConst.qnXbrliStringItemType and modelConcept.periodType != "duration": val.modelXbrl.log( "ERROR-SEMANTIC", ("EFM.6.08.17", "GFM.2.03.17"), _("Concept %(concept)s of type xbrli:stringItemType must have periodType duration" ), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.8 check for embedded linkbase for e in modelDocument.xmlRootElement.iterdescendants( tag="{http://www.xbrl.org/2003/linkbase}linkbase"): if isinstance(e, ModelObject): val.modelXbrl.error( ("EFM.6.07.08", "GFM.1.03.08"), _("Taxonomy schema %(schema)s contains an embedded linkbase" ), modelObject=e, schema=modelDocument.basename) break requiredUsedOns = { XbrlConst.qnLinkPresentationLink, XbrlConst.qnLinkCalculationLink, XbrlConst.qnLinkDefinitionLink } standardUsedOns = { XbrlConst.qnLinkLabel, XbrlConst.qnLinkReference, XbrlConst.qnLinkDefinitionArc, XbrlConst.qnLinkCalculationArc, XbrlConst.qnLinkPresentationArc, XbrlConst.qnLinkLabelArc, XbrlConst.qnLinkReferenceArc, # per WH, private footnote arc and footnore resource roles are not allowed XbrlConst.qnLinkFootnoteArc, XbrlConst.qnLinkFootnote, } # 6.7.9 role types authority for e in modelDocument.xmlRootElement.iterdescendants( tag="{http://www.xbrl.org/2003/linkbase}roleType"): if isinstance(e, ModelObject): roleURI = e.get("roleURI") if targetNamespaceAuthority != UrlUtil.authority(roleURI): val.modelXbrl.error( ("EFM.6.07.09", "GFM.1.03.09"), _("RoleType %(roleType)s does not match authority %(targetNamespaceAuthority)s" ), modelObject=e, roleType=roleURI, targetNamespaceAuthority=targetNamespaceAuthority, targetNamespace=modelDocument.targetNamespace) # 6.7.9 end with .../role/lc3 name if not roleTypePattern.match(roleURI): val.modelXbrl.warning( ("EFM.6.07.09.roleEnding", "GFM.1.03.09"), "RoleType %(roleType)s should end with /role/{LC3name}", modelObject=e, roleType=roleURI) # 6.7.10 only one role type declaration in DTS modelRoleTypes = val.modelXbrl.roleTypes.get(roleURI) if modelRoleTypes is not None: modelRoleType = modelRoleTypes[0] definition = modelRoleType.definitionNotStripped usedOns = modelRoleType.usedOns if len(modelRoleTypes) == 1: # 6.7.11 used on's for pre, cal, def if any has a used on if not usedOns.isdisjoint(requiredUsedOns) and len( requiredUsedOns - usedOns) > 0: val.modelXbrl.error( ("EFM.6.07.11", "GFM.1.03.11"), _("RoleType %(roleType)s missing used on %(usedOn)s" ), modelObject=e, roleType=roleURI, usedOn=requiredUsedOns - usedOns) # 6.7.12 definition match pattern if (val.disclosureSystem.roleDefinitionPattern is not None and (definition is None or not val.disclosureSystem. roleDefinitionPattern.match(definition))): val.modelXbrl.error( ("EFM.6.07.12", "GFM.1.03.12-14"), _("RoleType %(roleType)s definition \"%(definition)s\" must match {Sortcode} - {Type} - {Title}" ), modelObject=e, roleType=roleURI, definition=(definition or "")) if usedOns & standardUsedOns: # semantics check val.modelXbrl.log( "ERROR-SEMANTIC", ("EFM.6.08.03", "GFM.2.03.03"), _("RoleType %(roleuri)s is defined using role types already defined by standard roles for: %(qnames)s" ), modelObject=e, roleuri=roleURI, qnames=', '.join( str(qn) for qn in usedOns & standardUsedOns)) # 6.7.13 arcrole types authority for e in modelDocument.xmlRootElement.iterdescendants( tag="{http://www.xbrl.org/2003/linkbase}arcroleType"): if isinstance(e, ModelObject): arcroleURI = e.get("arcroleURI") if targetNamespaceAuthority != UrlUtil.authority(arcroleURI): val.modelXbrl.error( ("EFM.6.07.13", "GFM.1.03.15"), _("ArcroleType %(arcroleType)s does not match authority %(targetNamespaceAuthority)s" ), modelObject=e, arcroleType=arcroleURI, targetNamespaceAuthority=targetNamespaceAuthority, targetNamespace=modelDocument.targetNamespace) # 6.7.13 end with .../arcrole/lc3 name if not arcroleTypePattern.match(arcroleURI): val.modelXbrl.warning( ("EFM.6.07.13.arcroleEnding", "GFM.1.03.15"), _("ArcroleType %(arcroleType)s should end with /arcrole/{LC3name}" ), modelObject=e, arcroleType=arcroleURI) # 6.7.15 definition match pattern modelRoleTypes = val.modelXbrl.arcroleTypes[arcroleURI] definition = modelRoleTypes[0].definition if definition is None or not arcroleDefinitionPattern.match( definition): val.modelXbrl.error( ("EFM.6.07.15", "GFM.1.03.17"), _("ArcroleType %(arcroleType)s definition must be non-empty" ), modelObject=e, arcroleType=arcroleURI) # semantic checks usedOns = modelRoleTypes[0].usedOns if usedOns & standardUsedOns: # semantics check val.modelXbrl.log( "ERROR-SEMANTIC", ("EFM.6.08.03", "GFM.2.03.03"), _("ArcroleType %(arcroleuri)s is defined using role types already defined by standard arcroles for: %(qnames)s" ), modelObject=e, arcroleuri=arcroleURI, qnames=', '.join( str(qn) for qn in usedOns & standardUsedOns)) #6.3.3 filename check m = re.match(r"^\w+-([12][0-9]{3}[01][0-9][0-3][0-9]).xsd$", modelDocument.basename) if m: try: # check date value datetime.datetime.strptime(m.group(1), "%Y%m%d").date() # date and format are ok, check "should" part of 6.3.3 if val.fileNameBasePart: expectedFilename = "{0}-{1}.xsd".format( val.fileNameBasePart, val.fileNameDatePart) if modelDocument.basename != expectedFilename: val.modelXbrl.log( "WARNING-SEMANTIC", ("EFM.6.03.03.matchInstance", "GFM.1.01.01.matchInstance"), _('Schema file name warning: %(filename)s, should match %(expectedFilename)s' ), modelObject=modelDocument, filename=modelDocument.basename, expectedFilename=expectedFilename) except ValueError: val.modelXbrl.error( (val.EFM60303, "GFM.1.01.01"), _('Invalid schema file base name part (date) in "{base}-{yyyymmdd}.xsd": %(filename)s' ), modelObject=modelDocument, filename=modelDocument.basename, messageCodes=("EFM.6.03.03", "EFM.6.23.01", "GFM.1.01.01")) else: val.modelXbrl.error( (val.EFM60303, "GFM.1.01.01"), _('Invalid schema file name, must match "{base}-{yyyymmdd}.xsd": %(filename)s' ), modelObject=modelDocument, filename=modelDocument.basename, messageCodes=("EFM.6.03.03", "EFM.6.23.01", "GFM.1.01.01")) elif modelDocument.type == ModelDocument.Type.LINKBASE: # if it is part of the submission (in same directory) check name labelRels = None if modelDocument.filepath.startswith( val.modelXbrl.modelDocument.filepathdir): #6.3.3 filename check extLinkElt = XmlUtil.descendant( modelDocument.xmlRootElement, XbrlConst.link, "*", "{http://www.w3.org/1999/xlink}type", "extended") if extLinkElt is None: # no ext link element val.modelXbrl.error( (val.EFM60303 + ".noLinkElement", "GFM.1.01.01.noLinkElement"), _('Invalid linkbase file name: %(filename)s, has no extended link element, cannot determine link type.' ), modelObject=modelDocument, filename=modelDocument.basename, messageCodes=("EFM.6.03.03.noLinkElement", "EFM.6.23.01.noLinkElement", "GFM.1.01.01.noLinkElement")) elif extLinkElt.localName not in extLinkEltFileNameEnding: val.modelXbrl.error( "EFM.6.03.02", _('Invalid linkbase link element %(linkElement)s in %(filename)s' ), modelObject=modelDocument, linkElement=extLinkElt.localName, filename=modelDocument.basename) else: m = re.match( r"^\w+-([12][0-9]{3}[01][0-9][0-3][0-9])(_[a-z]{3}).xml$", modelDocument.basename) expectedSuffix = extLinkEltFileNameEnding[extLinkElt.localName] if m and m.group(2) == expectedSuffix: try: # check date value datetime.datetime.strptime(m.group(1), "%Y%m%d").date() # date and format are ok, check "should" part of 6.3.3 if val.fileNameBasePart: expectedFilename = "{0}-{1}{2}.xml".format( val.fileNameBasePart, val.fileNameDatePart, expectedSuffix) if modelDocument.basename != expectedFilename: val.modelXbrl.log( "WARNING-SEMANTIC", ("EFM.6.03.03.matchInstance", "GFM.1.01.01.matchInstance"), _('Linkbase name warning: %(filename)s should match %(expectedFilename)s' ), modelObject=modelDocument, filename=modelDocument.basename, expectedFilename=expectedFilename) except ValueError: val.modelXbrl.error( (val.EFM60303, "GFM.1.01.01"), _('Invalid linkbase base file name part (date) in "{base}-{yyyymmdd}_{suffix}.xml": %(filename)s' ), modelObject=modelDocument, filename=modelDocument.basename, messageCodes=("EFM.6.03.03", "EFM.6.23.01", "GFM.1.01.01")) else: val.modelXbrl.error( (val.EFM60303, "GFM.1.01.01"), _('Invalid linkbase name, must match "{base}-{yyyymmdd}%(expectedSuffix)s.xml": %(filename)s' ), modelObject=modelDocument, filename=modelDocument.basename, expectedSuffix=expectedSuffix, messageCodes=("EFM.6.03.03", "EFM.6.23.01", "GFM.1.01.01")) if extLinkElt.localName == "labelLink": if labelRels is None: labelRels = val.modelXbrl.relationshipSet( XbrlConst.conceptLabel) for labelElt in XmlUtil.children(extLinkElt, XbrlConst.link, "label"): # 6.10.9 if XbrlConst.isNumericRole(labelElt.role): for rel in labelRels.toModelObject(labelElt): if rel.fromModelObject is not None and not rel.fromModelObject.isNumeric: val.modelXbrl.error( "EFM.6.10.09", _("Label of non-numeric concept %(concept)s has a numeric role: %(role)s" ), modelObject=(labelElt, rel.fromModelObject), concept=rel.fromModelObject.qname, role=labelElt.role)
def saveLoadableExcel(dts, excelFile): from arelle import ModelDocument, XmlUtil, xlwt workbook = xlwt.Workbook(encoding="utf-8") sheet1 = workbook.add_sheet("Sheet1") sheet2 = workbook.add_sheet("Sheet2") # identify type of taxonomy sheet1Headers = None cellFont = None for doc in dts.urlDocs.values(): if doc.type == ModelDocument.Type.SCHEMA and doc.inDTS: for i in range(len(headersStyles)): if re.match(headersStyles[i][0], doc.targetNamespace): cellFont = headersStyles[i][1] sheet1Headers = headersStyles[i][2] break if sheet1Headers is None: dts.info("error:saveLoadableExcel", _("Referenced taxonomy style not identified, assuming general pattern."), modelObject=dts) cellFont = headersStyles[-1][1] sheet1Headers = headersStyles[-1][2] hdrCellFmt = xlwt.easyxf(cellFont + "align: wrap on, vert center, horiz center; " "pattern: pattern solid_fill, fore_color light_orange; " "border: top thin, right thin, bottom thin, left thin; ") # sheet 1 col widths for i, hdr in enumerate(sheet1Headers): sheet1.col(i).width = 256 * headerWidths.get(hdr[1], 40) # sheet 2 headers for i, hdr in enumerate(sheet2Headers): sheet2.col(i).width = 256 * hdr[1] sheet2.write(0, i, hdr[0], hdrCellFmt) # referenced taxonomies sheet1row = 0 sheet2row = 2 cellFmt = xlwt.easyxf(cellFont + "border: top thin, right thin, bottom thin, left thin; " "align: wrap on, vert top, horiz left;") cellFmtIndented = dict((i, xlwt.easyxf(cellFont + "border: top thin, right thin, bottom thin, left thin; " "align: wrap on, vert top, horiz left, indent {0};" .format(i))) for i in range(16)) # identify extension schema extensionSchemaDoc = None if dts.modelDocument.type == ModelDocument.Type.SCHEMA: extensionSchemaDoc = dts.modelDocument elif dts.modelDocument.type == ModelDocument.Type.INSTANCE: for doc, docReference in dts.modelDocument.referencesDocument.items(): if docReference.referenceType == "href": extensionSchemaDoc = doc break if extensionSchemaDoc is None: dts.info("error:saveLoadableExcel", _("Unable to identify extension taxonomy."), modelObject=dts) return for doc, docReference in extensionSchemaDoc.referencesDocument.items(): if docReference.referenceType == "import" and doc.targetNamespace != XbrlConst.xbrli: sheet2.write(sheet2row, 0, "import", cellFmt) sheet2.write(sheet2row, 1, "schema", cellFmt) sheet2.write(sheet2row, 2, XmlUtil.xmlnsprefix(doc.xmlRootElement, doc.targetNamespace), cellFmt) sheet2.write(sheet2row, 3, doc.uri, cellFmt) sheet2.write(sheet2row, 4, doc.targetNamespace, cellFmt) sheet2row += 1 sheet2row += 1 doc = extensionSchemaDoc sheet2.write(sheet2row, 0, "extension", cellFmt) sheet2.write(sheet2row, 1, "schema", cellFmt) sheet2.write(sheet2row, 2, XmlUtil.xmlnsprefix(doc.xmlRootElement, doc.targetNamespace), cellFmt) sheet2.write(sheet2row, 3, os.path.basename(doc.uri), cellFmt) sheet2.write(sheet2row, 4, doc.targetNamespace, cellFmt) sheet2row += 1 for doc, docReference in extensionSchemaDoc.referencesDocument.items(): if docReference.referenceType == "href" and doc.type == ModelDocument.Type.LINKBASE: linkbaseType = "" role = docReference.referringModelObject.get("{http://www.w3.org/1999/xlink}role") or "" if role.startswith("http://www.xbrl.org/2003/role/") and role.endswith("LinkbaseRef"): linkbaseType = os.path.basename(role)[0:-11] sheet2.write(sheet2row, 0, "extension", cellFmt) sheet2.write(sheet2row, 1, "linkbase", cellFmt) sheet2.write(sheet2row, 2, linkbaseType, cellFmt) sheet2.write(sheet2row, 3, os.path.basename(doc.uri), cellFmt) sheet2.write(sheet2row, 4, "", cellFmt) sheet2row += 1 sheet2row += 1 # extended link roles defined in this document for roleURI, roleTypes in sorted(dts.roleTypes.items(), # sort on definition if any else URI key=lambda item: (item[1][0].definition or item[0])): for roleType in roleTypes: if roleType.modelDocument == extensionSchemaDoc: sheet2.write(sheet2row, 0, "extension", cellFmt) sheet2.write(sheet2row, 1, "role", cellFmt) sheet2.write(sheet2row, 2, "", cellFmt) sheet2.write(sheet2row, 3, roleType.definition, cellFmt) sheet2.write(sheet2row, 4, roleURI, cellFmt) sheet2row += 1 # tree walk recursive function def treeWalk(row, depth, concept, preferredLabel, arcrole, preRelSet, visited): if concept is not None: # calc parents calcRelSet = dts.relationshipSet(XbrlConst.summationItem, preRelSet.linkrole) calcRel = None for modelRel in calcRelSet.toModelObject(concept): calcRel = modelRel break for i, hdr in enumerate(sheet1Headers): colType = hdr[1] value = "" if colType == "name": value = str(concept.qname.localName) elif colType == "prefix": value = concept.qname.prefix elif colType == "type": value = str(concept.type.qname) elif colType == "substitutionGroup": value = str(concept.substitutionGroupQname) elif colType == "abstract": value = "true" if concept.isAbstract else "false" elif colType == "periodType": value = concept.periodType elif colType == "balance": value = concept.balance elif colType == "label": role = hdr[2] lang = hdr[3] value = concept.label(preferredLabel if role == XbrlConst.standardLabel else role, linkroleHint=preRelSet.linkrole, lang=lang) elif colType == "preferredLabel" and preferredLabel: if preferredLabel.startswith("http://www.xbrl.org/2003/role/"): value = os.path.basename(preferredLabel) else: value = preferredLabel elif colType == "calculationParent" and calcRel is not None: calcParent = calcRel.fromModelObject if calcParent is not None: value = str(calcParent.qname) elif colType == "calculationWeight" and calcRel is not None: value = calcRel.weight elif colType == "depth": value = depth if "indented" in hdr: style = cellFmtIndented[min(depth, len(cellFmtIndented) - 1)] else: style = cellFmt sheet1.write(row, i, value, style) row += 1 if concept not in visited: visited.add(concept) for modelRel in preRelSet.fromModelObject(concept): if modelRel.toModelObject is not None: row = treeWalk(row, depth + 1, modelRel.toModelObject, modelRel.preferredLabel, arcrole, preRelSet, visited) visited.remove(concept) return row # use presentation relationships for Sheet1 arcrole = XbrlConst.parentChild # sort URIs by definition linkroleUris = [] relationshipSet = dts.relationshipSet(arcrole) if relationshipSet: for linkroleUri in relationshipSet.linkRoleUris: modelRoleTypes = dts.roleTypes.get(linkroleUri) if modelRoleTypes: roledefinition = (modelRoleTypes[0].genLabel(strip=True) or modelRoleTypes[0].definition or linkroleUri) else: roledefinition = linkroleUri linkroleUris.append((roledefinition, linkroleUri)) linkroleUris.sort() # for each URI in definition order for roledefinition, linkroleUri in linkroleUris: # write linkrole sheet1.write(sheet1row, 0, (roledefinition or linkroleUri), xlwt.easyxf(cellFont)) # ELR has no boarders, just font specified sheet1row += 1 # write header row for i, hdr in enumerate(sheet1Headers): sheet1.write(sheet1row, i, hdr[0], hdrCellFmt) sheet1row += 1 # elr relationships for tree walk linkRelationshipSet = dts.relationshipSet(arcrole, linkroleUri) for rootConcept in linkRelationshipSet.rootConcepts: sheet1row = treeWalk(sheet1row, 0, rootConcept, None, arcrole, linkRelationshipSet, set()) sheet1row += 1 # double space rows between tables try: workbook.save(excelFile) dts.info("info:saveLoadableExcel", _("Saved Excel file: %(excelFile)s"), excelFile=os.path.basename(excelFile), modelXbrl=dts) except Exception as ex: dts.error("exception:saveLoadableExcel", _("File saving exception: %(error)s"), error=ex, modelXbrl=dts)
def checkDTS(val, modelDocument, visited): global targetNamespaceDatePattern, efmFilenamePattern, roleTypePattern, arcroleTypePattern, \ arcroleDefinitionPattern, namePattern, linkroleDefinitionBalanceIncomeSheet, \ namespacesConflictPattern if targetNamespaceDatePattern is None: targetNamespaceDatePattern = re.compile(r"/([12][0-9]{3})-([01][0-9])-([0-3][0-9])|" r"/([12][0-9]{3})([01][0-9])([0-3][0-9])|") efmFilenamePattern = re.compile(r"^[a-z0-9][a-z0-9_\.\-]*(\.xsd|\.xml)$") roleTypePattern = re.compile(r"^.*/role/[^/\s]+$") arcroleTypePattern = re.compile(r"^.*/arcrole/[^/\s]+$") arcroleDefinitionPattern = re.compile(r"^.*[^\\s]+.*$") # at least one non-whitespace character namePattern = re.compile("[][()*+?\\\\/^{}|@#%^=~`\"';:,<>&$\u00a3\u20ac]") # u20ac=Euro, u00a3=pound sterling linkroleDefinitionBalanceIncomeSheet = re.compile(r"[^-]+-\s+Statement\s+-\s+.*(income|balance|financial\W+position)", re.IGNORECASE) namespacesConflictPattern = re.compile(r"http://(xbrl\.us|fasb\.org|xbrl\.sec\.gov)/(dei|us-types|us-roles|rr)/([0-9]{4}-[0-9]{2}-[0-9]{2})$") nonDomainItemNameProblemPattern = re.compile( r"({0})|(FirstQuarter|SecondQuarter|ThirdQuarter|FourthQuarter|[1-4]Qtr|Qtr[1-4]|ytd|YTD|HalfYear)(?:$|[A-Z\W])" .format(re.sub(r"\W", "", (val.entityRegistrantName or "").title()))) visited.append(modelDocument) definesLabelLinkbase = False for referencedDocument in modelDocument.referencesDocument.items(): #6.07.01 no includes if referencedDocument[1] == "include": val.modelXbrl.error(("EFM.6.07.01", "GFM.1.03.01", "SBR.NL.2.2.0.18"), _("Taxonomy schema %(schema)s includes %(include)s, only import is allowed"), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), include=os.path.basename(referencedDocument[0].uri)) if referencedDocument[0] not in visited: checkDTS(val, referencedDocument[0], visited) if val.disclosureSystem.standardTaxonomiesDict is None: pass if val.validateEFM: if modelDocument.uri in val.disclosureSystem.standardTaxonomiesDict: if modelDocument.targetNamespace: # check for duplicates of us-types, dei, and rr taxonomies match = namespacesConflictPattern.match(modelDocument.targetNamespace) if match is not None: val.standardNamespaceConflicts[match.group(2)].add(modelDocument) else: if len(modelDocument.basename) > 32: val.modelXbrl.error("EFM.5.01.01.tooManyCharacters", _("Document file name %(filename)s must not exceed 32 characters."), modelObject=modelDocument, filename=modelDocument.basename) if not efmFilenamePattern.match(modelDocument.basename): val.modelXbrl.error("EFM.5.01.01", _("Document file name %(filename)s must start with a-z or 0-9, contain lower case letters, ., -, _, and end with .xsd or .xml."), modelObject=modelDocument, filename=modelDocument.basename) if (modelDocument.type == ModelDocument.Type.SCHEMA and modelDocument.targetNamespace not in val.disclosureSystem.baseTaxonomyNamespaces and modelDocument.uri.startswith(val.modelXbrl.uriDir)): # check schema contents types if val.validateSBRNL: definesLinkroles = False definesArcroles = False definesLinkParts = False definesAbstractItems = False definesNonabstractItems = False definesConcepts = False definesTuples = False definesPresentationTuples = False definesSpecificationTuples = False definesTypes = False definesEnumerations = False definesDimensions = False definesDomains = False definesHypercubes = False # 6.7.3 check namespace for standard authority targetNamespaceAuthority = UrlUtil.authority(modelDocument.targetNamespace) if targetNamespaceAuthority in val.disclosureSystem.standardAuthorities: val.modelXbrl.error(("EFM.6.07.03", "GFM.1.03.03"), _("Taxonomy schema %(schema)s namespace %(targetNamespace)s is a disallowed authority"), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace, targetNamespaceAuthority=targetNamespaceAuthority) # 6.7.4 check namespace format if modelDocument.targetNamespace is None or not modelDocument.targetNamespace.startswith("http://"): match = None elif val.validateEFMorGFM: targetNamespaceDate = modelDocument.targetNamespace[len(targetNamespaceAuthority):] match = targetNamespaceDatePattern.match(targetNamespaceDate) else: match = None if match is not None: try: if match.lastindex == 3: date = datetime.date(int(match.group(1)),int(match.group(2)),int(match.group(3))) elif match.lastindex == 6: date = datetime.date(int(match.group(4)),int(match.group(5)),int(match.group(6))) else: match = None except ValueError: match = None if match is None: val.modelXbrl.error(("EFM.6.07.04", "GFM.1.03.04"), _("Taxonomy schema %(schema)s namespace %(targetNamespace)s must have format http://{authority}/{versionDate}"), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace) elif val.fileNameDate and date > val.fileNameDate: val.modelXbrl.info(("EFM.6.07.06", "GFM.1.03.06"), _("Warning: Taxonomy schema %(schema)s namespace %(targetNamespace)s has date later than document name date %(docNameDate)s"), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace, docNameDate=val.fileNameDate) if modelDocument.targetNamespace is not None: # 6.7.5 check prefix for _ authority = UrlUtil.authority(modelDocument.targetNamespace) if not re.match(r"(http://|https://|ftp://|urn:)\w+",authority): val.modelXbrl.error(("EFM.6.07.05", "GFM.1.03.05"), _("Taxonomy schema %(schema)s namespace %(targetNamespace)s must be a valid URL with a valid authority for the namespace."), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace) prefix = XmlUtil.xmlnsprefix(modelDocument.xmlRootElement,modelDocument.targetNamespace) if not prefix: val.modelXbrl.error(("EFM.6.07.07", "GFM.1.03.07"), _("Taxonomy schema %(schema)s namespace %(targetNamespace)s missing prefix for the namespace."), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace) elif "_" in prefix: val.modelXbrl.error(("EFM.6.07.07", "GFM.1.03.07"), _("Taxonomy schema %(schema)s namespace %(targetNamespace)s prefix %(prefix)s must not have an '_'"), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace, prefix=prefix) if val.validateSBRNL: genrlSpeclRelSet = val.modelXbrl.relationshipSet(XbrlConst.generalSpecial) for modelConcept in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/2001/XMLSchema}element"): if isinstance(modelConcept,ModelConcept): # 6.7.16 name not duplicated in standard taxonomies name = modelConcept.get("name") if name is None: name = "" if modelConcept.get("ref") is not None: continue # don't validate ref's here for c in val.modelXbrl.nameConcepts.get(name, []): if c.modelDocument != modelDocument: if (val.validateEFMorGFM and not c.modelDocument.uri.startswith(val.modelXbrl.uriDir)): val.modelXbrl.error(("EFM.6.07.16", "GFM.1.03.18"), _("Concept %(concept)s is also defined in standard taxonomy schema schema %(standardSchema)s"), modelObject=c, concept=modelConcept.qname, standardSchema=os.path.basename(c.modelDocument.uri)) elif val.validateSBRNL: if not (genrlSpeclRelSet.isRelated(modelConcept, "child", c) or genrlSpeclRelSet.isRelated(c, "child", modelConcept)): val.modelXbrl.error("SBR.NL.2.2.2.02", _("Concept %(concept)s is also defined in standard taxonomy schema %(standardSchema)s without a general-special relationship"), modelObject=c, concept=modelConcept.qname, standardSchema=os.path.basename(c.modelDocument.uri)) ''' removed RH 2011-12-23 corresponding set up of table in ValidateFiling if val.validateSBRNL and name in val.nameWordsTable: if not any( any( genrlSpeclRelSet.isRelated(c, "child", modelConcept) for c in val.modelXbrl.nameConcepts.get(partialWordName, [])) for partialWordName in val.nameWordsTable[name]): val.modelXbrl.error("SBR.NL.2.3.2.01", _("Concept %(specialName)s is appears to be missing a general-special relationship to %(generalNames)s"), modelObject=c, specialName=modelConcept.qname, generalNames=', or to '.join(val.nameWordsTable[name])) ''' # 6.7.17 id properly formed id = modelConcept.id requiredId = (prefix if prefix is not None else "") + "_" + name if val.validateEFMorGFM and id != requiredId: val.modelXbrl.error(("EFM.6.07.17", "GFM.1.03.19"), _("Concept %(concept)s id %(id)s should be $(requiredId)s"), modelObject=modelConcept, concept=modelConcept.qname, id=id, requiredId=requiredId) # 6.7.18 nillable is true nillable = modelConcept.get("nillable") if nillable != "true": val.modelXbrl.error(("EFM.6.07.18", "GFM.1.03.20"), _("Taxonomy schema %(schema)s element %(concept)s nillable %(nillable)s should be 'true'"), modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=name, nillable=nillable) # 6.7.19 not tuple if modelConcept.isTuple: if val.validateEFMorGFM: val.modelXbrl.error(("EFM.6.07.19", "GFM.1.03.21"), _("Concept %(concept)s is a tuple"), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.20 no typed domain ref if modelConcept.isTypedDimension: val.modelXbrl.error(("EFM.6.07.20", "GFM.1.03.22"), _("Concept %(concept)s has typedDomainRef %(typedDomainRef)s"), modelObject=modelConcept, concept=modelConcept.qname, typedDomainRef=modelConcept.typedDomainElement.qname if modelConcept.typedDomainElement is not None else modelConcept.typedDomainRef) # 6.7.21 abstract must be duration isDuration = modelConcept.periodType == "duration" if modelConcept.abstract == "true" and not isDuration: val.modelXbrl.error(("EFM.6.07.21", "GFM.1.03.23"), _("Taxonomy schema %(schema)s element %(concept)s is abstract but period type is not duration"), modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=name) # 6.7.22 abstract must be stringItemType ''' removed SEC EFM v.17, Edgar release 10.4, and GFM 2011-04-08 if modelConcept.abstract == "true" and modelConcept.typeQname != XbrlConst. qnXbrliStringItemType: val.modelXbrl.error(("EFM.6.07.22", "GFM.1.03.24"), _("Concept %(concept)s is abstract but type is not xbrli:stringItemType"), modelObject=modelConcept, concept=modelConcept.qname) ''' substititutionGroupQname = modelConcept.substitutionGroupQname # 6.7.23 Axis must be subs group dimension if name.endswith("Axis") ^ (substititutionGroupQname == XbrlConst.qnXbrldtDimensionItem): val.modelXbrl.error(("EFM.6.07.23", "GFM.1.03.25"), _("Concept %(concept)s must end in Axis to be in xbrldt:dimensionItem substitution group"), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.24 Table must be subs group hypercube if name.endswith("Table") ^ (substititutionGroupQname == XbrlConst.qnXbrldtHypercubeItem): val.modelXbrl.error(("EFM.6.07.24", "GFM.1.03.26"), _("Concept %(concept)s must end in Table to be in xbrldt:hypercubeItem substitution group"), modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=modelConcept.qname) # 6.7.25 if neither hypercube or dimension, substitution group must be item if substititutionGroupQname not in (None, XbrlConst.qnXbrldtDimensionItem, XbrlConst.qnXbrldtHypercubeItem, XbrlConst.qnXbrliItem): val.modelXbrl.error(("EFM.6.07.25", "GFM.1.03.27"), _("Concept %(concept)s has disallowed substitution group %(substitutionGroup)s"), modelObject=modelConcept, concept=modelConcept.qname, substitutionGroup=modelConcept.substitutionGroupQname) # 6.7.26 Table must be subs group hypercube if name.endswith("LineItems") and modelConcept.abstract != "true": val.modelXbrl.error(("EFM.6.07.26", "GFM.1.03.28"), _("Concept %(concept)s is a LineItems but not abstract"), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.27 type domainMember must end with Domain or Member conceptType = modelConcept.type isDomainItemType = conceptType is not None and conceptType.isDomainItemType endsWithDomainOrMember = name.endswith("Domain") or name.endswith("Member") if isDomainItemType != endsWithDomainOrMember: val.modelXbrl.error(("EFM.6.07.27", "GFM.1.03.29"), _("Concept %(concept)s must end with Domain or Member for type of domainItemType"), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.28 domainItemType must be duration if isDomainItemType and not isDuration: val.modelXbrl.error(("EFM.6.07.28", "GFM.1.03.30"), _("Concept %(concept)s is a domainItemType and must be periodType duration"), modelObject=modelConcept, concept=modelConcept.qname) # 6.8.5 semantic check, check LC3 name if not name[0].isupper(): val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.05.firstLetter", "GFM.2.03.05.firstLetter"), _("Concept %(concept)s name must start with a capital letter"), modelObject=modelConcept, concept=modelConcept.qname) if namePattern.search(name): val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.05.disallowedCharacter", "GFM.2.03.05.disallowedCharacter"), _("Concept %(concept)s has disallowed name character"), modelObject=modelConcept, concept=modelConcept.qname) if len(name) > 200: val.modelXbrl.log("ERROR-SEMANTIC", "EFM.6.08.05.nameLength", _("Concept %(concept)s name length %(namelength)s exceeds 200 characters"), modelObject=modelConcept, concept=modelConcept.qname, namelength=len(name)) if val.validateEFM: label = modelConcept.label(lang="en-US", fallbackToQname=False) if label: # allow Joe's Bar, N.A. to be JoesBarNA -- remove ', allow A. as not article "a" lc3name = ''.join(re.sub(r"['.-]", "", (w[0] or w[2] or w[3] or w[4])).title() for w in re.findall(r"((\w+')+\w+)|(A[.-])|([.-]A(?=\W|$))|(\w+)", label) # EFM implies this should allow - and . re.findall(r"[\w\-\.]+", label) if w[4].lower() not in ("the", "a", "an")) if not(name == lc3name or (name and lc3name and lc3name[0].isdigit() and name[1:] == lc3name and (name[0].isalpha() or name[0] == '_'))): val.modelXbrl.log("WARNING-SEMANTIC", "EFM.6.08.05.LC3", _("Concept %(concept)s should match expected LC3 composition %(lc3name)s"), modelObject=modelConcept, concept=modelConcept.qname, lc3name=lc3name) if conceptType is not None: # 6.8.6 semantic check if not isDomainItemType and conceptType.qname != XbrlConst.qnXbrliDurationItemType: nameProblems = nonDomainItemNameProblemPattern.findall(name) if any(any(t) for t in nameProblems): # list of tuples with possibly nonempty strings val.modelXbrl.log("WARNING-SEMANTIC", ("EFM.6.08.06", "GFM.2.03.06"), _("Concept %(concept)s should not contain company or period information, found: %(matches)s"), modelObject=modelConcept, concept=modelConcept.qname, matches=", ".join(''.join(t) for t in nameProblems)) if conceptType.qname == XbrlConst.qnXbrliMonetaryItemType: if not modelConcept.balance: # 6.8.11 may not appear on a income or balance statement if any(linkroleDefinitionBalanceIncomeSheet.match(roleType.definition) for rel in val.modelXbrl.relationshipSet(XbrlConst.parentChild).toModelObject(modelConcept) for roleType in val.modelXbrl.roleTypes.get(rel.linkrole,())): val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.11", "GFM.2.03.11"), _("Concept %(concept)s must have a balance because it appears in a statement of income or balance sheet"), modelObject=modelConcept, concept=modelConcept.qname) # 6.11.5 semantic check, must have a documentation label stdLabel = modelConcept.label(lang="en-US", fallbackToQname=False) defLabel = modelConcept.label(preferredLabel=XbrlConst.documentationLabel, lang="en-US", fallbackToQname=False) if not defLabel or ( # want different words than std label stdLabel and re.findall(r"\w+", stdLabel) == re.findall(r"\w+", defLabel)): val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.11.05", "GFM.2.04.04"), _("Concept %(concept)s is monetary without a balance and must have a documentation label that disambiguates its sign"), modelObject=modelConcept, concept=modelConcept.qname) # 6.8.16 semantic check if conceptType.qname == XbrlConst.qnXbrliDateItemType and modelConcept.periodType != "duration": val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.16", "GFM.2.03.16"), _("Concept %(concept)s of type xbrli:dateItemType must have periodType duration"), modelObject=modelConcept, concept=modelConcept.qname) # 6.8.17 semantic check if conceptType.qname == XbrlConst.qnXbrliStringItemType and modelConcept.periodType != "duration": val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.17", "GFM.2.03.17"), _("Concept %(concept)s of type xbrli:stringItemType must have periodType duration"), modelObject=modelConcept, concept=modelConcept.qname) if val.validateSBRNL: if modelConcept.isTuple: if modelConcept.substitutionGroupQname.localName == "presentationTuple" and modelConcept.substitutionGroupQname.namespaceURI.endswith("/basis/sbr/xbrl/xbrl-syntax-extension"): # namespace may change each year definesPresentationTuples = True elif modelConcept.substitutionGroupQname.localName == "specificationTuple" and modelConcept.substitutionGroupQname.namespaceURI.endswith("/basis/sbr/xbrl/xbrl-syntax-extension"): # namespace may change each year definesSpecificationTuples = True else: definesTuples = True definesConcepts = True if modelConcept.isAbstract: val.modelXbrl.error("SBR.NL.2.2.2.03", _("Concept %(concept)s is an abstract tuple"), modelObject=modelConcept, concept=modelConcept.qname) if tupleCycle(val,modelConcept): val.modelXbrl.error("SBR.NL.2.2.2.07", _("Tuple %(concept)s has a tuple cycle"), modelObject=modelConcept, concept=modelConcept.qname) if modelConcept.get("nillable") != "false" and modelConcept.isRoot: val.modelXbrl.error("SBR.NL.2.2.2.17", #don't want default, just what was really there _("Tuple %(concept)s must have nillable='false'"), modelObject=modelConcept, concept=modelConcept.qname) elif modelConcept.isItem: definesConcepts = True if modelConcept.abstract == "true": if modelConcept.isRoot: if modelConcept.get("nillable") != "false": #don't want default, just what was really there val.modelXbrl.error("SBR.NL.2.2.2.16", _("Abstract root concept %(concept)s must have nillable='false'"), modelObject=modelConcept, concept=modelConcept.qname) if modelConcept.typeQname != XbrlConst.qnXbrliStringItemType: val.modelXbrl.error("SBR.NL.2.2.2.21", _("Abstract root concept %(concept)s must have type='xbrli:stringItemType'"), modelObject=modelConcept, concept=modelConcept.qname) if modelConcept.balance: val.modelXbrl.error("SBR.NL.2.2.2.22", _("Abstract concept %(concept)s must not have a balance attribute"), modelObject=modelConcept, concept=modelConcept.qname) if modelConcept.isHypercubeItem: definesHypercubes = True elif modelConcept.isDimensionItem: definesDimensions = True elif substititutionGroupQname and substititutionGroupQname.localName in ("domainItem","domainMemberItem"): definesDomains = True elif modelConcept.isItem: definesAbstractItems = True else: # not abstract if modelConcept.isItem: definesNonabstractItems = True if not (modelConcept.label(preferredLabel=XbrlConst.documentationLabel,fallbackToQname=False,lang="nl") or val.modelXbrl.relationshipSet(XbrlConst.conceptReference).fromModelObject(c) or modelConcept.genLabel(role=XbrlConst.genDocumentationLabel,lang="nl") or val.modelXbrl.relationshipSet(XbrlConst.elementReference).fromModelObject(c)): val.modelXbrl.error("SBR.NL.2.2.2.28", _("Concept %(concept)s must have a documentation label or reference"), modelObject=modelConcept, concept=modelConcept.qname) if modelConcept.balance and not modelConcept.instanceOfType(XbrlConst.qnXbrliMonetaryItemType): val.modelXbrl.error("SBR.NL.2.2.2.24", _("Non-monetary concept %(concept)s must not have a balance attribute"), modelObject=modelConcept, concept=modelConcept.qname) if modelConcept.isLinkPart: definesLinkParts = True val.modelXbrl.error("SBR.NL.2.2.5.01", _("Link:part concept %(concept)s is not allowed"), modelObject=modelConcept, concept=modelConcept.qname) if not modelConcept.genLabel(fallbackToQname=False,lang="nl"): val.modelXbrl.error("SBR.NL.2.2.5.02", _("Link part definition %(concept)s must have a generic label in language 'nl'"), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.8 check for embedded linkbase for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}linkbase"): if isinstance(e,ModelObject): val.modelXbrl.error(("EFM.6.07.08", "GFM.1.03.08"), _("Taxonomy schema %(schema)s contains an embedded linkbase"), modelObject=e, schema=os.path.basename(modelDocument.uri)) break requiredUsedOns = {XbrlConst.qnLinkPresentationLink, XbrlConst.qnLinkCalculationLink, XbrlConst.qnLinkDefinitionLink} standardUsedOns = {XbrlConst.qnLinkLabel, XbrlConst.qnLinkReference, XbrlConst.qnLinkFootnote, XbrlConst.qnLinkDefinitionArc, XbrlConst.qnLinkCalculationArc, XbrlConst.qnLinkPresentationArc, XbrlConst.qnLinkLabelArc, XbrlConst.qnLinkReferenceArc # XbrlConst.qnLinkFootnoteArc note: not in EFM table for 6.8.3 and causes tests 6.5.30 to fail } # 6.7.9 role types authority for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}roleType"): if isinstance(e,ModelObject): roleURI = e.get("roleURI") if targetNamespaceAuthority != UrlUtil.authority(roleURI): val.modelXbrl.error(("EFM.6.07.09", "GFM.1.03.09"), _("RoleType %(roleType)s does not match authority %(targetNamespaceAuthority)s"), modelObject=e, roleType=roleURI, targetNamespaceAuthority=targetNamespaceAuthority) # 6.7.9 end with .../role/lc3 name if not roleTypePattern.match(roleURI): val.modelXbrl.warning(("EFM.6.07.09.roleEnding", "GFM.1.03.09"), "RoleType %(roleType)s should end with /role/{LC3name}", modelObject=e, roleType=roleURI) # 6.7.10 only one role type declaration in DTS modelRoleTypes = val.modelXbrl.roleTypes.get(roleURI) if modelRoleTypes is not None: modelRoleType = modelRoleTypes[0] definition = modelRoleType.definitionNotStripped usedOns = modelRoleType.usedOns if len(modelRoleTypes) > 1: val.modelXbrl.error(("EFM.6.07.10", "GFM.1.03.10"), _("RoleType %(roleType)s is defined in multiple taxonomies"), modelObject=modelRoleTypes, roleType=roleURI, numberOfDeclarations=len(modelRoleTypes)) elif len(modelRoleTypes) == 1: # 6.7.11 used on's for pre, cal, def if any has a used on if not usedOns.isdisjoint(requiredUsedOns) and len(requiredUsedOns - usedOns) > 0: val.modelXbrl.error(("EFM.6.07.11", "GFM.1.03.11"), _("RoleType %(roleType)s missing used on %(usedOn)s"), modelObject=e, roleType=roleURI, usedOn=requiredUsedOns - usedOns) # 6.7.12 definition match pattern if (val.disclosureSystem.roleDefinitionPattern is not None and (definition is None or not val.disclosureSystem.roleDefinitionPattern.match(definition))): val.modelXbrl.error(("EFM.6.07.12", "GFM.1.03.12-14"), _("RoleType %(roleType)s definition \"%(definition)s\" must match {Sortcode} - {Type} - {Title}"), modelObject=e, roleType=roleURI, definition=definition) if usedOns & standardUsedOns: # semantics check val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.03", "GFM.2.03.03"), _("RoleType %(roleuri)s is defined using role types already defined by standard roles for: %(qnames)s"), modelObject=e, roleuri=roleURI, qnames=', '.join(str(qn) for qn in usedOns & standardUsedOns)) if val.validateSBRNL: if usedOns & XbrlConst.standardExtLinkQnames or XbrlConst.qnGenLink in usedOns: definesLinkroles = True if not e.genLabel(): val.modelXbrl.error("SBR.NL.2.2.3.03", _("Link RoleType %(roleType)s missing a generic standard label"), modelObject=e, roleType=roleURI) nlLabel = e.genLabel(lang="nl") if definition != nlLabel: val.modelXbrl.error("SBR.NL.2.2.3.04", _("Link RoleType %(roleType)s definition does not match NL standard generic label, \ndefinition: %(definition)s \nNL label: %(label)s"), modelObject=e, roleType=roleURI, definition=definition, label=nlLabel) if definition and (definition[0].isspace() or definition[-1].isspace()): val.modelXbrl.error("SBR.NL.2.2.3.07", _('Link RoleType %(roleType)s definition has leading or trailing spaces: "%(definition)s"'), modelObject=e, roleType=roleURI, definition=definition) # 6.7.13 arcrole types authority for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}arcroleType"): if isinstance(e,ModelObject): arcroleURI = e.get("arcroleURI") if targetNamespaceAuthority != UrlUtil.authority(arcroleURI): val.modelXbrl.error(("EFM.6.07.13", "GFM.1.03.15"), _("ArcroleType %(arcroleType)s does not match authority %(targetNamespaceAuthority)s"), modelObject=e, arcroleType=arcroleURI, targetNamespaceAuthority=targetNamespaceAuthority) # 6.7.13 end with .../arcrole/lc3 name if not arcroleTypePattern.match(arcroleURI): val.modelXbrl.warning(("EFM.6.07.13.arcroleEnding", "GFM.1.03.15"), _("ArcroleType %(arcroleType)s should end with /arcrole/{LC3name}"), modelObject=e, arcroleType=arcroleURI) # 6.7.14 only one arcrole type declaration in DTS modelRoleTypes = val.modelXbrl.arcroleTypes[arcroleURI] if len(modelRoleTypes) > 1: val.modelXbrl.error(("EFM.6.07.14", "GFM.1.03.16"), _("ArcroleType %(arcroleType)s is defined in multiple taxonomies"), modelObject=e, arcroleType=arcroleURI, numberOfDeclarations=len(modelRoleTypes) ) # 6.7.15 definition match pattern definition = modelRoleTypes[0].definition if definition is None or not arcroleDefinitionPattern.match(definition): val.modelXbrl.error(("EFM.6.07.15", "GFM.1.03.17"), _("ArcroleType %(arcroleType)s definition must be non-empty"), modelObject=e, arcroleType=arcroleURI) # semantic checks usedOns = modelRoleTypes[0].usedOns if usedOns & standardUsedOns: # semantics check val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.03", "GFM.2.03.03"), _("ArcroleType %(arcroleuri)s is defined using role types already defined by standard arcroles for: %(qnames)s"), modelObject=e, arcroleuri=arcroleURI, qnames=', '.join(str(qn) for qn in usedOns & standardUsedOns)) if val.validateSBRNL: definesArcroles = True val.modelXbrl.error("SBR.NL.2.2.4.01", _("Arcrole type definition is not allowed: %(arcroleURI)s"), modelObject=e, arcroleURI=arcroleURI) if val.validateSBRNL: for appinfoElt in modelDocument.xmlRootElement.iter(tag="{http://www.w3.org/2001/XMLSchema}appinfo"): for nonLinkElt in appinfoElt.iterdescendants(): if isinstance(nonLinkElt, ModelObject) and nonLinkElt.namespaceURI != XbrlConst.link: val.modelXbrl.error("SBR.NL.2.2.11.05", _("Appinfo contains disallowed non-link element %(element)s"), modelObject=nonLinkElt, element=nonLinkElt.qname) for cplxTypeElt in modelDocument.xmlRootElement.iter(tag="{http://www.w3.org/2001/XMLSchema}complexType"): choiceElt = cplxTypeElt.find("{http://www.w3.org/2001/XMLSchema}choice") if choiceElt is not None: val.modelXbrl.error("SBR.NL.2.2.11.09", _("ComplexType contains disallowed xs:choice element"), modelObject=choiceElt) for cplxContentElt in modelDocument.xmlRootElement.iter(tag="{http://www.w3.org/2001/XMLSchema}complexContent"): if XmlUtil.descendantAttr(cplxContentElt, "http://www.w3.org/2001/XMLSchema", ("extension","restriction"), "base") != "sbr:placeholder": val.modelXbrl.error("SBR.NL.2.2.11.10", _("ComplexContent is disallowed"), modelObject=cplxContentElt) definesTypes = (modelDocument.xmlRootElement.find("{http://www.w3.org/2001/XMLSchema}complexType") is not None or modelDocument.xmlRootElement.find("{http://www.w3.org/2001/XMLSchema}simpleType") is not None) if (definesLinkroles + definesArcroles + definesLinkParts + definesAbstractItems + definesNonabstractItems + definesTuples + definesPresentationTuples + definesSpecificationTuples + definesTypes + definesEnumerations + definesDimensions + definesDomains + definesHypercubes) != 1: schemaContents = [] if definesLinkroles: schemaContents.append(_("linkroles")) if definesArcroles: schemaContents.append(_("arcroles")) if definesLinkParts: schemaContents.append(_("link parts")) if definesAbstractItems: schemaContents.append(_("abstract items")) if definesNonabstractItems: schemaContents.append(_("nonabstract items")) if definesTuples: schemaContents.append(_("tuples")) if definesPresentationTuples: schemaContents.append(_("sbrPresentationTuples")) if definesSpecificationTuples: schemaContents.append(_("sbrSpecificationTuples")) if definesTypes: schemaContents.append(_("types")) if definesEnumerations: schemaContents.append(_("enumerations")) if definesDimensions: schemaContents.append(_("dimensions")) if definesDomains: schemaContents.append(_("domains")) if definesHypercubes: schemaContents.append(_("hypercubes")) if schemaContents: val.modelXbrl.error("SBR.NL.2.2.1.01", _("Taxonomy schema may only define one of these: %(contents)s"), modelObject=modelDocument, contents=', '.join(schemaContents)) elif not any(refDoc.inDTS and refDoc.targetNamespace not in val.disclosureSystem.baseTaxonomyNamespaces for refDoc in modelDocument.referencesDocument.keys()): # no linkbase ref or includes val.modelXbrl.error("SBR.NL.2.2.1.01", _("Taxonomy schema must be a DTS entrypoint OR define linkroles OR arcroles OR link:parts OR context fragments OR abstract items OR tuples OR non-abstract elements OR types OR enumerations OR dimensions OR domains OR hypercubes"), modelObject=modelDocument) if definesConcepts ^ any( # xor so either concepts and no label LB or no concepts and has label LB (refDoc.type == ModelDocument.Type.LINKBASE and XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link, "labelLink") is not None) for refDoc in modelDocument.referencesDocument.keys()): # no label linkbase val.modelXbrl.error("SBR.NL.2.2.1.02", _("A schema that defines concepts MUST have a linked 2.1 label linkbase"), modelObject=modelDocument) if (definesNonabstractItems or definesTuples) and not any( # was xor but changed to and not per RH 1/11/12 (refDoc.type == ModelDocument.Type.LINKBASE and (XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link, "referenceLink") is not None or XmlUtil.descendant(refDoc.xmlRootElement, XbrlConst.link, "label", "{http://www.w3.org/1999/xlink}role", "http://www.xbrl.org/2003/role/documentation" ) is not None)) for refDoc in modelDocument.referencesDocument.keys()): val.modelXbrl.error("SBR.NL.2.2.1.03", _("A schema that defines non-abstract items MUST have a linked (2.1) reference linkbase AND/OR a label linkbase with @xlink:role=documentation"), modelObject=modelDocument) if val.fileNameBasePart: #6.3.3 filename check expectedFilename = "{0}-{1}.xsd".format(val.fileNameBasePart, val.fileNameDatePart) if modelDocument.basename != expectedFilename and not ( # skip if an edgar testcase re.match("e[0-9]{8}(gd|ng)", val.fileNameBasePart) and re.match("e.*-[0-9]{8}.*", modelDocument.basename)): val.modelXbrl.error(("EFM.6.03.03", "GFM.1.01.01"), _('Invalid schema file name: %(filename)s, expected %(expectedFilename)s'), modelObject=modelDocument, filename=modelDocument.basename, expectedFilename=expectedFilename) elif modelDocument.type == ModelDocument.Type.LINKBASE: # if it is part of the submission (in same directory) check name if modelDocument.filepath.startswith(val.modelXbrl.modelDocument.filepathdir): #6.3.3 filename check extLinkElt = XmlUtil.descendant(modelDocument.xmlRootElement, XbrlConst.link, "*", "{http://www.w3.org/1999/xlink}type", "extended") if val.fileNameBasePart: if extLinkElt is not None: if extLinkElt.localName in extLinkEltFileNameEnding: expectedFilename = "{0}-{1}_{2}.xml".format(val.fileNameBasePart, val.fileNameDatePart, extLinkEltFileNameEnding[extLinkElt.localName]) if modelDocument.basename != expectedFilename and not ( # skip if an edgar testcase re.match("e[0-9]{8}(gd|ng)", val.fileNameBasePart) and re.match("e.*-[0-9]{8}.*", modelDocument.basename)): val.modelXbrl.error(("EFM.6.03.03", "GFM.1.01.01"), _('Invalid linkbase file name: %(filename)s, expected %(expectedFilename)s'), modelObject=modelDocument, filename=modelDocument.basename, expectedFilename=expectedFilename) else: # no ext link element val.modelXbrl.error(("EFM.6.03.03.noLinkElement", "GFM.1.01.01.noLinkElement"), _('Invalid linkbase file name: %(filename)s, has no extended link element, cannot determine link type.'), modelObject=modelDocument, filename=modelDocument.basename) visited.remove(modelDocument)
def checkDTSdocument(val, modelDocument, *args, **kwargs): modelXbrl = val.modelXbrl if modelDocument.type in (ModelDocument.Type.SCHEMA, ModelDocument.Type.LINKBASE): isSchema = modelDocument.type == ModelDocument.Type.SCHEMA docinfo = modelDocument.xmlDocument.docinfo if docinfo and docinfo.xml_version != "1.0": modelXbrl.error("SBR.NL.2.2.0.02" if isSchema else "SBR.NL.2.3.0.02", _('%(docType)s xml version must be "1.0" but is "%(xmlVersion)s"'), modelObject=modelDocument, docType=modelDocument.gettype().title(), xmlVersion=docinfo.xml_version) if modelDocument.documentEncoding.lower() != "utf-8": modelXbrl.error("SBR.NL.2.2.0.03" if isSchema else "SBR.NL.2.3.0.03", _('%(docType)s encoding must be "utf-8" but is "%(xmlEncoding)s"'), modelObject=modelDocument, docType=modelDocument.gettype().title(), xmlEncoding=modelDocument.documentEncoding) lookingForPrecedingComment = True for commentNode in modelDocument.xmlRootElement.itersiblings(preceding=True): if isinstance(commentNode, etree._Comment): if lookingForPrecedingComment: lookingForPrecedingComment = False else: modelXbrl.error("SBR.NL.2.2.0.05" if isSchema else "SBR.NL.2.3.0.05", _('%(docType)s must have only one comment node before schema element'), modelObject=modelDocument, docType=modelDocument.gettype().title()) if lookingForPrecedingComment: modelXbrl.error("SBR.NL.2.2.0.04" if isSchema else "SBR.NL.2.3.0.04", _('%(docType)s must have comment node only on line 2'), modelObject=modelDocument, docType=modelDocument.gettype().title()) # check namespaces are used for prefix, ns in modelDocument.xmlRootElement.nsmap.items(): if ((prefix not in val.valUsedPrefixes) and (modelDocument.type != ModelDocument.Type.SCHEMA or ns != modelDocument.targetNamespace)): modelXbrl.error("SBR.NL.2.2.0.11" if modelDocument.type == ModelDocument.Type.SCHEMA else "SBR.NL.2.3.0.08", _('%(docType)s namespace declaration "%(declaration)s" is not used'), modelObject=modelDocument, docType=modelDocument.gettype().title(), declaration=("xmlns" + (":" + prefix if prefix else "") + "=" + ns)) if isSchema and val.annotationsCount > 1: modelXbrl.error("SBR.NL.2.2.0.22", _('Schema has %(annotationsCount)s xs:annotation elements, only 1 allowed'), modelObject=modelDocument, annotationsCount=val.annotationsCount) if modelDocument.type == ModelDocument.Type.LINKBASE: if not val.containsRelationship: modelXbrl.error("SBR.NL.2.3.0.12", "Linkbase has no relationships", modelObject=modelDocument) # check file name suffixes extLinkElt = XmlUtil.descendant(modelDocument.xmlRootElement, XbrlConst.link, "*", "{http://www.w3.org/1999/xlink}type", "extended") if extLinkElt is not None: expectedSuffix = None if extLinkElt.localName == "labelLink": anyLabel = XmlUtil.descendant(modelDocument.xmlRootElement, XbrlConst.link, "label", "{http://www.w3.org/XML/1998/namespace}lang", "*") if anyLabel is not None: xmlLang = anyLabel.get("{http://www.w3.org/XML/1998/namespace}lang") expectedSuffix = "-lab-{0}.xml".format(xmlLang) else: expectedSuffix = {"referenceLink": "-ref.xml", "presentationLink": "-pre.xml", "definitionLink": "-def.xml"}.get(extLinkElt.localName, None) if expectedSuffix: if not modelDocument.uri.endswith(expectedSuffix): modelXbrl.error("SBR.NL.3.2.1.09", "Linkbase filename expected to end with %(expectedSuffix)s: %(filename)s", modelObject=modelDocument, expectedSuffix=expectedSuffix, filename=modelDocument.uri) elif extLinkElt.qname == XbrlConst.qnGenLink: anyLabel = XmlUtil.descendant(modelDocument.xmlRootElement, XbrlConst.link, "label", "{http://www.w3.org/XML/1998/namespace}lang", "*") if anyLabel is not None: xmlLang = anyLabel.get("{http://www.w3.org/XML/1998/namespace}lang") expectedSuffix = "-generic-lab-{0}.xml".format(xmlLang) elif XmlUtil.descendant(modelDocument.xmlRootElement, XbrlConst.link, "reference") is not None: expectedSuffix = "-generic-ref.xml" if expectedSuffix and not modelDocument.uri.endswith(expectedSuffix): modelXbrl.error("SBR.NL.3.2.1.10", "Generic linkbase filename expected to end with %(expectedSuffix)s: %(filename)s", modelObject=modelDocument, expectedSuffix=expectedSuffix, filename=modelDocument.uri) # label checks for qnLabel in (XbrlConst.qnLinkLabel, XbrlConst.qnGenLabel): for modelLabel in modelDocument.xmlRootElement.iterdescendants(tag=qnLabel.clarkNotation): if isinstance(modelLabel, ModelResource): if not modelLabel.text or not modelLabel.text[:1].isupper(): modelXbrl.error("SBR.NL.3.2.7.05", _("Labels MUST have a capital first letter, label %(label)s: %(text)s"), modelObject=modelLabel, label=modelLabel.xlinkLabel, text=modelLabel.text[:64]) if modelLabel.role in (XbrlConst.standardLabel, XbrlConst.genStandardLabel): if len(modelLabel.text) > 255: modelXbrl.error("SBR.NL.3.2.7.06", _("Labels with the 'standard' role MUST NOT exceed 255 characters, label %(label)s: %(text)s"), modelObject=modelLabel, label=modelLabel.xlinkLabel, text=modelLabel.text[:64]) if modelLabel.role in (XbrlConst.standardLabel, XbrlConst.genStandardLabel): if len(modelLabel.text) > 255: modelXbrl.error("SBR.NL.3.2.7.06", _("Labels with the 'standard' role MUST NOT exceed 255 characters, label %(label)s: %(text)s"), modelObject=modelLabel, label=modelLabel.xlinkLabel, text=modelLabel.text[:64]) for modelResource in modelDocument.xmlRootElement.iter(): # locator checks if isinstance(modelResource, ModelLocator): hrefModelObject = modelResource.dereference() if isinstance(hrefModelObject, ModelObject): expectedLocLabel = hrefModelObject.id + "_loc" if modelResource.xlinkLabel != expectedLocLabel: modelXbrl.error("SBR.NL.3.2.11.01", _("Locator @xlink:label names MUST be concatenated from: @id from the XML node, underscore, 'loc', expected %(expectedLocLabel)s, found %(foundLocLabel)s"), modelObject=modelResource, expectedLocLabel=expectedLocLabel, foundLocLabel=modelResource.xlinkLabel) # xlinkLabel checks if isinstance(modelResource, ModelResource): if re.match(r"[^a-zA-Z0-9_-]", modelResource.xlinkLabel): modelXbrl.error("SBR.NL.3.2.11.03", _("@xlink:label names MUST use a-zA-Z0-9_- characters only: %(xlinkLabel)s"), modelObject=modelResource, xlinkLabel=modelResource.xlinkLabel) elif modelDocument.targetNamespace: # SCHEMA with targetNamespace # check for unused imports for referencedDocument in modelDocument.referencesDocument.keys(): if (referencedDocument.type == ModelDocument.Type.SCHEMA and referencedDocument.targetNamespace not in {XbrlConst.xbrli, XbrlConst.link} and referencedDocument.targetNamespace not in val.referencedNamespaces): modelXbrl.error("SBR.NL.2.2.0.15", _("A schema import schemas of which no content is being addressed: %(importedFile)s"), modelObject=modelDocument, importedFile=referencedDocument.basename) if modelDocument.targetNamespace != modelDocument.targetNamespace.lower(): modelXbrl.error("SBR.NL.3.2.3.02", _("Namespace URI's MUST be lower case: %(namespaceURI)s"), modelObject=modelDocument, namespaceURI=modelDocument.targetNamespace) if len(modelDocument.targetNamespace) > 255: modelXbrl.error("SBR.NL.3.2.3.03", _("Namespace URI's MUST NOT be longer than 255 characters: %(namespaceURI)s"), modelObject=modelDocument, namespaceURI=modelDocument.targetNamespace) if re.match(r"[^a-z0-9_/-]", modelDocument.targetNamespace): modelXbrl.error("SBR.NL.3.2.3.04", _("Namespace URI's MUST use only signs from a-z0-9_-/: %(namespaceURI)s"), modelObject=modelDocument, namespaceURI=modelDocument.targetNamespace) if not modelDocument.targetNamespace.startswith('http://www.nltaxonomie.nl'): modelXbrl.error("SBR.NL.3.2.3.05", _("Namespace URI's MUST start with 'http://www.nltaxonomie.nl': %(namespaceURI)s"), modelObject=modelDocument, namespaceURI=modelDocument.targetNamespace) namespacePrefix = XmlUtil.xmlnsprefix(modelDocument.xmlRootElement, modelDocument.targetNamespace) if not namespacePrefix: modelXbrl.error("SBR.NL.3.2.4.01", _("TargetNamespaces MUST have a prefix: %(namespaceURI)s"), modelObject=modelDocument, namespaceURI=modelDocument.targetNamespace) elif namespacePrefix in val.prefixNamespace: modelXbrl.error("SBR.NL.3.2.4.02", _("Namespace prefix MUST be unique within the NT but prefix '%(prefix)s' is used by both %(namespaceURI)s and %(namespaceURI2)s."), modelObject=modelDocument, prefix=namespacePrefix, namespaceURI=modelDocument.targetNamespace, namespaceURI2=val.prefixNamespace[namespacePrefix]) else: val.prefixNamespace[namespacePrefix] = modelDocument.targetNamespace val.namespacePrefix[modelDocument.targetNamespace] = namespacePrefix if namespacePrefix in {"xsi", "xsd", "xs", "xbrli", "link", "xlink", "xbrldt", "xbrldi", "gen", "xl"}: modelXbrl.error("SBR.NL.3.2.4.03", _("Namespace prefix '%(prefix)s' reserved by organizations for international specifications is used %(namespaceURI)s."), modelObject=modelDocument, prefix=namespacePrefix, namespaceURI=modelDocument.targetNamespace) if len(namespacePrefix) > 20: modelXbrl.warning("SBR.NL.3.2.4.06", _("Namespace prefix '%(prefix)s' SHOULD not exceed 20 characters %(namespaceURI)s."), modelObject=modelDocument, prefix=namespacePrefix, namespaceURI=modelDocument.targetNamespace) # check every non-targetnamespace prefix against its schema requiredLinkrole = None # only set for extension taxonomies # check folder names if modelDocument.filepathdir.startswith(modelXbrl.uriDir): partnerPrefix = None pathDir = modelDocument.filepathdir[len(modelXbrl.uriDir) + 1:].replace("\\", "/") lastPathSegment = None for pathSegment in pathDir.split("/"): if pathSegment.lower() != pathSegment: modelXbrl.error("SBR.NL.3.2.1.02", _("Folder names must be in lower case: %(folder)s"), modelObject=modelDocument, folder=pathSegment) if len(pathSegment) >= 15 : modelXbrl.error("SBR.NL.3.2.1.03", _("Folder names must be less than 15 characters: %(folder)s"), modelObject=modelDocument, folder=pathSegment) if pathSegment in ("bd", "kvk", "cbs"): partnerPrefix = pathSegment + '-' lastPathSegment = pathSegment if modelDocument.basename.lower() != modelDocument.basename: modelXbrl.error("SBR.NL.3.2.1.05", _("File names must be in lower case: %(file)s"), modelObject=modelDocument, file=modelDocument.basename) if partnerPrefix and not modelDocument.basename.startswith(partnerPrefix): modelXbrl.error("SBR.NL.3.2.1.14", "NT Partner DTS files MUST start with %(partnerPrefix)s consistently: %(filename)s", modelObject=modelDocument, partnerPrefix=partnerPrefix, filename=modelDocument.uri) if modelDocument.type == ModelDocument.Type.SCHEMA: if modelDocument.targetNamespace: nsParts = modelDocument.targetNamespace.split("/") # [0] = https, [1] = // [2] = nl.taxonomie [3] = year or version nsYrOrVer = nsParts[3] requiredNamespace = "http://www.nltaxonomie.nl/" + nsYrOrVer + "/" + pathDir + "/" + modelDocument.basename[:-4] requiredLinkrole = "http://www.nltaxonomie.nl/" + nsYrOrVer + "/" + pathDir + "/" if modelDocument == modelXbrl.modelDocument: # entry point nsYr = "{year}" if '2009' <= nsParts[3] < '2020': # must be a year, use as year nsYr = nsParts[3] else: # look for year in parts of basename of required namespace for nsPart in nsParts: for baseNamePart in nsPart.split('-'): if '2009' <= baseNamePart < '2020': nsYr = baseNamePart break if not requiredNamespace.endswith('-' + nsYr): requiredNamespace += '-' + nsYr if not modelDocument.targetNamespace.startswith(requiredNamespace): modelXbrl.error("SBR.NL.3.2.3.06", _("Namespace URI's MUST be constructed like %(requiredNamespace)s: %(namespaceURI)s"), modelObject=modelDocument, requiredNamespace=requiredNamespace, namespaceURI=modelDocument.targetNamespace) else: requiredLinkrole = '' # concept checks for modelConcept in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/2001/XMLSchema}element"): if isinstance(modelConcept, ModelConcept): # 6.7.16 name not duplicated in standard taxonomies name = modelConcept.get("name") if name: ''' removed per RH 2013-03-25 substititutionGroupQname = modelConcept.substitutionGroupQname if substititutionGroupQname: if name.endswith("Member") ^ (substititutionGroupQname.localName == "domainMemberItem" and substititutionGroupQname.namespaceURI.endswith("/xbrl/xbrl-syntax-extension")): modelXbrl.error("SBR.NL.3.2.5.11", _("Concept %(concept)s must end in Member to be in sbr:domainMemberItem substitution group"), modelObject=modelConcept, concept=modelConcept.qname) if name.endswith("Domain") ^ (substititutionGroupQname.localName == "domainItem" and substititutionGroupQname.namespaceURI.endswith("/xbrl/xbrl-syntax-extension")): modelXbrl.error("SBR.NL.3.2.5.12", _("Concept %(concept)s must end in Domain to be in sbr:domainItem substitution group"), modelObject=modelConcept, concept=modelConcept.qname) if name.endswith("TypedAxis") ^ (substititutionGroupQname == XbrlConst.qnXbrldtDimensionItem and modelConcept.isTypedDimension): modelXbrl.error("SBR.NL.3.2.5.14", _("Concept %(concept)s must end in TypedAxis to be in xbrldt:dimensionItem substitution group if they represent a typed dimension"), modelObject=modelConcept, concept=modelConcept.qname) if (name.endswith("Axis") and not name.endswith("TypedAxis")) ^ (substititutionGroupQname == XbrlConst.qnXbrldtDimensionItem and modelConcept.isExplicitDimension): modelXbrl.error("SBR.NL.3.2.5.13", _("Concept %(concept)s must end in Axis to be in xbrldt:dimensionItem substitution group if they represent an explicit dimension"), modelObject=modelConcept, concept=modelConcept.qname) if name.endswith("Table") ^ (substititutionGroupQname == XbrlConst.qnXbrldtHypercubeItem): modelXbrl.error("SBR.NL.3.2.5.15", _("Concept %(concept)s must end in Table to be in xbrldt:hypercubeItem substitution group"), modelObject=modelConcept, concept=modelConcept.qname) if name.endswith("Title") ^ (substititutionGroupQname.localName == "presentationItem" and substititutionGroupQname.namespaceURI.endswith("/xbrl/xbrl-syntax-extension")): modelXbrl.error("SBR.NL.3.2.5.16", _("Concept %(concept)s must end in Title to be in sbr:presentationItem substitution group"), modelObject=modelConcept, concept=modelConcept.qname) ''' if len(name) > 200: modelXbrl.error("SBR.NL.3.2.12.02" if modelConcept.isLinkPart else "SBR.NL.3.2.5.21" if (modelConcept.isItem or modelConcept.isTuple) else "SBR.NL.3.2.14.01", _("Concept %(concept)s name length %(namelength)s exceeds 200 characters"), modelObject=modelConcept, concept=modelConcept.qname, namelength=len(name)) # type checks for typeType in ("simpleType", "complexType"): for modelType in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/2001/XMLSchema}" + typeType): if isinstance(modelType, ModelType): name = modelType.get("name") if name is None: name = "" if modelType.get("ref") is not None: continue # don't validate ref's here if len(name) > 200: modelXbrl.error("SBR.NL.3.2.5.21", _("Type %(type)s name length %(namelength)s exceeds 200 characters"), modelObject=modelType, type=modelType.qname, namelength=len(name)) if modelType.qnameDerivedFrom and modelType.qnameDerivedFrom.namespaceURI != XbrlConst.xbrli: modelXbrl.error("SBR.NL.3.2.8.01", _("Custom datatypes MUST be a restriction from XII defined datatypes: %(type)s"), modelObject=modelType, type=modelType.qname) if re.match(r"[^a-zA-Z0-9_-]", name): modelXbrl.error("SBR.NL.3.2.8.02", _("Datatype names MUST use characters a-zA-Z0-9_- only: %(type)s"), modelObject=modelDocument, type=modelType.qname) if modelType.facets and "enumeration" in modelType.facets: if not modelType.qnameDerivedFrom == XbrlConst.qnXbrliStringItemType: modelXbrl.error("SBR.NL.3.2.13.01", _("Enumerations MUST use a restriction on xbrli:stringItemType: %(type)s"), modelObject=modelDocument, type=modelType.qname) if lastPathSegment == "entrypoints": if not modelDocument.xmlRootElement.id: modelXbrl.error("SBR.NL.2.2.0.23", _("xs:schema/@id MUST be present in schema files in the reports/{NT partner}/entrypoints/ folder"), modelObject=modelDocument) # check for idObject conflicts for id, modelObject in modelDocument.idObjects.items(): if id in val.idObjects: modelXbrl.error("SBR.NL.3.2.6.01", _("ID %(id)s must be unique in the DTS but is present on two elements."), modelObject=(modelObject, val.idObjects[id]), id=id) else: val.idObjects[id] = modelObject for roleURI, modelRoleTypes in modelXbrl.roleTypes.items(): if not roleURI.startswith("http://www.xbrl.org"): usedOns = set.union(*[modelRoleType.usedOns for modelRoleType in modelRoleTypes]) # check roletypes for linkroles (only) if usedOns & {XbrlConst.qnLinkPresentationLink, XbrlConst.qnLinkCalculationLink, XbrlConst.qnLinkDefinitionLink, XbrlConst.qnLinkLabel, XbrlConst.qnLinkReference, XbrlConst.qnLinkFootnote}: if len(modelRoleTypes) > 1: modelXbrl.error("SBR.NL.3.2.9.01", _("Linkrole URI's MUST be unique in the NT: %(linkrole)s"), modelObject=modelRoleTypes, linkrole=roleURI) if roleURI.lower() != roleURI: modelXbrl.error("SBR.NL.3.2.9.02", _("Linkrole URI's MUST be in lowercase: %(linkrole)s"), modelObject=modelRoleTypes, linkrole=roleURI) if re.match(r"[^a-z0-9_/-]", roleURI): modelXbrl.error("SBR.NL.3.2.9.03", _("Linkrole URI's MUST use characters a-z0-9_-/ only: %(linkrole)s"), modelObject=modelRoleTypes, linkrole=roleURI) if len(roleURI) > 255: modelXbrl.error("SBR.NL.3.2.9.04", _("Linkrole URI's MUST NOT be longer than 255 characters, length is %(len)s: %(linkrole)s"), modelObject=modelRoleTypes, len=len(roleURI), linkrole=roleURI) ''' removed per RH 2013-03-13 e-mail if not roleURI.startswith('http://www.nltaxonomie.nl'): modelXbrl.error("SBR.NL.3.2.9.05", _("Linkrole URI's MUST start with 'http://www.nltaxonomie.nl': %(linkrole)s"), modelObject=modelRoleTypes, linkrole=roleURI) if (requiredLinkrole and not roleURI.startswith(requiredLinkrole) and re.match(r".*(domain$|axis$|table$|lineitem$)", roleURI)): modelXbrl.error("SBR.NL.3.2.9.06", _("Linkrole URI's MUST have the following construct: http://www.nltaxonomie.nl / {folder path} / {functional name} - {domain or axis or table or lineitem}: %(linkrole)s"), modelObject=modelRoleTypes, linkrole=roleURI) ''' for modelRoleType in modelRoleTypes: if len(modelRoleType.id) > 255: modelXbrl.error("SBR.NL.3.2.10.02", _("Linkrole @id MUST NOT exceed 255 characters, length is %(length)s: %(linkroleID)s"), modelObject=modelRoleType, length=len(modelRoleType.id), linkroleID=modelRoleType.id) partnerPrefix = modelRoleTypes[0].modelDocument.basename.split('-') if partnerPrefix: # first element before dash is prefix urnPartnerLinkroleStart = "urn:{0}:linkrole:".format(partnerPrefix[0]) if not roleURI.startswith(urnPartnerLinkroleStart): modelXbrl.error("SBR.NL.3.2.9.10", _("Linkrole MUST start with urn:{NT partner code}:linkrole:, \nexpecting: %(expectedStart)s..., \nfound: %(linkrole)s"), modelObject=modelRoleType, expectedStart=urnPartnerLinkroleStart, linkrole=roleURI)
def final(val): if not (val.validateEBA or val.validateEIOPA): return modelXbrl = val.modelXbrl modelDocument = modelXbrl.modelDocument _statusMsg = _("validating {0} filing rules").format(val.disclosureSystem.name) modelXbrl.profileActivity() modelXbrl.modelManager.showStatus(_statusMsg) if modelDocument.type == ModelDocument.Type.INSTANCE and (val.validateEBA or val.validateEIOPA): if not modelDocument.uri.endswith(".xbrl"): modelXbrl.warning("EBA.1.1", _('XBRL instance documents SHOULD use the extension ".xbrl" but it is "%(extension)s"'), modelObject=modelDocument, extension=os.path.splitext(modelDocument.basename)[1]) if modelDocument.documentEncoding.lower() not in ("utf-8", "utf-8-sig"): modelXbrl.error("EBA.1.4", _('XBRL instance documents MUST use "UTF-8" encoding but is "%(xmlEncoding)s"'), modelObject=modelDocument, xmlEncoding=modelDocument.documentEncoding) schemaRefElts = [] schemaRefFileNames = [] for doc, docRef in modelDocument.referencesDocument.items(): if docRef.referenceType == "href": if docRef.referringModelObject.localName == "schemaRef": schemaRefElts.append(docRef.referringModelObject) schemaRefFileNames.append(doc.basename) if not UrlUtil.isAbsolute(doc.uri): modelXbrl.error("EBA.2.2", _('The link:schemaRef element in submitted instances MUST resolve to the full published entry point URL: %(url)s.'), modelObject=docRef.referringModelObject, url=doc.uri) elif docRef.referringModelObject.localName == "linkbaseRef": modelXbrl.error("EBA.2.3", _('The link:linkbaseRef element is not allowed: %(fileName)s.'), modelObject=docRef.referringModelObject, fileName=doc.basename) if len(schemaRefFileNames) > 1: modelXbrl.error("EBA.1.5", _('XBRL instance documents MUST reference only one entry point schema but %(numEntryPoints)s were found: %(entryPointNames)s'), modelObject=modelDocument, numEntryPoints=len(schemaRefFileNames), entryPointNames=', '.join(sorted(schemaRefFileNames))) ### check entry point names appropriate for filing indicator (DPM DB?) if len(schemaRefElts) != 1: modelXbrl.error("EBA.2.3", _('Any reported XBRL instance document MUST contain only one xbrli:xbrl/link:schemaRef node, but %(entryPointCount)s.'), modelObject=schemaRefElts, entryPointCount=len(schemaRefElts)) # non-streaming EBA checks if not getattr(modelXbrl, "isStreamingMode", False): validateFacts(val, modelXbrl.facts) # check sum of fact md5s (otherwise checked in streaming process) xbrlFactsCheckVersion = None expectedSumOfFactMd5s = None for pi in modelDocument.xmlRootElement.getchildren(): if isinstance(pi, etree._ProcessingInstruction) and pi.target == "xbrl-facts-check": _match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']", pi.text) if _match: _matchGroups = _match.groups() if len(_matchGroups) == 2: if _matchGroups[0] == "version": xbrlFactsCheckVersion = _matchGroups[1] elif _matchGroups[0] == "sum-of-fact-md5s": try: expectedSumOfFactMd5s = Md5Sum(_matchGroups[1]) except ValueError: modelXbrl.error("EIOPA:xbrlFactsCheckError", _("Invalid sum-of-md5s %(sumOfMd5)s"), modelObject=modelXbrl, sumOfMd5=_matchGroups[1]) if xbrlFactsCheckVersion and expectedSumOfFactMd5s: sumOfFactMd5s = Md5Sum() for f in modelXbrl.factsInInstance: sumOfFactMd5s += f.md5sum if sumOfFactMd5s != expectedSumOfFactMd5s: modelXbrl.warning("EIOPA:xbrlFactsCheckWarning", _("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"), modelObject=modelXbrl, expectedMd5=expectedSumOfFactMd5s, actualMd5Sum=sumOfFactMd5s) else: modelXbrl.info("info", _("Successful XBRL facts sum of md5s."), modelObject=modelXbrl) if not val.filingIndicators: modelXbrl.error("EBA.1.6", _('Missing filing indicators. Reported XBRL instances MUST include appropriate filing indicator elements'), modelObject=modelDocument) if val.numFilingIndicatorTuples > 1: modelXbrl.info("EBA.1.6.2", _('Multiple filing indicators tuples when not in streaming mode (info).'), modelObject=modelXbrl.factsByQname[qnFIndicators]) if len(val.cntxDates) > 1: modelXbrl.error("EBA.2.13", _('Contexts must have the same date: %(dates)s.'), # when streaming values are no longer available, but without streaming they can be logged modelObject=set(_cntx for _cntxs in val.cntxDates.values() for _cntx in _cntxs), dates=', '.join(XmlUtil.dateunionValue(_dt, subtractOneDay=True) for _dt in val.cntxDates.keys())) if val.unusedCntxIDs: modelXbrl.warning("EBA.2.7", _('Unused xbrli:context nodes SHOULD NOT be present in the instance: %(unusedContextIDs)s.'), modelObject=[modelXbrl.contexts[unusedCntxID] for unusedCntxID in val.unusedCntxIDs if unusedCntxID in modelXbrl.contexts], unusedContextIDs=", ".join(sorted(val.unusedCntxIDs))) if len(val.cntxEntities) > 1: modelXbrl.warning("EBA.2.9", _('All entity identifiers and schemes must be the same, %(count)s found: %(entities)s.'), modelObject=modelDocument, count=len(val.cntxEntities), entities=", ".join(sorted(str(cntxEntity) for cntxEntity in val.cntxEntities))) if val.unusedUnitIDs: modelXbrl.warning("EBA.2.21", _('Unused xbrli:unit nodes SHOULD NOT be present in the instance: %(unusedUnitIDs)s.'), modelObject=[modelXbrl.units[unusedUnitID] for unusedUnitID in val.unusedUnitIDs if unusedUnitID in modelXbrl.units], unusedUnitIDs=", ".join(sorted(val.unusedUnitIDs))) if len(val.currenciesUsed) > 1: modelXbrl.error("EBA.3.1", _("There MUST be only one currency but %(numCurrencies)s were found: %(currencies)s.'"), modelObject=val.currenciesUsed.values(), numCurrencies=len(val.currenciesUsed), currencies=", ".join(str(c) for c in val.currenciesUsed.keys())) if val.prefixesUnused: modelXbrl.warning("EBA.3.4", _("There SHOULD be no unused prefixes but these were declared: %(unusedPrefixes)s.'"), modelObject=modelDocument, unusedPrefixes=', '.join(sorted(val.prefixesUnused))) for ns, prefixes in val.namespacePrefixesUsed.items(): nsDocs = modelXbrl.namespaceDocs.get(ns) if nsDocs: for nsDoc in nsDocs: nsDocPrefix = XmlUtil.xmlnsprefix(nsDoc.xmlRootElement, ns) if any(prefix != nsDocPrefix for prefix in prefixes if prefix is not None): modelXbrl.warning("EBA.3.5", _("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s"), modelObject=modelDocument, namespace=ns, declaredPrefix=nsDocPrefix, foundPrefixes=', '.join(sorted(prefixes - {None}))) modelXbrl.profileActivity(_statusMsg, minTimeToShow=0.0) modelXbrl.modelManager.showStatus(None) del val.prefixNamespace, val.namespacePrefix, val.idObjects, val.typedDomainElements del val.utrValidator
def checkFilingDTS(val, modelDocument, isEFM, isGFM, visited): global targetNamespaceDatePattern, efmFilenamePattern, htmlFileNamePattern, roleTypePattern, arcroleTypePattern, \ arcroleDefinitionPattern, namePattern, linkroleDefinitionBalanceIncomeSheet, \ usNamespacesConflictPattern, ifrsNamespacesConflictPattern if targetNamespaceDatePattern is None: targetNamespaceDatePattern = re.compile(r"/([12][0-9]{3})-([01][0-9])-([0-3][0-9])|" r"/([12][0-9]{3})([01][0-9])([0-3][0-9])|") efmFilenamePattern = re.compile(r"^[a-z0-9][a-zA-Z0-9_\.\-]*(\.xsd|\.xml|\.htm)$") htmlFileNamePattern = re.compile(r"^[a-zA-Z0-9][._a-zA-Z0-9-]*(\.htm)$") roleTypePattern = re.compile(r"^.*/role/[^/\s]+$") arcroleTypePattern = re.compile(r"^.*/arcrole/[^/\s]+$") arcroleDefinitionPattern = re.compile(r"^.*[^\\s]+.*$") # at least one non-whitespace character namePattern = re.compile("[][()*+?\\\\/^{}|@#%^=~`\"';:,<>&$\u00a3\u20ac]") # u20ac=Euro, u00a3=pound sterling linkroleDefinitionBalanceIncomeSheet = re.compile(r"[^-]+-\s+Statement\s+-\s+.*(income|balance|financial\W+position)", re.IGNORECASE) usNamespacesConflictPattern = re.compile(r"http://(xbrl\.us|fasb\.org|xbrl\.sec\.gov)/(dei|us-types|us-roles|rr)/([0-9]{4}-[0-9]{2}-[0-9]{2})$") ifrsNamespacesConflictPattern = re.compile(r"http://xbrl.ifrs.org/taxonomy/([0-9]{4}-[0-9]{2}-[0-9]{2})/(ifrs[\w-]*)$") nonDomainItemNameProblemPattern = re.compile( r"({0})|(FirstQuarter|SecondQuarter|ThirdQuarter|FourthQuarter|[1-4]Qtr|Qtr[1-4]|ytd|YTD|HalfYear)(?:$|[A-Z\W])" .format(re.sub(r"\W", "", (val.entityRegistrantName or "").title()))) visited.append(modelDocument) for referencedDocument, modelDocumentReference in modelDocument.referencesDocument.items(): #6.07.01 no includes if modelDocumentReference.referenceType == "include": val.modelXbrl.error(("EFM.6.07.01", "GFM.1.03.01"), _("Taxonomy schema %(schema)s includes %(include)s, only import is allowed"), modelObject=modelDocumentReference.referringModelObject, schema=os.path.basename(modelDocument.uri), include=os.path.basename(referencedDocument.uri)) if referencedDocument not in visited and referencedDocument.inDTS: # ignore EdgarRenderer added non-DTS documents checkFilingDTS(val, referencedDocument, isEFM, isGFM, visited) if val.disclosureSystem.standardTaxonomiesDict is None: pass if isEFM: if modelDocument.uri in val.disclosureSystem.standardTaxonomiesDict: if modelDocument.targetNamespace: # check for duplicates of us-types, dei, and rr taxonomies for pattern, indexGroup in ((usNamespacesConflictPattern,2),(ifrsNamespacesConflictPattern,2)): match = pattern.match(modelDocument.targetNamespace) if match is not None: val.standardNamespaceConflicts[match.group(indexGroup)].add(modelDocument) else: if len(modelDocument.basename) > 32: val.modelXbrl.error("EFM.5.01.01.tooManyCharacters", _("Document file name %(filename)s must not exceed 32 characters."), modelObject=modelDocument, filename=modelDocument.basename) if modelDocument.type == ModelDocument.Type.INLINEXBRL: if not htmlFileNamePattern.match(modelDocument.basename): val.modelXbrl.error("EFM.5.01.01", _("Document file name %(filename)s must start with a-z or 0-9, contain upper or lower case letters, ., -, _, and end with .htm."), modelObject=modelDocument, filename=modelDocument.basename) elif not efmFilenamePattern.match(modelDocument.basename): val.modelXbrl.error("EFM.5.01.01", _("Document file name %(filename)s must start with a-z or 0-9, contain upper or lower case letters, ., -, _, and end with .xsd or .xml."), modelObject=modelDocument, filename=modelDocument.basename) if (modelDocument.type == ModelDocument.Type.SCHEMA and modelDocument.targetNamespace not in val.disclosureSystem.baseTaxonomyNamespaces and modelDocument.uri.startswith(val.modelXbrl.uriDir)): val.hasExtensionSchema = True # check schema contents types # 6.7.3 check namespace for standard authority targetNamespaceAuthority = UrlUtil.authority(modelDocument.targetNamespace) if targetNamespaceAuthority in val.disclosureSystem.standardAuthorities: val.modelXbrl.error(("EFM.6.07.03", "GFM.1.03.03"), _("Taxonomy schema %(schema)s namespace %(targetNamespace)s is a disallowed authority"), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace, targetNamespaceAuthority=UrlUtil.authority(modelDocument.targetNamespace, includeScheme=False)) # 6.7.4 check namespace format if modelDocument.targetNamespace is None or not modelDocument.targetNamespace.startswith("http://"): match = None else: targetNamespaceDate = modelDocument.targetNamespace[len(targetNamespaceAuthority):] match = targetNamespaceDatePattern.match(targetNamespaceDate) if match is not None: try: if match.lastindex == 3: date = datetime.date(int(match.group(1)),int(match.group(2)),int(match.group(3))) elif match.lastindex == 6: date = datetime.date(int(match.group(4)),int(match.group(5)),int(match.group(6))) else: match = None except ValueError: match = None if match is None: val.modelXbrl.error(("EFM.6.07.04", "GFM.1.03.04"), _("Taxonomy schema %(schema)s namespace %(targetNamespace)s must have format http://{authority}/{versionDate}"), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace) elif val.fileNameDate and date > val.fileNameDate: val.modelXbrl.info(("EFM.6.07.06", "GFM.1.03.06"), _("Warning: Taxonomy schema %(schema)s namespace %(targetNamespace)s has date later than document name date %(docNameDate)s"), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace, docNameDate=val.fileNameDate) if modelDocument.targetNamespace is not None: # 6.7.5 check prefix for _ authority = UrlUtil.authority(modelDocument.targetNamespace) if not re.match(r"(http://|https://|ftp://|urn:)\w+",authority): val.modelXbrl.error(("EFM.6.07.05", "GFM.1.03.05"), _("Taxonomy schema %(schema)s namespace %(targetNamespace)s must be a valid URL with a valid authority for the namespace."), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace) prefix = XmlUtil.xmlnsprefix(modelDocument.xmlRootElement,modelDocument.targetNamespace) if not prefix: val.modelXbrl.error(("EFM.6.07.07", "GFM.1.03.07"), _("Taxonomy schema %(schema)s namespace %(targetNamespace)s missing prefix for the namespace."), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace) elif "_" in prefix: val.modelXbrl.error(("EFM.6.07.07", "GFM.1.03.07"), _("Taxonomy schema %(schema)s namespace %(targetNamespace)s prefix %(prefix)s must not have an '_'"), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace, prefix=prefix) for modelConcept in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/2001/XMLSchema}element"): if isinstance(modelConcept,ModelConcept): # 6.7.16 name not duplicated in standard taxonomies name = modelConcept.get("name") if name is None: name = "" if modelConcept.get("ref") is not None: continue # don't validate ref's here for c in val.modelXbrl.nameConcepts.get(name, []): if c.modelDocument != modelDocument: if not c.modelDocument.uri.startswith(val.modelXbrl.uriDir): val.modelXbrl.error(("EFM.6.07.16", "GFM.1.03.18"), _("Concept %(concept)s is also defined in standard taxonomy schema schema %(standardSchema)s"), modelObject=(modelConcept,c), concept=modelConcept.qname, standardSchema=os.path.basename(c.modelDocument.uri), standardConcept=c.qname) # 6.7.17 id properly formed _id = modelConcept.id requiredId = (prefix if prefix is not None else "") + "_" + name if _id != requiredId: val.modelXbrl.error(("EFM.6.07.17", "GFM.1.03.19"), _("Concept %(concept)s id %(id)s should be %(requiredId)s"), modelObject=modelConcept, concept=modelConcept.qname, id=_id, requiredId=requiredId) # 6.7.18 nillable is true nillable = modelConcept.get("nillable") if nillable != "true" and modelConcept.isItem: val.modelXbrl.error(("EFM.6.07.18", "GFM.1.03.20"), _("Taxonomy schema %(schema)s element %(concept)s nillable %(nillable)s should be 'true'"), modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=name, nillable=nillable) # 6.7.19 not tuple if modelConcept.isTuple: val.modelXbrl.error(("EFM.6.07.19", "GFM.1.03.21"), _("Concept %(concept)s is a tuple"), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.20 no typed domain ref if modelConcept.isTypedDimension: val.modelXbrl.error(("EFM.6.07.20", "GFM.1.03.22"), _("Concept %(concept)s has typedDomainRef %(typedDomainRef)s"), modelObject=modelConcept, concept=modelConcept.qname, typedDomainRef=modelConcept.typedDomainElement.qname if modelConcept.typedDomainElement is not None else modelConcept.typedDomainRef) # 6.7.21 abstract must be duration isDuration = modelConcept.periodType == "duration" if modelConcept.isAbstract and not isDuration: val.modelXbrl.error(("EFM.6.07.21", "GFM.1.03.23"), _("Taxonomy schema %(schema)s element %(concept)s is abstract but period type is not duration"), modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=modelConcept.qname) # 6.7.22 abstract must be stringItemType ''' removed SEC EFM v.17, Edgar release 10.4, and GFM 2011-04-08 if modelConcept.abstract == "true" and modelConcept.typeQname != XbrlConst. qnXbrliStringItemType: val.modelXbrl.error(("EFM.6.07.22", "GFM.1.03.24"), _("Concept %(concept)s is abstract but type is not xbrli:stringItemType"), modelObject=modelConcept, concept=modelConcept.qname) ''' substitutionGroupQname = modelConcept.substitutionGroupQname # 6.7.23 Axis must be subs group dimension if name.endswith("Axis") ^ (substitutionGroupQname == XbrlConst.qnXbrldtDimensionItem): val.modelXbrl.error(("EFM.6.07.23", "GFM.1.03.25"), _("Concept %(concept)s must end in Axis to be in xbrldt:dimensionItem substitution group"), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.24 Table must be subs group hypercube if name.endswith("Table") ^ (substitutionGroupQname == XbrlConst.qnXbrldtHypercubeItem): val.modelXbrl.error(("EFM.6.07.24", "GFM.1.03.26"), _("Concept %(concept)s must end in Table to be in xbrldt:hypercubeItem substitution group"), modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=modelConcept.qname) # 6.7.25 if neither hypercube or dimension, substitution group must be item if substitutionGroupQname not in (None, XbrlConst.qnXbrldtDimensionItem, XbrlConst.qnXbrldtHypercubeItem, XbrlConst.qnXbrliItem): val.modelXbrl.error(("EFM.6.07.25", "GFM.1.03.27"), _("Concept %(concept)s has disallowed substitution group %(substitutionGroup)s"), modelObject=modelConcept, concept=modelConcept.qname, substitutionGroup=modelConcept.substitutionGroupQname) # 6.7.26 Table must be subs group hypercube if name.endswith("LineItems") and modelConcept.abstract != "true": val.modelXbrl.error(("EFM.6.07.26", "GFM.1.03.28"), _("Concept %(concept)s is a LineItems but not abstract"), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.27 type domainMember must end with Domain or Member conceptType = modelConcept.type isDomainItemType = conceptType is not None and conceptType.isDomainItemType endsWithDomainOrMember = name.endswith("Domain") or name.endswith("Member") if isDomainItemType != endsWithDomainOrMember: val.modelXbrl.error(("EFM.6.07.27", "GFM.1.03.29"), _("Concept %(concept)s must end with Domain or Member for type of domainItemType"), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.28 domainItemType must be duration if isDomainItemType and not isDuration: val.modelXbrl.error(("EFM.6.07.28", "GFM.1.03.30"), _("Concept %(concept)s is a domainItemType and must be periodType duration"), modelObject=modelConcept, concept=modelConcept.qname) #6.7.31 (version 27) fractions if modelConcept.isFraction: val.modelXbrl.error("EFM.6.07.31", _("Concept %(concept)s is a fraction"), modelObject=modelConcept, concept=modelConcept.qname) #6.7.32 (version 27) instant non numeric if modelConcept.isItem and (not modelConcept.isNumeric and not isDuration and not modelConcept.isAbstract and not isDomainItemType): val.modelXbrl.error("EFM.6.07.32", _("Taxonomy schema %(schema)s element %(concept)s is non-numeric but period type is not duration"), modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=modelConcept.qname) # 6.8.5 semantic check, check LC3 name if name: if not name[0].isupper(): val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.05.firstLetter", "GFM.2.03.05.firstLetter"), _("Concept %(concept)s name must start with a capital letter"), modelObject=modelConcept, concept=modelConcept.qname) if namePattern.search(name): val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.05.disallowedCharacter", "GFM.2.03.05.disallowedCharacter"), _("Concept %(concept)s has disallowed name character"), modelObject=modelConcept, concept=modelConcept.qname) if len(name) > 200: val.modelXbrl.log("ERROR-SEMANTIC", "EFM.6.08.05.nameLength", _("Concept %(concept)s name length %(namelength)s exceeds 200 characters"), modelObject=modelConcept, concept=modelConcept.qname, namelength=len(name)) if isEFM: label = modelConcept.label(lang="en-US", fallbackToQname=False) if label: # allow Joe's Bar, N.A. to be JoesBarNA -- remove ', allow A. as not article "a" lc3name = ''.join(re.sub(r"['.-]", "", (w[0] or w[2] or w[3] or w[4])).title() for w in re.findall(r"((\w+')+\w+)|(A[.-])|([.-]A(?=\W|$))|(\w+)", label) # EFM implies this should allow - and . re.findall(r"[\w\-\.]+", label) if w[4].lower() not in ("the", "a", "an")) if not(name == lc3name or (name and lc3name and lc3name[0].isdigit() and name[1:] == lc3name and (name[0].isalpha() or name[0] == '_'))): val.modelXbrl.log("WARNING-SEMANTIC", "EFM.6.08.05.LC3", _("Concept %(concept)s should match expected LC3 composition %(lc3name)s"), modelObject=modelConcept, concept=modelConcept.qname, lc3name=lc3name) if conceptType is not None: # 6.8.6 semantic check if not isDomainItemType and conceptType.qname != XbrlConst.qnXbrliDurationItemType: nameProblems = nonDomainItemNameProblemPattern.findall(name) if any(any(t) for t in nameProblems): # list of tuples with possibly nonempty strings val.modelXbrl.log("WARNING-SEMANTIC", ("EFM.6.08.06", "GFM.2.03.06"), _("Concept %(concept)s should not contain company or period information, found: %(matches)s"), modelObject=modelConcept, concept=modelConcept.qname, matches=", ".join(''.join(t) for t in nameProblems)) if conceptType.qname == XbrlConst.qnXbrliMonetaryItemType: if not modelConcept.balance: # 6.8.11 may not appear on a income or balance statement if any(linkroleDefinitionBalanceIncomeSheet.match(roleType.definition) for rel in val.modelXbrl.relationshipSet(XbrlConst.parentChild).toModelObject(modelConcept) for roleType in val.modelXbrl.roleTypes.get(rel.linkrole,())): val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.11", "GFM.2.03.11"), _("Concept %(concept)s must have a balance because it appears in a statement of income or balance sheet"), modelObject=modelConcept, concept=modelConcept.qname) # 6.11.5 semantic check, must have a documentation label stdLabel = modelConcept.label(lang="en-US", fallbackToQname=False) defLabel = modelConcept.label(preferredLabel=XbrlConst.documentationLabel, lang="en-US", fallbackToQname=False) if not defLabel or ( # want different words than std label stdLabel and re.findall(r"\w+", stdLabel) == re.findall(r"\w+", defLabel)): val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.11.05", "GFM.2.04.04"), _("Concept %(concept)s is monetary without a balance and must have a documentation label that disambiguates its sign"), modelObject=modelConcept, concept=modelConcept.qname) # 6.8.16 semantic check if conceptType.qname == XbrlConst.qnXbrliDateItemType and modelConcept.periodType != "duration": val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.16", "GFM.2.03.16"), _("Concept %(concept)s of type xbrli:dateItemType must have periodType duration"), modelObject=modelConcept, concept=modelConcept.qname) # 6.8.17 semantic check if conceptType.qname == XbrlConst.qnXbrliStringItemType and modelConcept.periodType != "duration": val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.17", "GFM.2.03.17"), _("Concept %(concept)s of type xbrli:stringItemType must have periodType duration"), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.8 check for embedded linkbase for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}linkbase"): if isinstance(e,ModelObject): val.modelXbrl.error(("EFM.6.07.08", "GFM.1.03.08"), _("Taxonomy schema %(schema)s contains an embedded linkbase"), modelObject=e, schema=modelDocument.basename) break requiredUsedOns = {XbrlConst.qnLinkPresentationLink, XbrlConst.qnLinkCalculationLink, XbrlConst.qnLinkDefinitionLink} standardUsedOns = {XbrlConst.qnLinkLabel, XbrlConst.qnLinkReference, XbrlConst.qnLinkDefinitionArc, XbrlConst.qnLinkCalculationArc, XbrlConst.qnLinkPresentationArc, XbrlConst.qnLinkLabelArc, XbrlConst.qnLinkReferenceArc, # per WH, private footnote arc and footnore resource roles are not allowed XbrlConst.qnLinkFootnoteArc, XbrlConst.qnLinkFootnote, } # 6.7.9 role types authority for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}roleType"): if isinstance(e,ModelObject): roleURI = e.get("roleURI") if targetNamespaceAuthority != UrlUtil.authority(roleURI): val.modelXbrl.error(("EFM.6.07.09", "GFM.1.03.09"), _("RoleType %(roleType)s does not match authority %(targetNamespaceAuthority)s"), modelObject=e, roleType=roleURI, targetNamespaceAuthority=targetNamespaceAuthority, targetNamespace=modelDocument.targetNamespace) # 6.7.9 end with .../role/lc3 name if not roleTypePattern.match(roleURI): val.modelXbrl.warning(("EFM.6.07.09.roleEnding", "GFM.1.03.09"), "RoleType %(roleType)s should end with /role/{LC3name}", modelObject=e, roleType=roleURI) # 6.7.10 only one role type declaration in DTS modelRoleTypes = val.modelXbrl.roleTypes.get(roleURI) if modelRoleTypes is not None: modelRoleType = modelRoleTypes[0] definition = modelRoleType.definitionNotStripped usedOns = modelRoleType.usedOns if len(modelRoleTypes) == 1: # 6.7.11 used on's for pre, cal, def if any has a used on if not usedOns.isdisjoint(requiredUsedOns) and len(requiredUsedOns - usedOns) > 0: val.modelXbrl.error(("EFM.6.07.11", "GFM.1.03.11"), _("RoleType %(roleType)s missing used on %(usedOn)s"), modelObject=e, roleType=roleURI, usedOn=requiredUsedOns - usedOns) # 6.7.12 definition match pattern if (val.disclosureSystem.roleDefinitionPattern is not None and (definition is None or not val.disclosureSystem.roleDefinitionPattern.match(definition))): val.modelXbrl.error(("EFM.6.07.12", "GFM.1.03.12-14"), _("RoleType %(roleType)s definition \"%(definition)s\" must match {Sortcode} - {Type} - {Title}"), modelObject=e, roleType=roleURI, definition=(definition or "")) if usedOns & standardUsedOns: # semantics check val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.03", "GFM.2.03.03"), _("RoleType %(roleuri)s is defined using role types already defined by standard roles for: %(qnames)s"), modelObject=e, roleuri=roleURI, qnames=', '.join(str(qn) for qn in usedOns & standardUsedOns)) # 6.7.13 arcrole types authority for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}arcroleType"): if isinstance(e,ModelObject): arcroleURI = e.get("arcroleURI") if targetNamespaceAuthority != UrlUtil.authority(arcroleURI): val.modelXbrl.error(("EFM.6.07.13", "GFM.1.03.15"), _("ArcroleType %(arcroleType)s does not match authority %(targetNamespaceAuthority)s"), modelObject=e, arcroleType=arcroleURI, targetNamespaceAuthority=targetNamespaceAuthority, targetNamespace=modelDocument.targetNamespace) # 6.7.13 end with .../arcrole/lc3 name if not arcroleTypePattern.match(arcroleURI): val.modelXbrl.warning(("EFM.6.07.13.arcroleEnding", "GFM.1.03.15"), _("ArcroleType %(arcroleType)s should end with /arcrole/{LC3name}"), modelObject=e, arcroleType=arcroleURI) # 6.7.15 definition match pattern modelRoleTypes = val.modelXbrl.arcroleTypes[arcroleURI] definition = modelRoleTypes[0].definition if definition is None or not arcroleDefinitionPattern.match(definition): val.modelXbrl.error(("EFM.6.07.15", "GFM.1.03.17"), _("ArcroleType %(arcroleType)s definition must be non-empty"), modelObject=e, arcroleType=arcroleURI) # semantic checks usedOns = modelRoleTypes[0].usedOns if usedOns & standardUsedOns: # semantics check val.modelXbrl.log("ERROR-SEMANTIC", ("EFM.6.08.03", "GFM.2.03.03"), _("ArcroleType %(arcroleuri)s is defined using role types already defined by standard arcroles for: %(qnames)s"), modelObject=e, arcroleuri=arcroleURI, qnames=', '.join(str(qn) for qn in usedOns & standardUsedOns)) #6.3.3 filename check m = re.match(r"^\w+-([12][0-9]{3}[01][0-9][0-3][0-9]).xsd$", modelDocument.basename) if m: try: # check date value datetime.datetime.strptime(m.group(1),"%Y%m%d").date() # date and format are ok, check "should" part of 6.3.3 if val.fileNameBasePart: expectedFilename = "{0}-{1}.xsd".format(val.fileNameBasePart, val.fileNameDatePart) if modelDocument.basename != expectedFilename: val.modelXbrl.log("WARNING-SEMANTIC", ("EFM.6.03.03.matchInstance", "GFM.1.01.01.matchInstance"), _('Schema file name warning: %(filename)s, should match %(expectedFilename)s'), modelObject=modelDocument, filename=modelDocument.basename, expectedFilename=expectedFilename) except ValueError: val.modelXbrl.error((val.EFM60303, "GFM.1.01.01"), _('Invalid schema file base name part (date) in "{base}-{yyyymmdd}.xsd": %(filename)s'), modelObject=modelDocument, filename=modelDocument.basename, messageCodes=("EFM.6.03.03", "EFM.6.23.01", "GFM.1.01.01")) else: val.modelXbrl.error((val.EFM60303, "GFM.1.01.01"), _('Invalid schema file name, must match "{base}-{yyyymmdd}.xsd": %(filename)s'), modelObject=modelDocument, filename=modelDocument.basename, messageCodes=("EFM.6.03.03", "EFM.6.23.01", "GFM.1.01.01")) elif modelDocument.type == ModelDocument.Type.LINKBASE: # if it is part of the submission (in same directory) check name labelRels = None if modelDocument.filepath.startswith(val.modelXbrl.modelDocument.filepathdir): #6.3.3 filename check extLinkElt = XmlUtil.descendant(modelDocument.xmlRootElement, XbrlConst.link, "*", "{http://www.w3.org/1999/xlink}type", "extended") if extLinkElt is None:# no ext link element val.modelXbrl.error((val.EFM60303 + ".noLinkElement", "GFM.1.01.01.noLinkElement"), _('Invalid linkbase file name: %(filename)s, has no extended link element, cannot determine link type.'), modelObject=modelDocument, filename=modelDocument.basename, messageCodes=("EFM.6.03.03.noLinkElement", "EFM.6.23.01.noLinkElement", "GFM.1.01.01.noLinkElement")) elif extLinkElt.localName not in extLinkEltFileNameEnding: val.modelXbrl.error("EFM.6.03.02", _('Invalid linkbase link element %(linkElement)s in %(filename)s'), modelObject=modelDocument, linkElement=extLinkElt.localName, filename=modelDocument.basename) else: m = re.match(r"^\w+-([12][0-9]{3}[01][0-9][0-3][0-9])(_[a-z]{3}).xml$", modelDocument.basename) expectedSuffix = extLinkEltFileNameEnding[extLinkElt.localName] if m and m.group(2) == expectedSuffix: try: # check date value datetime.datetime.strptime(m.group(1),"%Y%m%d").date() # date and format are ok, check "should" part of 6.3.3 if val.fileNameBasePart: expectedFilename = "{0}-{1}{2}.xml".format(val.fileNameBasePart, val.fileNameDatePart, expectedSuffix) if modelDocument.basename != expectedFilename: val.modelXbrl.log("WARNING-SEMANTIC", ("EFM.6.03.03.matchInstance", "GFM.1.01.01.matchInstance"), _('Linkbase name warning: %(filename)s should match %(expectedFilename)s'), modelObject=modelDocument, filename=modelDocument.basename, expectedFilename=expectedFilename) except ValueError: val.modelXbrl.error((val.EFM60303, "GFM.1.01.01"), _('Invalid linkbase base file name part (date) in "{base}-{yyyymmdd}_{suffix}.xml": %(filename)s'), modelObject=modelDocument, filename=modelDocument.basename, messageCodes=("EFM.6.03.03", "EFM.6.23.01", "GFM.1.01.01")) else: val.modelXbrl.error((val.EFM60303, "GFM.1.01.01"), _('Invalid linkbase name, must match "{base}-{yyyymmdd}%(expectedSuffix)s.xml": %(filename)s'), modelObject=modelDocument, filename=modelDocument.basename, expectedSuffix=expectedSuffix, messageCodes=("EFM.6.03.03", "EFM.6.23.01", "GFM.1.01.01")) if extLinkElt.localName == "labelLink": if labelRels is None: labelRels = val.modelXbrl.relationshipSet(XbrlConst.conceptLabel) for labelElt in XmlUtil.children(extLinkElt, XbrlConst.link, "label"): # 6.10.9 if XbrlConst.isNumericRole(labelElt.role): for rel in labelRels.toModelObject(labelElt): if rel.fromModelObject is not None and not rel.fromModelObject.isNumeric: val.modelXbrl.error("EFM.6.10.09", _("Label of non-numeric concept %(concept)s has a numeric role: %(role)s"), modelObject=(labelElt, rel.fromModelObject), concept=rel.fromModelObject.qname, role=labelElt.role)
def checkHierarchyConstraints(elt): constraints = ixHierarchyConstraints.get(elt.localName) if constraints: for _rel, names in constraints: reqt = _rel[0] rel = _rel[1:] if reqt in ('&', '^', '1'): nameFilter = ('*', ) else: nameFilter = names if nameFilter == ('*', ): namespaceFilter = namespacePrefix = '*' elif len(nameFilter) == 1 and "}" in nameFilter[ 0] and nameFilter[0][0] == "{": namespaceFilter, _sep, nameFilter = nameFilter[0][ 1:].partition("}") namespacePrefix = XmlUtil.xmlnsprefix(elt, namespaceFilter) else: namespaceFilter = elt.namespaceURI namespacePrefix = elt.prefix relations = { "ancestor": XmlUtil.ancestor, "parent": XmlUtil.parent, "child-choice": XmlUtil.children, "child-sequence": XmlUtil.children, "child-or-text": XmlUtil.children, "descendant": XmlUtil.descendants }[rel](elt, namespaceFilter, nameFilter) if rel in ("ancestor", "parent"): if relations is None: relations = [] else: relations = [relations] if rel == "child-or-text": relations += XmlUtil.innerTextNodes(elt, ixExclude=True, ixEscape=False, ixContinuation=False) issue = '' if reqt in ('^', ): if not any(r.localName in names and r.namespaceURI == elt.namespaceURI for r in relations): issue = " and is missing one of " + ', '.join(names) if reqt in ('1', ) and not elt.isNil: if sum(r.localName in names and r.namespaceURI == elt.namespaceURI for r in relations) != 1: issue = " and must have exactly one of " + ', '.join( names) if reqt in ('&', '^'): disallowed = [ str(r.elementQname) for r in relations if not (r.tag in names or (r.localName in names and r.namespaceURI == elt.namespaceURI)) ] if disallowed: issue += " and may not have " + ", ".join(disallowed) elif rel == "child-sequence": sequencePosition = 0 for i, r in enumerate(relations): rPos = names.index(str(r.localName)) if rPos < sequencePosition: issue += " and is out of sequence: " + str( r.elementQname) else: sequencePosition = rPos if reqt == '?' and len(relations) > 1: issue = " may only have 0 or 1 but {0} present ".format( len(relations)) if reqt == '+' and len(relations) == 0: issue = " must have at least 1 but none present " disallowedChildText = bool( reqt == '&' and rel in ("child-sequence", "child-choice") and elt.textValue.strip()) if ((reqt == '+' and not relations) or (reqt == '-' and relations) or (issue) or disallowedChildText): code = "{}:{}".format( ixSect[elt.namespaceURI].get(elt.localName, "other")["constraint"], { 'ancestor': "ancestorNode", 'parent': "parentNode", 'child-choice': "childNodes", 'child-sequence': "childNodes", 'child-or-text': "childNodesOrText", 'descendant': "descendantNodes" }[rel] + { '+': "Required", '-': "Disallowed", '&': "Allowed", '^': "Specified", '1': "Specified" }.get(reqt, "Specified")) msg = _("Inline XBRL ix:{0} {1} {2} {3} {4} element{5}" ).format( elt.localName, { '+': "must", '-': "may not", '&': "may only", '?': "may", '+': "must", '^': "must", '1': "must" }[reqt], { 'ancestor': "be nested in", 'parent': "have parent", 'child-choice': "have child", 'child-sequence': "have child", 'child-or-text': "have child or text,", 'descendant': "have as descendant" }[rel], '' if rel == 'child-or-text' else ', '.join( str(r.elementQname) for r in relations) if names == ('*', ) and relations else ", ".join( "{}:{}".format(namespacePrefix, n) for n in names), issue, " and no child text (\"{}\")".format( elt.textValue.strip()[:32]) if disallowedChildText else "") modelXbrl.error( code, msg, modelObject=[elt] + relations, requirement=reqt, messageCodes= ("ix{ver.sect}:ancestorNode{Required|Disallowed}", "ix{ver.sect}:childNodesOrTextRequired", "ix{ver.sect}:childNodes{Required|Disallowed|Allowed}", "ix{ver.sect}:descendantNodesDisallowed", "ix{ver.sect}:parentNodeRequired")) # other static element checks (that don't require a complete object model, context, units, etc if elt.localName == "nonFraction": childElts = XmlUtil.children(elt, '*', '*') hasText = (elt.text or "") or any( (childElt.tail or "") for childElt in childElts) if elt.isNil: ancestorNonFractions = XmlUtil.ancestors( elt, _ixNS, elt.localName) if ancestorNonFractions: modelXbrl.error( ixMsgCode("nonFractionAncestors", elt), _("Fact %(fact)s is a nil nonFraction and MUST not have an ancestor ix:nonFraction" ), modelObject=[elt] + ancestorNonFractions, fact=elt.qname) if childElts or hasText: modelXbrl.error( ixMsgCode("nonFractionTextAndElementChildren", elt), _("Fact %(fact)s is a nil nonFraction and MUST not have an child elements or text" ), modelObject=[elt] + childElts, fact=elt.qname) elt.setInvalid( ) # prevent further validation or cascading errors else: if ((childElts and (len(childElts) != 1 or childElts[0].namespaceURI != _ixNS or childElts[0].localName != "nonFraction")) or (childElts and hasText)): modelXbrl.error( ixMsgCode("nonFractionTextAndElementChildren", elt), _("Fact %(fact)s is a non-nil nonFraction and MUST have exactly one ix:nonFraction child element or text." ), modelObject=[elt] + childElts, fact=elt.qname) elt.setInvalid() if elt.localName == "fraction": if elt.isNil: ancestorFractions = XmlUtil.ancestors(elt, _ixNS, elt.localName) if ancestorFractions: modelXbrl.error( ixMsgCode("fractionAncestors", elt), _("Fact %(fact)s is a nil fraction and MUST not have an ancestor ix:fraction" ), modelObject=[elt] + ancestorFractions, fact=elt.qname) else: nonFrChildren = [ e for e in XmlUtil.children(elt, _ixNS, '*') if e.localName not in ("fraction", "numerator", "denominator") ] if nonFrChildren: modelXbrl.error( ixMsgCode("fractionElementChildren", elt), _("Fact %(fact)s is a non-nil fraction and not have any child elements except ix:fraction, ix:numerator and ix:denominator: %(children)s" ), modelObject=[elt] + nonFrChildren, fact=elt.qname, children=", ".join(e.localName for e in nonFrChildren)) for ancestorFraction in XmlUtil.ancestors( elt, XbrlConst.ixbrl11, "fraction"): # only ix 1.1 if normalizeSpace(elt.get("unitRef")) != normalizeSpace( ancestorFraction.get("unitRef")): modelXbrl.error( ixMsgCode("fractionNestedUnitRef", elt), _("Fact %(fact)s fraction and ancestor fractions must have matching unitRefs: %(unitRef)s, %(unitRef2)s" ), modelObject=[elt] + nonFrChildren, fact=elt.qname, unitRef=elt.get("unitRef"), unitRef2=ancestorFraction.get("unitRef")) if elt.localName in ("nonFraction", "numerator", "denominator", "nonNumeric"): fmt = elt.format if fmt: if fmt in _customTransforms: pass elif fmt.namespaceURI not in FunctionIxt.ixtNamespaceFunctions: modelXbrl.error( ixMsgCode("invalidTransformation", elt, sect="validation"), _("Fact %(fact)s has unrecognized transformation namespace %(namespace)s" ), modelObject=elt, fact=elt.qname, transform=fmt, namespace=fmt.namespaceURI) elt.setInvalid() elif fmt.localName not in FunctionIxt.ixtNamespaceFunctions[ fmt.namespaceURI]: modelXbrl.error( ixMsgCode("invalidTransformation", elt, sect="validation"), _("Fact %(fact)s has unrecognized transformation name %(name)s" ), modelObject=elt, fact=elt.qname, transform=fmt, name=fmt.localName) elt.setInvalid()
def checkDTS(val, modelDocument, visited): global targetNamespaceDatePattern, roleTypePattern, arcroleTypePattern, arcroleDefinitionPattern if targetNamespaceDatePattern is None: targetNamespaceDatePattern = re.compile(r"/([12][0-9]{3})-([01][0-9])-([0-3][0-9])|" r"/([12][0-9]{3})([01][0-9])([0-3][0-9])|") roleTypePattern = re.compile(r".*/role/[^/]+") arcroleTypePattern = re.compile(r".*/arcrole/[^/]+") arcroleDefinitionPattern = re.compile(r"^.*[^\\s]+.*$") # at least one non-whitespace character visited.append(modelDocument) definesLabelLinkbase = False for referencedDocument in modelDocument.referencesDocument.items(): #6.07.01 no includes if referencedDocument[1] == "include": val.modelXbrl.error(("EFM.6.07.01", "GFM.1.03.01", "SBR.NL.2.2.0.18"), _("Taxonomy schema %(schema)s includes %(include)s, only import is allowed"), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), include=os.path.basename(referencedDocument[0].uri)) if referencedDocument[0] not in visited: checkDTS(val, referencedDocument[0], visited) if val.disclosureSystem.standardTaxonomiesDict is None: pass if (modelDocument.type == ModelDocument.Type.SCHEMA and modelDocument.targetNamespace not in val.disclosureSystem.baseTaxonomyNamespaces and modelDocument.uri.startswith(val.modelXbrl.uriDir)): # check schema contents types if val.validateSBRNL: definesConcepts = False definesLinkroles = False definesArcroles = False definesLinkParts = False definesAbstractItems = False definesNonabstractItems = False definesTuples = False definesEnumerations = False definesDimensions = False definesDomains = False definesHypercubes = False # 6.7.3 check namespace for standard authority targetNamespaceAuthority = UrlUtil.authority(modelDocument.targetNamespace) if targetNamespaceAuthority in val.disclosureSystem.standardAuthorities: val.modelXbrl.error(("EFM.6.07.03", "GFM.1.03.03"), _("Taxonomy schema %(schema)s namespace %(targetNamespace)s is a disallowed authority"), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace) # 6.7.4 check namespace format if modelDocument.targetNamespace is None: match = None elif val.validateEFMorGFM: targetNamespaceDate = modelDocument.targetNamespace[len(targetNamespaceAuthority):] match = targetNamespaceDatePattern.match(targetNamespaceDate) else: match = None if match is not None: try: if match.lastindex == 3: datetime.date(int(match.group(1)),int(match.group(2)),int(match.group(3))) elif match.lastindex == 6: datetime.date(int(match.group(4)),int(match.group(5)),int(match.group(6))) else: match = None except ValueError: match = None if match is None: val.modelXbrl.error(("EFM.6.07.04", "GFM.1.03.04"), _("Taxonomy schema %(schema)s namespace %(targetNamespace)s must have format http://{authority}/{versionDate}"), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace) if modelDocument.targetNamespace is not None: # 6.7.5 check prefix for _ prefix = XmlUtil.xmlnsprefix(modelDocument.xmlRootElement,modelDocument.targetNamespace) if prefix and "_" in prefix: val.modelXbrl.error(("EFM.6.07.07", "GFM.1.03.07"), _("Taxonomy schema %(schema)s namespace %(targetNamespace)s prefix %(prefix)s must not have an '_'"), modelObject=modelDocument, schema=os.path.basename(modelDocument.uri), targetNamespace=modelDocument.targetNamespace, prefix=prefix) for modelConcept in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/2001/XMLSchema}element"): if isinstance(modelConcept,ModelConcept): # 6.7.16 name not duplicated in standard taxonomies name = modelConcept.get("name") if name is None: name = "" if modelConcept.get("ref") is not None: continue # don't validate ref's here concepts = val.modelXbrl.nameConcepts.get(name) if concepts is not None: for c in concepts: if c.modelDocument != modelDocument: if (val.validateEFMorGFM and not c.modelDocument.uri.startswith(val.modelXbrl.uriDir)): val.modelXbrl.error(("EFM.6.07.16", "GFM.1.03.18"), _("Concept %(concept)s is also defined in standard taxonomy schema schema %(standardSchema)s"), modelObject=c, concept=modelConcept.qname, standardSchema=os.path.basename(c.modelDocument.uri)) elif val.validateSBRNL and c.modelDocument != modelDocument: relSet = val.modelXbrl.relationshipSet(XbrlConst.generalSpecial) if not (relSet.isRelated(modelConcept, "child", c) or relSet.isRelated(modelConcept, "child", c)): val.modelXbrl.error("SBR.NL.2.2.2.02", _("Concept %(concept)s is also defined in standard taxonomy schema %(standardSchema)s without a general-special relationship"), modelObject=c, concept=modelConcept.qname, standardSchema=os.path.basename(c.modelDocument.uri)) # 6.7.17 id properly formed id = modelConcept.id requiredId = (prefix if prefix is not None else "") + "_" + name if val.validateEFMorGFM and id != requiredId: val.modelXbrl.error(("EFM.6.07.17", "GFM.1.03.19"), _("Concept %(concept)s id %(id)s should be $(requiredId)s"), modelObject=modelConcept, concept=modelConcept.qname, id=id, requiredId=requiredId) # 6.7.18 nillable is true nillable = modelConcept.get("nillable") if nillable != "true": val.modelXbrl.error(("EFM.6.07.18", "GFM.1.03.20"), _("Taxonomy schema %(schema)s element %(concept)s nillable %(nillable)s should be 'true'"), modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=name, nillable=nillable) if modelConcept is not None: # 6.7.19 not tuple if modelConcept.isTuple: if val.validateEFMorGFM: val.modelXbrl.error(("EFM.6.07.19", "GFM.1.03.21"), _("Concept %(concept)s is a tuple"), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.20 no typed domain ref if modelConcept.isTypedDimension: val.modelXbrl.error(("EFM.6.07.20", "GFM.1.03.22"), _("Concept %(concept)s has typedDomainRef %(typedDomainRef)s"), modelObject=modelConcept, concept=modelConcept.qname, typedDomainRef=modelConcept.typedDomainElement.qname if modelConcept.typedDomainElement is not None else modelConcept.typedDomainRef) # 6.7.21 abstract must be duration isDuration = modelConcept.periodType == "duration" if modelConcept.abstract == "true" and not isDuration: val.modelXbrl.error(("EFM.6.07.21", "GFM.1.03.23"), _("Taxonomy schema %(schema)s element %(concept)s is abstract but period type is not duration"), modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=name) # 6.7.22 abstract must be stringItemType ''' removed SEC EFM v.17, Edgar release 10.4, and GFM 2011-04-08 if modelConcept.abstract == "true" and modelConcept.typeQname != XbrlConst. qnXbrliStringItemType: val.modelXbrl.error(("EFM.6.07.22", "GFM.1.03.24"), _("Concept %(concept)s is abstract but type is not xbrli:stringItemType"), modelObject=modelConcept, concept=modelConcept.qname) ''' substititutionGroupQname = modelConcept.substitutionGroupQname # 6.7.23 Axis must be subs group dimension if name.endswith("Axis") ^ (substititutionGroupQname == XbrlConst.qnXbrldtDimensionItem): val.modelXbrl.error(("EFM.6.07.23", "GFM.1.03.25"), _("Concept %(concept)s must end in Axis to be in dimensionItem substitution group"), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.24 Table must be subs group hypercube if name.endswith("Table") ^ (substititutionGroupQname == XbrlConst.qnXbrldtHypercubeItem): val.modelXbrl.error(("EFM.6.07.24", "GFM.1.03.26"), _("Concept %(concept)s is an Axis but not in hypercubeItem substitution group"), modelObject=modelConcept, schema=os.path.basename(modelDocument.uri), concept=modelConcept.qname) # 6.7.25 if neither hypercube or dimension, substitution group must be item if substititutionGroupQname not in (None, XbrlConst.qnXbrldtDimensionItem, XbrlConst.qnXbrldtHypercubeItem, XbrlConst.qnXbrliItem): val.modelXbrl.error(("EFM.6.07.25", "GFM.1.03.27"), _("Concept %(concept)s has disallowed substitution group %(substitutionGroup)s"), modelObject=modelConcept, concept=modelConcept.qname, substitutionGroup=modelConcept.substitutionGroupQname) # 6.7.26 Table must be subs group hypercube if name.endswith("LineItems") and modelConcept.abstract != "true": val.modelXbrl.error(("EFM.6.07.26", "GFM.1.03.28"), _("Concept %(concept)s is a LineItems but not abstract"), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.27 type domainMember must end with Domain or Member conceptType = modelConcept.type isDomainItemType = conceptType is not None and conceptType.isDomainItemType endsWithDomainOrMember = name.endswith("Domain") or name.endswith("Member") if isDomainItemType != endsWithDomainOrMember: val.modelXbrl.error(("EFM.6.07.27", "GFM.1.03.29"), _("Concept %(concept)s must end with Domain or Member for type of domainItemType"), modelObject=modelConcept, concept=modelConcept.qname) # 6.7.28 domainItemType must be duration if isDomainItemType and not isDuration: val.modelXbrl.error(("EFM.6.07.28", "GFM.1.03.30"), _("Concept %(concept)s is a domainItemType and must be periodType duration"), modelObject=modelConcept, concept=modelConcept.qname) if val.validateSBRNL: definesConcepts = True if modelConcept.isTuple: definesTuples = True if modelConcept.abstract == "true": val.modelXbrl.error("SBR.NL.2.2.2.03", _("Concept %(concept)s is an abstract tuple"), modelObject=modelConcept, concept=modelConcept.qname) if tupleCycle(val,modelConcept): val.modelXbrl.error("SBR.NL.2.2.2.07", _("Tuple %(concept)s has a tuple cycle"), modelObject=modelConcept, concept=modelConcept.qname) if modelConcept.nillable != "false" and modelConcept.isRoot: val.modelXbrl.error("SBR.NL.2.2.2.17", _("Tuple %(concept)s must have nillable='false'"), modelObject=modelConcept, concept=modelConcept.qname) if modelConcept.abstract == "true": if modelConcept.isRoot: if modelConcept.nillable != "false": val.modelXbrl.error("SBR.NL.2.2.2.16", _("Abstract root concept %(concept)s must have nillable='false'"), modelObject=modelConcept, concept=modelConcept.qname) if modelConcept.typeQname != XbrlConst.qnXbrliStringItemType: val.modelXbrl.error("SBR.NL.2.2.2.21", _("Abstract root concept %(concept)s must have type='xbrli:stringItemType'"), modelObject=modelConcept, concept=modelConcept.qname) else: # not root if modelConcept.isItem: val.modelXbrl.error("SBR.NL.2.2.2.31", _("Taxonomy schema {0} abstract item %(concept)s must not be a child of a tuple"), modelObject=modelConcept, concept=modelConcept.qname) if modelConcept.balance: val.modelXbrl.error("SBR.NL.2.2.2.22", _("Abstract concept %(concept)s must not have a balance attribute"), modelObject=modelConcept, concept=modelConcept.qname) if modelConcept.isTuple: val.modelXbrl.error("SBR.NL.2.2.2.31", _("Tuple %(concept)s must not be abstract"), modelObject=modelConcept, concept=modelConcept.qname) if modelConcept.isHypercubeItem: definesHypercubes = True elif modelConcept.isDimensionItem: definesDimensions = True elif substititutionGroupQname.localName == "domainItem": definesDomains = True elif modelConcept.isItem: definesAbstractItems = True else: # not abstract if not (modelConcept.label(preferredLabel=XbrlConst.documentationLabel,fallbackToQname=False,lang="nl") or val.modelXbrl.relationshipSet(XbrlConst.conceptReference).fromModelObject(c)): val.modelXbrl.error("SBR.NL.2.2.2.28", _("Concept %(concept)s must have a documentation label or reference"), modelObject=modelConcept, concept=modelConcept.qname) if modelConcept.balance and not modelConcept.instanceOfType(XbrlConst.qnXbrliMonetaryItemType): val.modelXbrl.error("SBR.NL.2.2.2.24", _("Non-monetary concept %(concept)s must not have a balance attribute"), modelObject=modelConcept, concept=modelConcept.qname) if not modelConcept.label(fallbackToQname=False,lang="nl"): val.modelXbrl.error("SBR.NL.2.2.2.26", _("Concept %(concept)s must have a standard label in language 'nl'"), modelObject=modelConcept, concept=modelConcept.qname) if not modelConcept.isRoot: # tuple child if modelConcept.get("maxOccurs") is not None and modelConcept.get("maxOccurs") != "1": val.modelXbrl.error("SBR.NL.2.2.2.30", _("Tuple concept %(concept)s must have maxOccurs='1'"), modelObject=modelConcept, concept=modelConcept.qname) if modelConcept.isLinkPart: definesLinkParts = True val.modelXbrl.error("SBR.NL.2.2.5.01", _("Link:part concept %(concept)s is not allowed"), modelObject=modelConcept, concept=modelConcept.qname) if modelConcept.isTypedDimension: domainElt = modelConcept.typedDomainElement if domainElt is not None and domainElt.localName == "complexType": val.modelXbrl.error("SBR.NL.2.2.8.02", _("Typed dimension %(concept)s domain element %(typedDomainElement)s has disallowed complex content"), modelObject=modelConcept, concept=modelConcept.qname, typedDomainElement=domainElt.qname) # 6.7.8 check for embedded linkbase for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}linkbase"): if isinstance(e,ModelObject): val.modelXbrl.error(("EFM.6.07.08", "GFM.1.03.08"), _("Taxonomy schema %(schema)s contains an embedded linkbase"), modelObject=e, schema=os.path.basename(modelDocument.uri)) break requiredUsedOns = {XbrlConst.qnLinkPresentationLink, XbrlConst.qnLinkCalculationLink, XbrlConst.qnLinkDefinitionLink} # 6.7.9 role types authority for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}roleType"): if isinstance(e,ModelObject): roleURI = e.get("roleURI") if targetNamespaceAuthority != UrlUtil.authority(roleURI): val.modelXbrl.error(("EFM.6.07.09", "GFM.1.03.09"), _("RoleType %(roleType)s does not match authority %(targetNamespaceAuthority)s"), modelObject=e, roleType=roleURI, targetNamespaceAuthority=targetNamespaceAuthority) # 6.7.9 end with .../role/lc3 name if not roleTypePattern.match(roleURI): val.modelXbrl.warning(("EFM.6.07.09", "GFM.1.03.09"), "RoleType %(roleType)s should end with /role/{LC3name}", modelObject=e, roleType=roleURI) # 6.7.10 only one role type declaration in DTS modelRoleTypes = val.modelXbrl.roleTypes.get(roleURI) if modelRoleTypes is not None: if len(modelRoleTypes) > 1: val.modelXbrl.error(("EFM.6.07.10", "GFM.1.03.10"), _("RoleType %(roleType)s is defined in multiple taxonomies"), modelObject=e, roleType=roleURI) elif len(modelRoleTypes) == 1: # 6.7.11 used on's for pre, cal, def if any has a used on usedOns = modelRoleTypes[0].usedOns if not usedOns.isdisjoint(requiredUsedOns) and len(requiredUsedOns - usedOns) > 0: val.modelXbrl.error(("EFM.6.07.11", "GFM.1.03.11"), _("RoleType %(roleType)s missing used on %(usedOn)s"), modelObject=e, roleType=roleURI, usedOn=requiredUsedOns - usedOns) # 6.7.12 definition match pattern definition = modelRoleTypes[0].definitionNotStripped if (val.disclosureSystem.roleDefinitionPattern is not None and (definition is None or not val.disclosureSystem.roleDefinitionPattern.match(definition))): val.modelXbrl.error(("EFM.6.07.12", "GFM.1.03.12-14"), _("RoleType %(roleType)s definition \"%(definition)s\" must match {Sortcode} - {Type} - {Title}"), modelObject=e, roleType=roleURI, definition=definition) if val.validateSBRNL and (usedOns & XbrlConst.standardExtLinkQnames): definesLinkroles = True # 6.7.13 arcrole types authority for e in modelDocument.xmlRootElement.iterdescendants(tag="{http://www.xbrl.org/2003/linkbase}arcroleType"): if isinstance(e,ModelObject): arcroleURI = e.get("arcroleURI") if targetNamespaceAuthority != UrlUtil.authority(arcroleURI): val.modelXbrl.error(("EFM.6.07.13", "GFM.1.03.15"), _("ArcroleType %(arcroleType)s does not match authority %(targetNamespaceAuthority)s"), modelObject=e, arcroleType=arcroleURI, targetNamespaceAuthority=targetNamespaceAuthority) # 6.7.13 end with .../arcrole/lc3 name if not arcroleTypePattern.match(arcroleURI): val.modelXbrl.warning(("EFM.6.07.13", "GFM.1.03.15"), _("ArcroleType %(arcroleType)s should end with /arcrole/{LC3name}"), modelObject=e, arcroleType=arcroleURI) # 6.7.14 only one arcrole type declaration in DTS modelRoleTypes = val.modelXbrl.arcroleTypes[arcroleURI] if len(modelRoleTypes) > 1: val.modelXbrl.error(("EFM.6.07.14", "GFM.1.03.16"), _("ArcroleType %(arcroleType)s is defined in multiple taxonomies"), modelObject=e, arcroleType=arcroleURI) # 6.7.15 definition match pattern definition = modelRoleTypes[0].definition if definition is None or not arcroleDefinitionPattern.match(definition): val.modelXbrl.error(("EFM.6.07.15", "GFM.1.03.17"), _("ArcroleType %(arcroleType)s definition must be non-empty"), modelObject=e, arcroleType=arcroleURI) if val.validateSBRNL: definesArcroles = True if val.validateSBRNL and (definesLinkroles + definesArcroles + definesLinkParts + definesAbstractItems + definesNonabstractItems + definesTuples + definesEnumerations + definesDimensions + definesDomains + definesHypercubes) > 1: schemaContents = [] if definesLinkroles: schemaContents.append(_("linkroles")) if definesArcroles: schemaContents.append(_("arcroles")) if definesLinkParts: schemaContents.append(_("link parts")) if definesAbstractItems: schemaContents.append(_("abstract items")) if definesNonabstractItems: schemaContents.append(_("nonabstract items")) if definesTuples: schemaContents.append(_("tuples")) if definesEnumerations: schemaContents.append(_("enumerations")) if definesDimensions: schemaContents.append(_("dimensions")) if definesDomains: schemaContents.append(_("domains")) if definesHypercubes: schemaContents.append(_("hypercubes")) val.modelXbrl.error("SBR.NL.2.2.1.01", _("Taxonomy schema %(schema)s may only define one of these: %(contents)s"), modelObject=val.modelXbrl, schema=os.path.basename(modelDocument.uri), contents=', '.join(schemaContents)) visited.remove(modelDocument)