def cfcnCall(self): # tuple of (expression, element holding the expression) try: return self._cfcnCall except AttributeError: self._cfcnCall = None if self.localName == "test-case": #xpath testcase queryElement = XmlUtil.descendant(self, None, "query") if queryElement is not None: filepath = (self.modelDocument.filepathdir + "/" + "Queries/XQuery/" + self.get("FilePath") + queryElement.get("name") + '.xq') if os.sep != "/": filepath = filepath.replace("/", os.sep) with io.open(filepath, 'rt', encoding='utf-8') as f: self._cfcnCall = (f.read(), self) else: for callElement in XmlUtil.descendants(self, XbrlConst.cfcn, "call"): self._cfcnCall = (XmlUtil.innerText(callElement), callElement) break if self._cfcnCall is None and self.namespaceURI == "http://xbrl.org/2011/conformance-rendering/transforms": name = self.getparent().get("name") input = self.get("input") if name and input: self._cfcnCall = ("{0}('{1}')".format( name, input.replace("'", "''")), self) return self._cfcnCall
def cfcnTest(self): # tuple of (expression, element holding the expression) try: return self._cfcnTest except AttributeError: self._cfcnTest = None if self.localName == "test-case": #xpath testcase outputFileElement = XmlUtil.descendant(self, None, "output-file") if outputFileElement is not None and outputFileElement.get( "compare") == "Text": filepath = (self.modelDocument.filepathdir + "/" + "ExpectedTestResults/" + self.get("FilePath") + outputFileElement.text) if os.sep != "/": filepath = filepath.replace("/", os.sep) with io.open(filepath, 'rt', encoding='utf-8') as f: self._cfcnTest = ("xs:string($result) eq '{0}'".format( f.read()), self) else: testElement = XmlUtil.descendant(self, XbrlConst.cfcn, "test") if testElement is not None: self._cfcnTest = (XmlUtil.innerText(testElement), testElement) elif self.namespaceURI == "http://xbrl.org/2011/conformance-rendering/transforms": output = self.get("output") if output: self._cfcnTest = ("$result eq '{0}'".format( output.replace("'", "''")), self) return self._cfcnTest
def readMeFirstUris(self): try: return self._readMeFirstUris except AttributeError: self._readMeFirstUris = [] for anElement in self.iterdescendants(): if isinstance(anElement, ModelObject) and anElement.get( "readMeFirst") == "true": if anElement.get("{http://www.w3.org/1999/xlink}href"): uri = anElement.get( "{http://www.w3.org/1999/xlink}href") else: uri = XmlUtil.innerText(anElement) if anElement.get("name"): self._readMeFirstUris.append( (ModelValue.qname(anElement, anElement.get("name")), uri)) elif anElement.get("dts"): self._readMeFirstUris.append( (anElement.get("dts"), uri)) else: self._readMeFirstUris.append(uri) if not self._readMeFirstUris: # provide a dummy empty instance document self._readMeFirstUris.append( os.path.join(self.modelXbrl.modelManager.cntlr.configDir, "empty-instance.xml")) return self._readMeFirstUris
def cfcnCall(self): # tuple of (expression, element holding the expression) try: return self._cfcnCall except AttributeError: self._cfcnCall = None if self.localName == "test-case": # xpath testcase queryElement = XmlUtil.descendant(self, None, "query") if queryElement is not None: filepath = ( self.modelDocument.filepathdir + "/" + "Queries/XQuery/" + self.get("FilePath") + queryElement.get("name") + ".xq" ) if os.sep != "/": filepath = filepath.replace("/", os.sep) with io.open(filepath, "rt", encoding="utf-8") as f: self._cfcnCall = (f.read(), self) else: for callElement in XmlUtil.descendants(self, XbrlConst.cfcn, "call"): self._cfcnCall = (XmlUtil.innerText(callElement), callElement) break if self._cfcnCall is None and self.namespaceURI == "http://xbrl.org/2011/conformance-rendering/transforms": name = self.getparent().get("name") input = self.get("input") if name and input: self._cfcnCall = ("{0}('{1}')".format(name, input.replace("'", "''")), self) return self._cfcnCall
def cfcnTest(self): # tuple of (expression, element holding the expression) try: return self._cfcnTest except AttributeError: self._cfcnTest = None if self.localName == "test-case": # xpath testcase outputFileElement = XmlUtil.descendant(self, None, "output-file") if outputFileElement is not None and outputFileElement.get("compare") == "Text": filepath = ( self.modelDocument.filepathdir + "/" + "ExpectedTestResults/" + self.get("FilePath") + outputFileElement.text ) if os.sep != "/": filepath = filepath.replace("/", os.sep) with io.open(filepath, "rt", encoding="utf-8") as f: self._cfcnTest = ("xs:string($result) eq '{0}'".format(f.read()), self) else: testElement = XmlUtil.descendant(self, XbrlConst.cfcn, "test") if testElement is not None: self._cfcnTest = (XmlUtil.innerText(testElement), testElement) elif self.namespaceURI == "http://xbrl.org/2011/conformance-rendering/transforms": output = self.get("output") if output: self._cfcnTest = ("$result eq '{0}'".format(output.replace("'", "''")), self) return self._cfcnTest
def readMeFirstUris(self): try: return self._readMeFirstUris except AttributeError: self._readMeFirstUris = [] # first look if any plugin method to get readme first URIs if not any( pluginXbrlMethod(self) for pluginXbrlMethod in pluginClassMethods( "ModelTestcaseVariation.ReadMeFirstUris")): if self.localName == "testGroup": #w3c testcase instanceTestElement = XmlUtil.descendant( self, None, "instanceTest") if instanceTestElement is not None: # take instance first self._readMeFirstUris.append( XmlUtil.descendantAttr( instanceTestElement, None, "instanceDocument", "{http://www.w3.org/1999/xlink}href")) else: schemaTestElement = XmlUtil.descendant( self, None, "schemaTest") if schemaTestElement is not None: self._readMeFirstUris.append( XmlUtil.descendantAttr( schemaTestElement, None, "schemaDocument", "{http://www.w3.org/1999/xlink}href")) elif self.localName == "test-case": #xpath testcase inputFileElement = XmlUtil.descendant( self, None, "input-file") if inputFileElement is not None: # take instance first self._readMeFirstUris.append("TestSources/" + inputFileElement.text + ".xml") else: # default built-in method for readme first uris for anElement in self.iterdescendants(): if isinstance(anElement, ModelObject) and anElement.get( "readMeFirst") == "true": if anElement.get( "{http://www.w3.org/1999/xlink}href"): uri = anElement.get( "{http://www.w3.org/1999/xlink}href") else: uri = XmlUtil.innerText(anElement) if anElement.get("name"): self._readMeFirstUris.append( (ModelValue.qname(anElement, anElement.get("name")), uri)) elif anElement.get("dts"): self._readMeFirstUris.append( (anElement.get("dts"), uri)) else: self._readMeFirstUris.append(uri) if not self._readMeFirstUris: # provide a dummy empty instance document self._readMeFirstUris.append( os.path.join(self.modelXbrl.modelManager.cntlr.configDir, "empty-instance.xml")) return self._readMeFirstUris
def propertyView(self): if self.localName == "nonFraction" or self.localName == "fraction": numProperties = (("format", self.format), ("scale", self.scale), ("html value", XmlUtil.innerText(self))) else: numProperties = () return super(ModelInlineFact,self).propertyView + \ numProperties
def readMeFirstUris(self): try: return self._readMeFirstUris except AttributeError: self._readMeFirstUris = [] # first look if any plugin method to get readme first URIs if not any( pluginXbrlMethod(self) for pluginXbrlMethod in pluginClassMethods( "ModelTestcaseVariation.ReadMeFirstUris")): if self.localName == "testGroup": #w3c testcase instanceTestElement = XmlUtil.descendant( self, None, "instanceTest") if instanceTestElement is not None: # take instance first self._readMeFirstUris.append( XmlUtil.descendantAttr( instanceTestElement, None, "instanceDocument", "{http://www.w3.org/1999/xlink}href")) else: schemaTestElement = XmlUtil.descendant( self, None, "schemaTest") if schemaTestElement is not None: self._readMeFirstUris.append( XmlUtil.descendantAttr( schemaTestElement, None, "schemaDocument", "{http://www.w3.org/1999/xlink}href")) elif self.localName == "test-case": #xpath testcase inputFileElement = XmlUtil.descendant( self, None, "input-file") if inputFileElement is not None: # take instance first self._readMeFirstUris.append( "TestSources/" + inputFileElement.text + ".xml") else: # default built-in method for readme first uris for anElement in self.iterdescendants(): if isinstance(anElement, ModelObject) and anElement.get( "readMeFirst") == "true": if anElement.get( "{http://www.w3.org/1999/xlink}href"): uri = anElement.get( "{http://www.w3.org/1999/xlink}href") else: uri = XmlUtil.innerText(anElement) if anElement.get("name"): self._readMeFirstUris.append((ModelValue.qname( anElement, anElement.get("name")), uri)) elif anElement.get("dts"): self._readMeFirstUris.append( (anElement.get("dts"), uri)) else: self._readMeFirstUris.append(uri) if not self._readMeFirstUris: # provide a dummy empty instance document self._readMeFirstUris.append( os.path.join(self.modelXbrl.modelManager.cntlr.configDir, "empty-instance.xml")) return self._readMeFirstUris
def propertyView(self): if self.localName == "nonFraction" or self.localName == "fraction": numProperties = (("format", self.format), ("scale", self.scale), ("html value", XmlUtil.innerText(self))) else: numProperties = () return (("file", self.modelDocument.basename), ("line", self.sourceline)) + \ super(ModelInlineFact,self).propertyView + \ numProperties
def value(self): v = XmlUtil.innerText(self, ixExclude=True) f = self.format if f is not None: if (f.namespaceURI.startswith("http://www.xbrl.org/inlineXBRL/transformation") and f.localName in FunctionIxt.ixtFunctions): v = FunctionIxt.ixtFunctions[f.localName](v) if self.localName == "nonNumeric" or self.localName == "tuple": return v else: return self.transformedValue(v)
def loadStandardTaxonomiesDict(self): if self.selection: self.standardTaxonomiesDict = {} self.standardAuthorities = set() if not self.standardTaxonomiesUrl: return basename = os.path.basename(self.standardTaxonomiesUrl) self.modelManager.cntlr.showStatus( _("parsing {0}").format(basename)) try: for file in (self.modelManager.cntlr.webCache.getfilename( self.standardTaxonomiesUrl), os.path.join(self.modelManager.cntlr.configDir, "xbrlschemafiles.xml")): xmldoc = xml.dom.minidom.parse(file) for locElt in xmldoc.getElementsByTagName("Loc"): href = None localHref = None namespaceUri = None attType = None family = None for childElt in locElt.childNodes: if childElt.nodeType == 1: #element ln = childElt.localName value = XmlUtil.innerText(childElt) if ln == "Href": href = value elif ln == "LocalHref": localHref = value elif ln == "Namespace": namespaceUri = value elif ln == "AttType": attType = value elif ln == "Family": family = value if href: if namespaceUri and (attType == "SCH" or attType == "ENT"): if namespaceUri not in self.standardTaxonomiesDict: self.standardTaxonomiesDict[ namespaceUri] = (href, localHref) authority = UrlUtil.authority(namespaceUri) self.standardAuthorities.add(authority) if family == "BASE": self.baseTaxonomyNamespaces.add( namespaceUri) if href not in self.standardTaxonomiesDict: self.standardTaxonomiesDict[ href] = "Allowed" + attType except (EnvironmentError, xml.parsers.expat.ExpatError, xml.dom.DOMException) as err: self.modelManager.cntlr.addToLog( "{0}: import error: {1}".format(basename, err))
def name(self): try: return self._name except AttributeError: if self.get("name"): self._name = self.get("name") else: nameElement = XmlUtil.descendant(self, None, "name" if self.localName != "testcase" else "number") if nameElement is not None: self._name = XmlUtil.innerText(nameElement) else: self._name = None return self._name
def cfcnTest(self): # tuple of (expression, element holding the expression) try: return self._cfcnTest except AttributeError: self._cfcnTest = None testElement = XmlUtil.descendant(self, XbrlConst.cfcn, "test") if testElement is not None: self._cfcnTest = (XmlUtil.innerText(testElement), testElement) elif self.namespaceURI == "http://xbrl.org/2011/conformance-rendering/transforms": output = self.get("output") if output: self._cfcnTest = ("$result eq '{0}'".format(output.replace("'","''")), self) return self._cfcnTest
def cfcnCall(self): # tuple of (expression, element holding the expression) try: return self._cfcnCall except AttributeError: self._cfcnCall = None for callElement in XmlUtil.descendants(self, XbrlConst.cfcn, "call"): self._cfcnCall = (XmlUtil.innerText(callElement), callElement) break if self._cfcnCall is None and self.namespaceURI == "http://xbrl.org/2011/conformance-rendering/transforms": name = self.getparent().get("name") input = self.get("input") if name and input: self._cfcnCall = ("{0}('{1}')".format(name, input.replace("'","''")), self) return self._cfcnCall
def value(self): try: return self._ixValue except AttributeError: v = XmlUtil.innerText(self, ixExclude=True, strip=False) f = self.format if f is not None: if (f.namespaceURI in FunctionIxt.ixtNamespaceURIs and f.localName in FunctionIxt.ixtFunctions): v = FunctionIxt.ixtFunctions[f.localName](v) if self.localName == "nonNumeric" or self.localName == "tuple": self._ixValue = v else: self._ixValue = self.transformedValue(v) return self._ixValue
def cfcnTest(self): # tuple of (expression, element holding the expression) try: return self._cfcnTest except AttributeError: self._cfcnTest = None testElement = XmlUtil.descendant(self, XbrlConst.cfcn, "test") if testElement is not None: self._cfcnTest = (XmlUtil.innerText(testElement), testElement) elif self.namespaceURI == "http://xbrl.org/2011/conformance-rendering/transforms": output = self.get("output") if output: self._cfcnTest = ("$result eq '{0}'".format( output.replace("'", "''")), self) return self._cfcnTest
def cfcnCall(self): # tuple of (expression, element holding the expression) try: return self._cfcnCall except AttributeError: self._cfcnCall = None for callElement in XmlUtil.descendants(self, XbrlConst.cfcn, "call"): self._cfcnCall = (XmlUtil.innerText(callElement), callElement) break if self._cfcnCall is None and self.namespaceURI == "http://xbrl.org/2011/conformance-rendering/transforms": name = self.getparent().get("name") input = self.get("input") if name and input: self._cfcnCall = ("{0}('{1}')".format( name, input.replace("'", "''")), self) return self._cfcnCall
def loadStandardTaxonomiesDict(self): if self.selection: self.standardTaxonomiesDict = {} self.standardAuthorities = set() if not self.standardTaxonomiesUrl: return basename = os.path.basename(self.standardTaxonomiesUrl) self.modelManager.cntlr.showStatus(_("parsing {0}").format(basename)) try: for file in (self.modelManager.cntlr.webCache.getfilename(self.standardTaxonomiesUrl), os.path.join(self.modelManager.cntlr.configDir,"xbrlschemafiles.xml")): xmldoc = xml.dom.minidom.parse(file) for locElt in xmldoc.getElementsByTagName("Loc"): href = None localHref = None namespaceUri = None attType = None family = None for childElt in locElt.childNodes: if childElt.nodeType == 1: #element ln = childElt.localName value = XmlUtil.innerText(childElt) if ln == "Href": href = value elif ln == "LocalHref": localHref = value elif ln == "Namespace": namespaceUri = value elif ln == "AttType": attType = value elif ln == "Family": family = value if href: if namespaceUri and (attType == "SCH" or attType == "ENT"): if namespaceUri not in self.standardTaxonomiesDict: self.standardTaxonomiesDict[namespaceUri] = (href, localHref) authority = UrlUtil.authority(namespaceUri) self.standardAuthorities.add(authority) if family == "BASE": self.baseTaxonomyNamespaces.add(namespaceUri) if href not in self.standardTaxonomiesDict: self.standardTaxonomiesDict[href] = "Allowed" + attType except (EnvironmentError, xml.parsers.expat.ExpatError, xml.dom.DOMException) as err: self.modelManager.cntlr.addToLog("{0}: import error: {1}".format(basename,err))
def readMeFirstUris(self): try: return self._readMeFirstUris except AttributeError: self._readMeFirstUris = [] for anElement in self.iterdescendants(): if isinstance(anElement,ModelObject) and anElement.get("readMeFirst") == "true": if anElement.get("{http://www.w3.org/1999/xlink}href"): uri = anElement.get("{http://www.w3.org/1999/xlink}href") else: uri = XmlUtil.innerText(anElement) if anElement.get("name"): self._readMeFirstUris.append( (ModelValue.qname(anElement, anElement.get("name")), uri) ) elif anElement.get("dts"): self._readMeFirstUris.append( (anElement.get("dts"), uri) ) else: self._readMeFirstUris.append(uri) if not self._readMeFirstUris: # provide a dummy empty instance document self._readMeFirstUris.append(os.path.join(self.modelXbrl.modelManager.cntlr.configDir, "empty-instance.xml")) return self._readMeFirstUris
def value(self): """(str) -- Overrides and corresponds to value property of ModelFact, for relevant inner text nodes aggregated and transformed as needed.""" try: return self._ixValue except AttributeError: v = XmlUtil.innerText(self, ixExclude=True, strip=True) # transforms are whitespace-collapse f = self.format if f is not None: if (f.namespaceURI in FunctionIxt.ixtNamespaceURIs and f.localName in FunctionIxt.ixtFunctions): try: v = FunctionIxt.ixtFunctions[f.localName](v) except Exception as err: self._ixValue = ModelValue.INVALIDixVALUE raise err if self.localName == "nonNumeric" or self.localName == "tuple": self._ixValue = v else: self._ixValue = self.transformedValue(v) return self._ixValue
def view(self, modelDocument): relationshipSet = self.modelXbrl.relationshipSet(self.arcrole, self.linkrole, self.linkqname, self.arcqname) if relationshipSet: # sort URIs by definition linkroleUris = [] for linkroleUri in relationshipSet.linkRoleUris: modelRoleTypes = self.modelXbrl.roleTypes.get(linkroleUri) if modelRoleTypes: roledefinition = (modelRoleTypes[0].definition or linkroleUri) else: roledefinition = linkroleUri linkroleUris.append((roledefinition, linkroleUri)) linkroleUris.sort() for roledefinition, linkroleUri in linkroleUris: linkRelationshipSet = self.modelXbrl.relationshipSet(self.arcrole, linkroleUri, self.linkqname, self.arcqname) for rootConcept in linkRelationshipSet.rootConcepts: self.treeDepth(rootConcept, rootConcept, 2, self.arcrole, linkRelationshipSet, set()) # set up facts self.conceptFacts = defaultdict(list) for fact in self.modelXbrl.facts: self.conceptFacts[fact.qname].append(fact) # sort contexts by period self.periodContexts = defaultdict(set) contextStartDatetimes = {} for context in self.modelXbrl.contexts.values(): if self.type in (CSV, HTML): if self.ignoreDims: if context.isForeverPeriod: contextkey = datetime.datetime(datetime.MINYEAR,1,1) else: contextkey = context.endDatetime else: if context.isForeverPeriod: contextkey = "forever" else: contextkey = (context.endDatetime - datetime.timedelta(days=1)).strftime("%Y-%m-%d") values = [] dims = context.qnameDims if len(dims) > 0: for dimQname in sorted(dims.keys(), key=lambda d: str(d)): dimvalue = dims[dimQname] if dimvalue.isExplicit: values.append(dimvalue.member.label(self.labelrole,lang=self.lang)) else: values.append(XmlUtil.innerText(dimvalue.typedMember)) nonDimensions = context.nonDimValues("segment") + context.nonDimValues("scenario") if len(nonDimensions) > 0: for element in sorted(nonDimensions, key=lambda e: e.localName): values.append(XmlUtil.innerText(element)) if len(values) > 0: contextkey += " - " + ', '.join(values) else: contextkey = context.id objectId = context.objectId() self.periodContexts[contextkey].add(objectId) if context.isStartEndPeriod: contextStartDatetimes[objectId] = context.startDatetime self.periodKeys = list(self.periodContexts.keys()) self.periodKeys.sort() # set up treeView widget and tabbed pane heading = ["Concept"] columnHeadings = [] self.contextColId = {} self.startdatetimeColId = {} self.numCols = 1 for periodKey in self.periodKeys: columnHeadings.append(periodKey) for contextId in self.periodContexts[periodKey]: self.contextColId[contextId] = self.numCols if contextId in contextStartDatetimes: self.startdatetimeColId[contextStartDatetimes[contextId]] = self.numCols self.numCols += 1 for colHeading in columnHeadings: if self.ignoreDims: if colHeading.year == datetime.MINYEAR: date = "forever" else: date = (colHeading - datetime.timedelta(days=1)).strftime("%Y-%m-%d") heading.append(date) else: heading.append(colHeading) self.addRow(heading, asHeader=True) # must do after determining tree depth if relationshipSet: # for each URI in definition order for roledefinition, linkroleUri in linkroleUris: attr = {"role": linkroleUri} self.addRow([roledefinition], treeIndent=0, colSpan=len(heading), xmlRowElementName="linkRole", xmlRowEltAttr=attr, xmlCol0skipElt=True) linkRelationshipSet = self.modelXbrl.relationshipSet(self.arcrole, linkroleUri, self.linkqname, self.arcqname) for rootConcept in linkRelationshipSet.rootConcepts: self.viewConcept(rootConcept, rootConcept, "", self.labelrole, 1, linkRelationshipSet, set()) return True
def validate(modelXbrl, elt, recurse=True, attrQname=None, ixFacts=False): global ModelInlineValueObject, ixMsgCode if ModelInlineValueObject is None: from arelle.ModelInstanceObject import ModelInlineValueObject from arelle.XhtmlValidate import ixMsgCode isIxFact = isinstance(elt, ModelInlineValueObject) facets = None # attrQname can be provided for attributes that are global and LAX if (getattr(elt,"xValid", UNVALIDATED) == UNVALIDATED) and (not isIxFact or ixFacts): qnElt = elt.qname if ixFacts and isIxFact else elt.elementQname modelConcept = modelXbrl.qnameConcepts.get(qnElt) isAbstract = False if modelConcept is not None: isNillable = modelConcept.isNillable type = modelConcept.type if modelConcept.isAbstract: baseXsdType = "noContent" isAbstract = True elif modelConcept.isFraction: baseXsdType = "fraction" else: baseXsdType = modelConcept.baseXsdType facets = modelConcept.facets elif qnElt == XbrlConst.qnXbrldiExplicitMember: # not in DTS baseXsdType = "QName" type = None isNillable = False elif qnElt == XbrlConst.qnXbrldiTypedMember: # not in DTS baseXsdType = "noContent" type = None isNillable = False else: baseXsdType = None type = None isNillable = True # allow nil if no schema definition isNil = elt.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1") if attrQname is None: if isNil and not isNillable: if ModelInlineValueObject is not None and isinstance(elt, ModelInlineValueObject): errElt = "{0} fact {1}".format(elt.elementQname, elt.qname) else: errElt = elt.elementQname modelXbrl.error("xmlSchema:nilNonNillableElement", _("Element %(element)s fact %(fact)s type %(typeName)s is nil but element has not been defined nillable"), modelObject=elt, element=errElt, fact=elt.qname, transform=elt.format, typeName=modelConcept.baseXsdType if modelConcept is not None else "unknown", value=XmlUtil.innerText(elt, ixExclude=True)) try: if isAbstract: raise ValueError("element is abstract") if isNil: text = "" elif baseXsdType == "noContent": text = elt.textValue # no descendant text nodes else: text = elt.stringValue # include descendant text nodes if len(text) == 0 and modelConcept is not None: if modelConcept.default is not None: text = modelConcept.default elif modelConcept.fixed is not None: text = modelConcept.fixed except Exception as err: if ModelInlineValueObject is not None and isinstance(elt, ModelInlineValueObject): errElt = "{0} fact {1}".format(elt.elementQname, elt.qname) else: errElt = elt.elementQname if isIxFact and err.__class__.__name__ == "FunctionArgType": modelXbrl.error(ixMsgCode("transformValueError", elt), _("Inline element %(element)s fact %(fact)s type %(typeName)s transform %(transform)s value error: %(value)s"), modelObject=elt, element=errElt, fact=elt.qname, transform=elt.format, typeName=modelConcept.baseXsdType if modelConcept is not None else "unknown", value=XmlUtil.innerText(elt, ixExclude=True, ixContinuation=elt.namespaceURI==XbrlConst.ixbrl11)) elif isIxFact and err.__class__.__name__ == "ixtFunctionNotAvailable": modelXbrl.error(ixMsgCode("formatCodeUndefined", elt), _("Inline element %(element)s fact %(fact)s type %(typeName)s transform %(transform)s not available, value: %(value)s"), modelObject=elt, element=errElt, fact=elt.qname, transform=elt.format, typeName=modelConcept.baseXsdType if modelConcept is not None else "unknown", value=XmlUtil.innerText(elt, ixExclude=True, ixContinuation=elt.namespaceURI==XbrlConst.ixbrl11)) elif isAbstract: modelXbrl.error("xmlSchema:abstractElement", _("Element %(element)s has abstract declaration, value: %(value)s"), modelObject=elt, element=errElt, error=str(err), value=elt.text) else: modelXbrl.error("xmlSchema:valueError", _("Element %(element)s error %(error)s value: %(value)s"), modelObject=elt, element=errElt, error=str(err), value=elt.text) elt.sValue = elt.xValue = text = INVALIDixVALUE elt.xValid = INVALID if text is not INVALIDixVALUE: validateValue(modelXbrl, elt, None, baseXsdType, text, isNillable, isNil, facets) # note that elt.sValue and elt.xValue are not innerText but only text elements on specific element (or attribute) if type is not None: definedAttributes = type.attributes else: definedAttributes = {} presentAttributes = set() # validate attributes # find missing attributes for default values for attrTag, attrValue in elt.items(): qn = qnameClarkName(attrTag) #qn = qname(attrTag, noPrefixIsNoNamespace=True) baseXsdAttrType = None facets = None if attrQname is not None: # validate all attributes and element if attrQname != qn: continue elif type is not None: presentAttributes.add(qn) if qn in definedAttributes: # look for concept-type-specific attribute definition modelAttr = definedAttributes[qn] elif qn.namespaceURI: # may be a globally defined attribute modelAttr = modelXbrl.qnameAttributes.get(qn) else: modelAttr = None if modelAttr is not None: baseXsdAttrType = modelAttr.baseXsdType facets = modelAttr.facets if baseXsdAttrType is None: # look for global attribute definition attrObject = modelXbrl.qnameAttributes.get(qn) if attrObject is not None: baseXsdAttrType = attrObject.baseXsdType facets = attrObject.facets elif attrTag == "{http://xbrl.org/2006/xbrldi}dimension": # some fallbacks? baseXsdAttrType = "QName" elif attrTag == "id": baseXsdAttrType = "ID" elif elt.namespaceURI == "http://www.w3.org/2001/XMLSchema": if attrTag in {"type", "ref", "base", "refer", "itemType"}: baseXsdAttrType = "QName" elif attrTag in {"name"}: baseXsdAttrType = "NCName" elif attrTag in {"default", "fixed", "form"}: baseXsdAttrType = "string" elif elt.namespaceURI == "http://xbrl.org/2006/xbrldi": if attrTag == "dimension": baseXsdAttrType = "QName" elif qn in predefinedAttributeTypes: baseXsdAttrType, facets = predefinedAttributeTypes[qn] validateValue(modelXbrl, elt, attrTag, baseXsdAttrType, attrValue, facets=facets) # if no attributes assigned above, there won't be an xAttributes, if so assign a shared dict to save memory try: elt.xAttributes except AttributeError: elt.xAttributes = xAttributesSharedEmptyDict if type is not None: if attrQname is None: missingAttributes = type.requiredAttributeQnames - presentAttributes - elt.slottedAttributesNames if missingAttributes: modelXbrl.error("xmlSchema:attributesRequired", _("Element %(element)s type %(typeName)s missing required attributes: %(attributes)s"), modelObject=elt, element=qnElt, typeName=baseXsdType, attributes=','.join(str(a) for a in missingAttributes)) extraAttributes = presentAttributes - _DICT_SET(definedAttributes.keys()) - XbrlConst.builtinAttributes if extraAttributes: attributeWildcards = type.attributeWildcards extraAttributes -= set(a for a in extraAttributes if validateAnyWildcard(qnElt, a, attributeWildcards)) if isIxFact: extraAttributes -= XbrlConst.ixAttributes if extraAttributes: modelXbrl.error("xmlSchema:attributesExtraneous", _("Element %(element)s type %(typeName)s extraneous attributes: %(attributes)s"), modelObject=elt, element=qnElt, typeName=baseXsdType, attributes=','.join(str(a) for a in extraAttributes)) # add default attribute values for attrQname in (type.defaultAttributeQnames - presentAttributes): modelAttr = type.attributes[attrQname] validateValue(modelXbrl, elt, attrQname.clarkNotation, modelAttr.baseXsdType, modelAttr.default, facets=modelAttr.facets) if recurse: global validateElementSequence, modelGroupCompositorTitle if validateElementSequence is None: from arelle.XmlValidateParticles import validateElementSequence, modelGroupCompositorTitle try: #childElts = list(elt) # uses __iter__ for inline facts childElts = [e for e in elt if isinstance(e, ModelObject)] if isNil: if childElts or elt.text: modelXbrl.error("xmlSchema:nilElementHasContent", _("Element %(element)s is nil but has contents"), modelObject=elt, element=qnElt) else: errResult = validateElementSequence(modelXbrl, type, childElts, ixFacts) if errResult is not None and errResult[2]: iElt, occured, errDesc, errArgs = errResult errElt = childElts[iElt] if iElt < len(childElts) else elt errArgs["modelObject"] = errElt errArgs["element"] = errElt.qname errArgs["parentElement"] = elt.qname if "compositor" in errArgs: # compositor is an object, provide friendly string errArgs["compositor"] = modelGroupCompositorTitle(errArgs["compositor"]) modelXbrl.error(*errDesc,**errArgs) # when error is in an xbrli element, check any further unvalidated children if qnElt.namespaceURI == XbrlConst.xbrli and iElt < len(childElts): for childElt in childElts[iElt:]: if (getattr(childElt,"xValid", UNVALIDATED) == UNVALIDATED): validate(modelXbrl, childElt, ixFacts=ixFacts) recurse = False # cancel child element validation below, recursion was within validateElementSequence except AttributeError as ex: raise ex #pass # HF Why is this here???? if recurse: # if there is no complex or simple type (such as xbrli:measure) then this code is used for child in (elt.modelTupleFacts if ixFacts and isIxFact else elt): if isinstance(child, ModelObject): validate(modelXbrl, child, recurse, attrQname, ixFacts)
def validateFiling(val, modelXbrl): linkroleDefinitionStatementSheet = re.compile( r"[^-]+-\s+Statement\s+-\s+.*", # no restriction to type of statement re.IGNORECASE) if not hasattr(modelXbrl.modelDocument, "xmlDocument"): # not parsed return val._isStandardUri = {} modelXbrl.modelManager.disclosureSystem.loadStandardTaxonomiesDict() # find typedDomainRefs before validateXBRL pass val.typedDomainQnames = set() val.typedDomainElements = set() for modelConcept in modelXbrl.qnameConcepts.values(): if modelConcept.isTypedDimension: typedDomainElement = modelConcept.typedDomainElement if isinstance(typedDomainElement, ModelConcept): val.typedDomainQnames.add(typedDomainElement.qname) val.typedDomainElements.add(typedDomainElement) # note that some XFM tests are done by ValidateXbrl to prevent multiple node walks xbrlInstDoc = modelXbrl.modelDocument.xmlDocument.getroot() disclosureSystem = val.disclosureSystem disclosureSystemVersion = disclosureSystem.version modelXbrl.modelManager.showStatus( _("validating {0}").format(disclosureSystem.name)) val.modelXbrl.profileActivity() conceptsUsed = { } # key=concept object value=True if has presentation label labelsRelationshipSet = modelXbrl.relationshipSet(XbrlConst.conceptLabel) genLabelsRelationshipSet = modelXbrl.relationshipSet( XbrlConst.elementLabel) presentationRelationshipSet = modelXbrl.relationshipSet( XbrlConst.parentChild) referencesRelationshipSetWithProhibits = modelXbrl.relationshipSet( XbrlConst.conceptReference, includeProhibits=True) val.modelXbrl.profileActivity("... cache lbl, pre, ref relationships", minTimeToShow=1.0) val.validateLoggingSemantic = validateLoggingSemantic = ( modelXbrl.isLoggingEffectiveFor(level="WARNING-SEMANTIC") or modelXbrl.isLoggingEffectiveFor(level="ERROR-SEMANTIC")) # instance checks val.fileNameBasePart = None # prevent testing on fileNameParts if not instance or invalid val.fileNameDate = None val.entityRegistrantName = None val.requiredContext = None val.standardNamespaceConflicts = defaultdict(set) val.exhibitType = None # e.g., EX-101, EX-201 # entry point schema checks print("Starting entrypoiny schema checks") if modelXbrl.modelDocument.type == ModelDocument.Type.SCHEMA: # entry must have a P-link if not any(hrefElt.localName == "linkbaseRef" and hrefElt.get("{http://www.w3.org/1999/xlink}role") == "http://www.xbrl.org/2003/role/presentationLinkbaseRef" for hrefElt, hrefDoc, hrefId in modelXbrl.modelDocument.hrefObjects): modelXbrl.error( "SBR.NL.2.02.10.01", 'Entrypoint schema must have a presentation linkbase', modelObject=modelXbrl.modelDocument) # all-labels and references checks print("Starting labels and reference checks") for concept in modelXbrl.qnameConcepts.values(): conceptHasDefaultLangStandardLabel = False for modelLabelRel in labelsRelationshipSet.fromModelObject(concept): modelLabel = modelLabelRel.toModelObject role = modelLabel.role text = modelLabel.text lang = modelLabel.xmlLang if role == XbrlConst.documentationLabel: if concept.modelDocument.targetNamespace in disclosureSystem.standardTaxonomiesDict: modelXbrl.error( "SBR.NL.2.01.00.08", _("Concept %(concept)s of a standard taxonomy cannot have a documentation label: %(text)s" ), modelObject=modelLabel, concept=concept.qname, text=text) elif text and lang and disclosureSystem.defaultXmlLang and lang.startswith( disclosureSystem.defaultXmlLang): if role == XbrlConst.standardLabel: # merge of pre-plugin code per LOGIUS conceptHasDefaultLangStandardLabel = True match = modelXbrl.modelManager.disclosureSystem.labelCheckPattern.search( text) if match: modelXbrl.error( "SBR.NL.2.03.08.07", 'Label for concept %(concept)s role %(role)s has disallowed characters: "%(text)s"', modelObject=modelLabel, concept=concept.qname, role=role, text=match.group()) for modelRefRel in referencesRelationshipSetWithProhibits.fromModelObject( concept): modelReference = modelRefRel.toModelObject text = XmlUtil.innerText(modelReference) #6.18.1 no reference to company extension concepts if (concept.modelDocument.targetNamespace in disclosureSystem.standardTaxonomiesDict and not isStandardUri(val, modelRefRel.modelDocument.uri) ): # merge of pre-plugin code per LOGIUS #6.18.2 no extension to add or remove references to standard concepts modelXbrl.error( "SBR.NL.2.01.00.08", _("References for standard taxonomy concept %(concept)s are not allowed in an extension linkbase: %(text)s" ), modelObject=modelReference, concept=concept.qname, text=text, xml=XmlUtil.xmlstring(modelReference, stripXmlns=True, contentsOnly=True)) if concept.isItem or concept.isTuple: if concept.modelDocument.targetNamespace not in disclosureSystem.standardTaxonomiesDict: ''' Only continue if the concept's namespace is not a standard namespace. When using "--disclosureSystem SBR-NL" as load argument, this list is populated from 'config/sbr-nl-taxonomies.xml' ''' if not conceptHasDefaultLangStandardLabel: modelXbrl.error( "SBR.NL.2.02.02.26", _("Concept %(concept)s missing standard label in local language." ), modelObject=concept, concept=concept.qname) subsGroup = concept.get("substitutionGroup") if not concept.isAbstract: if subsGroup == "sbr:presentationItem" and not ( presentationRelationshipSet.toModelObject(concept) or presentationRelationshipSet.fromModelObject( concept)): modelXbrl.error( "SBR.NL.2.02.02.04", _("Concept %(concept)s not referred to by presentation relationship." ), modelObject=concept, concept=concept.qname) elif ((concept.isDimensionItem or (subsGroup and (subsGroup.endswith(":domainItem") or subsGroup.endswith(":domainMemberItem")))) and not (presentationRelationshipSet.toModelObject(concept) or presentationRelationshipSet.fromModelObject( concept))): modelXbrl.error( "SBR.NL.2.02.10.03", _("DTS concept %(concept)s not referred to by presentation relationship." ), modelObject=concept, concept=concept.qname) if (concept.substitutionGroupQname and concept.substitutionGroupQname.namespaceURI not in disclosureSystem.baseTaxonomyNamespaces): modelXbrl.error( "SBR.NL.2.02.02.05", _("Concept %(concept)s has a substitutionGroup of a non-standard concept." ), modelObject=concept, concept=concept.qname) if concept.isTuple: # verify same presentation linkbase nesting if concept.type is not None: for missingQname in set( concept.type.elements ) ^ pLinkedNonAbstractDescendantQnames( modelXbrl, concept): modelXbrl.error( "SBR.NL.2.03.04.01", _("Tuple %(concept)s has mismatch between content and presentation children: %(missingQname)s." ), modelObject=concept, concept=concept.qname, missingQname=missingQname) else: print(f"{concept.elementNamespaceURI}") checkConceptLabels(val, modelXbrl, labelsRelationshipSet, disclosureSystem, concept) checkConceptLabels(val, modelXbrl, genLabelsRelationshipSet, disclosureSystem, concept) val.modelXbrl.profileActivity("... filer concepts checks", minTimeToShow=1.0) # checks on all documents: instance, schema, instance checkFilingDTS(val, modelXbrl.modelDocument, []) ''' removed RH 2011-12-23, corresponding use of nameWordsTable in ValidateFilingDTS if val.validateSBRNL: del val.nameWordsTable ''' val.modelXbrl.profileActivity("... filer DTS checks", minTimeToShow=1.0) conceptRelsUsedWithPreferredLabels = defaultdict(list) usedCalcsPresented = defaultdict( set) # pairs of concepts objectIds used in calc usedCalcFromTosELR = {} localPreferredLabels = defaultdict(set) drsELRs = set() # do calculation, then presentation, then other arcroles val.summationItemRelsSetAllELRs = modelXbrl.relationshipSet( XbrlConst.summationItem) for arcroleFilter in (XbrlConst.summationItem, XbrlConst.parentChild, "*"): for baseSetKey, baseSetModelLinks in modelXbrl.baseSets.items(): arcrole, ELR, linkqname, arcqname = baseSetKey if ELR and linkqname and arcqname and not arcrole.startswith( "XBRL-"): # assure summationItem, then parentChild, then others if not (arcroleFilter == arcrole or arcroleFilter == "*" and arcrole not in (XbrlConst.summationItem, XbrlConst.parentChild)): continue if arcrole == XbrlConst.parentChild: ineffectiveArcs = ModelRelationshipSet.ineffectiveArcs( baseSetModelLinks, arcrole) #validate ineffective arcs for modelRel in ineffectiveArcs: if isinstance(modelRel.fromModelObject, ModelObject) and isinstance( modelRel.toModelObject, ModelObject): modelXbrl.error( "SBR.NL.2.03.04.06", _("Ineffective arc %(arc)s in \nlink role %(linkrole)s \narcrole %(arcrole)s \nfrom %(conceptFrom)s \nto %(conceptTo)s \n%(ineffectivity)s" ), modelObject=modelRel, arc=modelRel.qname, arcrole=modelRel.arcrole, linkrole=modelRel.linkrole, linkroleDefinition=modelXbrl. roleTypeDefinition(modelRel.linkrole), conceptFrom=modelRel.fromModelObject.qname, conceptTo=modelRel.toModelObject.qname, ineffectivity=modelRel.ineffectivity) if arcrole == XbrlConst.parentChild: isStatementSheet = any( linkroleDefinitionStatementSheet.match( roleType.definition or '') for roleType in val.modelXbrl.roleTypes.get(ELR, ())) conceptsPresented = set() # 6.12.2 check for distinct order attributes parentChildRels = modelXbrl.relationshipSet(arcrole, ELR) for relFrom, siblingRels in parentChildRels.fromModelObjects( ).items(): targetConceptPreferredLabels = defaultdict(dict) orderRels = {} firstRel = True relFromUsed = True for rel in siblingRels: if firstRel: firstRel = False if relFrom in conceptsUsed: conceptsUsed[ relFrom] = True # 6.12.3, has a pres relationship relFromUsed = True relTo = rel.toModelObject preferredLabel = rel.preferredLabel if relTo in conceptsUsed: conceptsUsed[ relTo] = True # 6.12.3, has a pres relationship if preferredLabel and preferredLabel != "": conceptRelsUsedWithPreferredLabels[ relTo].append(rel) if preferredLabel in ("periodStart", "periodEnd"): modelXbrl.error( "SBR.NL.2.03.04.03", _("Preferred label on presentation relationships not allowed" ), modelObject=modelRel) # 6.12.5 distinct preferred labels in base set preferredLabels = targetConceptPreferredLabels[ relTo] if (preferredLabel in preferredLabels or (not relFrom.isTuple and (not preferredLabel or None in preferredLabels))): if preferredLabel in preferredLabels: rel2, relTo2 = preferredLabels[ preferredLabel] else: rel2 = relTo2 = None modelXbrl.error( "SBR.NL.2.03.04.06", _("Concept %(concept)s has duplicate preferred label %(preferredLabel)s in link role %(linkrole)s" ), modelObject=(rel, relTo, rel2, relTo2), concept=relTo.qname, fromConcept=rel.fromModelObject.qname, preferredLabel=preferredLabel, linkrole=rel.linkrole, linkroleDefinition=modelXbrl. roleTypeDefinition(rel.linkrole)) else: preferredLabels[preferredLabel] = (rel, relTo) if relFromUsed: # 6.14.5 conceptsPresented.add(relFrom.objectIndex) conceptsPresented.add(relTo.objectIndex) order = rel.order if order in orderRels: modelXbrl.error( "SBR.NL.2.03.04.05", _("Duplicate presentation relations from concept %(conceptFrom)s for order %(order)s in base set role %(linkrole)s to concept %(conceptTo)s and to concept %(conceptTo2)s" ), modelObject=(rel, orderRels[order]), conceptFrom=relFrom.qname, order=rel.arcElement.get("order"), linkrole=rel.linkrole, linkroleDefinition=modelXbrl. roleTypeDefinition(rel.linkrole), conceptTo=rel.toModelObject.qname, conceptTo2=orderRels[order].toModelObject. qname) else: orderRels[order] = rel if not relFrom.isTuple: if relTo in localPreferredLabels: if {None, preferredLabel } & localPreferredLabels[relTo]: val.modelXbrl.error( "SBR.NL.2.03.04.06", _("Non-distinguished preferredLabel presentation relations from concept %(conceptFrom)s in base set role %(linkrole)s" ), modelObject=rel, conceptFrom=relFrom.qname, linkrole=rel.linkrole, conceptTo=relTo.qname) localPreferredLabels[relTo].add(preferredLabel) targetConceptPreferredLabels.clear() orderRels.clear() localPreferredLabels.clear() # clear for next relationship for conceptPresented in conceptsPresented: if conceptPresented in usedCalcsPresented: usedCalcPairingsOfConcept = usedCalcsPresented[ conceptPresented] if len(usedCalcPairingsOfConcept & conceptsPresented) > 0: usedCalcPairingsOfConcept -= conceptsPresented elif arcrole == XbrlConst.summationItem: # find a calc relationship to get the containing document name for modelRel in val.modelXbrl.relationshipSet( arcrole, ELR).modelRelationships: val.modelXbrl.error( "SBR.NL.2.03.09.01", _("Calculation linkbase linkrole %(linkrole)s"), modelObject=modelRel, linkrole=ELR) break elif arcrole == XbrlConst.all or arcrole == XbrlConst.notAll: drsELRs.add(ELR) else: if arcrole == XbrlConst.dimensionDefault: for modelRel in val.modelXbrl.relationshipSet( arcrole).modelRelationships: val.modelXbrl.error( "SBR.NL.2.03.06.05", _("Dimension-default in from %(conceptFrom)s to %(conceptTo)s in role %(linkrole)s is not allowed" ), modelObject=modelRel, conceptFrom=modelRel.fromModelObject.qname, conceptTo=modelRel.toModelObject.qname, linkrole=modelRel.linkrole) ''' removed per RH 2013-01-11 if not (XbrlConst.isStandardArcrole(arcrole) or XbrlConst.isDefinitionOrXdtArcrole(arcrole)): for modelRel in val.modelXbrl.relationshipSet(arcrole).modelRelationships: relTo = modelRel.toModelObject relFrom = modelRel.fromModelObject if not ((isinstance(relFrom,ModelConcept) and isinstance(relTo,ModelConcept)) or (relFrom.modelDocument.inDTS and (relTo.qname == XbrlConst.qnGenLabel and modelRel.arcrole == XbrlConst.elementLabel) or (relTo.qname == XbrlConst.qnGenReference and modelRel.arcrole == XbrlConst.elementReference) or (relTo.qname == val.qnSbrLinkroleorder))): val.modelXbrl.error("SBR.NL.2.3.2.07", _("The source and target of an arc must be in the DTS from %(elementFrom)s to %(elementTo)s, in linkrole %(linkrole)s, arcrole %(arcrole)s"), modelObject=modelRel, elementFrom=relFrom.qname, elementTo=relTo.qname, linkrole=modelRel.linkrole, arcrole=arcrole) ''' del localPreferredLabels # dereference del usedCalcFromTosELR del val.summationItemRelsSetAllELRs val.modelXbrl.profileActivity("... filer relationships checks", minTimeToShow=1.0) # checks on dimensions checkFilingDimensions(val, drsELRs) val.modelXbrl.profileActivity("... filer dimensions checks", minTimeToShow=1.0) del conceptRelsUsedWithPreferredLabels # 6 16 4, 1.16.5 Base sets of Domain Relationship Sets testing val.modelXbrl.profileActivity("... filer preferred label checks", minTimeToShow=1.0) # moved from original validateSBRnl finally for qname, modelType in modelXbrl.qnameTypes.items(): if qname.namespaceURI not in val.disclosureSystem.baseTaxonomyNamespaces: facets = modelType.facets if facets: lengthFacets = _DICT_SET( facets.keys()) & {"minLength", "maxLength", "length"} if lengthFacets: modelXbrl.error( "SBR.NL.2.02.07.02", _("Type %(typename)s has length restriction facets %(facets)s" ), modelObject=modelType, typename=modelType.qname, facets=", ".join(lengthFacets)) if "enumeration" in facets and not modelType.isDerivedFrom( XbrlConst.qnXbrliStringItemType): modelXbrl.error( "SBR.NL.2.02.07.04", _("Concept %(concept)s has enumeration and is not based on stringItemType" ), modelObject=modelType, concept=modelType.qname) # check presentation link roles for generic linkbase order number ordersRelationshipSet = modelXbrl.relationshipSet( "http://www.nltaxonomie.nl/2011/arcrole/linkrole-order") presLinkroleNumberURI = {} presLinkrolesCount = 0 for countLinkroles in (True, False): for _roleURI, modelRoleTypes in modelXbrl.roleTypes.items(): for modelRoleType in modelRoleTypes: if XbrlConst.qnLinkPresentationLink in modelRoleType.usedOns: if countLinkroles: presLinkrolesCount += 1 else: if not ordersRelationshipSet: modelXbrl.error( "SBR.NL.2.02.03.06", _("Presentation linkrole %(linkrole)s missing order number relationship set" ), modelObject=modelRoleType, linkrole=modelRoleType.roleURI) else: order = None for orderNumRel in ordersRelationshipSet.fromModelObject( modelRoleType): order = getattr(orderNumRel.toModelObject, "xValue", "(noPSVIvalue)") if order in presLinkroleNumberURI: modelXbrl.error( "SBR.NL.2.02.03.06", _("Presentation linkrole order number %(order)s of %(linkrole)s also used in %(otherLinkrole)s" ), modelObject=modelRoleType, order=order, linkrole=modelRoleType.roleURI, otherLinkrole=presLinkroleNumberURI[ order]) else: presLinkroleNumberURI[ order] = modelRoleType.roleURI if not order: modelXbrl.error( "SBR.NL.2.02.03.06", _("Presentation linkrole %(linkrole)s missing order number" ), modelObject=modelRoleType, linkrole=modelRoleType.roleURI) if countLinkroles and presLinkrolesCount < 2: break # don't check order numbers if only one presentation linkrole # check arc role definitions for labels for arcroleURI, modelRoleTypes in modelXbrl.arcroleTypes.items(): for modelRoleType in modelRoleTypes: if (not arcroleURI.startswith("http://xbrl.org/") and modelRoleType.modelDocument.targetNamespace not in val.disclosureSystem.baseTaxonomyNamespaces and (not modelRoleType.genLabel(lang="nl") or not modelRoleType.genLabel(lang="en"))): modelXbrl.error( "SBR.NL.2.02.04.02", _("ArcroleType missing nl or en generic label: %(arcrole)s" ), modelObject=modelRoleType, arcrole=arcroleURI) for domainElt in val.typedDomainElements: if domainElt.modelDocument.targetNamespace not in val.disclosureSystem.baseTaxonomyNamespaces: if not domainElt.genLabel(fallbackToQname=False, lang="nl"): modelXbrl.error( "SBR.NL.2.02.08.01", _("Typed dimension domain element %(concept)s must have a generic label" ), modelObject=domainElt, concept=domainElt.qname) if domainElt.type is not None and domainElt.type.localName == "complexType": modelXbrl.error( "SBR.NL.2.02.08.02", _("Typed dimension domain element %(concept)s has disallowed complex content" ), modelObject=domainElt, concept=domainElt.qname) modelXbrl.profileActivity("... SBR role types and type facits checks", minTimeToShow=1.0) # end moved from ValidateFiling # 3.2.4.4 check each using prefix against taxonomy declaring the prefix for docs in modelXbrl.namespaceDocs.values(): for doc in docs: for prefix, NS in doc.xmlRootElement.nsmap.items(): if NS in val.namespacePrefix and prefix != val.namespacePrefix[ NS]: modelXbrl.error( "SBR.NL.3.02.04.04", _("The assigned namespace prefix %(assignedPrefix)s for the schema that declares the targetnamespace %(namespace)s, MUST be adhired by all other NT schemas, referencedPrefix: %(referencedPrefix)s" ), modelObject=doc.xmlRootElement, namespace=NS, assignedPrefix=val.namespacePrefix.get(NS, ''), referencedPrefix=prefix) # check non-concept elements that can appear in elements for labels (concepts checked by labelsRelationshipSet = modelXbrl.relationshipSet( (XbrlConst.conceptLabel, XbrlConst.elementLabel)) baseTaxonomyNamespaces = val.disclosureSystem.baseTaxonomyNamespaces for eltDef in modelXbrl.qnameConcepts.values(): if (not (eltDef.isItem or eltDef.isTuple or eltDef.isLinkPart) and eltDef.qname.namespaceURI not in baseTaxonomyNamespaces): eltDefHasDefaultLangStandardLabel = False for modelLabelRel in labelsRelationshipSet.fromModelObject(eltDef): modelLabel = modelLabelRel.toModelObject role = modelLabel.role text = modelLabel.text lang = modelLabel.xmlLang if text and lang and val.disclosureSystem.defaultXmlLang and lang.startswith( val.disclosureSystem.defaultXmlLang): if role in (XbrlConst.standardLabel, XbrlConst.genStandardLabel): eltDefHasDefaultLangStandardLabel = True if not eltDefHasDefaultLangStandardLabel: modelXbrl.error( "SBR.NL.3.02.15.01", _("XML nodes that can appear in instances MUST have standard labels in the local language: %(element)s" ), modelObject=eltDef, element=eltDef.qname) val.modelXbrl.profileStat( _("validate{0}").format( modelXbrl.modelManager.disclosureSystem.validationType)) modelXbrl.modelManager.showStatus(_("ready"), 2000)
def view(self): self.blockSelectEvent = 1 self.blockViewModelObject = 0 self.tag_has = defaultdict(list) # temporary until Tk 8.6 # relationship set based on linkrole parameter, to determine applicable linkroles relationshipSet = self.modelXbrl.relationshipSet(self.arcrole, self.linkrole, self.linkqname, self.arcqname) if not relationshipSet: self.modelXbrl.modelManager.addToLog(_("no relationships for {0}").format(self.arcrole)) return False # set up facts self.conceptFacts = defaultdict(list) for fact in self.modelXbrl.facts: self.conceptFacts[fact.qname].append(fact) # sort contexts by period self.periodContexts = defaultdict(set) contextStartDatetimes = {} ignoreDims = self.ignoreDims.get() showDimDefaults = self.showDimDefaults.get() for context in self.modelXbrl.contexts.values(): if ignoreDims: if context.isForeverPeriod: contextkey = datetime.datetime(datetime.MINYEAR,1,1) else: contextkey = context.endDatetime else: if context.isForeverPeriod: contextkey = "forever" else: contextkey = (context.endDatetime - datetime.timedelta(days=1)).strftime("%Y-%m-%d") values = [] dims = context.qnameDims if len(dims) > 0: for dimQname in sorted(dims.keys(), key=lambda d: str(d)): dimvalue = dims[dimQname] if dimvalue.isExplicit: values.append(dimvalue.member.label(self.labelrole,lang=self.lang)) else: values.append(XmlUtil.innerText(dimvalue.typedMember)) nonDimensions = context.nonDimValues("segment") + context.nonDimValues("scenario") if len(nonDimensions) > 0: for element in sorted(nonDimensions, key=lambda e: e.localName): values.append(XmlUtil.innerText(element)) if len(values) > 0: contextkey += " - " + ', '.join(values) objectId = context.objectId() self.periodContexts[contextkey].add(objectId) if context.isStartEndPeriod: contextStartDatetimes[objectId] = context.startDatetime self.periodKeys = list(self.periodContexts.keys()) self.periodKeys.sort() # set up treeView widget and tabbed pane self.treeView.column("#0", width=300, anchor="w") self.treeView.heading("#0", text="Concept") columnIds = [] columnIdHeadings = [] self.contextColId = {} self.startdatetimeColId = {} self.numCols = 1 for periodKey in self.periodKeys: colId = "#{0}".format(self.numCols) columnIds.append(colId) columnIdHeadings.append((colId,periodKey)) for contextId in self.periodContexts[periodKey]: self.contextColId[contextId] = colId if contextId in contextStartDatetimes: self.startdatetimeColId[contextStartDatetimes[contextId]] = colId self.numCols += 1 self.treeView["columns"] = columnIds for colId, colHeading in columnIdHeadings: self.treeView.column(colId, width=60, anchor="w") if ignoreDims: if colHeading.year == datetime.MINYEAR: date = "forever" else: date = (colHeading - datetime.timedelta(days=1)).strftime("%Y-%m-%d") self.treeView.heading(colId, text=date) else: self.treeView.heading(colId, text=colHeading) # fact rendering for previousNode in self.treeView.get_children(""): self.treeView.delete(previousNode) self.rowColFactId = {} # root node for tree view self.id = 1 # sort URIs by definition linkroleUris = [] relationshipSet = self.modelXbrl.relationshipSet(self.arcrole, self.linkrole, self.linkqname, self.arcqname) if self.linkrole: roleType = self.modelXbrl.roleTypes[self.linkrole] linkroleUris.append((self.linkrole, roleType.definition, roleType.objectId(self.id))) self.id += 1 else: for linkroleUri in relationshipSet.linkRoleUris: modelRoleTypes = self.modelXbrl.roleTypes.get(linkroleUri) if modelRoleTypes is not None and len(modelRoleTypes) > 0: roledefinition = modelRoleTypes[0].definition if roledefinition is None or roledefinition == "": roledefinition = linkroleUri roleId = modelRoleTypes[0].objectId(self.id) else: roledefinition = linkroleUri roleId = "node{0}".format(self.id) self.id += 1 linkroleUris.append((roledefinition, linkroleUri, roleId)) linkroleUris.sort() # for each URI in definition order for linkroleUriTuple in linkroleUris: linknode = self.treeView.insert("", "end", linkroleUriTuple[2], text=linkroleUriTuple[0], tags=("ELR",)) linkRelationshipSet = self.modelXbrl.relationshipSet(self.arcrole, linkroleUriTuple[1], self.linkqname, self.arcqname) for rootConcept in linkRelationshipSet.rootConcepts: node = self.viewConcept(rootConcept, rootConcept, "", self.labelrole, linknode, 1, linkRelationshipSet, set()) return True
def description(self): nameElement = XmlUtil.descendant(self, None, ("description", "documentation")) if nameElement is not None: return XmlUtil.innerText(nameElement) return None
def view(self, modelDocument): relationshipSet = self.modelXbrl.relationshipSet(self.arcrole, self.linkrole, self.linkqname, self.arcqname) if relationshipSet: # sort URIs by definition linkroleUris = [] for linkroleUri in relationshipSet.linkRoleUris: modelRoleTypes = self.modelXbrl.roleTypes.get(linkroleUri) if modelRoleTypes: roledefinition = (modelRoleTypes[0].genLabel(lang=self.lang, strip=True) or modelRoleTypes[0].definition or linkroleUri) else: roledefinition = linkroleUri linkroleUris.append((roledefinition, linkroleUri)) linkroleUris.sort() for roledefinition, linkroleUri in linkroleUris: linkRelationshipSet = self.modelXbrl.relationshipSet(self.arcrole, linkroleUri, self.linkqname, self.arcqname) for rootConcept in linkRelationshipSet.rootConcepts: self.treeDepth(rootConcept, rootConcept, 2, self.arcrole, linkRelationshipSet, set()) # allocate facts to table structure for US-GAAP-style filings if not self.modelXbrl.hasTableIndexing: from arelle import TableStructure TableStructure.evaluateTableIndex(self.modelXbrl, lang=self.lang) # set up facts self.conceptFacts = defaultdict(list) for fact in self.modelXbrl.facts: self.conceptFacts[fact.qname].append(fact) # sort contexts by period self.periodContexts = defaultdict(set) contextStartDatetimes = {} for context in self.modelXbrl.contexts.values(): if self.type in (CSV, XLSX, HTML): if self.ignoreDims: if context.isForeverPeriod: contextkey = datetime.datetime(datetime.MINYEAR,1,1) else: contextkey = context.endDatetime else: if context.isForeverPeriod: contextkey = "forever" else: contextkey = (context.endDatetime - datetime.timedelta(days=1)).strftime("%Y-%m-%d") values = [] dims = context.qnameDims if len(dims) > 0: for dimQname in sorted(dims.keys(), key=lambda d: str(d)): dimvalue = dims[dimQname] if dimvalue.isExplicit: values.append(dimvalue.member.label(self.labelrole,lang=self.lang) if dimvalue.member is not None else str(dimvalue.memberQname)) else: values.append(XmlUtil.innerText(dimvalue.typedMember)) nonDimensions = context.nonDimValues("segment") + context.nonDimValues("scenario") if len(nonDimensions) > 0: for element in sorted(nonDimensions, key=lambda e: e.localName): values.append(XmlUtil.innerText(element)) if len(values) > 0: contextkey += " - " + ', '.join(values) else: contextkey = context.id objectId = context.objectId() self.periodContexts[contextkey].add(objectId) if context.isStartEndPeriod: contextStartDatetimes[objectId] = context.startDatetime self.periodKeys = list(self.periodContexts.keys()) self.periodKeys.sort() # set up treeView widget and tabbed pane heading = ["Concept"] columnHeadings = [] self.contextColId = {} self.startdatetimeColId = {} self.numCols = 1 for periodKey in self.periodKeys: columnHeadings.append(periodKey) for contextId in self.periodContexts[periodKey]: self.contextColId[contextId] = self.numCols if contextId in contextStartDatetimes: self.startdatetimeColId[contextStartDatetimes[contextId]] = self.numCols self.numCols += 1 for colHeading in columnHeadings: if self.ignoreDims: if colHeading.year == datetime.MINYEAR: date = "forever" else: date = (colHeading - datetime.timedelta(days=1)).strftime("%Y-%m-%d") heading.append(date) else: heading.append(colHeading) self.setColWidths([(70 if iCol==0 else 24) for iCol, col in enumerate(heading)]) self.setColWrapText([True for col in heading]) self.addRow(heading, asHeader=True) # must do after determining tree depth if relationshipSet: # for each URI in definition order for roledefinition, linkroleUri in linkroleUris: attr = {"role": linkroleUri} self.addRow([roledefinition], treeIndent=0, colSpan=len(heading), xmlRowElementName="linkRole", xmlRowEltAttr=attr, xmlCol0skipElt=True) linkRelationshipSet = self.modelXbrl.relationshipSet(self.arcrole, linkroleUri, self.linkqname, self.arcqname) # set up concepts which apply to linkrole for us-gaap style filings self.conceptFacts.clear() if linkroleUri and self.modelXbrl.roleTypes[linkroleUri] and hasattr(self.modelXbrl.roleTypes[linkroleUri][0], "_tableFacts"): for fact in self.modelXbrl.roleTypes[linkroleUri][0]._tableFacts: self.conceptFacts[fact.qname].append(fact) else: for fact in self.modelXbrl.facts: if linkRelationshipSet.fromModelObject(fact.concept) or linkRelationshipSet.toModelObject(fact.concept): self.conceptFacts[fact.qname].append(fact) # view root and descendant for rootConcept in linkRelationshipSet.rootConcepts: self.viewConcept(rootConcept, rootConcept, "", self.labelrole, 1, linkRelationshipSet, set()) return True
def backgroundFind(self, expr, logViewLines): exprType = self.options["exprType"] inConceptLabel = self.options["conceptLabel"] inConceptName = self.options["conceptName"] inConceptType = self.options["conceptType"] inConceptSubs = self.options["conceptSubs"] inConceptPer = self.options["conceptPer"] inConceptBal = self.options["conceptBal"] inFactLabel = self.options["factLabel"] inFactName = self.options["factName"] inFactValue = self.options["factValue"] inFactCntx = self.options["factCntx"] inFactUnit = self.options["factUnit"] inMessagesLog = self.options["messagesLog"] nextIsDown = self.options["direction"] == "down" objsFound = set() try: if exprType == "text": # escape regex metacharacters pattern = re.compile( ''.join([(('\\' + c) if c in reMetaChars else c) for c in expr]), re.IGNORECASE) isRE = True isXP = False elif exprType == "regex": pattern = re.compile(expr, re.IGNORECASE) isRE = True isXP = False elif exprType == "xpath": isRE = False isXP = True self.resultText.setValue(_("Compiling xpath expression...")) XPathParser.initializeParser(self.modelManager) self.modelManager.showStatus( _("Compiling xpath expression...")) xpProg = XPathParser.parse( self, expr, XPathParser.staticExpressionFunctionContext(), "find expression", Trace.CALL) xpCtx = XPathContext.create(self.modelXbrl, sourceElement=None) else: return # nothing to do if inMessagesLog: for lineNumber, line in enumerate(logViewLines): if pattern.search(line): objsFound.add(lineNumber) elif self.modelXbrl.modelDocument.type == ModelDocument.Type.RSSFEED: for rssItem in self.modelXbrl.modelDocument.items: if any( pattern.search(str(value)) for name, value in rssItem.propertyView): objsFound.add(rssItem) else: # DTS search if inConceptLabel or inConceptName or inConceptType or inConceptSubs or inConceptPer or inConceptBal: self.modelManager.cntlr.uiThreadQueue.put( (self.resultText.setValue, [_("Matching concepts...") ])) self.modelManager.showStatus(_("Matching concepts...")) for conceptName, concepts in self.modelXbrl.nameConcepts.items( ): for concept in concepts: if ((isXP and xpCtx.evaluateBooleanValue( xpProg, contextItem=concept.qname)) or (isRE and (inConceptLabel and pattern.search(concept.label())) or (inConceptName and pattern.search(conceptName)) or (inConceptType and pattern.search(str(concept.typeQname))) or (inConceptSubs and pattern.search( str(concept.substitutionGroupQname))) or (inConceptPer and concept.periodType and pattern.search(concept.periodType)) or (inConceptBal and concept.balance and pattern.search(concept.balance)))): objsFound.add(concept) if inFactLabel or inFactName or inFactValue or inFactCntx or inFactUnit: self.modelManager.cntlr.uiThreadQueue.put( (self.resultText.setValue, [_("Matching facts...")])) self.modelManager.showStatus(_("Matching facts...")) for fact in self.modelXbrl.facts: if ((isXP and xpCtx.evaluateBooleanValue( xpProg, contextItem=fact)) or (isRE and (inFactName and pattern.search(fact.concept.name) or (inFactLabel and pattern.search(fact.concept.label())) or (inFactValue and pattern.search(fact.value)) or (inFactCntx and pattern.search( XmlUtil.innerText(fact.context.element))) or (inFactUnit and pattern.search( XmlUtil.innerText(fact.unit.element)))))): objsFound.add(fact) except XPathContext.XPathException as err: err = _("Find expression error: {0} \n{1}").format( err.message, err.sourceErrorIndication) self.modelManager.addToLog(err) self.modelManager.cntlr.uiThreadQueue.put( (self.resultText.setValue, [err])) self.modelManager.showStatus(_("Completed with errors"), 5000) numConcepts = 0 numFacts = 0 numRssItems = 0 numMessages = 0 self.objsList = [] for obj in objsFound: if inMessagesLog: numMessages += 1 self.objsList.append(('m', "{0:06}".format(obj), obj)) elif isinstance(obj, ModelConcept): numConcepts += 1 self.objsList.append(('c', obj.localName, obj.objectId())) elif isinstance(obj, ModelFact): numFacts += 1 self.objsList.append(('f', obj.__hash__(), obj.objectId())) elif isinstance(obj, ModelRssItem): numRssItems += 1 self.objsList.append(('r', obj.__hash__(), obj.objectId())) self.objsList.sort() self.result = "Found " if numConcepts: self.result += "{0} concepts".format(numConcepts) if numFacts: self.result += ", " if numFacts: self.result += "{0} facts".format(numFacts) if numRssItems: self.result += "{0} RSS items".format(numRssItems) if numMessages: self.result += "{0} Messages".format(numMessages) if numConcepts + numFacts + numRssItems + numMessages == 0: self.result += "no matches" self.foundIndex = -1 self.modelManager.cntlr.uiThreadQueue.put( (self.resultText.setValue, [self.result])) else: self.foundIndex = 0 if nextIsDown else (len(self.objsList) - 1) self.modelManager.cntlr.uiThreadQueue.put((self.next, [])) self.modelManager.showStatus(_("Ready..."), 2000)
def validateFiling(val, modelXbrl): linkroleDefinitionStatementSheet = re.compile(r"[^-]+-\s+Statement\s+-\s+.*", # no restriction to type of statement re.IGNORECASE) if not hasattr(modelXbrl.modelDocument, "xmlDocument"): # not parsed return val._isStandardUri = {} modelXbrl.modelManager.disclosureSystem.loadStandardTaxonomiesDict() # find typedDomainRefs before validateXBRL pass val.typedDomainQnames = set() val.typedDomainElements = set() for modelConcept in modelXbrl.qnameConcepts.values(): if modelConcept.isTypedDimension: typedDomainElement = modelConcept.typedDomainElement if isinstance(typedDomainElement, ModelConcept): val.typedDomainQnames.add(typedDomainElement.qname) val.typedDomainElements.add(typedDomainElement) # note that some XFM tests are done by ValidateXbrl to prevent multiple node walks xbrlInstDoc = modelXbrl.modelDocument.xmlDocument.getroot() disclosureSystem = val.disclosureSystem disclosureSystemVersion = disclosureSystem.version modelXbrl.modelManager.showStatus(_("validating {0}").format(disclosureSystem.name)) val.modelXbrl.profileActivity() conceptsUsed = {} # key=concept object value=True if has presentation label labelsRelationshipSet = modelXbrl.relationshipSet(XbrlConst.conceptLabel) genLabelsRelationshipSet = modelXbrl.relationshipSet(XbrlConst.elementLabel) presentationRelationshipSet = modelXbrl.relationshipSet(XbrlConst.parentChild) referencesRelationshipSetWithProhibits = modelXbrl.relationshipSet(XbrlConst.conceptReference, includeProhibits=True) val.modelXbrl.profileActivity("... cache lbl, pre, ref relationships", minTimeToShow=1.0) val.validateLoggingSemantic = validateLoggingSemantic = ( modelXbrl.isLoggingEffectiveFor(level="WARNING-SEMANTIC") or modelXbrl.isLoggingEffectiveFor(level="ERROR-SEMANTIC")) # instance checks val.fileNameBasePart = None # prevent testing on fileNameParts if not instance or invalid val.fileNameDate = None val.entityRegistrantName = None val.requiredContext = None val.standardNamespaceConflicts = defaultdict(set) val.exhibitType = None # e.g., EX-101, EX-201 # entry point schema checks if modelXbrl.modelDocument.type == ModelDocument.Type.SCHEMA: # entry must have a P-link if not any(hrefElt.localName == "linkbaseRef" and hrefElt.get("{http://www.w3.org/1999/xlink}role") == "http://www.xbrl.org/2003/role/presentationLinkbaseRef" for hrefElt, hrefDoc, hrefId in modelXbrl.modelDocument.hrefObjects): modelXbrl.error("SBR.NL.2.2.10.01", 'Entrypoint schema must have a presentation linkbase', modelObject=modelXbrl.modelDocument) # all-labels and references checks for concept in modelXbrl.qnameConcepts.values(): conceptHasDefaultLangStandardLabel = False for modelLabelRel in labelsRelationshipSet.fromModelObject(concept): modelLabel = modelLabelRel.toModelObject role = modelLabel.role text = modelLabel.text lang = modelLabel.xmlLang if role == XbrlConst.documentationLabel: if concept.modelDocument.targetNamespace in disclosureSystem.standardTaxonomiesDict: modelXbrl.error("SBR.NL.2.1.0.08", _("Concept %(concept)s of a standard taxonomy cannot have a documentation label: %(text)s"), modelObject=modelLabel, concept=concept.qname, text=text) elif text and lang and disclosureSystem.defaultXmlLang and lang.startswith(disclosureSystem.defaultXmlLang): if role == XbrlConst.standardLabel: # merge of pre-plugin code per LOGIUS conceptHasDefaultLangStandardLabel = True match = modelXbrl.modelManager.disclosureSystem.labelCheckPattern.search(text) if match: modelXbrl.error("SBR.NL.2.3.8.07", 'Label for concept %(concept)s role %(role)s has disallowed characters: "%(text)s"', modelObject=modelLabel, concept=concept.qname, role=role, text=match.group()) for modelRefRel in referencesRelationshipSetWithProhibits.fromModelObject(concept): modelReference = modelRefRel.toModelObject text = XmlUtil.innerText(modelReference) #6.18.1 no reference to company extension concepts if (concept.modelDocument.targetNamespace in disclosureSystem.standardTaxonomiesDict and not isStandardUri(val, modelRefRel.modelDocument.uri)): # merge of pre-plugin code per LOGIUS #6.18.2 no extension to add or remove references to standard concepts modelXbrl.error("SBR.NL.2.1.0.08", _("References for standard taxonomy concept %(concept)s are not allowed in an extension linkbase: %(text)s"), modelObject=modelReference, concept=concept.qname, text=text, xml=XmlUtil.xmlstring(modelReference, stripXmlns=True, contentsOnly=True)) if concept.isItem or concept.isTuple: if concept.modelDocument.targetNamespace not in disclosureSystem.standardTaxonomiesDict: if not conceptHasDefaultLangStandardLabel: modelXbrl.error("SBR.NL.2.2.2.26", _("Concept %(concept)s missing standard label in local language."), modelObject=concept, concept=concept.qname) subsGroup = concept.get("substitutionGroup") if ((not concept.isAbstract or subsGroup == "sbr:presentationItem") and not (presentationRelationshipSet.toModelObject(concept) or presentationRelationshipSet.fromModelObject(concept))): modelXbrl.error("SBR.NL.2.2.2.04", _("Concept %(concept)s not referred to by presentation relationship."), modelObject=concept, concept=concept.qname) elif ((concept.isDimensionItem or (subsGroup and (subsGroup.endswith(":domainItem") or subsGroup.endswith(":domainMemberItem")))) and not (presentationRelationshipSet.toModelObject(concept) or presentationRelationshipSet.fromModelObject(concept))): modelXbrl.error("SBR.NL.2.2.10.03", _("DTS concept %(concept)s not referred to by presentation relationship."), modelObject=concept, concept=concept.qname) if (concept.substitutionGroupQname and concept.substitutionGroupQname.namespaceURI not in disclosureSystem.baseTaxonomyNamespaces): modelXbrl.error("SBR.NL.2.2.2.05", _("Concept %(concept)s has a substitutionGroup of a non-standard concept."), modelObject=concept, concept=concept.qname) if concept.isTuple: # verify same presentation linkbase nesting for missingQname in set(concept.type.elements) ^ pLinkedNonAbstractDescendantQnames(modelXbrl, concept): modelXbrl.error("SBR.NL.2.3.4.01", _("Tuple %(concept)s has mismatch between content and presentation children: %(missingQname)s."), modelObject=concept, concept=concept.qname, missingQname=missingQname) checkConceptLabels(val, modelXbrl, labelsRelationshipSet, disclosureSystem, concept) checkConceptLabels(val, modelXbrl, genLabelsRelationshipSet, disclosureSystem, concept) val.modelXbrl.profileActivity("... filer concepts checks", minTimeToShow=1.0) # checks on all documents: instance, schema, instance checkFilingDTS(val, modelXbrl.modelDocument, []) ''' removed RH 2011-12-23, corresponding use of nameWordsTable in ValidateFilingDTS if val.validateSBRNL: del val.nameWordsTable ''' val.modelXbrl.profileActivity("... filer DTS checks", minTimeToShow=1.0) conceptRelsUsedWithPreferredLabels = defaultdict(list) usedCalcsPresented = defaultdict(set) # pairs of concepts objectIds used in calc usedCalcFromTosELR = {} localPreferredLabels = defaultdict(set) drsELRs = set() # do calculation, then presentation, then other arcroles val.summationItemRelsSetAllELRs = modelXbrl.relationshipSet(XbrlConst.summationItem) for arcroleFilter in (XbrlConst.summationItem, XbrlConst.parentChild, "*"): for baseSetKey, baseSetModelLinks in modelXbrl.baseSets.items(): arcrole, ELR, linkqname, arcqname = baseSetKey if ELR and linkqname and arcqname and not arcrole.startswith("XBRL-"): # assure summationItem, then parentChild, then others if not (arcroleFilter == arcrole or arcroleFilter == "*" and arcrole not in (XbrlConst.summationItem, XbrlConst.parentChild)): continue if arcrole == XbrlConst.parentChild: ineffectiveArcs = ModelRelationshipSet.ineffectiveArcs(baseSetModelLinks, arcrole) #validate ineffective arcs for modelRel in ineffectiveArcs: if modelRel.fromModelObject is not None and modelRel.toModelObject is not None: modelXbrl.error("SBR.NL.2.3.4.06", _("Ineffective arc %(arc)s in \nlink role %(linkrole)s \narcrole %(arcrole)s \nfrom %(conceptFrom)s \nto %(conceptTo)s \n%(ineffectivity)s"), modelObject=modelRel, arc=modelRel.qname, arcrole=modelRel.arcrole, linkrole=modelRel.linkrole, linkroleDefinition=modelXbrl.roleTypeDefinition(modelRel.linkrole), conceptFrom=modelRel.fromModelObject.qname, conceptTo=modelRel.toModelObject.qname, ineffectivity=modelRel.ineffectivity) if arcrole == XbrlConst.parentChild: isStatementSheet = any(linkroleDefinitionStatementSheet.match(roleType.definition or '') for roleType in val.modelXbrl.roleTypes.get(ELR,())) conceptsPresented = set() # 6.12.2 check for distinct order attributes parentChildRels = modelXbrl.relationshipSet(arcrole, ELR) for relFrom, siblingRels in parentChildRels.fromModelObjects().items(): targetConceptPreferredLabels = defaultdict(dict) orderRels = {} firstRel = True relFromUsed = True for rel in siblingRels: if firstRel: firstRel = False if relFrom in conceptsUsed: conceptsUsed[relFrom] = True # 6.12.3, has a pres relationship relFromUsed = True relTo = rel.toModelObject preferredLabel = rel.preferredLabel if relTo in conceptsUsed: conceptsUsed[relTo] = True # 6.12.3, has a pres relationship if preferredLabel and preferredLabel != "": conceptRelsUsedWithPreferredLabels[relTo].append(rel) if preferredLabel in ("periodStart","periodEnd"): modelXbrl.error("SBR.NL.2.3.4.03", _("Preferred label on presentation relationships not allowed"), modelObject=modelRel) # 6.12.5 distinct preferred labels in base set preferredLabels = targetConceptPreferredLabels[relTo] if (preferredLabel in preferredLabels or (not relFrom.isTuple and (not preferredLabel or None in preferredLabels))): if preferredLabel in preferredLabels: rel2, relTo2 = preferredLabels[preferredLabel] else: rel2 = relTo2 = None modelXbrl.error("SBR.NL.2.3.4.06", _("Concept %(concept)s has duplicate preferred label %(preferredLabel)s in link role %(linkrole)s"), modelObject=(rel, relTo, rel2, relTo2), concept=relTo.qname, fromConcept=rel.fromModelObject.qname, preferredLabel=preferredLabel, linkrole=rel.linkrole, linkroleDefinition=modelXbrl.roleTypeDefinition(rel.linkrole)) else: preferredLabels[preferredLabel] = (rel, relTo) if relFromUsed: # 6.14.5 conceptsPresented.add(relFrom.objectIndex) conceptsPresented.add(relTo.objectIndex) order = rel.order if order in orderRels: modelXbrl.error("SBR.NL.2.3.4.05", _("Duplicate presentation relations from concept %(conceptFrom)s for order %(order)s in base set role %(linkrole)s to concept %(conceptTo)s and to concept %(conceptTo2)s"), modelObject=(rel, orderRels[order]), conceptFrom=relFrom.qname, order=rel.arcElement.get("order"), linkrole=rel.linkrole, linkroleDefinition=modelXbrl.roleTypeDefinition(rel.linkrole), conceptTo=rel.toModelObject.qname, conceptTo2=orderRels[order].toModelObject.qname) else: orderRels[order] = rel if not relFrom.isTuple: if relTo in localPreferredLabels: if {None, preferredLabel} & localPreferredLabels[relTo]: val.modelXbrl.error("SBR.NL.2.3.4.06", _("Non-distinguished preferredLabel presentation relations from concept %(conceptFrom)s in base set role %(linkrole)s"), modelObject=rel, conceptFrom=relFrom.qname, linkrole=rel.linkrole, conceptTo=relTo.qname) localPreferredLabels[relTo].add(preferredLabel) targetConceptPreferredLabels.clear() orderRels.clear() localPreferredLabels.clear() # clear for next relationship for conceptPresented in conceptsPresented: if conceptPresented in usedCalcsPresented: usedCalcPairingsOfConcept = usedCalcsPresented[conceptPresented] if len(usedCalcPairingsOfConcept & conceptsPresented) > 0: usedCalcPairingsOfConcept -= conceptsPresented elif arcrole == XbrlConst.summationItem: # find a calc relationship to get the containing document name for modelRel in val.modelXbrl.relationshipSet(arcrole, ELR).modelRelationships: val.modelXbrl.error("SBR.NL.2.3.9.01", _("Calculation linkbase linkrole %(linkrole)s"), modelObject=modelRel, linkrole=ELR) break elif arcrole == XbrlConst.all or arcrole == XbrlConst.notAll: drsELRs.add(ELR) else: if arcrole == XbrlConst.dimensionDefault: for modelRel in val.modelXbrl.relationshipSet(arcrole).modelRelationships: val.modelXbrl.error("SBR.NL.2.3.6.05", _("Dimension-default in from %(conceptFrom)s to %(conceptTo)s in role %(linkrole)s is not allowed"), modelObject=modelRel, conceptFrom=modelRel.fromModelObject.qname, conceptTo=modelRel.toModelObject.qname, linkrole=modelRel.linkrole) ''' removed per RH 2013-01-11 if not (XbrlConst.isStandardArcrole(arcrole) or XbrlConst.isDefinitionOrXdtArcrole(arcrole)): for modelRel in val.modelXbrl.relationshipSet(arcrole).modelRelationships: relTo = modelRel.toModelObject relFrom = modelRel.fromModelObject if not ((isinstance(relFrom,ModelConcept) and isinstance(relTo,ModelConcept)) or (relFrom.modelDocument.inDTS and (relTo.qname == XbrlConst.qnGenLabel and modelRel.arcrole == XbrlConst.elementLabel) or (relTo.qname == XbrlConst.qnGenReference and modelRel.arcrole == XbrlConst.elementReference) or (relTo.qname == val.qnSbrLinkroleorder))): val.modelXbrl.error("SBR.NL.2.3.2.07", _("The source and target of an arc must be in the DTS from %(elementFrom)s to %(elementTo)s, in linkrole %(linkrole)s, arcrole %(arcrole)s"), modelObject=modelRel, elementFrom=relFrom.qname, elementTo=relTo.qname, linkrole=modelRel.linkrole, arcrole=arcrole) ''' del localPreferredLabels # dereference del usedCalcFromTosELR del val.summationItemRelsSetAllELRs val.modelXbrl.profileActivity("... filer relationships checks", minTimeToShow=1.0) # checks on dimensions checkFilingDimensions(val, drsELRs) val.modelXbrl.profileActivity("... filer dimensions checks", minTimeToShow=1.0) del conceptRelsUsedWithPreferredLabels # 6 16 4, 1.16.5 Base sets of Domain Relationship Sets testing val.modelXbrl.profileActivity("... filer preferred label checks", minTimeToShow=1.0) # moved from original validateSBRnl finally for qname, modelType in modelXbrl.qnameTypes.items(): if qname.namespaceURI not in val.disclosureSystem.baseTaxonomyNamespaces: facets = modelType.facets if facets: lengthFacets = _DICT_SET(facets.keys()) & {"minLength", "maxLength", "length"} if lengthFacets: modelXbrl.error("SBR.NL.2.2.7.02", _("Type %(typename)s has length restriction facets %(facets)s"), modelObject=modelType, typename=modelType.qname, facets=", ".join(lengthFacets)) if "enumeration" in facets and not modelType.isDerivedFrom(XbrlConst.qnXbrliStringItemType): modelXbrl.error("SBR.NL.2.2.7.04", _("Concept %(concept)s has enumeration and is not based on stringItemType"), modelObject=modelType, concept=modelType.qname) ''' removed RH 2011-12-23, corresponding use of nameWordsTable in ValidateFilingDTS # build camelCasedNamesTable self.nameWordsTable = {} for name in modelXbrl.nameConcepts.keys(): words = [] wordChars = [] lastchar = "" for c in name: if c.isupper() and lastchar.islower(): # it's another word partialName = ''.join(wordChars) if partialName in modelXbrl.nameConcepts: words.append(partialName) wordChars.append(c) lastchar = c if words: self.nameWordsTable[name] = words self.modelXbrl.profileActivity("... build name words table", minTimeToShow=1.0) ''' # check presentation link roles for generic linkbase order number ordersRelationshipSet = modelXbrl.relationshipSet("http://www.nltaxonomie.nl/2011/arcrole/linkrole-order") presLinkroleNumberURI = {} presLinkrolesCount = 0 for countLinkroles in (True, False): for _roleURI, modelRoleTypes in modelXbrl.roleTypes.items(): for modelRoleType in modelRoleTypes: if XbrlConst.qnLinkPresentationLink in modelRoleType.usedOns: if countLinkroles: presLinkrolesCount += 1 else: if not ordersRelationshipSet: modelXbrl.error("SBR.NL.2.2.3.06", _("Presentation linkrole %(linkrole)s missing order number relationship set"), modelObject=modelRoleType, linkrole=modelRoleType.roleURI) else: order = None for orderNumRel in ordersRelationshipSet.fromModelObject(modelRoleType): order = getattr(orderNumRel.toModelObject, "xValue", "(noPSVIvalue)") if order in presLinkroleNumberURI: modelXbrl.error("SBR.NL.2.2.3.06", _("Presentation linkrole order number %(order)s of %(linkrole)s also used in %(otherLinkrole)s"), modelObject=modelRoleType, order=order, linkrole=modelRoleType.roleURI, otherLinkrole=presLinkroleNumberURI[order]) else: presLinkroleNumberURI[order] = modelRoleType.roleURI if not order: modelXbrl.error("SBR.NL.2.2.3.06", _("Presentation linkrole %(linkrole)s missing order number"), modelObject=modelRoleType, linkrole=modelRoleType.roleURI) if countLinkroles and presLinkrolesCount < 2: break # don't check order numbers if only one presentation linkrole # check arc role definitions for labels for arcroleURI, modelRoleTypes in modelXbrl.arcroleTypes.items(): for modelRoleType in modelRoleTypes: if (not arcroleURI.startswith("http://xbrl.org/") and modelRoleType.modelDocument.targetNamespace not in val.disclosureSystem.baseTaxonomyNamespaces and (not modelRoleType.genLabel(lang="nl") or not modelRoleType.genLabel(lang="en"))): modelXbrl.error("SBR.NL.2.2.4.02", _("ArcroleType missing nl or en generic label: %(arcrole)s"), modelObject=modelRoleType, arcrole=arcroleURI) for domainElt in val.typedDomainElements: if domainElt.modelDocument.targetNamespace not in val.disclosureSystem.baseTaxonomyNamespaces: if not domainElt.genLabel(fallbackToQname=False, lang="nl"): modelXbrl.error("SBR.NL.2.2.8.01", _("Typed dimension domain element %(concept)s must have a generic label"), modelObject=domainElt, concept=domainElt.qname) if domainElt.type is not None and domainElt.type.localName == "complexType": modelXbrl.error("SBR.NL.2.2.8.02", _("Typed dimension domain element %(concept)s has disallowed complex content"), modelObject=domainElt, concept=domainElt.qname) modelXbrl.profileActivity("... SBR role types and type facits checks", minTimeToShow=1.0) # end moved from ValidateFiling # 3.2.4.4 check each using prefix against taxonomy declaring the prefix for docs in modelXbrl.namespaceDocs.values(): for doc in docs: for prefix, NS in doc.xmlRootElement.nsmap.items(): if NS in val.namespacePrefix and prefix != val.namespacePrefix[NS]: modelXbrl.error("SBR.NL.3.2.4.04", _("The assigned namespace prefix %(assignedPrefix)s for the schema that declares the targetnamespace %(namespace)s, MUST be adhired by all other NT schemas, referencedPrefix: %(referencedPrefix)s"), modelObject=doc.xmlRootElement, namespace=NS, assignedPrefix=val.namespacePrefix.get(NS, ''), referencedPrefix=prefix) # check non-concept elements that can appear in elements for labels (concepts checked by labelsRelationshipSet = modelXbrl.relationshipSet((XbrlConst.conceptLabel, XbrlConst.elementLabel)) baseTaxonomyNamespaces = val.disclosureSystem.baseTaxonomyNamespaces for eltDef in modelXbrl.qnameConcepts.values(): if (not (eltDef.isItem or eltDef.isTuple or eltDef.isLinkPart) and eltDef.qname.namespaceURI not in baseTaxonomyNamespaces): eltDefHasDefaultLangStandardLabel = False for modelLabelRel in labelsRelationshipSet.fromModelObject(eltDef): modelLabel = modelLabelRel.toModelObject role = modelLabel.role text = modelLabel.text lang = modelLabel.xmlLang if text and lang and val.disclosureSystem.defaultXmlLang and lang.startswith(val.disclosureSystem.defaultXmlLang): if role in (XbrlConst.standardLabel, XbrlConst.genStandardLabel): eltDefHasDefaultLangStandardLabel = True if not eltDefHasDefaultLangStandardLabel: modelXbrl.error("SBR.NL.3.2.15.01", _("XML nodes that can appear in instances MUST have standard labels in the local language: %(element)s"), modelObject=eltDef, element=eltDef.qname) val.modelXbrl.profileStat(_("validate{0}").format(modelXbrl.modelManager.disclosureSystem.validationType)) modelXbrl.modelManager.showStatus(_("ready"), 2000)
def backgroundFind(self, expr, logViewLines): exprType = self.options["exprType"] inConceptLabel = self.options["conceptLabel"] inConceptName = self.options["conceptName"] inConceptType = self.options["conceptType"] inConceptSubs = self.options["conceptSubs"] inConceptPer = self.options["conceptPer"] inConceptBal = self.options["conceptBal"] inFactLabel = self.options["factLabel"] inFactName = self.options["factName"] inFactValue = self.options["factValue"] inFactCntx = self.options["factCntx"] inFactUnit = self.options["factUnit"] inMessagesLog = self.options["messagesLog"] nextIsDown = self.options["direction"] == "down" objsFound = set() try: if exprType == "text": # escape regex metacharacters pattern = re.compile(''.join( [(('\\' + c) if c in reMetaChars else c) for c in expr]), re.IGNORECASE) isRE = True isXP = False elif exprType == "regex": pattern = re.compile(expr, re.IGNORECASE) isRE = True isXP = False elif exprType == "xpath": isRE = False isXP = True self.resultText.setValue(_("Compiling xpath expression...")) XPathParser.initializeParser(self) self.modelManager.showStatus(_("Compiling xpath expression...")) xpProg= XPathParser.parse(self, expr, XPathParser.staticExpressionFunctionContext(), "find expression", Trace.CALL) xpCtx = XPathContext.create(self.modelXbrl, sourceElement=None) else: return # nothing to do if inMessagesLog: for lineNumber, line in enumerate(logViewLines): if pattern.search(line): objsFound.add(lineNumber) elif self.modelXbrl.modelDocument.type == ModelDocument.Type.RSSFEED: for rssItem in self.modelXbrl.modelDocument.items: if any(pattern.search(str(value)) for name, value in rssItem.propertyView): objsFound.add(rssItem) else: # DTS search if inConceptLabel or inConceptName or inConceptType or inConceptSubs or inConceptPer or inConceptBal: self.modelManager.cntlr.uiThreadQueue.put((self.resultText.setValue, [_("Matching concepts...")])) self.modelManager.showStatus(_("Matching concepts...")) for conceptName, concepts in self.modelXbrl.nameConcepts.items(): for concept in concepts: if ((isXP and xpCtx.evaluateBooleanValue(xpProg, contextItem=concept.qname)) or (isRE and (inConceptLabel and pattern.search(concept.label())) or (inConceptName and pattern.search(conceptName)) or (inConceptType and pattern.search(str(concept.typeQname))) or (inConceptSubs and pattern.search(str(concept.substitutionGroupQname))) or (inConceptPer and concept.periodType and pattern.search(concept.periodType)) or (inConceptBal and concept.balance and pattern.search(concept.balance)) ) ): objsFound.add(concept) if inFactLabel or inFactName or inFactValue or inFactCntx or inFactUnit: self.modelManager.cntlr.uiThreadQueue.put((self.resultText.setValue, [_("Matching facts...")])) self.modelManager.showStatus(_("Matching facts...")) for fact in self.modelXbrl.facts: if ((isXP and xpCtx.evaluateBooleanValue(xpProg, contextItem=fact)) or (isRE and (inFactName and pattern.search(fact.concept.name) or (inFactLabel and pattern.search(fact.concept.label())) or (inFactValue and pattern.search(fact.value)) or (inFactCntx and pattern.search(XmlUtil.innerText(fact.context.element))) or (inFactUnit and pattern.search(XmlUtil.innerText(fact.unit.element)))) ) ): objsFound.add(fact) except XPathContext.XPathException as err: err = _("Find expression error: {0} \n{1}").format(err.message, err.sourceErrorIndication) self.modelManager.addToLog(err) self.modelManager.cntlr.uiThreadQueue.put((self.resultText.setValue, [err])) self.modelManager.showStatus(_("Completed with errors"), 5000) numConcepts = 0 numFacts = 0 numRssItems = 0 numMessages = 0 self.objsList = [] for obj in objsFound: if inMessagesLog: numMessages += 1 self.objsList.append( ('m', "{0:06}".format(obj), obj) ) elif isinstance(obj,ModelConcept): numConcepts += 1 self.objsList.append( ('c', obj.localName, obj.objectId()) ) elif isinstance(obj,ModelFact): numFacts += 1 self.objsList.append( ('f', obj.__hash__(), obj.objectId()) ) elif isinstance(obj,ModelRssItem): numRssItems += 1 self.objsList.append( ('r', obj.__hash__(), obj.objectId()) ) self.objsList.sort() self.result = "Found " if numConcepts: self.result += "{0} concepts".format(numConcepts) if numFacts: self.result += ", " if numFacts: self.result += "{0} facts".format(numFacts) if numRssItems: self.result += "{0} RSS items".format(numRssItems) if numMessages: self.result += "{0} Messages".format(numMessages) if numConcepts + numFacts + numRssItems + numMessages == 0: self.result += "no matches" self.foundIndex = -1 self.modelManager.cntlr.uiThreadQueue.put((self.resultText.setValue, [self.result])) else: self.foundIndex = 0 if nextIsDown else (len(self.objsList) - 1) self.modelManager.cntlr.uiThreadQueue.put((self.next, [])) self.modelManager.showStatus(_("Ready..."), 2000)
def validate(val, modelXbrl, infosetModelXbrl): infoset = infosetModelXbrl.modelDocument if infoset.type == Type.INSTANCE: # compare facts (assumed out of order) infosetFacts = defaultdict(list) for fact in infosetModelXbrl.facts: infosetFacts[fact.qname].append(fact) if len(modelXbrl.factsInInstance) != len( infosetModelXbrl.factsInInstance): modelXbrl.error( "arelle:infosetTest", _("Fact counts mismatch, testcase instance %(foundFactCount)s, infoset instance %(expectedFactCount)s" ), modelObject=(modelXbrl.modelDocument, infosetModelXbrl.modelDocument), foundFactCount=len(modelXbrl.factsInInstance), expectedFactCount=len(infosetModelXbrl.factsInInstance)) else: for i, instFact in enumerate(modelXbrl.facts): infosetFact = None for fact in infosetFacts[instFact.qname]: if fact.isTuple and fact.isDuplicateOf(instFact, deemP0Equal=True): infosetFact = fact break elif fact.isItem and fact.isVEqualTo(instFact, deemP0Equal=True): infosetFact = fact break if infosetFact is None: # takes precision/decimals into account if fact is not None: fact.isVEqualTo(instFact, deemP0Equal=True) modelXbrl.error( "arelle:infosetTest", _("Fact %(factNumber)s mismatch %(concept)s"), modelObject=instFact, factNumber=(i + 1), concept=instFact.qname) else: ptvPeriodType = infosetFact.get( "{http://www.xbrl.org/2003/ptv}periodType") ptvBalance = infosetFact.get( "{http://www.xbrl.org/2003/ptv}balance") ptvDecimals = infosetFact.get( "{http://www.xbrl.org/2003/ptv}decimals") ptvPrecision = infosetFact.get( "{http://www.xbrl.org/2003/ptv}precision") if ptvPeriodType and ptvPeriodType != instFact.concept.periodType: modelXbrl.error( "arelle:infosetTest", _("Fact %(factNumber)s periodType mismatch %(concept)s expected %(expectedPeriodType)s found %(foundPeriodType)s" ), modelObject=(instFact, infosetFact), factNumber=(i + 1), concept=instFact.qname, expectedPeriodType=ptvPeriodType, foundPeriodType=instFact.concept.periodType) if ptvBalance and ptvBalance != instFact.concept.balance: modelXbrl.error( "arelle:infosetTest", _("Fact %(factNumber)s balance mismatch %(concept)s expected %(expectedBalance)s found %(foundBalance)s" ), modelObject=(instFact, infosetFact), factNumber=(i + 1), concept=instFact.qname, expectedBalance=ptvBalance, foundBalance=instFact.concept.balance) if ptvDecimals and ptvDecimals != str( inferredDecimals(fact)): modelXbrl.error( "arelle:infosetTest", _("Fact %(factNumber)s inferred decimals mismatch %(concept)s expected %(expectedDecimals)s found %(inferredDecimals)s" ), modelObject=(instFact, infosetFact), factNumber=(i + 1), concept=instFact.qname, expectedDecimals=ptvDecimals, inferredDecimals=str(inferredDecimals(fact))) if ptvPrecision and ptvPrecision != str( inferredPrecision(fact)): modelXbrl.error( "arelle:infosetTest", _("Fact %(factNumber)s inferred precision mismatch %(concept)s expected %(expectedPrecision)s found %(inferredPrecision)s" ), modelObject=(instFact, infosetFact), factNumber=(i + 1), concept=instFact.qname, expectedPrecisions=ptvPrecision, inferredPrecision=str(inferredPrecision(fact))) elif infoset.type == Type.ARCSINFOSET: # compare arcs for arcElt in XmlUtil.children(infoset.xmlRootElement, "http://www.xbrl.org/2003/ptv", "arc"): linkType = arcElt.get("linkType") arcRole = arcElt.get("arcRole") extRole = arcElt.get("extRole") fromObj = resolvePath(modelXbrl, arcElt.get("fromPath")) if fromObj is None: modelXbrl.error("arelle:infosetTest", _("Arc fromPath not found: %(fromPath)s"), modelObject=arcElt, fromPath=arcElt.get("fromPath")) continue if linkType in ("label", "reference"): labelLang = arcElt.get("labelLang") resRole = arcElt.get("resRole") if linkType == "label": expectedLabel = XmlUtil.text(arcElt) foundLabel = fromObj.label(preferredLabel=resRole, fallbackToQname=False, lang=None, strip=True, linkrole=extRole) if foundLabel != expectedLabel: modelXbrl.error( "arelle:infosetTest", _("Label expected='%(expectedLabel)s', found='%(foundLabel)s'" ), modelObject=arcElt, expectedLabel=expectedLabel, foundLabel=foundLabel) continue elif linkType == "reference": expectedRef = XmlUtil.innerText(arcElt) referenceFound = False for refrel in modelXbrl.relationshipSet( XbrlConst.conceptReference, extRole).fromModelObject(fromObj): ref = refrel.toModelObject if resRole == ref.role: foundRef = XmlUtil.innerText(ref) if foundRef != expectedRef: modelXbrl.error( "arelle:infosetTest", _("Reference inner text expected='%(expectedRef)s, found='%(foundRef)s'" ), modelObject=arcElt, expectedRef=expectedRef, foundRef=foundRef) referenceFound = True break if referenceFound: continue modelXbrl.error( "arelle:infosetTest", _("%(linkType)s not found containing '%(text)s' linkRole %(linkRole)s" ), modelObject=arcElt, linkType=linkType.title(), text=XmlUtil.innerText(arcElt), linkRole=extRole) else: toObj = resolvePath(modelXbrl, arcElt.get("toPath")) if toObj is None: modelXbrl.error("arelle:infosetTest", _("Arc toPath not found: %(toPath)s"), modelObject=arcElt, toPath=arcElt.get("toPath")) continue weight = arcElt.get("weight") if weight is not None: weight = float(weight) order = arcElt.get("order") if order is not None: order = float(order) preferredLabel = arcElt.get("preferredLabel") found = False for rel in modelXbrl.relationshipSet( arcRole, extRole).fromModelObject(fromObj): if (rel.toModelObject == toObj and (weight is None or rel.weight == weight) and (order is None or rel.order == order)): found = True if not found: modelXbrl.error( "arelle:infosetTest", _("Arc not found: from %(fromPath)s, to %(toPath)s, role %(arcRole)s, linkRole $(extRole)s" ), modelObject=arcElt, fromPath=arcElt.get("fromPath"), toPath=arcElt.get("toPath"), arcRole=arcRole, linkRole=extRole) continue # validate dimensions of each fact factElts = XmlUtil.children(modelXbrl.modelDocument.xmlRootElement, None, "*") for itemElt in XmlUtil.children(infoset.xmlRootElement, None, "item"): try: qnElt = XmlUtil.child(itemElt, None, "qnElement") factQname = qname(qnElt, XmlUtil.text(qnElt)) sPointer = int(XmlUtil.child(itemElt, None, "sPointer").text) factElt = factElts[sPointer - 1] # 1-based xpath indexing if factElt.qname != factQname: modelXbrl.error( "arelle:infosetTest", _("Fact %(sPointer)s mismatch Qname, expected %(qnElt)s, observed %(factQname)s" ), modelObject=itemElt, sPointer=sPointer, qnElt=factQname, factQname=factElt.qname) elif not factElt.isItem or factElt.context is None: modelXbrl.error( "arelle:infosetTest", _("Fact %(sPointer)s has no context: %(qnElt)s"), modelObject=(itemElt, factElt), sPointer=sPointer, qnElt=factQname) else: context = factElt.context memberElts = XmlUtil.children(itemElt, None, "member") numNonDefaults = 0 for memberElt in memberElts: dimElt = XmlUtil.child(memberElt, None, "qnDimension") qnDim = qname(dimElt, XmlUtil.text(dimElt)) isDefault = XmlUtil.text( XmlUtil.child(memberElt, None, "bDefaulted")) == "true" if not isDefault: numNonDefaults += 1 if not ( (qnDim in context.qnameDims and not isDefault) or (qnDim in factElt.modelXbrl.qnameDimensionDefaults and isDefault)): modelXbrl.error( "arelle:infosetTest", _("Fact %(sPointer)s (qnElt)s dimension mismatch %(qnDim)s" ), modelObject=(itemElt, factElt, context), sPointer=sPointer, qnElt=factQname, qnDim=qnDim) if numNonDefaults != len(context.qnameDims): modelXbrl.error( "arelle:infosetTest", _("Fact %(sPointer)s (qnElt)s dimensions count mismatch" ), modelObject=(itemElt, factElt, context), sPointer=sPointer, qnElt=factQname) except (IndexError, ValueError, AttributeError) as err: modelXbrl.error( "arelle:infosetTest", _("Invalid entity fact dimensions infoset sPointer: %(test)s, error details: %(error)s" ), modelObject=itemElt, test=XmlUtil.innerTextList(itemElt), error=str(err))
def view(self, modelDocument): relationshipSet = self.modelXbrl.relationshipSet(self.arcrole, self.linkrole, self.linkqname, self.arcqname) if relationshipSet: # sort URIs by definition linkroleUris = [] for linkroleUri in relationshipSet.linkRoleUris: modelRoleTypes = self.modelXbrl.roleTypes.get(linkroleUri) if modelRoleTypes: roledefinition = (modelRoleTypes[0].genLabel(lang=self.lang, strip=True) or modelRoleTypes[0].definition or linkroleUri) else: roledefinition = linkroleUri linkroleUris.append((roledefinition, linkroleUri)) linkroleUris.sort() for roledefinition, linkroleUri in linkroleUris: linkRelationshipSet = self.modelXbrl.relationshipSet(self.arcrole, linkroleUri, self.linkqname, self.arcqname) for rootConcept in linkRelationshipSet.rootConcepts: self.treeDepth(rootConcept, rootConcept, 2, self.arcrole, linkRelationshipSet, set()) # set up facts self.conceptFacts = defaultdict(list) for fact in self.modelXbrl.facts: self.conceptFacts[fact.qname].append(fact) # sort contexts by period self.periodContexts = defaultdict(set) contextStartDatetimes = {} for context in self.modelXbrl.contexts.values(): if self.type in (CSV, HTML): if self.ignoreDims: if context.isForeverPeriod: contextkey = datetime.datetime(datetime.MINYEAR,1,1) else: contextkey = context.endDatetime else: if context.isForeverPeriod: contextkey = u"forever" else: contextkey = (context.endDatetime - datetime.timedelta(days=1)).strftime(u"%Y-%m-%d") values = [] dims = context.qnameDims if len(dims) > 0: for dimQname in sorted(dims.keys(), key=lambda d: unicode(d)): dimvalue = dims[dimQname] if dimvalue.isExplicit: values.append(dimvalue.member.label(self.labelrole,lang=self.lang) if dimvalue.member is not None else unicode(dimvalue.memberQname)) else: values.append(XmlUtil.innerText(dimvalue.typedMember)) nonDimensions = context.nonDimValues(u"segment") + context.nonDimValues(u"scenario") if len(nonDimensions) > 0: for element in sorted(nonDimensions, key=lambda e: e.localName): values.append(XmlUtil.innerText(element)) if len(values) > 0: contextkey += u" - " + u', '.join(values) else: contextkey = context.id objectId = context.objectId() self.periodContexts[contextkey].add(objectId) if context.isStartEndPeriod: contextStartDatetimes[objectId] = context.startDatetime self.periodKeys = list(self.periodContexts.keys()) self.periodKeys.sort() # set up treeView widget and tabbed pane heading = [u"Concept"] columnHeadings = [] self.contextColId = {} self.startdatetimeColId = {} self.numCols = 1 for periodKey in self.periodKeys: columnHeadings.append(periodKey) for contextId in self.periodContexts[periodKey]: self.contextColId[contextId] = self.numCols if contextId in contextStartDatetimes: self.startdatetimeColId[contextStartDatetimes[contextId]] = self.numCols self.numCols += 1 for colHeading in columnHeadings: if self.ignoreDims: if colHeading.year == datetime.MINYEAR: date = u"forever" else: date = (colHeading - datetime.timedelta(days=1)).strftime(u"%Y-%m-%d") heading.append(date) else: heading.append(colHeading) self.addRow(heading, asHeader=True) # must do after determining tree depth if relationshipSet: # for each URI in definition order for roledefinition, linkroleUri in linkroleUris: attr = {u"role": linkroleUri} self.addRow([roledefinition], treeIndent=0, colSpan=len(heading), xmlRowElementName=u"linkRole", xmlRowEltAttr=attr, xmlCol0skipElt=True) linkRelationshipSet = self.modelXbrl.relationshipSet(self.arcrole, linkroleUri, self.linkqname, self.arcqname) for rootConcept in linkRelationshipSet.rootConcepts: self.viewConcept(rootConcept, rootConcept, u"", self.labelrole, 1, linkRelationshipSet, set()) return True
def validate(val, modelXbrl, infosetModelXbrl): infoset = infosetModelXbrl.modelDocument if infoset.type == Type.INSTANCE: # compare facts (assumed out of order) infosetFacts = defaultdict(list) for fact in infosetModelXbrl.facts: infosetFacts[fact.qname].append(fact) if len(modelXbrl.factsInInstance) != len(infosetModelXbrl.factsInInstance): modelXbrl.error("arelle:infosetTest", _("Fact counts mismatch, testcase instance %(foundFactCount)s, infoset instance %(expectedFactCount)s"), modelObject=(modelXbrl.modelDocument, infosetModelXbrl.modelDocument), foundFactCount=len(modelXbrl.factsInInstance), expectedFactCount=len(infosetModelXbrl.factsInInstance)) else: for i, instFact in enumerate(modelXbrl.facts): infosetFact = None for fact in infosetFacts[instFact.qname]: if fact.isTuple and fact.isDuplicateOf(instFact, deemP0Equal=True): infosetFact = fact break elif fact.isItem and fact.isVEqualTo(instFact, deemP0Equal=True): infosetFact = fact break if infosetFact is None: # takes precision/decimals into account if fact is not None: fact.isVEqualTo(instFact, deemP0Equal=True) modelXbrl.error("arelle:infosetTest", _("Fact %(factNumber)s mismatch %(concept)s"), modelObject=instFact, factNumber=(i+1), concept=instFact.qname) else: ptvPeriodType = infosetFact.get("{http://www.xbrl.org/2003/ptv}periodType") ptvBalance = infosetFact.get("{http://www.xbrl.org/2003/ptv}balance") if ptvPeriodType and ptvPeriodType != instFact.concept.periodType: modelXbrl.error("arelle:infosetTest", _("Fact %(factNumber)s periodType mismatch %(concept)s expected %(expectedPeriodType)s found %(foundPeriodType)s"), modelObject=(instFact, infosetFact), factNumber=(i+1), concept=instFact.qname, expectedPeriodType=ptvPeriodType, foundPeriodType=instFact.concept.periodType) if ptvBalance and ptvBalance != instFact.concept.balance: modelXbrl.error("arelle:infosetTest", _("Fact %(factNumber)s balance mismatch %(concept)s expected %(expectedBalance)s found %(foundBalance)s"), modelObject=(instFact, infosetFact), factNumber=(i+1), concept=instFact.qname, expectedBalance=ptvBalance, foundBalance=instFact.concept.balance) elif infoset.type == Type.ARCSINFOSET: # compare arcs for arcElt in XmlUtil.children(infoset.xmlRootElement, "http://www.xbrl.org/2003/ptv", "arc"): linkType = arcElt.get("linkType") arcRole = arcElt.get("arcRole") extRole = arcElt.get("extRole") fromObj = resolvePath(modelXbrl, arcElt.get("fromPath")) if fromObj is None: modelXbrl.error("arelle:infosetTest", _("Arc fromPath not found: %(fromPath)s"), modelObject=arcElt, fromPath=arcElt.get("fromPath")) continue if linkType in ("label", "reference"): labelLang = arcElt.get("labelLang") resRole = arcElt.get("resRole") if linkType == "label": expectedLabel = XmlUtil.text(arcElt) foundLabel = fromObj.label(preferredLabel=resRole,fallbackToQname=False,lang=None,strip=True,linkrole=extRole) if foundLabel != expectedLabel: modelXbrl.error("arelle:infosetTest", _("Label expected='%(expectedLabel)s', found='%(foundLabel)s'"), modelObject=arcElt, expectedLabel=expectedLabel, foundLabel=foundLabel) continue elif linkType == "reference": expectedRef = XmlUtil.innerText(arcElt) referenceFound = False for refrel in modelXbrl.relationshipSet(XbrlConst.conceptReference,extRole).fromModelObject(fromObj): ref = refrel.toModelObject if resRole == ref.role: foundRef = XmlUtil.innerText(ref) if foundRef != expectedRef: modelXbrl.error("arelle:infosetTest", _("Reference inner text expected='%(expectedRef)s, found='%(foundRef)s'"), modelObject=arcElt, expectedRef=expectedRef, foundRef=foundRef) referenceFound = True break if referenceFound: continue modelXbrl.error("arelle:infosetTest", _("%(linkType)s not found containing '%(text)s' linkRole %(linkRole)s"), modelObject=arcElt, linkType=linkType.title(), text=XmlUtil.innerText(arcElt), linkRole=extRole) else: toObj = resolvePath(modelXbrl, arcElt.get("toPath")) if toObj is None: modelXbrl.error("arelle:infosetTest", _("Arc toPath not found: %(toPath)s"), modelObject=arcElt, toPath=arcElt.get("toPath")) continue weight = arcElt.get("weight") if weight is not None: weight = float(weight) order = arcElt.get("order") if order is not None: order = float(order) preferredLabel = arcElt.get("preferredLabel") found = False for rel in modelXbrl.relationshipSet(arcRole, extRole).fromModelObject(fromObj): if (rel.toModelObject == toObj and (weight is None or rel.weight == weight) and (order is None or rel.order == order)): found = True if not found: modelXbrl.error("arelle:infosetTest", _("Arc not found: from %(toPath)s, to %(toPath)s, role %(arcRole)s, linkRole $(extRole)s"), modelObject=arcElt, fromPath=arcElt.get("fromPath"), toPath=arcElt.get("toPath"), arcRole=arcRole, linkRole=extRole) continue # validate dimensions of each fact factElts = XmlUtil.children(modelXbrl.modelDocument.xmlRootElement, None, "*") for itemElt in XmlUtil.children(infoset.xmlRootElement, None, "item"): try: qnElt = XmlUtil.child(itemElt,None,"qnElement") factQname = qname(qnElt, XmlUtil.text(qnElt)) sPointer = int(XmlUtil.child(itemElt,None,"sPointer").text) factElt = factElts[sPointer - 1] # 1-based xpath indexing if factElt.qname != factQname: modelXbrl.error("arelle:infosetTest", _("Fact %(sPointer)s mismatch Qname, expected %(qnElt)s, observed %(factQname)s"), modelObject=itemElt, sPointer=sPointer, qnElt=factQname, factQname=factElt.qname) elif not factElt.isItem or factElt.context is None: modelXbrl.error("arelle:infosetTest", _("Fact %(sPointer)s has no context: %(qnElt)s"), modelObject=(itemElt,factElt), sPointer=sPointer, qnElt=factQname) else: context = factElt.context memberElts = XmlUtil.children(itemElt,None,"member") numNonDefaults = 0 for memberElt in memberElts: dimElt = XmlUtil.child(memberElt, None, "qnDimension") qnDim = qname(dimElt, XmlUtil.text(dimElt)) isDefault = XmlUtil.text(XmlUtil.child(memberElt, None, "bDefaulted")) == "true" if not isDefault: numNonDefaults += 1 if not ((qnDim in context.qnameDims and not isDefault) or (qnDim in factElt.modelXbrl.qnameDimensionDefaults and isDefault)): modelXbrl.error("arelle:infosetTest", _("Fact %(sPointer)s (qnElt)s dimension mismatch %(qnDim)s"), modelObject=(itemElt, factElt, context), sPointer=sPointer, qnElt=factQname, qnDim=qnDim) if numNonDefaults != len(context.qnameDims): modelXbrl.error("arelle:infosetTest", _("Fact %(sPointer)s (qnElt)s dimensions count mismatch"), modelObject=(itemElt, factElt, context), sPointer=sPointer, qnElt=factQname) except (IndexError, ValueError, AttributeError) as err: modelXbrl.error("arelle:infosetTest", _("Invalid entity fact dimensions infoset sPointer: %(test)s, error details: %(error)s"), modelObject=itemElt, test=XmlUtil.innerTextList(itemElt), error=str(err))
def validate(modelXbrl, elt, recurse=True, attrQname=None, ixFacts=False): global ModelInlineFact if ModelInlineFact is None: from arelle.ModelInstanceObject import ModelInlineFact isIxFact = isinstance(elt, ModelInlineFact) # attrQname can be provided for attributes that are global and LAX if (not hasattr(elt,"xValid") or elt.xValid == UNVALIDATED) and (not isIxFact or ixFacts): qnElt = elt.qname if ixFacts and isIxFact else elt.elementQname modelConcept = modelXbrl.qnameConcepts.get(qnElt) try: text = elt.elementText except Exception as err: if isIxFact and err.__class__.__name__ == "FunctionArgType": modelXbrl.error("ixTransform:valueError", _("Inline element %(element)s fact %(fact)s type %(typeName)s transform %(transform)s value error: %(value)s"), modelObject=elt, element=elt.elementQname, fact=elt.qname, transform=elt.format, typeName=modelConcept.baseXsdType if modelConcept is not None else "unknown", value=XmlUtil.innerText(elt, ixExclude=True)) else: modelXbrl.error("xmlValidation:valueError", _("Element %(element)s error %(error)s value: %(value)s"), modelObject=elt, element=elt.elementQname, error=str(err), value=elt.text) elt.sValue = elt.xValue = text = INVALIDixVALUE elt.xValid = INVALID facets = None if modelConcept is not None: isNillable = modelConcept.isNillable type = modelConcept.type if modelConcept.isAbstract: baseXsdType = "noContent" else: baseXsdType = modelConcept.baseXsdType facets = modelConcept.facets if len(text) == 0: if modelConcept.default is not None: text = modelConcept.default elif modelConcept.fixed is not None: text = modelConcept.fixed elif qnElt == XbrlConst.qnXbrldiExplicitMember: # not in DTS baseXsdType = "QName" type = None isNillable = False elif qnElt == XbrlConst.qnXbrldiTypedMember: # not in DTS baseXsdType = "noContent" type = None isNillable = False else: baseXsdType = None type = None isNillable = False isNil = isNillable and elt.get("{http://www.w3.org/2001/XMLSchema-instance}nil") == "true" if attrQname is None: if text is not INVALIDixVALUE: validateValue(modelXbrl, elt, None, baseXsdType, text, isNillable, isNil, facets) if type is not None: definedAttributes = type.attributes else: definedAttributes = {} presentAttributes = set() # validate attributes # find missing attributes for default values for attrTag, attrValue in elt.items(): qn = qname(attrTag, noPrefixIsNoNamespace=True) baseXsdAttrType = None facets = None if attrQname is not None: # validate all attributes and element if attrQname != qn: continue elif type is not None: presentAttributes.add(qn) if qn in definedAttributes: # look for concept-type-specific attribute definition modelAttr = definedAttributes[qn] elif qn.namespaceURI: # may be a globally defined attribute modelAttr = modelXbrl.qnameAttributes.get(qn) else: modelAttr = None if modelAttr is not None: baseXsdAttrType = modelAttr.baseXsdType facets = modelAttr.facets if baseXsdAttrType is None: # look for global attribute definition attrObject = modelXbrl.qnameAttributes.get(qn) if attrObject is not None: baseXsdAttrType = attrObject.baseXsdType facets = attrObject.facets elif attrTag == "{http://xbrl.org/2006/xbrldi}dimension": # some fallbacks? baseXsdAttrType = "QName" elif attrTag == "id": baseXsdAttrType = "ID" elif elt.namespaceURI == "http://www.w3.org/2001/XMLSchema": if attrTag in {"type", "ref", "base", "refer", "itemType"}: baseXsdAttrType = "QName" elif attrTag in {"name"}: baseXsdAttrType = "NCName" elif attrTag in {"default", "fixed", "form"}: baseXsdAttrType = "string" elif qn in predefinedAttributeTypes: baseXsdAttrType, facets = predefinedAttributeTypes[qn] validateValue(modelXbrl, elt, attrTag, baseXsdAttrType, attrValue, facets=facets) # if no attributes assigned above, there won't be an xAttributes, if so assign a shared dict to save memory try: elt.xAttributes except AttributeError: elt.xAttributes = xAttributesSharedEmptyDict if type is not None: if attrQname is None: missingAttributes = type.requiredAttributeQnames - presentAttributes - elt.slottedAttributesNames if missingAttributes: modelXbrl.error("xmlSchema:attributesRequired", _("Element %(element)s type %(typeName)s missing required attributes: %(attributes)s"), modelObject=elt, element=qnElt, typeName=baseXsdType, attributes=','.join(str(a) for a in missingAttributes)) # add default attribute values for attrQname in (type.defaultAttributeQnames - presentAttributes): modelAttr = type.attributes[attrQname] validateValue(modelXbrl, elt, attrQname.clarkNotation, modelAttr.baseXsdType, modelAttr.default, facets=modelAttr.facets) if recurse: global validateElementSequence, modelGroupCompositorTitle if validateElementSequence is None: from arelle.XmlValidateParticles import validateElementSequence, modelGroupCompositorTitle try: childElts = elt.modelTupleFacts if ixFacts and isIxFact else list(elt) if isNil: if childElts and any(True for e in childElts if isinstance(e, ModelObject)) or elt.text: modelXbrl.error("xmlSchema:nilElementHasContent", _("Element %(element)s is nil but has contents"), modelObject=elt, element=qnElt) else: errResult = validateElementSequence(modelXbrl, type, childElts, ixFacts) if errResult is not None and errResult[2]: iElt, occured, errDesc, errArgs = errResult errElt = childElts[iElt] if iElt < len(childElts) else elt errArgs["modelObject"] = errElt errArgs["element"] = errElt.qname errArgs["parentElement"] = elt.qname if "compositor" in errArgs: # compositor is an object, provide friendly string errArgs["compositor"] = modelGroupCompositorTitle(errArgs["compositor"]) modelXbrl.error(*errDesc,**errArgs) recurse = False # cancel child element validation below, recursion was within validateElementSequence except AttributeError as ex: pass if recurse: # if there is no complex or simple type (such as xbrli:measure) then this code is used for child in (elt.modelTupleFacts if ixFacts and isIxFact else elt): if isinstance(child, ModelObject): validate(modelXbrl, child, recurse, attrQname, ixFacts)
def view(self): self.blockSelectEvent = 1 self.blockViewModelObject = 0 self.tag_has = defaultdict(list) # temporary until Tk 8.6 # relationship set based on linkrole parameter, to determine applicable linkroles relationshipSet = self.modelXbrl.relationshipSet( self.arcrole, self.linkrole, self.linkqname, self.arcqname) if not relationshipSet: self.modelXbrl.modelManager.addToLog( _("no relationships for {0}").format(self.arcrole)) return False # consider facts in the relationshipSet (only) contexts = set() self.conceptFacts = defaultdict(list) if self.linkrole and self.modelXbrl.roleTypes[ self.linkrole] and hasattr( self.modelXbrl.roleTypes[self.linkrole][0], "_tableFacts"): for fact in self.modelXbrl.roleTypes[self.linkrole][0]._tableFacts: self.conceptFacts[fact.qname].append(fact) if fact.context is not None: contexts.add(fact.context) else: for fact in self.modelXbrl.facts: if relationshipSet.fromModelObject( fact.concept) or relationshipSet.toModelObject( fact.concept): self.conceptFacts[fact.qname].append(fact) if fact.context is not None: contexts.add(fact.context) # sort contexts by period self.periodContexts = defaultdict(set) contextStartDatetimes = {} ignoreDims = self.ignoreDims.get() showDimDefaults = self.showDimDefaults.get() for context in contexts: if context is None or context.endDatetime is None: contextkey = "missing period" elif ignoreDims: if context.isForeverPeriod: contextkey = datetime.datetime(datetime.MINYEAR, 1, 1) else: contextkey = context.endDatetime else: if context.isForeverPeriod: contextkey = "forever" else: contextkey = ( context.endDatetime - datetime.timedelta(days=1)).strftime("%Y-%m-%d") values = [] dims = context.qnameDims if len(dims) > 0: for dimQname in sorted(dims.keys(), key=lambda d: str(d)): dimvalue = dims[dimQname] if dimvalue.isExplicit: values.append( dimvalue.member. label(self.labelrole, lang=self.lang ) if dimvalue.member is not None else str(dimvalue.memberQname)) else: values.append( XmlUtil.innerText(dimvalue.typedMember)) nonDimensions = context.nonDimValues( "segment") + context.nonDimValues("scenario") if len(nonDimensions) > 0: for element in sorted(nonDimensions, key=lambda e: e.localName): values.append(XmlUtil.innerText(element)) if len(values) > 0: contextkey += " - " + ', '.join(values) objectId = context.objectId() self.periodContexts[contextkey].add(objectId) if context.isStartEndPeriod: contextStartDatetimes[objectId] = context.startDatetime self.periodKeys = list(self.periodContexts.keys()) self.periodKeys.sort() # set up treeView widget and tabbed pane self.treeView.column("#0", width=300, anchor="w") self.treeView.heading("#0", text="Concept") columnIds = [] columnIdHeadings = [] self.contextColId = {} self.startdatetimeColId = {} self.numCols = 1 for periodKey in self.periodKeys: colId = "#{0}".format(self.numCols) columnIds.append(colId) columnIdHeadings.append((colId, periodKey)) for contextId in self.periodContexts[periodKey]: self.contextColId[contextId] = colId if contextId in contextStartDatetimes: self.startdatetimeColId[ contextStartDatetimes[contextId]] = colId self.numCols += 1 self.treeView["columns"] = columnIds for colId, colHeading in columnIdHeadings: self.treeView.column(colId, width=100, anchor="w") if ignoreDims: if colHeading.year == datetime.MINYEAR: date = "forever" else: date = (colHeading - datetime.timedelta(days=1)).strftime("%Y-%m-%d") self.treeView.heading(colId, text=date) else: self.treeView.heading(colId, text=colHeading) # fact rendering self.clearTreeView() self.rowColFactId = {} # root node for tree view self.id = 1 # sort URIs by definition linkroleUris = [] relationshipSet = self.modelXbrl.relationshipSet( self.arcrole, self.linkrole, self.linkqname, self.arcqname) if self.linkrole: roleType = self.modelXbrl.roleTypes[self.linkrole][0] linkroleUris.append( ((roleType.genLabel(lang=self.lang, strip=True) or roleType.definition or linkroleUri), self.linkrole, roleType.objectId(self.id))) self.id += 1 else: for linkroleUri in relationshipSet.linkRoleUris: modelRoleTypes = self.modelXbrl.roleTypes.get(linkroleUri) if modelRoleTypes: roledefinition = (modelRoleTypes[0].genLabel( lang=self.lang, strip=True) or modelRoleTypes[0].definition or linkroleUri) roleId = modelRoleTypes[0].objectId(self.id) else: roledefinition = linkroleUri roleId = "node{0}".format(self.id) self.id += 1 linkroleUris.append((roledefinition, linkroleUri, roleId)) linkroleUris.sort() # for each URI in definition order for linkroleUriTuple in linkroleUris: linknode = self.treeView.insert("", "end", linkroleUriTuple[2], text=linkroleUriTuple[0], tags=("ELR", )) linkRelationshipSet = self.modelXbrl.relationshipSet( self.arcrole, linkroleUriTuple[1], self.linkqname, self.arcqname) for rootConcept in linkRelationshipSet.rootConcepts: node = self.viewConcept(rootConcept, rootConcept, "", self.labelrole, linknode, 1, linkRelationshipSet, set()) if self.expandAllOnFirstDisplay: self.expandAll() return True
def validate(modelXbrl, elt, recurse=True, attrQname=None, ixFacts=False): global ModelInlineValueObject, ixMsgCode if ModelInlineValueObject is None: from arelle.ModelInstanceObject import ModelInlineValueObject from arelle.XhtmlValidate import ixMsgCode isIxFact = isinstance(elt, ModelInlineValueObject) facets = None # attrQname can be provided for attributes that are global and LAX if (getattr(elt, "xValid", UNVALIDATED) == UNVALIDATED) and (not isIxFact or ixFacts): qnElt = elt.qname if ixFacts and isIxFact else elt.elementQname modelConcept = modelXbrl.qnameConcepts.get(qnElt) isAbstract = False if modelConcept is not None: isNillable = modelConcept.isNillable type = modelConcept.type if modelConcept.isAbstract: baseXsdType = "noContent" isAbstract = True elif modelConcept.isFraction: baseXsdType = "fraction" else: baseXsdType = modelConcept.baseXsdType facets = modelConcept.facets elif qnElt == XbrlConst.qnXbrldiExplicitMember: # not in DTS baseXsdType = "QName" type = None isNillable = False elif qnElt == XbrlConst.qnXbrldiTypedMember: # not in DTS baseXsdType = "noContent" type = None isNillable = False else: baseXsdType = None type = None isNillable = True # allow nil if no schema definition isNil = elt.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ( "true", "1") if attrQname is None: if isNil and not isNillable: if ModelInlineValueObject is not None and isinstance( elt, ModelInlineValueObject): errElt = "{0} fact {1}".format(elt.elementQname, elt.qname) else: errElt = elt.elementQname modelXbrl.error( "xmlSchema:nilNonNillableElement", _("Element %(element)s fact %(fact)s type %(typeName)s is nil but element has not been defined nillable" ), modelObject=elt, element=errElt, fact=elt.qname, typeName=modelConcept.baseXsdType if modelConcept is not None else "unknown") try: if isAbstract: raise ValueError("element is abstract") if isNil: text = "" elif baseXsdType == "noContent": text = elt.textValue # no descendant text nodes else: text = elt.stringValue # include descendant text nodes if modelConcept is not None: if len(text) == 0: if modelConcept.default is not None: text = modelConcept.default elif modelConcept.fixed is not None: text = modelConcept.fixed if baseXsdType == "token" and modelConcept.isEnumeration: if modelConcept.instanceOfType( XbrlConst.qnEnumeration2ItemTypes): baseXsdType = "enumerationHrefs" else: baseXsdType = "enumerationQNames" except Exception as err: if ModelInlineValueObject is not None and isinstance( elt, ModelInlineValueObject): errElt = "{0} fact {1}".format(elt.elementQname, elt.qname) else: errElt = elt.elementQname if isIxFact and err.__class__.__name__ == "FunctionArgType": modelXbrl.error( ixMsgCode("transformValueError", elt), _("Inline element %(element)s fact %(fact)s type %(typeName)s transform %(transform)s value error: %(value)s" ), modelObject=elt, element=errElt, fact=elt.qname, transform=elt.format, typeName=modelConcept.baseXsdType if modelConcept is not None else "unknown", value=XmlUtil.innerText(elt, ixExclude=True, ixContinuation=elt.namespaceURI == XbrlConst.ixbrl11)) elif isIxFact and err.__class__.__name__ == "ixtFunctionNotAvailable": modelXbrl.error( ixMsgCode("invalidTransformation", elt, sect="validation"), _("Fact %(fact)s has unrecognized transformation %(transform)s, value: %(value)s" ), modelObject=elt, element=errElt, fact=elt.qname, transform=elt.format, typeName=modelConcept.baseXsdType if modelConcept is not None else "unknown", value=XmlUtil.innerText(elt, ixExclude=True, ixContinuation=elt.namespaceURI == XbrlConst.ixbrl11)) elif isAbstract: modelXbrl.error( "xmlSchema:abstractElement", _("Element %(element)s has abstract declaration, value: %(value)s" ), modelObject=elt, element=errElt, error=str(err), value=elt.text) else: modelXbrl.error( "xmlSchema:valueError", _("Element %(element)s error %(error)s value: %(value)s" ), modelObject=elt, element=errElt, error=str(err), value=elt.text) elt.sValue = elt.xValue = text = INVALIDixVALUE elt.xValid = INVALID if text is not INVALIDixVALUE: validateValue(modelXbrl, elt, None, baseXsdType, text, isNillable, isNil, facets) # note that elt.sValue and elt.xValue are not innerText but only text elements on specific element (or attribute) if type is not None: definedAttributes = type.attributes else: definedAttributes = {} presentAttributes = set() # validate attributes # find missing attributes for default values for attrTag, attrValue in elt.items(): qn = qnameClarkName(attrTag) #qn = qname(attrTag, noPrefixIsNoNamespace=True) baseXsdAttrType = None facets = None if attrQname is not None: # validate all attributes and element if attrQname != qn: continue elif type is not None: presentAttributes.add(qn) if qn in definedAttributes: # look for concept-type-specific attribute definition modelAttr = definedAttributes[qn] elif qn.namespaceURI: # may be a globally defined attribute modelAttr = modelXbrl.qnameAttributes.get(qn) else: modelAttr = None if modelAttr is not None: baseXsdAttrType = modelAttr.baseXsdType facets = modelAttr.facets if baseXsdAttrType is None: # look for global attribute definition attrObject = modelXbrl.qnameAttributes.get(qn) if attrObject is not None: baseXsdAttrType = attrObject.baseXsdType facets = attrObject.facets elif attrTag == "{http://xbrl.org/2006/xbrldi}dimension": # some fallbacks? baseXsdAttrType = "QName" elif attrTag == "id": baseXsdAttrType = "ID" elif elt.namespaceURI == "http://www.w3.org/2001/XMLSchema": if attrTag in {"type", "ref", "base", "refer", "itemType"}: baseXsdAttrType = "QName" elif attrTag in {"name"}: baseXsdAttrType = "NCName" elif attrTag in {"default", "fixed", "form"}: baseXsdAttrType = "string" elif elt.namespaceURI == "http://xbrl.org/2006/xbrldi": if attrTag == "dimension": baseXsdAttrType = "QName" elif qn in predefinedAttributeTypes: baseXsdAttrType, facets = predefinedAttributeTypes[qn] validateValue(modelXbrl, elt, attrTag, baseXsdAttrType, attrValue, facets=facets) # if no attributes assigned above, there won't be an xAttributes, if so assign a shared dict to save memory try: elt.xAttributes except AttributeError: elt.xAttributes = xAttributesSharedEmptyDict if type is not None: if attrQname is None: missingAttributes = type.requiredAttributeQnames - presentAttributes - elt.slottedAttributesNames if missingAttributes: modelXbrl.error( "xmlSchema:attributesRequired", _("Element %(element)s type %(typeName)s missing required attributes: %(attributes)s" ), modelObject=elt, element=qnElt, typeName=baseXsdType, attributes=','.join(str(a) for a in missingAttributes)) extraAttributes = presentAttributes - _DICT_SET( definedAttributes.keys()) - XbrlConst.builtinAttributes if extraAttributes: attributeWildcards = type.attributeWildcards extraAttributes -= set( a for a in extraAttributes if validateAnyWildcard(qnElt, a, attributeWildcards)) if isIxFact: extraAttributes -= XbrlConst.ixAttributes if extraAttributes: modelXbrl.error( "xmlSchema:attributesExtraneous", _("Element %(element)s type %(typeName)s extraneous attributes: %(attributes)s" ), modelObject=elt, element=qnElt, typeName=baseXsdType, attributes=','.join( str(a) for a in extraAttributes)) # add default attribute values for attrQname in (type.defaultAttributeQnames - presentAttributes): modelAttr = type.attributes[attrQname] validateValue(modelXbrl, elt, attrQname.clarkNotation, modelAttr.baseXsdType, modelAttr.default, facets=modelAttr.facets) if recurse: global validateElementSequence, modelGroupCompositorTitle if validateElementSequence is None: from arelle.XmlValidateParticles import validateElementSequence, modelGroupCompositorTitle try: #childElts = list(elt) # uses __iter__ for inline facts childElts = [e for e in elt if isinstance(e, ModelObject)] if isNil: if childElts or elt.text: modelXbrl.error( "xmlSchema:nilElementHasContent", _("Element %(element)s is nil but has contents" ), modelObject=elt, element=qnElt) else: errResult = validateElementSequence( modelXbrl, type, childElts, ixFacts) if errResult is not None and errResult[2]: iElt, occured, errDesc, errArgs = errResult errElt = childElts[iElt] if iElt < len( childElts) else elt errArgs["modelObject"] = errElt errArgs["element"] = errElt.qname errArgs["parentElement"] = elt.qname if "compositor" in errArgs: # compositor is an object, provide friendly string errArgs[ "compositor"] = modelGroupCompositorTitle( errArgs["compositor"]) modelXbrl.error(*errDesc, **errArgs) # when error is in an xbrli element, check any further unvalidated children if qnElt.namespaceURI == XbrlConst.xbrli and iElt < len( childElts): for childElt in childElts[iElt:]: if (getattr(childElt, "xValid", UNVALIDATED) == UNVALIDATED): validate(modelXbrl, childElt, ixFacts=ixFacts) recurse = False # cancel child element validation below, recursion was within validateElementSequence except AttributeError as ex: raise ex #pass # HF Why is this here???? if recurse: # if there is no complex or simple type (such as xbrli:measure) then this code is used for child in (elt.modelTupleFacts if ixFacts and isIxFact else elt): if isinstance(child, ModelObject): validate(modelXbrl, child, recurse, attrQname, ixFacts)
def view(self, modelDocument): if self.cols: if isinstance(self.cols, str): self.cols = self.cols.replace(',', ' ').split() unrecognizedCols = [] for col in self.cols: if col not in COL_WIDTHS: unrecognizedCols.append(col) if unrecognizedCols: self.modelXbrl.error("arelle:unrecognizedFactListColumn", _("Unrecognized columns: %(cols)s"), modelXbrl=self.modelXbrl, cols=','.join(unrecognizedCols)) if "Period" in self.cols: i = self.cols.index("Period") self.cols[i:i + 1] = ["Start", "End/Instant"] else: self.cols = ["Concept", "Facts"] col0 = self.cols[0] try: colIdxFacts = self.cols.index("Facts") except ValueError: self.modelXbrl.error("arelle:factTableFactsColumn", _("A columns entry for Facts is required"), modelXbrl=self.modelXbrl) colIdxFacts = len(self.cols) self.cols.append("Facts") if col0 not in ("Concept", "Label", "Name", "LocalName"): self.modelXbrl.error( "arelle:firstFactTableColumn", _("First column must be Concept, Label, Name or LocalName: %(col1)s" ), modelXbrl=self.modelXbrl, col1=col0) self.isCol0Label = col0 in ("Concept", "Label") relationshipSet = self.modelXbrl.relationshipSet( self.arcrole, self.linkrole, self.linkqname, self.arcqname) if relationshipSet: # sort URIs by definition linkroleUris = [] for linkroleUri in relationshipSet.linkRoleUris: modelRoleTypes = self.modelXbrl.roleTypes.get(linkroleUri) if modelRoleTypes: roledefinition = (modelRoleTypes[0].genLabel( lang=self.lang, strip=True) or modelRoleTypes[0].definition or linkroleUri) else: roledefinition = linkroleUri linkroleUris.append((roledefinition, linkroleUri)) linkroleUris.sort() for roledefinition, linkroleUri in linkroleUris: linkRelationshipSet = self.modelXbrl.relationshipSet( self.arcrole, linkroleUri, self.linkqname, self.arcqname) for rootConcept in linkRelationshipSet.rootConcepts: self.treeDepth(rootConcept, rootConcept, 2, self.arcrole, linkRelationshipSet, set()) self.linkRoleDefintions = dict( (linkroleUri, roledefinition) for roledefinition, linkroleUri in linkroleUris) # allocate facts to table structure for US-GAAP-style filings if not self.modelXbrl.hasTableIndexing: from arelle import TableStructure TableStructure.evaluateTableIndex(self.modelXbrl, lang=self.lang) # set up facts self.conceptFacts = defaultdict(list) for fact in self.modelXbrl.facts: self.conceptFacts[fact.qname].append(fact) # sort contexts by period self.periodContexts = defaultdict(set) contextStartDatetimes = {} for context in self.modelXbrl.contexts.values(): if self.type in (CSV, XLSX, HTML): if context is None or context.endDatetime is None: contextkey = "missing period" elif self.ignoreDims: if context.isForeverPeriod: contextkey = datetime.datetime(datetime.MINYEAR, 1, 1) else: contextkey = context.endDatetime else: if context.isForeverPeriod: contextkey = "forever" else: contextkey = ( context.endDatetime - datetime.timedelta(days=1)).strftime("%Y-%m-%d") values = [] dims = context.qnameDims if len(dims) > 0: for dimQname in sorted(dims.keys(), key=lambda d: str(d)): dimvalue = dims[dimQname] if dimvalue.isExplicit: values.append( dimvalue.member. label(self.labelrole, lang=self.lang ) if dimvalue.member is not None else str(dimvalue.memberQname)) else: values.append( XmlUtil.innerText(dimvalue.typedMember)) nonDimensions = context.nonDimValues( "segment") + context.nonDimValues("scenario") if len(nonDimensions) > 0: for element in sorted(nonDimensions, key=lambda e: e.localName): values.append(XmlUtil.innerText(element)) if len(values) > 0: contextkey += " - " + ', '.join(values) else: contextkey = context.id objectId = context.objectId() self.periodContexts[contextkey].add(objectId) if context.isStartEndPeriod: contextStartDatetimes[objectId] = context.startDatetime self.periodKeys = list(self.periodContexts.keys()) self.periodKeys.sort() # set up treeView widget and tabbed pane heading = self.cols[0:colIdxFacts] columnHeadings = [] self.contextColId = {} self.startdatetimeColId = {} self.numCols = len(heading) for periodKey in self.periodKeys: columnHeadings.append(periodKey) for contextId in self.periodContexts[periodKey]: self.contextColId[contextId] = self.numCols if contextId in contextStartDatetimes: self.startdatetimeColId[ contextStartDatetimes[contextId]] = self.numCols self.numCols += 1 for colHeading in columnHeadings: if self.ignoreDims: if colHeading.year == datetime.MINYEAR: date = "forever" else: date = (colHeading - datetime.timedelta(days=1)).strftime("%Y-%m-%d") heading.append(date) else: heading.append(colHeading) heading += self.cols[colIdxFacts + 1:] self.numCols = len(heading) self.setColWidths([ COL_WIDTHS[col] if col in COL_WIDTHS else COL_WIDTHS["Facts"] for col in enumerate(heading) ]) self.setColWrapText([True for col in heading]) self.addRow(heading, asHeader=True) # must do after determining tree depth if relationshipSet: # for each URI in definition order for roledefinition, linkroleUri in linkroleUris: attr = {"role": linkroleUri, "definition": roledefinition} self.addRow([roledefinition], treeIndent=0, colSpan=len(heading), xmlRowElementName="linkRole", xmlRowEltAttr=attr, xmlCol0skipElt=True) linkRelationshipSet = self.modelXbrl.relationshipSet( self.arcrole, linkroleUri, self.linkqname, self.arcqname) # set up concepts which apply to linkrole for us-gaap style filings self.conceptFacts.clear() if linkroleUri and self.modelXbrl.roleTypes[ linkroleUri] and hasattr( self.modelXbrl.roleTypes[linkroleUri][0], "_tableFacts"): for fact in self.modelXbrl.roleTypes[linkroleUri][ 0]._tableFacts: self.conceptFacts[fact.qname].append(fact) else: for fact in self.modelXbrl.facts: if linkRelationshipSet.fromModelObject( fact.concept ) or linkRelationshipSet.toModelObject(fact.concept): self.conceptFacts[fact.qname].append(fact) # view root and descendant for rootConcept in linkRelationshipSet.rootConcepts: self.viewConcept(rootConcept, linkroleUri, "", self.labelrole, 1, linkRelationshipSet, set()) return True