def dir(self): self.open() if not self.isOpen: return None elif self.filesDir is not None: return self.filesDir elif self.isZip: files = [] for zipinfo in self.fs.infolist(): files.append(zipinfo.filename) self.filesDir = files elif self.isTarGz: self.filesDir = self.fs.getnames() elif self.isEis: files = [] for docElt in self.eisDocument.iter(tag=u"{http://www.sec.gov/edgar/common}document"): outfn = docElt.findtext(u"{http://www.sec.gov/edgar/common}conformedName") if outfn: files.append(outfn); self.filesDir = files elif self.isXfd: files = [] for data in self.xfdDocument.iter(tag=u"data"): outfn = data.findtext(u"filename") if outfn: if len(outfn) > 2 and outfn[0].isalpha() and \ outfn[1] == u':' and outfn[2] == u'\\': continue files.append(outfn); self.filesDir = files elif self.isRss: files = [] # return title, descr, pubdate, linst doc edgr = u"http://www.sec.gov/Archives/edgar" try: for dsElt in XmlUtil.descendants(self.rssDocument, None, u"item"): instDoc = None for instDocElt in XmlUtil.descendants(dsElt, edgr, u"xbrlFile"): if instDocElt.get(u"(http://www.sec.gov/Archives/edgar}description").endswith(u"INSTANCE DOCUMENT"): instDoc = instDocElt.get(u"(http://www.sec.gov/Archives/edgar}url") break if not instDoc: continue files.append(( XmlUtil.text(XmlUtil.descendant(dsElt, None, u"title")), # tooltip u"{0}\n {1}\n {2}\n {3}\n {4}".format( XmlUtil.text(XmlUtil.descendant(dsElt, edgr, u"companyName")), XmlUtil.text(XmlUtil.descendant(dsElt, edgr, u"formType")), XmlUtil.text(XmlUtil.descendant(dsElt, edgr, u"filingDate")), XmlUtil.text(XmlUtil.descendant(dsElt, edgr, u"cikNumber")), XmlUtil.text(XmlUtil.descendant(dsElt, edgr, u"period"))), XmlUtil.text(XmlUtil.descendant(dsElt, None, u"description")), XmlUtil.text(XmlUtil.descendant(dsElt, None, u"pubDate")), instDoc)) self.filesDir = files except (EnvironmentError, etree.LxmlError), err: pass
def dataUris(self): try: return self._dataUris except AttributeError: self._dataUris = defaultdict(list) # may contain instances, schemas, linkbases for dataElement in XmlUtil.descendants(self, None, ("data", "input")): for elt in XmlUtil.descendants(dataElement, None, ("xsd", "schema", "linkbase", "instance")): self._dataUris["schema" if elt.localName == "xsd" else elt.localName].append(elt.textValue.strip()) return self._dataUris
def stepAxis(self, op, p, sourceSequence): targetSequence = [] for node in sourceSequence: if not isinstance(node,(ModelObject, etree._ElementTree, ModelAttribute)): raise XPathException(self.progHeader, 'err:XPTY0020', _('Axis step {0} context item is not a node: {1}').format(op, node)) targetNodes = [] if isinstance(p,QNameDef): ns = p.namespaceURI; localname = p.localName if p.isAttribute: if isinstance(node,ModelObject): attrTag = p.localName if p.unprefixed else p.clarkNotation modelAttribute = None try: modelAttribute = node.xAttributes[attrTag] except (AttributeError, TypeError, IndexError, KeyError): # may be lax or deferred validated try: validate(node.modelXbrl, node, p) modelAttribute = node.xAttributes[attrTag] except (AttributeError, TypeError, IndexError, KeyError): pass if modelAttribute is None: value = node.get(attrTag) if value is not None: targetNodes.append(ModelAttribute(node,p.clarkNotation,UNKNOWN,value,value,value)) elif modelAttribute.xValid >= VALID: targetNodes.append(modelAttribute) elif op == '/' or op is None: if isinstance(node,(ModelObject, etree._ElementTree)): targetNodes = XmlUtil.children(node, ns, localname) elif op == '//': if isinstance(node,(ModelObject, etree._ElementTree)): targetNodes = XmlUtil.descendants(node, ns, localname) elif op == '..': if isinstance(node,ModelAttribute): targetNodes = [ node.modelElement ] else: targetNodes = [ XmlUtil.parent(node) ] elif isinstance(p, OperationDef) and isinstance(p.name,QNameDef): if isinstance(node,ModelObject): if p.name.localName == "text": targetNodes = [XmlUtil.text(node)] # todo: add element, attribute, node, etc... elif p == '*': # wildcard if op == '/' or op is None: if isinstance(node,(ModelObject, etree._ElementTree)): targetNodes = XmlUtil.children(node, '*', '*') elif op == '//': if isinstance(node,(ModelObject, etree._ElementTree)): targetNodes = XmlUtil.descendants(node, '*', '*') targetSequence.extend(targetNodes) return targetSequence
def dir(self): self.open() if not self.isOpen: return None elif self.filesDir is not None: return self.filesDir elif self.isZip: files = [] for zipinfo in self.fs.infolist(): files.append(zipinfo.filename) self.filesDir = files elif self.isXfd: files = [] for data in self.xfdDocument.iter(tag="data"): outfn = data.findtext("filename") if outfn: if len(outfn) > 2 and outfn[0].isalpha() and \ outfn[1] == ':' and outfn[2] == '\\': continue files.append(outfn); self.filesDir = files elif self.isRss: files = [] # return title, descr, pubdate, linst doc edgr = "http://www.sec.gov/Archives/edgar" try: for dsElt in XmlUtil.descendants(self.rssDocument, None, "item"): instDoc = None for instDocElt in XmlUtil.descendants(dsElt, edgr, "xbrlFile"): if instDocElt.get("(http://www.sec.gov/Archives/edgar}description").endswith("INSTANCE DOCUMENT"): instDoc = instDocElt.get("(http://www.sec.gov/Archives/edgar}url") break if not instDoc: continue files.append(( XmlUtil.text(XmlUtil.descendant(dsElt, None, "title")), # tooltip "{0}\n {1}\n {2}\n {3}\n {4}".format( XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "companyName")), XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "formType")), XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "filingDate")), XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "cikNumber")), XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "period"))), XmlUtil.text(XmlUtil.descendant(dsElt, None, "description")), XmlUtil.text(XmlUtil.descendant(dsElt, None, "pubDate")), instDoc)) self.filesDir = files except (EnvironmentError, etree.LxmlError) as err: pass return self.filesDir
def propertyView(self): assertions = [] for assertionElement in XmlUtil.descendants(self, None, "assertionTests"): assertions.append(("assertion",assertionElement.get("assertionID"))) assertions.append((" satisfied", assertionElement.get("countSatisfied"))) assertions.append((" not sat.", assertionElement.get("countNotSatisfied"))) ''' for assertionElement in XmlUtil.descendants(self, None, "assert"): efmNum = assertionElement.get("num") assertions.append(("assertion", "EFM.{0}.{1}.{2}".format(efmNum[0], efmNum[1:2], efmNum[3:4]))) assertions.append((" not sat.", "1")) ''' readMeFirsts = [("readFirst", readMeFirstUri) for readMeFirstUri in self.readMeFirstUris] parameters = [] if len(self.parameters) > 0: parameters.append(("parameters", None)) for pName, pTypeValue in self.parameters.items(): parameters.append((pName,pTypeValue[1])) return [("id", self.id), ("name", self.name), ("description", self.description)] + \ readMeFirsts + \ parameters + \ [("status", self.status), ("call", self.cfcnCall[0]) if self.cfcnCall else (), ("test", self.cfcnTest[0]) if self.cfcnTest else (), ("infoset", self.resultInfosetUri) if self.resultIsInfoset else (), ("expected", self.expected) if self.expected else (), ("actual", " ".join(str(i) for i in self.actual) if len(self.actual) > 0 else ())] + \ assertions
def stepAxis(self, op, p, sourceSequence): targetSequence = [] for node in sourceSequence: if not isinstance(node,(ModelObject.ModelObject, xml.dom.Node)): raise XPathException(p, 'err:XPTY0020', _('Axis step {0} context item is not a node: {1}').format(op, node)) targetNodes = [] if isinstance(node, ModelObject.ModelObject): node = node.element if isinstance(p,QNameDef): ns = p.namespaceURI; localname = p.localName if p.isAttribute: if p.unprefixed: if node.hasAttribute(localname): targetNodes.append(node.getAttribute(localname)) else: if node.hasAttributeNS(ns,localname): targetNodes.append(node.getAttributeNS(ns,localname)) elif op == '/' or op is None: targetNodes = XmlUtil.children(node, ns, localname) elif op == '//': targetNodes = XmlUtil.descendants(node, ns, localname) elif op == '..': targetNodes = [ XmlUtil.parent(node) ] elif isinstance(p, OperationDef) and isinstance(p.name,QNameDef): if p.name.localName == "text": targetNodes = [XmlUtil.text(node)] # todo: add element, attribute, node, etc... targetSequence.extend(targetNodes) return targetSequence
def testcaseVariationXbrlLoaded(testcaseModelXbrl, instanceModelXbrl, modelTestcaseVariation, *args, **kwargs): # Validate of RSS feed item or testcase variation (simulates filing & cmd line load events modelManager = instanceModelXbrl.modelManager if (hasattr(testcaseModelXbrl, "efmOptions") and modelManager.validateDisclosureSystem and getattr(modelManager.disclosureSystem, "EFMplugin", False) and (instanceModelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE or instanceModelXbrl.modelDocument.type == ModelDocument.Type.INLINEXBRL)): cntlr = modelManager.cntlr options = testcaseModelXbrl.efmOptions entrypointFiles = [{"file":instanceModelXbrl.modelDocument.uri}] if not hasattr(modelManager, "efmFiling"): # first instance of filing modelManager.efmFiling = Filing(cntlr, options, instanceModelXbrl.fileSource, entrypointFiles, None, None, instanceModelXbrl.errorCaptureLevel) # this event is called for filings (of instances) as well as test cases, for test case it just keeps options accessible for pluginXbrlMethod in pluginClassMethods("EdgarRenderer.Filing.Start"): pluginXbrlMethod(cntlr, options, entrypointFiles, modelManager.efmFiling) modelManager.efmFiling.addReport(instanceModelXbrl) _report = modelManager.efmFiling.reports[-1] _report.entryPoint = entrypointFiles[0] modelManager.efmFiling.arelleUnitTests = instanceModelXbrl.arelleUnitTests.copy() # allow unit tests to be used after instance processing finished # check for parameters on instance for _instanceElt in XmlUtil.descendants(modelTestcaseVariation, "*", "instance", "readMeFirst", "true", False): if instanceModelXbrl.modelDocument.uri.endswith(_instanceElt.text): if _instanceElt.get("exhibitType"): _report.entryPoint["exhibitType"] = _report.exhibitType = _instanceElt.get("exhibitType") break
def rssFeedDiscover(self, rootElement): """Initiates discovery of RSS feed """ # add self to namespaced document self.xmlRootElement = rootElement for itemElt in XmlUtil.descendants(rootElement, None, "item"): self.rssItems.append(itemElt)
def cfcnCall(self): # tuple of (expression, element holding the expression) try: return self._cfcnCall except AttributeError: self._cfcnCall = None if self.localName == "test-case": # xpath testcase queryElement = XmlUtil.descendant(self, None, "query") if queryElement is not None: filepath = ( self.modelDocument.filepathdir + "/" + "Queries/XQuery/" + self.get("FilePath") + queryElement.get("name") + ".xq" ) if os.sep != "/": filepath = filepath.replace("/", os.sep) with io.open(filepath, "rt", encoding="utf-8") as f: self._cfcnCall = (f.read(), self) else: for callElement in XmlUtil.descendants(self, XbrlConst.cfcn, "call"): self._cfcnCall = (XmlUtil.innerText(callElement), callElement) break if self._cfcnCall is None and self.namespaceURI == "http://xbrl.org/2011/conformance-rendering/transforms": name = self.getparent().get("name") input = self.get("input") if name and input: self._cfcnCall = ("{0}('{1}')".format(name, input.replace("'", "''")), self) return self._cfcnCall
def cfcnCall(self): # tuple of (expression, element holding the expression) try: return self._cfcnCall except AttributeError: self._cfcnCall = None if self.localName == "test-case": #xpath testcase queryElement = XmlUtil.descendant(self, None, "query") if queryElement is not None: filepath = (self.modelDocument.filepathdir + "/" + "Queries/XQuery/" + self.get("FilePath") + queryElement.get("name") + '.xq') if os.sep != "/": filepath = filepath.replace("/", os.sep) with io.open(filepath, 'rt', encoding='utf-8') as f: self._cfcnCall = (f.read(), self) else: for callElement in XmlUtil.descendants(self, XbrlConst.cfcn, "call"): self._cfcnCall = (XmlUtil.innerText(callElement), callElement) break if self._cfcnCall is None and self.namespaceURI == "http://xbrl.org/2011/conformance-rendering/transforms": name = self.getparent().get("name") input = self.get("input") if name and input: self._cfcnCall = ("{0}('{1}')".format( name, input.replace("'", "''")), self) return self._cfcnCall
def contextDiscover(self, cntxElement): id = cntxElement.getAttribute("id") self.modelXbrl.contexts[id] = modelContext = ModelObject.createContext( self, cntxElement) for container in (("segment", modelContext.segDimValues, modelContext.segNonDimValues), ("scenario", modelContext.scenDimValues, modelContext.scenNonDimValues)): containerName, containerDimValues, containerNonDimValues = container for containerElement in XmlUtil.descendants( cntxElement, XbrlConst.xbrli, containerName): for sElt in containerElement.childNodes: if sElt.nodeType == 1: if sElt.namespaceURI == XbrlConst.xbrldi and sElt.localName in ( "explicitMember", "typedMember"): XmlValidate.validate(self.modelXbrl, sElt) modelDimValue = ModelObject.createDimensionValue( self, sElt) dimension = modelDimValue.dimension if dimension and dimension not in containerDimValues: containerDimValues[dimension] = modelDimValue else: modelContext.errorDimValues.append( modelDimValue) modelContext.qnameDims[ modelDimValue. dimensionQname] = modelDimValue # both seg and scen else: containerNonDimValues.append(sElt)
def testcaseDiscover(self, testcaseElement): isTransformTestcase = testcaseElement.namespaceURI == "http://xbrl.org/2011/conformance-rendering/transforms" if XmlUtil.xmlnsprefix(testcaseElement, XbrlConst.cfcn) or isTransformTestcase: self.type = Type.REGISTRYTESTCASE self.outpath = self.xmlRootElement.get("outpath") self.testcaseVariations = [] priorTransformName = None for modelVariation in XmlUtil.descendants(testcaseElement, testcaseElement.namespaceURI, "variation"): self.testcaseVariations.append(modelVariation) if isTransformTestcase and modelVariation.getparent().get( "name") is not None: transformName = modelVariation.getparent().get("name") if transformName != priorTransformName: priorTransformName = transformName variationNumber = 1 modelVariation._name = "{0} v-{1:02}".format( priorTransformName, variationNumber) variationNumber += 1 if len(self.testcaseVariations) == 0: # may be a inline test case if XbrlConst.ixbrl in testcaseElement.values(): self.testcaseVariations.append(testcaseElement)
def stepAxis(self, op, p, sourceSequence): targetSequence = [] for node in sourceSequence: if not isinstance(node, (ModelObject.ModelObject, xml.dom.Node)): raise XPathException( p, 'err:XPTY0020', _('Axis step {0} context item is not a node: {1}').format( op, node)) targetNodes = [] if isinstance(node, ModelObject.ModelObject): node = node.element if isinstance(p, QNameDef): ns = p.namespaceURI localname = p.localName if p.isAttribute: if p.unprefixed: if node.hasAttribute(localname): targetNodes.append(node.getAttribute(localname)) else: if node.hasAttributeNS(ns, localname): targetNodes.append( node.getAttributeNS(ns, localname)) elif op == '/' or op is None: targetNodes = XmlUtil.children(node, ns, localname) elif op == '//': targetNodes = XmlUtil.descendants(node, ns, localname) elif op == '..': targetNodes = [XmlUtil.parent(node)] elif isinstance(p, OperationDef) and isinstance(p.name, QNameDef): if p.name.localName == "text": targetNodes = [XmlUtil.text(node)] # todo: add element, attribute, node, etc... targetSequence.extend(targetNodes) return targetSequence
def parameters(self): try: return self._parameters except AttributeError: self._parameters = dict([ (ModelValue.qname(paramElt, paramElt.get("name")), # prefix-less parameter names take default namespace of element (ModelValue.qname(paramElt, paramElt.get("datatype")),paramElt.get("value"))) for paramElt in XmlUtil.descendants(self, self.namespaceURI, "parameter")]) return self._parameters
def htmURLs(self): try: return self._htmURLs except AttributeError: self._htmURLs = [ (instDocElt.get(edgrDescription),instDocElt.get(edgrUrl)) for instDocElt in XmlUtil.descendants(self, edgr, "xbrlFile") if instDocElt.get(edgrFile).endswith(".htm")] return self._htmURLs
def parameters(self): try: return self._parameters except AttributeError: self._parameters = dict([ (ModelValue.qname(paramElt, paramElt.get("name"),noPrefixIsNoNamespace=True), (ModelValue.qname(paramElt, paramElt.get("datatype")),paramElt.get("value"))) for paramElt in XmlUtil.descendants(self, self.namespaceURI, "parameter")]) return self._parameters
def htmURLs(self): try: return self._htmURLs except AttributeError: self._htmURLs = [ (instDocElt.getAttributeNS(edgr,"description"),instDocElt.getAttributeNS(edgr,"url")) for instDocElt in XmlUtil.descendants(self.element, edgr, "xbrlFile") if instDocElt.getAttributeNS(edgr,"file").endswith(".htm")] return self._htmURLs
def url(self): try: return self._url except AttributeError: self._url = None for instDocElt in XmlUtil.descendants(self.element, edgr, "xbrlFile"): if instDocElt.getAttributeNS(edgr,"type").endswith(".INS"): self._url = instDocElt.getAttributeNS(edgr,"url") break return self._url
def unit_denominator(xc, p, args): if len(args) != 1: raise XPathContext.FunctionNumArgs() if len(args[0]) != 1: raise XPathContext.FunctionArgType(1,"xbrl:unit") unit = args[0][0] if isinstance(unit,ModelObject) and \ unit.localName == "unit" and unit.namespaceURI == XbrlConst.xbrli: measuresParent = XmlUtil.descendant(unit, XbrlConst.xbrli, "unitDenominator") if measuresParent is None: return [] return XmlUtil.descendants(measuresParent, XbrlConst.xbrli, "measure") raise XPathContext.FunctionArgType(1,"xbrl:unit")
def url(self): try: return self._url except AttributeError: self._url = None for instDocElt in XmlUtil.descendants(self, edgr, "xbrlFile"): if instDocElt.get(edgrType).endswith(".INS"): self._url = instDocElt.get(edgrUrl) break return self._url
def blockedMessageCodes(self): blockedCodesRegex = XmlUtil.descendantAttr( self, None, "results", "blockedMessageCodes") # DQC 4/5 test suite if not blockedCodesRegex: ignoredCodes = XmlUtil.descendants( self, None, "ignore-error") # ESEF test suite if ignoredCodes: blockedCodesRegex = "|".join(".*" + c.stringValue for c in ignoredCodes) return blockedCodesRegex
def unit_numerator(xc, p, args): if len(args) != 1: raise XPathContext.FunctionNumArgs() if len(args[0]) != 1: raise XPathContext.FunctionArgType(1,"xbrl:unit") unit = args[0][0] if isinstance(unit,ModelObject.ModelObject): unit = unit.element if isinstance(unit,xml.dom.Node) and unit.nodeType == 1 and \ unit.localName == "unit" and unit.namespaceURI == XbrlConst.xbrli: measuresParent = XmlUtil.descendant(unit, XbrlConst.xbrli, "unitNumerator") if measuresParent is None: measuresParent = unit return XmlUtil.descendants(measuresParent, XbrlConst.xbrli, "measure") raise XPathContext.FunctionArgType(1,"xbrl:unit")
def primaryDocumentURL(self): try: return self._primaryDocumentURL except AttributeError: formType = self.formType self._primaryDocumentURL = None for instDocElt in XmlUtil.descendants(self, edgr, "xbrlFile"): if instDocElt.get(edgrType) == formType: self._primaryDocumentURL = instDocElt.get(edgrUrl) break return self._primaryDocumentURL
def url(self): try: return self._url except AttributeError: self._url = None for instDocElt in XmlUtil.descendants(self.element, edgr, "xbrlFile"): if instDocElt.getAttributeNS(edgr, "type").endswith(".INS"): self._url = instDocElt.getAttributeNS(edgr, "url") break return self._url
def primaryDocumentURL(self): try: return self._primaryDocumentURL except AttributeError: formType = self.formType self._primaryDocumentURL = None for instDocElt in XmlUtil.descendants(self, edgr, u"xbrlFile"): if instDocElt.get(edgrType) == formType: self._primaryDocumentURL = instDocElt.get(edgrUrl) break return self._primaryDocumentURL
def unit_denominator(xc, p, args): if len(args) != 1: raise XPathContext.FunctionNumArgs() if len(args[0]) != 1: raise XPathContext.FunctionArgType(1, "xbrl:unit") unit = args[0][0] if isinstance(unit, ModelObject.ModelObject): unit = unit.element if isinstance(unit,xml.dom.Node) and unit.nodeType == 1 and \ unit.localName == "unit" and unit.namespaceURI == XbrlConst.xbrli: measuresParent = XmlUtil.descendant(unit, XbrlConst.xbrli, "unitDenominator") if measuresParent is None: return [] return XmlUtil.descendants(measuresParent, XbrlConst.xbrli, "measure") raise XPathContext.FunctionArgType(1, "xbrl:unit")
def htmURLs(self): try: return self._htmURLs except AttributeError: self._htmURLs = [ (instDocElt.getAttributeNS(edgr, "description"), instDocElt.getAttributeNS(edgr, "url")) for instDocElt in XmlUtil.descendants(self.element, edgr, "xbrlFile") if instDocElt.getAttributeNS(edgr, "file").endswith(".htm") ] return self._htmURLs
def dir(self): self.open() if not self.isOpen: return None elif self.filesDir is not None: return self.filesDir elif self.isZip: files = [] for zipinfo in self.fs.infolist(): files.append(zipinfo.filename) self.filesDir = files elif self.isXfd: files = [] for data in self.xfdDocument.getElementsByTagName("data"): outfn = XmlUtil.text(data.getElementsByTagName("filename")[0]) if len(outfn) > 1: if len(outfn) > 2 and outfn[0].isalpha() and \ outfn[1] == ':' and outfn[2] == '\\': continue files.append(outfn); self.filesDir = files elif self.isRss: files = [] # return title, descr, pubdate, linst doc edgr = "http://www.sec.gov/Archives/edgar" try: for dsElt in self.rssDocument.getElementsByTagName("item"): instDoc = None for instDocElt in XmlUtil.descendants(dsElt, edgr, "xbrlFile"): if instDocElt.getAttributeNS(edgr,"description").endswith("INSTANCE DOCUMENT"): instDoc = instDocElt.getAttributeNS(edgr,"url") break if not instDoc: continue files.append(( XmlUtil.text(XmlUtil.descendant(dsElt, None, "title")), # tooltip "{0}\n {1}\n {2}\n {3}\n {4}".format( XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "companyName")), XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "formType")), XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "filingDate")), XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "cikNumber")), XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "period"))), XmlUtil.text(XmlUtil.descendant(dsElt, None, "description")), XmlUtil.text(XmlUtil.descendant(dsElt, None, "pubDate")), instDoc)) self.filesDir = files except (EnvironmentError, xml.parsers.expat.ExpatError, xml.dom.DOMException) as err: pass return self.filesDir
def cfcnCall(self): # tuple of (expression, element holding the expression) try: return self._cfcnCall except AttributeError: self._cfcnCall = None for callElement in XmlUtil.descendants(self, XbrlConst.cfcn, "call"): self._cfcnCall = (XmlUtil.innerText(callElement), callElement) break if self._cfcnCall is None and self.namespaceURI == "http://xbrl.org/2011/conformance-rendering/transforms": name = self.getparent().get("name") input = self.get("input") if name and input: self._cfcnCall = ("{0}('{1}')".format(name, input.replace("'","''")), self) return self._cfcnCall
def cfcnCall(self): # tuple of (expression, element holding the expression) try: return self._cfcnCall except AttributeError: self._cfcnCall = None for callElement in XmlUtil.descendants(self, XbrlConst.cfcn, "call"): self._cfcnCall = (XmlUtil.innerText(callElement), callElement) break if self._cfcnCall is None and self.namespaceURI == "http://xbrl.org/2011/conformance-rendering/transforms": name = self.getparent().get("name") input = self.get("input") if name and input: self._cfcnCall = ("{0}('{1}')".format( name, input.replace("'", "''")), self) return self._cfcnCall
def testcaseDiscover(self, testcaseElement): isTransformTestcase = testcaseElement.namespaceURI == "http://xbrl.org/2011/conformance-rendering/transforms" if XmlUtil.xmlnsprefix(testcaseElement, XbrlConst.cfcn) or isTransformTestcase: self.type = Type.REGISTRYTESTCASE self.testcaseVariations = [] priorTransformName = None for modelVariation in XmlUtil.descendants(testcaseElement, testcaseElement.namespaceURI, "variation"): self.testcaseVariations.append(modelVariation) if isTransformTestcase and modelVariation.getparent().get("name") is not None: transformName = modelVariation.getparent().get("name") if transformName != priorTransformName: priorTransformName = transformName variationNumber = 1 modelVariation._name = "{0} v-{1:02}".format(priorTransformName, variationNumber) variationNumber += 1 if len(self.testcaseVariations) == 0: # may be a inline test case if XbrlConst.ixbrl in testcaseElement.values(): self.testcaseVariations.append(testcaseElement)
def contextDiscover(self, cntxElement): id = cntxElement.getAttribute("id") self.modelXbrl.contexts[id] = modelContext = ModelObject.createContext(self,cntxElement) for container in (("segment", modelContext.segDimValues, modelContext.segNonDimValues), ("scenario", modelContext.scenDimValues, modelContext.scenNonDimValues)): containerName, containerDimValues, containerNonDimValues = container for containerElement in XmlUtil.descendants(cntxElement, XbrlConst.xbrli, containerName): for sElt in containerElement.childNodes: if sElt.nodeType == 1: if sElt.namespaceURI == XbrlConst.xbrldi and sElt.localName in ("explicitMember","typedMember"): XmlValidate.validate(self.modelXbrl, sElt) modelDimValue = ModelObject.createDimensionValue(self,sElt) dimension = modelDimValue.dimension if dimension and dimension not in containerDimValues: containerDimValues[dimension] = modelDimValue else: modelContext.errorDimValues.append(modelDimValue) modelContext.qnameDims[modelDimValue.dimensionQname] = modelDimValue # both seg and scen else: containerNonDimValues.append(sElt)
def checkFormulaRules(val, formula, nameVariables): from arelle.ModelFormulaObject import (Aspect) if not (formula.hasRule(Aspect.CONCEPT) or formula.source(Aspect.CONCEPT)): if XmlUtil.hasDescendant(formula.element, XbrlConst.formula, "concept"): val.modelXbrl.error(_("Formula {0} concept rule does not have a nearest source and does not have a child element").format(formula.xlinkLabel), "err", "xbrlfe:incompleteConceptRule") else: val.modelXbrl.error(_("Formula {0} omits a rule for the concept aspect").format(formula.xlinkLabel), "err", "xbrlfe:missingConceptRule") if (not (formula.hasRule(Aspect.SCHEME) or formula.source(Aspect.SCHEME)) or not (formula.hasRule(Aspect.VALUE) or formula.source(Aspect.VALUE))): if XmlUtil.hasDescendant(formula.element, XbrlConst.formula, "entityIdentifier"): val.modelXbrl.error(_("Formula {0} entity identifier rule does not have a nearest source and does not have either a @scheme or a @value attribute").format(formula.xlinkLabel), "err", "xbrlfe:incompleteEntityIdentifierRule") else: val.modelXbrl.error(_("Formula {0} omits a rule for the entity identifier aspect").format(formula.xlinkLabel), "err", "xbrlfe:missingEntityIdentifierRule") if not (formula.hasRule(Aspect.PERIOD_TYPE) or formula.source(Aspect.PERIOD_TYPE)): if XmlUtil.hasDescendant(formula.element, XbrlConst.formula, "period"): val.modelXbrl.error(_("Formula {0} period rule does not have a nearest source and does not have a child element").format(formula.xlinkLabel), "err", "xbrlfe:incompletePeriodRule") else: val.modelXbrl.error(_("Formula {0} omits a rule for the period aspect").format(formula.xlinkLabel), "err", "xbrlfe:missingPeriodRule") # for unit need to see if the qname is statically determinable to determine if numeric concept = val.modelXbrl.qnameConcepts.get(formula.evaluateRule(None, Aspect.CONCEPT)) if not concept: # is there a source with a static QName filter sourceFactVar = nameVariables.get(formula.source(Aspect.CONCEPT)) if isinstance(sourceFactVar, ModelFactVariable): for varFilterRels in (formula.groupFilterRelationships, sourceFactVar.filterRelationships): for varFilterRel in varFilterRels: filter = varFilterRel.toModelObject if isinstance(filter,ModelConceptName): # relationship not constrained to real filters for conceptQname in filter.conceptQnames: concept = val.modelXbrl.qnameConcepts.get(conceptQname) if concept and concept.isNumeric: break if concept: # from concept aspect rule or from source factVariable concept Qname filter if concept.isNumeric: if not (formula.hasRule(Aspect.MULTIPLY_BY) or formula.hasRule(Aspect.DIVIDE_BY) or formula.source(Aspect.UNIT)): if XmlUtil.hasDescendant(formula.element, XbrlConst.formula, "unit"): val.modelXbrl.error(_("Formula {0} unit rule does not have a source and does not have a child element").format(formula.xlinkLabel), "err", "xbrlfe:missingSAVForUnitRule") else: val.modelXbrl.error(_("Formula {0} omits a rule for the unit aspect").format(formula.xlinkLabel), "err", "xbrlfe:missingUnitRule") elif (formula.hasRule(Aspect.MULTIPLY_BY) or formula.hasRule(Aspect.DIVIDE_BY) or formula.source(Aspect.UNIT, acceptFormulaSource=False)): val.modelXbrl.error(_("Formula {0} has a rule for the unit aspect of a non-numeric concept {1}").format(formula.xlinkLabel, str(concept.qname)), "err", "xbrlfe:conflictingAspectRules") aspectPeriodType = formula.evaluateRule(None, Aspect.PERIOD_TYPE) if ((concept.periodType == "duration" and aspectPeriodType == "instant") or (concept.periodType == "instant" and aspectPeriodType in ("duration","forever"))): val.modelXbrl.error(_("Formula {0} has a rule for the {2} period aspect of a {3} concept {1}").format(formula.xlinkLabel, str(concept.qname), aspectPeriodType, concept.periodType), "err", "xbrlfe:conflictingAspectRules") # check dimension elements for eltName, dim, badUsageErr, missingSavErr in (("explicitDimension", "explicit", "xbrlfe:badUsageOfExplicitDimensionRule", "xbrlfe:missingSAVForExplicitDimensionRule"), ("typedDimension", "typed", "xbrlfe:badUsageOfTypedDimensionRule", "xbrlfe:missingSAVForTypedDimensionRule")): for dimElt in XmlUtil.descendants(formula.element, XbrlConst.formula, eltName): dimQname = qname(dimElt, dimElt.getAttribute("dimension")) dimConcept = val.modelXbrl.qnameConcepts.get(dimQname) if dimQname and (not dimConcept or (not dimConcept.isExplicitDimension if dim == "explicit" else not dimConcept.isTypedDimension)): val.modelXbrl.error(_("Formula {0} dimension attribute {1} on the {2} dimension rule contains a QName that does not identify an {2} dimension.").format(formula.xlinkLabel, dimQname, dim), "err", badUsageErr) elif not XmlUtil.hasChild(dimElt, XbrlConst.formula, "*") and not formula.source(Aspect.DIMENSIONS, dimElt): val.modelXbrl.error(_("Formula {0} {1} dimension rule does not have any child elements and does not have a SAV for the {2} dimension that is identified by its dimension attribute.").format(formula.xlinkLabel, dim, dimQname), "err", missingSavErr) # check aspect model expectations if formula.aspectModel == "non-dimensional": unexpectedElts = XmlUtil.descendants(formula.element, XbrlConst.formula, ("explicitDimension", "typedDimension")) if unexpectedElts: val.modelXbrl.error(_("Formula {0} aspect model, {1}, includes an rule for aspect not defined in this aspect model: {2}").format( formula.xlinkLabel, formula.aspectModel, ", ".join([elt.localName for elt in unexpectedElts])), "err", "xbrlfe:unrecognisedAspectRule") # check source qnames for sourceElt in ([formula.element] + XmlUtil.descendants(formula.element, XbrlConst.formula, "*", "source","*")): if sourceElt.hasAttribute("source"): qnSource = qname(sourceElt, sourceElt.getAttribute("source"), noPrefixIsNoNamespace=True) if qnSource == XbrlConst.qnFormulaUncovered: if formula.implicitFiltering != "true": val.modelXbrl.error(_("Formula {0}, not implicit filtering element has formulaUncovered source: {1}").format( formula.xlinkLabel, sourceElt.localName), "err", "xbrlfe:illegalUseOfUncoveredQName") elif qnSource not in nameVariables: val.modelXbrl.error(_("Variable set {0}, source {1} is not in the variable set").format( formula.xlinkLabel, qnSource), "err", "xbrlfe:nonexistentSourceVariable") else: factVariable = nameVariables.get(qnSource) if not isinstance(factVariable, ModelFactVariable): val.modelXbrl.error(_("Variable set {0}, source {1} not a factVariable but is a {2}").format( formula.xlinkLabel, qnSource, factVariable.localName), "err", "xbrlfe:nonexistentSourceVariable") elif factVariable.fallbackValue is not None: val.modelXbrl.error(_("Formula {0}: source {1} is a fact variable that has a fallback value").format(formula.xlinkLabel, str(qnSource)), "err", "xbrlfe:bindEmptySourceVariable") elif sourceElt.localName == "formula" and factVariable.bindAsSequence == "true": val.modelXbrl.error(_("Formula {0}: formula source {1} is a fact variable that binds as a sequence").format(formula.xlinkLabel, str(qnSource)), "err", "xbrlfe:defaultAspectValueConflicts")
def validateFacts(val, factsToCheck): # may be called in streaming batches or all at end (final) if not streaming modelXbrl = val.modelXbrl modelDocument = modelXbrl.modelDocument isStreamingMode = getattr(modelXbrl, "isStreamingMode", False) # note EBA 2.1 is in ModelDocument.py timelessDatePattern = re.compile(r"\s*([0-9]{4})-([0-9]{2})-([0-9]{2})\s*$") for cntx in modelXbrl.contexts.values(): if cntx is not None: if getattr(cntx, "_batchChecked", False) and isStreamingMode: continue # prior streaming batch already checked cntx._batchChecked = True val.cntxEntities.add(cntx.entityIdentifier) dateElts = XmlUtil.descendants(cntx, XbrlConst.xbrli, ("startDate","endDate","instant")) if any(not timelessDatePattern.match(e.textValue) for e in dateElts): modelXbrl.error(("EBA.2.10","EIOPA.2.10"), _('Period dates must be whole dates without time or timezone: %(dates)s.'), modelObject=cntx, dates=", ".join(e.text for e in dateElts)) if cntx.isForeverPeriod: modelXbrl.error(("EBA.2.11","EIOPA.N.2.11"), _('Forever context period is not allowed.'), modelObject=cntx) elif cntx.isStartEndPeriod: modelXbrl.error(("EBA.2.13","EIOPA.N.2.11"), _('Start-End (flow) context period is not allowed.'), modelObject=cntx) elif cntx.isInstantPeriod: # cannot pass context object to final() below, for error logging, if streaming mode val.cntxDates[cntx.instantDatetime].add(modelXbrl if getattr(val.modelXbrl, "isStreamingMode", False) else cntx) if cntx.hasSegment: modelXbrl.error(("EBA.2.14","EIOPA.N.2.14"), _("Contexts MUST NOT contain xbrli:segment values: %(cntx)s.'"), modelObject=cntx, cntx=cntx.id) if cntx.nonDimValues("scenario"): modelXbrl.error(("EBA.2.15","EIOPA.N.2.15"), _("Contexts MUST NOT contain non-dimensional xbrli:scenario values: %(cntx)s.'"), modelObject=cntx, cntx=cntx.id) val.unusedCntxIDs.add(cntx.id) for unit in modelXbrl.units.values(): if unit is None or (getattr(unit, "_batchChecked", False) and isStreamingMode): continue # prior streaming batch already checked unit._batchChecked = True val.unusedUnitIDs.add(unit.id) factsByQname = defaultdict(set) # top level for this for f in factsToCheck: factsByQname[f.qname].add(f) val.unusedCntxIDs.discard(f.contextID) val.unusedUnitIDs.discard(f.unitID) if f.objectIndex < val.firstFactObjectIndex: val.firstFactObjectIndex = f.objectIndex val.firstFact = f for fIndicators in factsByQname[qnFIndicators]: val.numFilingIndicatorTuples += 1 for fIndicator in fIndicators.modelTupleFacts: _value = (getattr(fIndicator, "xValue", None) or fIndicator.value) # use validated xValue if DTS else value for skipDTS if _value in val.filingIndicators: modelXbrl.error(("EBA.1.6.1", "EIOPA.1.6.1"), _('Multiple filing indicators facts for indicator %(filingIndicator)s.'), modelObject=(fIndicator, val.filingIndicators[_value]), filingIndicator=_value) val.filingIndicators[_value] = fIndicator.get("{http://www.eurofiling.info/xbrl/ext/filing-indicators}filed", "true") in ("true", "1") val.unusedCntxIDs.discard(fIndicator.contextID) cntx = fIndicator.context if cntx is not None and (cntx.hasSegment or cntx.hasScenario): modelXbrl.error("EIOPA.S.1.6.d", _('Filing indicators must not contain segment or scenario elements %(filingIndicator)s.'), modelObject=fIndicator, filingIndicator=_value) if fIndicators.objectIndex > val.firstFactObjectIndex: modelXbrl.warning("EIOPA.1.6.2", _('Filing indicators should precede first fact %(firstFact)s.'), modelObject=(fIndicators, val.firstFact), firstFact=val.firstFact.qname) otherFacts = {} # (contextHash, unitHash, xmlLangHash) : fact nilFacts = [] # removed in current draft: stringFactsWithoutXmlLang = [] nonMonetaryNonPureFacts = [] for qname, facts in factsByQname.items(): for f in facts: if modelXbrl.skipDTS: c = f.qname.localName[0] isNumeric = c in ('m', 'p', 'i') isMonetary = c == 'm' isInteger = c == 'i' isPercent = c == 'p' isString = c == 's' isEnum = c == 'e' else: concept = f.concept if concept is not None: isNumeric = concept.isNumeric isMonetary = concept.isMonetary isInteger = concept.baseXbrliType in integerItemTypes isPercent = concept.typeQname in (qnPercentItemType, qnPureItemType) isString = concept.baseXbrliType in ("stringItemType", "normalizedStringItemType") isEnum = concept.typeQname == qnEnumerationItemType else: isNumeric = isString = isEnum = False # error situation k = (f.getparent().objectIndex, f.qname, f.context.contextDimAwareHash if f.context is not None else None, f.unit.hash if f.unit is not None else None, hash(f.xmlLang)) if f.qname == qnFIndicators and val.validateEIOPA: pass elif k not in otherFacts: otherFacts[k] = {f} else: matches = [o for o in otherFacts[k] if (f.getparent().objectIndex == o.getparent().objectIndex and f.qname == o.qname and f.context.isEqualTo(o.context) if f.context is not None and o.context is not None else True) and # (f.unit.isEqualTo(o.unit) if f.unit is not None and o.unit is not None else True) and (f.xmlLang == o.xmlLang)] if matches: contexts = [f.contextID] + [o.contextID for o in matches] modelXbrl.error(("EBA.2.16", "EIOPA.S.2.16.a"), _('Facts are duplicates %(fact)s contexts %(contexts)s.'), modelObject=[f] + matches, fact=f.qname, contexts=', '.join(contexts)) else: otherFacts[k].add(f) if isNumeric: if f.precision: modelXbrl.error(("EBA.2.17", "EIOPA.2.18.a"), _("Numeric fact %(fact)s of context %(contextID)s has a precision attribute '%(precision)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, precision=f.precision) if f.decimals and not f.isNil: if f.decimals == "INF": modelXbrl.error("EIOPA.S.2.18.f", _("Monetary fact %(fact)s of context %(contextID)s has a decimal attribute INF: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) else: try: xValue = f.xValue dec = int(f.decimals) # will only work if getattr(f,"xValid", XmlValidate.UNVALIDATED) >= XmlValidate.VALID: if isMonetary: if dec < -3: modelXbrl.error(("EBA.2.18","EIOPA.S.2.18.c"), _("Monetary fact %(fact)s of context %(contextID)s has a decimals attribute < -3: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) else: # apply dynamic decimals check if -.1 < xValue < .1: dMin = 2 elif -1 < xValue < 1: dMin = 1 elif -10 < xValue < 10: dMin = 0 elif -100 < xValue < 100: dMin = -1 elif -1000 < xValue < 1000: dMin = -2 else: dMin = -3 if dMin > dec: modelXbrl.warning("EIOPA:factDecimalsWarning", _("Monetary fact %(fact)s of context %(contextID)s value %(value)s has an imprecise decimals attribute: %(decimals)s, minimum is %(mindec)s"), modelObject=f, fact=f.qname, contextID=f.contextID, value=xValue, decimals=f.decimals, mindec=dMin) elif isInteger: if dec != 0: modelXbrl.error(("EBA.2.18","EIOPA.S.2.18.d"), _("Integer fact %(fact)s of context %(contextID)s has a decimals attribute \u2260 0: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) elif isPercent: if dec < 4: modelXbrl.error(("EBA.2.18","EIOPA.S.2.18.e"), _("Percent fact %(fact)s of context %(contextID)s has a decimals attribute < 4: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) else: if -.001 < xValue < .001: dMin = 4 elif -.01 < xValue < .01: dMin = 3 elif -.1 < xValue < .1: dMin = 2 elif -1 < xValue < 1: dMin = 1 else: dMin = 0 if dMin > dec: modelXbrl.warning("EIOPA:factDecimalsWarning", _("Numeric fact %(fact)s of context %(contextID)s value %(value)s has an imprecise decimals attribute: %(decimals)s, minimum is %(mindec)s"), modelObject=f, fact=f.qname, contextID=f.contextID, value=xValue, decimals=f.decimals, mindec=dMin) except (AttributeError, ValueError, TypeError): pass # should have been reported as a schema error by loader '''' (not intended by EBA 2.18, paste here is from EFM) if not f.isNil and getattr(f,"xValid", 0) == 4: try: insignificance = insignificantDigits(f.xValue, decimals=f.decimals) if insignificance: # if not None, returns (truncatedDigits, insiginficantDigits) modelXbrl.error(("EFM.6.05.37", "GFM.1.02.26"), _("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s has nonzero digits in insignificant portion %(insignificantDigits)s."), modelObject=f1, fact=f1.qname, contextID=f1.contextID, decimals=f1.decimals, value=f1.xValue, truncatedDigits=insignificance[0], insignificantDigits=insignificance[1]) except (ValueError,TypeError): modelXbrl.error(("EBA.2.18"), _("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s causes Value Error exception."), modelObject=f1, fact=f1.qname, contextID=f1.contextID, decimals=f1.decimals, value=f1.value) ''' unit = f.unit if unit is not None: if isMonetary: if unit.measures[0]: val.currenciesUsed[unit.measures[0][0]] = unit elif not unit.isSingleMeasure or unit.measures[0][0] != XbrlConst.qnXbrliPure: nonMonetaryNonPureFacts.append(f) if isEnum: _eQn = getattr(f,"xValue", None) or qnameEltPfxName(f, f.value) if _eQn: val.namespacePrefixesUsed[_eQn.namespaceURI].add(_eQn.prefix) val.prefixesUnused.discard(_eQn.prefix) ''' removed in current draft elif isString: if not f.xmlLang: stringFactsWithoutXmlLang.append(f) ''' if f.isNil: nilFacts.append(f) if val.footnotesRelationshipSet.fromModelObject(f): modelXbrl.warning("EIOPA.S.19", _("Fact %(fact)s of context %(contextID)s has footnotes.'"), modelObject=f, fact=f.qname, contextID=f.contextID) if nilFacts: nilFactNames = [str(f.qname) for f in nilFacts] nilFactNames = sorted(nilFactNames) modelXbrl.error(("EBA.2.19", "EIOPA.S.2.19"), _('Nil facts MUST NOT be present in the instance: %(nilFacts)s.'), modelObject=nilFacts, nilFacts=", ".join(fname for fname in nilFactNames)) ''' removed in current draft if stringFactsWithoutXmlLang: modelXbrl.error("EBA.2.20", _("String facts need to report xml:lang: '%(langLessFacts)s'"), modelObject=stringFactsWithoutXmlLang, langLessFacts=", ".join(set(str(f.qname) for f in stringFactsWithoutXmlLang))) ''' if nonMonetaryNonPureFacts: modelXbrl.error(("EBA.3.2","EIOPA.3.2.a"), _("Non monetary (numeric) facts MUST use the pure unit: '%(langLessFacts)s'"), modelObject=nonMonetaryNonPureFacts, langLessFacts=", ".join(set(str(f.qname) for f in nonMonetaryNonPureFacts))) val.utrValidator.validateFacts() # validate facts for UTR at logLevel WARNING unitHashes = {} for unit in modelXbrl.units.values(): if unit is not None: h = unit.hash if h in unitHashes and unit.isEqualTo(unitHashes[h]): modelXbrl.warning("EBA.2.21", _("Duplicate units SHOULD NOT be reported, units %(unit1)s and %(unit2)s have same measures.'"), modelObject=(unit, unitHashes[h]), unit1=unit.id, unit2=unitHashes[h].id) if not getattr(modelXbrl, "isStreamingMode", False): modelXbrl.error("EIOPA.2.21", _("Duplicate units MUST NOT be reported, units %(unit1)s and %(unit2)s have same measures.'"), modelObject=(unit, unitHashes[h]), unit1=unit.id, unit2=unitHashes[h].id) else: unitHashes[h] = unit for _measures in unit.measures: for _measure in _measures: val.namespacePrefixesUsed[_measure.namespaceURI].add(_measure.prefix) val.prefixesUnused.discard(_measure.prefix) del unitHashes cntxHashes = {} for cntx in modelXbrl.contexts.values(): if cntx is not None: h = cntx.contextDimAwareHash if h in cntxHashes and cntx.isEqualTo(cntxHashes[h]): if not getattr(modelXbrl, "isStreamingMode", False): modelXbrl.error("EIOPA.S.2.7.b", _("Duplicate contexts MUST NOT be reported, contexts %(cntx1)s and %(cntx2)s are equivalent.'"), modelObject=(cntx, cntxHashes[h]), cntx1=cntx.id, cntx2=cntxHashes[h].id) else: cntxHashes[h] = cntx for _dim in cntx.qnameDims.values(): _dimQn = _dim.dimensionQname val.namespacePrefixesUsed[_dimQn.namespaceURI].add(_dimQn.prefix) val.prefixesUnused.discard(_dimQn.prefix) if _dim.isExplicit: _memQn = _dim.memberQname else: _memQn = _dim.typedMember.qname if _memQn: val.namespacePrefixesUsed[_memQn.namespaceURI].add(_memQn.prefix) val.prefixesUnused.discard(_memQn.prefix) for elt in modelDocument.xmlRootElement.iter(): if isinstance(elt, ModelObject): # skip comments and processing instructions val.namespacePrefixesUsed[elt.qname.namespaceURI].add(elt.qname.prefix) val.prefixesUnused.discard(elt.qname.prefix) for attrTag in elt.keys(): if attrTag.startswith("{"): _prefix, _NS, _localName = XmlUtil.clarkNotationToPrefixNsLocalname(elt, attrTag, isAttribute=True) if _prefix: val.namespacePrefixesUsed[_NS].add(_prefix) val.prefixesUnused.discard(_prefix)
def resultIsTaxonomyPackage(self): return any(e.localName for e in XmlUtil.descendants(self,None,TXMY_PKG_SRC_ELTS))
def stepAxis(self, op, p, sourceSequence): targetSequence = [] for node in sourceSequence: if not isinstance( node, (ModelObject, etree._ElementTree, ModelAttribute)): raise XPathException( self.progHeader, 'err:XPTY0020', _('Axis step {0} context item is not a node: {1}').format( op, node)) targetNodes = [] if isinstance(p, QNameDef): ns = p.namespaceURI localname = p.localName if p.isAttribute: if isinstance(node, ModelObject): attrTag = p.localName if p.unprefixed else p.clarkNotation modelAttribute = None try: modelAttribute = node.xAttributes[attrTag] except (AttributeError, TypeError, IndexError, KeyError): # may be lax or deferred validated try: validate(node.modelXbrl, node, p) modelAttribute = node.xAttributes[attrTag] except (AttributeError, TypeError, IndexError, KeyError): pass if modelAttribute is None: value = node.get(attrTag) if value is not None: targetNodes.append( ModelAttribute(node, p.clarkNotation, UNKNOWN, value, value, value)) elif modelAttribute.xValid >= VALID: targetNodes.append(modelAttribute) elif op == '/' or op is None: if isinstance(node, (ModelObject, etree._ElementTree)): targetNodes = XmlUtil.children(node, ns, localname) elif op == '//': if isinstance(node, (ModelObject, etree._ElementTree)): targetNodes = XmlUtil.descendants(node, ns, localname) elif op == '..': if isinstance(node, ModelAttribute): targetNodes = [node.modelElement] else: targetNodes = [XmlUtil.parent(node)] elif isinstance(p, OperationDef) and isinstance(p.name, QNameDef): if isinstance(node, ModelObject): if p.name.localName == "text": targetNodes = [XmlUtil.text(node)] # todo: add element, attribute, node, etc... elif p == '*': # wildcard if op == '/' or op is None: if isinstance(node, (ModelObject, etree._ElementTree)): targetNodes = XmlUtil.children(node, '*', '*') elif op == '//': if isinstance(node, (ModelObject, etree._ElementTree)): targetNodes = XmlUtil.descendants(node, '*', '*') targetSequence.extend(targetNodes) return targetSequence
def expected(self): for pluginXbrlMethod in pluginClassMethods( "ModelTestcaseVariation.ExpectedResult"): expected = pluginXbrlMethod(self) if expected: return expected # default behavior without plugins if self.localName == "testcase": return self.document.basename[:4] #starts with PASS or FAIL elif self.localName == "testGroup": #w3c testcase instanceTestElement = XmlUtil.descendant(self, None, "instanceTest") if instanceTestElement is not None: # take instance first return XmlUtil.descendantAttr(instanceTestElement, None, "expected", "validity") else: schemaTestElement = XmlUtil.descendant(self, None, "schemaTest") if schemaTestElement is not None: return XmlUtil.descendantAttr(schemaTestElement, None, "expected", "validity") resultElement = XmlUtil.descendant(self, None, "result") if resultElement is not None: expected = resultElement.get("expected") if expected and resultElement.get( "nonStandardErrorCodes") == "true": # if @expected and @nonStandardErrorCodes then use expected instead of error codes return expected errorElements = XmlUtil.descendants(self, None, "error") resultElement = XmlUtil.descendant(self, None, "result") if isinstance(errorElements, list) and len(errorElements) > 0: return [ ModelValue.qname(e, e.stringValue) if not e.get("nonStandardErrorCodes") else e.stringValue for e in errorElements ] #else: # errorElement = errorElements[1] # if errorElement is not None and not errorElement.get("nonStandardErrorCodes"): # _errorText = XmlUtil.text(errorElement) # if ' ' in _errorText: # list of tokens # return _errorText # return ModelValue.qname(errorElement, _errorText) # turn into a QName if resultElement is not None: if expected: return expected for assertElement in XmlUtil.children(resultElement, None, "assert"): num = assertElement.get("num") if num == "99999": # inline test, use name as expected return assertElement.get("name") if len(num) == 5: return "EFM.{0}.{1}.{2}".format(num[0], num[1:3], num[3:6]) asserTests = {} for atElt in XmlUtil.children(resultElement, None, "assertionTests"): try: asserTests[atElt.get("assertionID")] = ( _INT(atElt.get("countSatisfied")), _INT(atElt.get("countNotSatisfied"))) except ValueError: pass if asserTests: return asserTests elif self.get("result"): return self.get("result") return None
def dir(self): self.open() if not self.isOpen: return None elif self.filesDir is not None: return self.filesDir elif self.isZip: files = [] for zipinfo in self.fs.infolist(): files.append(zipinfo.filename) self.filesDir = files elif self.isTarGz: self.filesDir = self.fs.getnames() elif self.isEis: files = [] for docElt in self.eisDocument.iter(tag="{http://www.sec.gov/edgar/common}document"): outfn = docElt.findtext("{http://www.sec.gov/edgar/common}conformedName") if outfn: files.append(outfn); self.filesDir = files elif self.isXfd: files = [] for data in self.xfdDocument.iter(tag="data"): outfn = data.findtext("filename") if outfn: if len(outfn) > 2 and outfn[0].isalpha() and \ outfn[1] == ':' and outfn[2] == '\\': continue files.append(outfn); self.filesDir = files elif self.isRss: files = [] # return title, descr, pubdate, linst doc edgr = "http://www.sec.gov/Archives/edgar" try: for dsElt in XmlUtil.descendants(self.rssDocument, None, "item"): instDoc = None for instDocElt in XmlUtil.descendants(dsElt, edgr, "xbrlFile"): if instDocElt.get("(http://www.sec.gov/Archives/edgar}description").endswith("INSTANCE DOCUMENT"): instDoc = instDocElt.get("(http://www.sec.gov/Archives/edgar}url") break if not instDoc: continue files.append(( XmlUtil.text(XmlUtil.descendant(dsElt, None, "title")), # tooltip "{0}\n {1}\n {2}\n {3}\n {4}".format( XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "companyName")), XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "formType")), XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "filingDate")), XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "cikNumber")), XmlUtil.text(XmlUtil.descendant(dsElt, edgr, "period"))), XmlUtil.text(XmlUtil.descendant(dsElt, None, "description")), XmlUtil.text(XmlUtil.descendant(dsElt, None, "pubDate")), instDoc)) self.filesDir = files except (EnvironmentError, etree.LxmlError) as err: pass elif self.isInstalledTaxonomyPackage: files = [] baseurlPathLen = len(os.path.dirname(self.baseurl)) + 1 def packageDirsFiles(dir): for file in os.listdir(dir): path = dir + "/" + file # must have / and not \\ even on windows files.append(path[baseurlPathLen:]) if os.path.isdir(path): packageDirsFiles(path) packageDirsFiles(self.baseurl[0:baseurlPathLen - 1]) self.filesDir = files return self.filesDir
def dir(self): self.open() if not self.isOpen: return None elif self.filesDir is not None: return self.filesDir elif self.isZip: files = [] for zipinfo in self.fs.infolist(): files.append(zipinfo.filename) self.filesDir = files elif self.isTarGz: self.filesDir = self.fs.getnames() elif self.isEis: files = [] for docElt in self.eisDocument.iter( tag="{http://www.sec.gov/edgar/common}document"): outfn = docElt.findtext( "{http://www.sec.gov/edgar/common}conformedName") if outfn: files.append(outfn) self.filesDir = files elif self.isXfd: files = [] for data in self.xfdDocument.iter(tag="data"): outfn = data.findtext("filename") if outfn: if len(outfn) > 2 and outfn[0].isalpha() and \ outfn[1] == ':' and outfn[2] == '\\': continue files.append(outfn) self.filesDir = files elif self.isRss: files = [] # return title, descr, pubdate, linst doc edgr = "http://www.sec.gov/Archives/edgar" try: for dsElt in XmlUtil.descendants(self.rssDocument, None, "item"): instDoc = None for instDocElt in XmlUtil.descendants( dsElt, edgr, "xbrlFile"): if instDocElt.get( "(http://www.sec.gov/Archives/edgar}description" ).endswith("INSTANCE DOCUMENT"): instDoc = instDocElt.get( "(http://www.sec.gov/Archives/edgar}url") break if not instDoc: continue files.append(( XmlUtil.text(XmlUtil.descendant(dsElt, None, "title")), # tooltip "{0}\n {1}\n {2}\n {3}\n {4}".format( XmlUtil.text( XmlUtil.descendant(dsElt, edgr, "companyName")), XmlUtil.text( XmlUtil.descendant(dsElt, edgr, "formType")), XmlUtil.text( XmlUtil.descendant(dsElt, edgr, "filingDate")), XmlUtil.text( XmlUtil.descendant(dsElt, edgr, "cikNumber")), XmlUtil.text( XmlUtil.descendant(dsElt, edgr, "period"))), XmlUtil.text( XmlUtil.descendant(dsElt, None, "description")), XmlUtil.text(XmlUtil.descendant( dsElt, None, "pubDate")), instDoc)) self.filesDir = files except (EnvironmentError, etree.LxmlError) as err: pass elif self.isInstalledTaxonomyPackage: files = [] baseurlPathLen = len(os.path.dirname(self.baseurl)) + 1 def packageDirsFiles(dir): for file in os.listdir(dir): path = dir + "/" + file # must have / and not \\ even on windows files.append(path[baseurlPathLen:]) if os.path.isdir(path): packageDirsFiles(path) packageDirsFiles(self.baseurl[0:baseurlPathLen - 1]) self.filesDir = files return self.filesDir
def checkFormulaRules(val, formula, nameVariables): from arelle.ModelFormulaObject import (Aspect) if not (formula.hasRule(Aspect.CONCEPT) or formula.source(Aspect.CONCEPT)): if XmlUtil.hasDescendant(formula.element, XbrlConst.formula, "concept"): val.modelXbrl.error( _("Formula {0} concept rule does not have a nearest source and does not have a child element" ).format(formula.xlinkLabel), "err", "xbrlfe:incompleteConceptRule") else: val.modelXbrl.error( _("Formula {0} omits a rule for the concept aspect").format( formula.xlinkLabel), "err", "xbrlfe:missingConceptRule") if (not (formula.hasRule(Aspect.SCHEME) or formula.source(Aspect.SCHEME)) or not (formula.hasRule(Aspect.VALUE) or formula.source(Aspect.VALUE))): if XmlUtil.hasDescendant(formula.element, XbrlConst.formula, "entityIdentifier"): val.modelXbrl.error( _("Formula {0} entity identifier rule does not have a nearest source and does not have either a @scheme or a @value attribute" ).format(formula.xlinkLabel), "err", "xbrlfe:incompleteEntityIdentifierRule") else: val.modelXbrl.error( _("Formula {0} omits a rule for the entity identifier aspect" ).format(formula.xlinkLabel), "err", "xbrlfe:missingEntityIdentifierRule") if not (formula.hasRule(Aspect.PERIOD_TYPE) or formula.source(Aspect.PERIOD_TYPE)): if XmlUtil.hasDescendant(formula.element, XbrlConst.formula, "period"): val.modelXbrl.error( _("Formula {0} period rule does not have a nearest source and does not have a child element" ).format(formula.xlinkLabel), "err", "xbrlfe:incompletePeriodRule") else: val.modelXbrl.error( _("Formula {0} omits a rule for the period aspect").format( formula.xlinkLabel), "err", "xbrlfe:missingPeriodRule") # for unit need to see if the qname is statically determinable to determine if numeric concept = val.modelXbrl.qnameConcepts.get( formula.evaluateRule(None, Aspect.CONCEPT)) if not concept: # is there a source with a static QName filter sourceFactVar = nameVariables.get(formula.source(Aspect.CONCEPT)) if isinstance(sourceFactVar, ModelFactVariable): for varFilterRels in (formula.groupFilterRelationships, sourceFactVar.filterRelationships): for varFilterRel in varFilterRels: filter = varFilterRel.toModelObject if isinstance( filter, ModelConceptName ): # relationship not constrained to real filters for conceptQname in filter.conceptQnames: concept = val.modelXbrl.qnameConcepts.get( conceptQname) if concept and concept.isNumeric: break if concept: # from concept aspect rule or from source factVariable concept Qname filter if concept.isNumeric: if not (formula.hasRule(Aspect.MULTIPLY_BY) or formula.hasRule( Aspect.DIVIDE_BY) or formula.source(Aspect.UNIT)): if XmlUtil.hasDescendant(formula.element, XbrlConst.formula, "unit"): val.modelXbrl.error( _("Formula {0} unit rule does not have a source and does not have a child element" ).format(formula.xlinkLabel), "err", "xbrlfe:missingSAVForUnitRule") else: val.modelXbrl.error( _("Formula {0} omits a rule for the unit aspect" ).format(formula.xlinkLabel), "err", "xbrlfe:missingUnitRule") elif (formula.hasRule(Aspect.MULTIPLY_BY) or formula.hasRule(Aspect.DIVIDE_BY) or formula.source(Aspect.UNIT, acceptFormulaSource=False)): val.modelXbrl.error( _("Formula {0} has a rule for the unit aspect of a non-numeric concept {1}" ).format(formula.xlinkLabel, str(concept.qname)), "err", "xbrlfe:conflictingAspectRules") aspectPeriodType = formula.evaluateRule(None, Aspect.PERIOD_TYPE) if ((concept.periodType == "duration" and aspectPeriodType == "instant") or (concept.periodType == "instant" and aspectPeriodType in ("duration", "forever"))): val.modelXbrl.error( _("Formula {0} has a rule for the {2} period aspect of a {3} concept {1}" ).format(formula.xlinkLabel, str(concept.qname), aspectPeriodType, concept.periodType), "err", "xbrlfe:conflictingAspectRules") # check dimension elements for eltName, dim, badUsageErr, missingSavErr in ( ("explicitDimension", "explicit", "xbrlfe:badUsageOfExplicitDimensionRule", "xbrlfe:missingSAVForExplicitDimensionRule"), ("typedDimension", "typed", "xbrlfe:badUsageOfTypedDimensionRule", "xbrlfe:missingSAVForTypedDimensionRule")): for dimElt in XmlUtil.descendants(formula.element, XbrlConst.formula, eltName): dimQname = qname(dimElt, dimElt.getAttribute("dimension")) dimConcept = val.modelXbrl.qnameConcepts.get(dimQname) if dimQname and ( not dimConcept or (not dimConcept.isExplicitDimension if dim == "explicit" else not dimConcept.isTypedDimension)): val.modelXbrl.error( _("Formula {0} dimension attribute {1} on the {2} dimension rule contains a QName that does not identify an {2} dimension." ).format(formula.xlinkLabel, dimQname, dim), "err", badUsageErr) elif not XmlUtil.hasChild(dimElt, XbrlConst.formula, "*") and not formula.source( Aspect.DIMENSIONS, dimElt): val.modelXbrl.error( _("Formula {0} {1} dimension rule does not have any child elements and does not have a SAV for the {2} dimension that is identified by its dimension attribute." ).format(formula.xlinkLabel, dim, dimQname), "err", missingSavErr) # check aspect model expectations if formula.aspectModel == "non-dimensional": unexpectedElts = XmlUtil.descendants( formula.element, XbrlConst.formula, ("explicitDimension", "typedDimension")) if unexpectedElts: val.modelXbrl.error( _("Formula {0} aspect model, {1}, includes an rule for aspect not defined in this aspect model: {2}" ).format( formula.xlinkLabel, formula.aspectModel, ", ".join([elt.localName for elt in unexpectedElts])), "err", "xbrlfe:unrecognisedAspectRule") # check source qnames for sourceElt in ([formula.element] + XmlUtil.descendants( formula.element, XbrlConst.formula, "*", "source", "*")): if sourceElt.hasAttribute("source"): qnSource = qname(sourceElt, sourceElt.getAttribute("source"), noPrefixIsNoNamespace=True) if qnSource == XbrlConst.qnFormulaUncovered: if formula.implicitFiltering != "true": val.modelXbrl.error( _("Formula {0}, not implicit filtering element has formulaUncovered source: {1}" ).format(formula.xlinkLabel, sourceElt.localName), "err", "xbrlfe:illegalUseOfUncoveredQName") elif qnSource not in nameVariables: val.modelXbrl.error( _("Variable set {0}, source {1} is not in the variable set" ).format(formula.xlinkLabel, qnSource), "err", "xbrlfe:nonexistentSourceVariable") else: factVariable = nameVariables.get(qnSource) if not isinstance(factVariable, ModelFactVariable): val.modelXbrl.error( _("Variable set {0}, source {1} not a factVariable but is a {2}" ).format(formula.xlinkLabel, qnSource, factVariable.localName), "err", "xbrlfe:nonexistentSourceVariable") elif factVariable.fallbackValue is not None: val.modelXbrl.error( _("Formula {0}: source {1} is a fact variable that has a fallback value" ).format(formula.xlinkLabel, str(qnSource)), "err", "xbrlfe:bindEmptySourceVariable") elif sourceElt.localName == "formula" and factVariable.bindAsSequence == "true": val.modelXbrl.error( _("Formula {0}: formula source {1} is a fact variable that binds as a sequence" ).format(formula.xlinkLabel, str(qnSource)), "err", "xbrlfe:defaultAspectValueConflicts")
def stepAxis(self, op, p, sourceSequence): targetSequence = [] for node in sourceSequence: if not isinstance(node,(ModelObject, etree._ElementTree, ModelAttribute)): raise XPathException(self.progHeader, 'err:XPTY0020', _('Axis step {0} context item is not a node: {1}').format(op, node)) targetNodes = [] if isinstance(p,QNameDef): ns = p.namespaceURI; localname = p.localName; axis = p.axis if p.isAttribute: if isinstance(node,ModelObject): attrTag = p.localName if p.unprefixed else p.clarkNotation modelAttribute = None try: modelAttribute = node.xAttributes[attrTag] except (AttributeError, TypeError, IndexError, KeyError): # may be lax or deferred validated try: xmlValidate(node.modelXbrl, node, p) modelAttribute = node.xAttributes[attrTag] except (AttributeError, TypeError, IndexError, KeyError): pass if modelAttribute is None: value = node.get(attrTag) if value is not None: targetNodes.append(ModelAttribute(node,p.clarkNotation,UNKNOWN,value,value,value)) elif modelAttribute.xValid >= VALID: targetNodes.append(modelAttribute) elif op == '/' or op is None: if axis is None or axis == "child": if isinstance(node,(ModelObject, etree._ElementTree)): targetNodes = XmlUtil.children(node, ns, localname) elif axis == "parent": if isinstance(node,ModelAttribute): parentNode = [ node.modelElement ] else: parentNode = [ XmlUtil.parent(node) ] if (isinstance(node,ModelObject) and (not ns or ns == parentNode.namespaceURI or ns == "*") and (localname == parentNode.localName or localname == "*")): targetNodes = [ parentNode ] elif axis == "self": if (isinstance(node,ModelObject) and (not ns or ns == node.namespaceURI or ns == "*") and (localname == node.localName or localname == "*")): targetNodes = [ node ] elif axis.startswith("descendant"): if isinstance(node,(ModelObject, etree._ElementTree)): targetNodes = XmlUtil.descendants(node, ns, localname) if (axis.endswith("-or-self") and isinstance(node,ModelObject) and (not ns or ns == node.namespaceURI or ns == "*") and (localname == node.localName or localname == "*")): targetNodes.append(node) elif axis.startswith("ancestor"): if isinstance(node,ModelObject): targetNodes = [ancestor for ancestor in XmlUtil.ancestors(node) if ((not ns or ns == ancestor.namespaceURI or ns == "*") and (localname == ancestor.localName or localname == "*"))] if (axis.endswith("-or-self") and isinstance(node,ModelObject) and (not ns or ns == node.namespaceURI or ns == "*") and (localname == node.localName or localname == "*")): targetNodes.insert(0, node) elif axis.endswith("-sibling"): if isinstance(node,ModelObject): targetNodes = [sibling for sibling in node.itersiblings(preceding=axis.startswith("preceding")) if ((not ns or ns == sibling.namespaceURI or ns == "*") and (localname == sibling.localName or localname == "*"))] elif axis == "preceding": if isinstance(node,ModelObject): for preceding in node.getroottree().iter(): if preceding == node: break elif ((not ns or ns == preceding.namespaceURI or ns == "*") and (localname == preceding.localName or localname == "*")): targetNodes.append(preceding) elif axis == "following": if isinstance(node,ModelObject): foundNode = False for following in node.getroottree().iter(): if following == node: foundNode = True elif (foundNode and (not ns or ns == following.namespaceURI or ns == "*") and (localname == following.localName or localname == "*")): targetNodes.append(following) elif op == '//': if isinstance(node,(ModelObject, etree. _ElementTree)): targetNodes = XmlUtil.descendants(node, ns, localname) elif op == '..': if isinstance(node,ModelAttribute): targetNodes = [ node.modelElement ] else: targetNodes = [ XmlUtil.parent(node) ] elif isinstance(p, OperationDef) and isinstance(p.name,QNameDef): if isinstance(node,ModelObject): if p.name.localName == "text": # note this is not string value, just child text targetNodes = [node.textValue] # todo: add element, attribute, node, etc... elif p == '*': # wildcard if op == '/' or op is None: if isinstance(node,(ModelObject, etree._ElementTree)): targetNodes = XmlUtil.children(node, '*', '*') elif op == '//': if isinstance(node,(ModelObject, etree._ElementTree)): targetNodes = XmlUtil.descendants(node, '*', '*') targetSequence.extend(targetNodes) return targetSequence
def validateFacts(val, factsToCheck): # may be called in streaming batches or all at end (final) if not streaming modelXbrl = val.modelXbrl modelDocument = modelXbrl.modelDocument # note EBA 2.1 is in ModelDocument.py timelessDatePattern = re.compile( r"\s*([0-9]{4})-([0-9]{2})-([0-9]{2})\s*$") for cntx in modelXbrl.contexts.values(): if getattr(cntx, "_batchChecked", False): continue # prior streaming batch already checked cntx._batchChecked = True val.cntxEntities.add(cntx.entityIdentifier) dateElts = XmlUtil.descendants(cntx, XbrlConst.xbrli, ("startDate", "endDate", "instant")) if any(not timelessDatePattern.match(e.textValue) for e in dateElts): modelXbrl.error( "EBA.2.10", _('Period dates must be whole dates without time or timezone: %(dates)s.' ), modelObject=cntx, dates=", ".join(e.text for e in dateElts)) if cntx.isForeverPeriod: modelXbrl.error("EBA.2.11", _('Forever context period is not allowed.'), modelObject=cntx) elif cntx.isStartEndPeriod: modelXbrl.error( "EBA.2.13", _('Start-End (flow) context period is not allowed.'), modelObject=cntx) elif cntx.isInstantPeriod: # cannot pass context object to final() below, for error logging, if streaming mode val.cntxDates[cntx.instantDatetime].add(modelXbrl if getattr( val.modelXbrl, "isStreamingMode", False) else cntx) if XmlUtil.hasChild(cntx, XbrlConst.xbrli, "segment"): modelXbrl.error( "EBA.2.14", _("The segment element not allowed in context Id: %(context)s" ), modelObject=cntx, context=cntx.contextID) for scenElt in XmlUtil.descendants(cntx, XbrlConst.xbrli, "scenario"): childTags = ", ".join([ child.prefixedName for child in scenElt.iterchildren() if isinstance(child, ModelObject) and child.tag != "{http://xbrl.org/2006/xbrldi}explicitMember" and child.tag != "{http://xbrl.org/2006/xbrldi}typedMember" ]) if len(childTags) > 0: modelXbrl.error( "EBA.2.15", _("Scenario of context Id %(context)s has disallowed content: %(content)s" ), modelObject=cntx, context=cntx.id, content=childTags) val.unusedCntxIDs.add(cntx.id) for unit in modelXbrl.units.values(): if getattr(unit, "_batchChecked", False): continue # prior streaming batch already checked unit._batchChecked = True val.unusedUnitIDs.add(unit.id) factsByQname = defaultdict(set) # top level for this for f in factsToCheck: factsByQname[f.qname].add(f) val.unusedCntxIDs.discard(f.contextID) val.unusedUnitIDs.discard(f.unitID) for fIndicators in factsByQname[qnFIndicators]: val.numFilingIndicatorTuples += 1 for fIndicator in fIndicators.modelTupleFacts: _value = (fIndicator.xValue or fIndicator.value ) # use validated xValue if DTS else value for skipDTS if _value in val.filingIndicators: modelXbrl.error( "EBA.1.6.1", _('Multiple filing indicators facts for indicator %(filingIndicator)s.' ), modelObject=(fIndicator, val.filingIndicators[_value]), filingIndicator=_value) val.filingIndicators[_value] = fIndicator val.unusedCntxIDs.discard(fIndicator.contextID) otherFacts = {} # (contextHash, unitHash, xmlLangHash) : fact nilFacts = [] # removed in current draft: stringFactsWithoutXmlLang = [] nonMonetaryNonPureFacts = [] for qname, facts in factsByQname.items(): for f in facts: if modelXbrl.skipDTS: c = f.qname.localName[0] isNumeric = c in ('m', 'p', 'i') isMonetary = c == 'm' isInteger = c == 'i' isPercent = c == 'p' isString = c == 's' else: concept = f.concept if concept is not None: isNumeric = concept.isNumeric isMonetary = concept.isMonetary isInteger = concept.baseXbrliType in integerItemTypes isPercent = concept.typeQname == qnPercentItemType isString = concept.baseXbrliType in ( "stringItemType", "normalizedStringItemType") else: isNumeric = isString = False # error situation k = (f.getparent().objectIndex, f.qname, f.context.contextDimAwareHash if f.context is not None else None, f.unit.hash if f.unit is not None else None, hash(f.xmlLang)) if f.qname == qnFIndicators and val.validateEIOPA: pass elif k not in otherFacts: otherFacts[k] = {f} else: matches = [ o for o in otherFacts[k] if (f.getparent().objectIndex == o.getparent().objectIndex and f.qname == o.qname and f.context. isEqualTo(o.context) if f.context is not None and o.context is not None else True) and # (f.unit.isEqualTo(o.unit) if f.unit is not None and o.unit is not None else True) and (f.xmlLang == o.xmlLang) ] if matches: contexts = [f.contextID] + [o.contextID for o in matches] modelXbrl.error( "EBA.2.16", _('Facts are duplicates %(fact)s contexts %(contexts)s.' ), modelObject=[f] + matches, fact=f.qname, contexts=', '.join(contexts)) else: otherFacts[k].add(f) if isNumeric: if f.precision: modelXbrl.error( "EBA.2.17", _("Numeric fact %(fact)s of context %(contextID)s has a precision attribute '%(precision)s'" ), modelObject=f, fact=f.qname, contextID=f.contextID, precision=f.precision) if f.decimals and f.decimals != "INF": try: dec = int(f.decimals) if isMonetary: if dec < -3: modelXbrl.error( "EBA.2.17", _("Monetary fact %(fact)s of context %(contextID)s has a decimal attribute < -3: '%(decimals)s'" ), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) elif isInteger: if dec != 0: modelXbrl.error( "EBA.2.17", _("Integer fact %(fact)s of context %(contextID)s has a decimal attribute \u2260 0: '%(decimals)s'" ), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) elif isPercent: if dec < 4: modelXbrl.error( "EBA.2.17", _("Percent fact %(fact)s of context %(contextID)s has a decimal attribute < 4: '%(decimals)s'" ), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) except ValueError: pass # should have been reported as a schema error by loader '''' (not intended by EBA 2.18, paste here is from EFM) if not f.isNil and getattr(f,"xValid", 0) == 4: try: insignificance = insignificantDigits(f.xValue, decimals=f.decimals) if insignificance: # if not None, returns (truncatedDigits, insiginficantDigits) modelXbrl.error(("EFM.6.05.37", "GFM.1.02.26"), _("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s has nonzero digits in insignificant portion %(insignificantDigits)s."), modelObject=f1, fact=f1.qname, contextID=f1.contextID, decimals=f1.decimals, value=f1.xValue, truncatedDigits=insignificance[0], insignificantDigits=insignificance[1]) except (ValueError,TypeError): modelXbrl.error(("EBA.2.18"), _("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s causes Value Error exception."), modelObject=f1, fact=f1.qname, contextID=f1.contextID, decimals=f1.decimals, value=f1.value) ''' unit = f.unit if unit is not None: if isMonetary: if unit.measures[0]: val.currenciesUsed[unit.measures[0][0]] = unit elif not unit.isSingleMeasure or unit.measures[0][ 0] != XbrlConst.qnXbrliPure: nonMonetaryNonPureFacts.append(f) ''' removed in current draft elif isString: if not f.xmlLang: stringFactsWithoutXmlLang.append(f) ''' if f.isNil: nilFacts.append(f) if nilFacts: modelXbrl.error( "EBA.2.19", _('Nil facts MUST NOT be present in the instance: %(nilFacts)s.'), modelObject=nilFacts, nilFacts=", ".join(str(f.qname) for f in nilFacts)) ''' removed in current draft if stringFactsWithoutXmlLang: modelXbrl.error("EBA.2.20", _("String facts need to report xml:lang: '%(langLessFacts)s'"), modelObject=stringFactsWithoutXmlLang, langLessFacts=", ".join(set(str(f.qname) for f in stringFactsWithoutXmlLang))) ''' if nonMonetaryNonPureFacts: modelXbrl.error( "EBA.3.2", _("Non monetary (numeric) facts MUST use the pure unit: '%(langLessFacts)s'" ), modelObject=nonMonetaryNonPureFacts, langLessFacts=", ".join( set(str(f.qname) for f in nonMonetaryNonPureFacts))) val.utrValidator.validateFacts( ) # validate facts for UTR at logLevel WARNING unitHashes = {} for unit in modelXbrl.units.values(): h = hash(unit) if h in unitHashes and unit.isEqualTo(unitHashes[h]): modelXbrl.warning( "EBA.2.22", _("Duplicate units SHOULD NOT be reported, units %(unit1)s and %(unit2)s have same measures.'" ), modelObject=(unit, unitHashes[h]), unit1=unit.id, unit2=unitHashes[h].id) else: unitHashes[h] = unit for elt in modelDocument.xmlRootElement.iter(): if isinstance( elt, ModelObject): # skip comments and processing instructions val.namespacePrefixesUsed[elt.qname.namespaceURI].add( elt.qname.prefix) val.prefixesUnused.discard(elt.qname.prefix)
def doObject(self, fObj, fromRel, pIndent, visited): if fObj is None: return cIndent = pIndent + " " if isinstance(fObj, (ModelValueAssertion, ModelExistenceAssertion, ModelFormula)): varSetType = "formula" if isinstance(fObj, ModelFormula) else "assertion" eltNbr = self.eltTypeCount[varSetType] = self.eltTypeCount.get(varSetType, 0) + 1 _id = fObj.id or "{}{}".format(varSetType, eltNbr) self.xf = "{}{} {} {{".format(pIndent, varSetType, _id) for arcrole in (XbrlConst.elementLabel, XbrlConst.assertionSatisfiedMessage, XbrlConst.assertionUnsatisfiedMessage): for modelRel in self.modelXbrl.relationshipSet(arcrole).fromModelObject(fObj): self.doObject(modelRel.toModelObject, modelRel, cIndent, visited) if fObj.aspectModel == "non-dimensional": self.xf = "{}aspect-model-non-dimensional;".format(cIndent) if fObj.implicitFiltering == "false": self.xf = "{}no-implicit-filtering;".format(cIndent) if isinstance(fObj, ModelFormula): for attr in ("decimals", "precision", "value"): if fObj.get(attr): self.xf = "{}{} {{{}}};".format(cIndent, attr, fObj.get(attr)) if fObj.get("source"): self.xf = "{}source {};".format(cIndent, fObj.get("source")) for aspectsElt in XmlUtil.children(fObj, XbrlConst.formula, "aspects"): self.xf = "{}aspect-rules{} {{".format(cIndent, "source {}".format(aspectsElt.get("source")) if aspectsElt.get("source") else "") for ruleElt in XmlUtil.children(aspectsElt, XbrlConst.formula, "*"): self.doObject(ruleElt, None, cIndent + " ", visited) self.xf = "{}}};".format(cIndent) for arcrole in (XbrlConst.variableSetFilter, XbrlConst.variableSet, XbrlConst.variableSetPrecondition): for modelRel in self.modelXbrl.relationshipSet(arcrole).fromModelObject(fObj): self.doObject(modelRel.toModelObject, modelRel, cIndent, visited) if isinstance(fObj, ModelValueAssertion): self.xf = "{}test {{{}}}".format(cIndent, fObj.viewExpression) elif isinstance(fObj, ModelExistenceAssertion): self.xf = "{}evaluation-count {{{}}}".format(cIndent, fObj.viewExpression or ". gt 0") self.xf = "{}}};".format(pIndent) elif isinstance(fObj, ModelConsistencyAssertion): eltNbr = self.eltTypeCount["consistencyAssertion"] = self.eltTypeCount.get("consistencyAssertion", 0) + 1 _id = fObj.id or "{}{}".format("consistencyAssertion", eltNbr) self.xf = "{}consistency-assertion {} {{".format(pIndent, _id) for arcrole in (XbrlConst.elementLabel, XbrlConst.assertionSatisfiedMessage, XbrlConst.assertionUnsatisfiedMessage): for modelRel in self.modelXbrl.relationshipSet(arcrole).fromModelObject(fObj): self.doObject(modelRel.toModelObject, modelRel, cIndent, visited) if fObj.isStrict: self.xf = "{}strict;".format(cIndent) if fObj.get("proportionalAcceptanceRadius"): self.xf = "{}proportional-acceptance-radius {{{}}};".format(cIndent, fObj.get("proportionalAcceptanceRadius")) if fObj.get("absoluteAcceptanceRadius"): self.xf = "{}absolute-acceptance-radius {{{}}};".format(cIndent, fObj.get("absoluteAcceptanceRadius")) for modelRel in self.modelXbrl.relationshipSet(XbrlConst.consistencyAssertionFormula).fromModelObject(fObj): self.doObject(modelRel.toModelObject, modelRel, cIndent, visited) self.xf = "{}}};".format(pIndent) elif isinstance(fObj, ModelFactVariable) and fromRel is not None: self.xf = "{}variable ${} {{".format(pIndent, fromRel.variableQname) if fromRel.variableQname.prefix: self.xmlns[fromRel.variableQname.prefix] = fromRel.variableQname.namespaceURI if fObj.bindAsSequence == "true": self.xf = "{}bind-as-sequence".format(cIndent) if fObj.nils == "true": self.xf = "{}nils".format(cIndent) if fObj.matches == "true": self.xf = "{}matches".format(cIndent) if fObj.fallbackValue: self.xf = "{}fallback {{{}}}".format(cIndent, fObj.fallbackValue) for modelRel in self.modelXbrl.relationshipSet(XbrlConst.variableFilter).fromModelObject(fObj): self.doObject(modelRel.toModelObject, modelRel, cIndent, visited) self.xf = "{}}};".format(pIndent) elif isinstance(fObj, ModelGeneralVariable) and fromRel is not None: self.xf = "{}variable ${} {{".format(pIndent, fromRel.variableQname) if fromRel.variableQname.prefix: self.xmlns[fromRel.variableQname.prefix] = fromRel.variableQname.namespaceURI if fObj.bindAsSequence: self.xf = "{}bind-as-sequence".format(cIndent) self.xf = "{}select {{{}}}".format(cIndent, fObj.select) elif isinstance(fObj, ModelParameter): if fromRel is not None: # parameter is referenced by a different QName on arc if fromRel.variableQname.prefix: self.xmlns[fromRel.variableQname.prefix] = fromRel.variableQname.namespaceURI self.xf = "{}parameter ${} references ${};".format(pIndent, fromRel.variableQname, fObj.parameterQname) else: # root level parameter if fObj.parameterQname.prefix: self.xmlns[fObj.parameterQname.prefix] = fObj.parameterQname.namespaceURI self.xf = "{}parameter {} {{".format(pIndent, fObj.parameterQname) if fObj.isRequired: self.xf = "{}required".format(cIndent) self.xf = "{} select {{{}}}".format(cIndent, fObj.select) if fObj.asType: self.xf = "{} as {{{}}}".format(cIndent, fObj.asType) if fObj.asType.prefix: self.xmlns[fObj.asType.prefix] = fObj.asType.namespaceURI self.xf = "{}}};".format(pIndent) elif isinstance(fObj, ModelFilter): if fromRel.isComplemented: self.xf = "{}complemented".format(pIndent) if not fromRel.isCovered and fromRel.localName == "variableFilterArc": self.xf = "{}non-covering".format(pIndent) if isinstance(fObj, ModelConceptName): if len(fObj.conceptQnames) == 1 and not fObj.qnameExpressions: qn = next(iter(fObj.conceptQnames)) self.xmlns[qn.prefix] = qn.namespaceURI self.xf = "{}concept-name {};".format(pIndent, qn) elif len(fObj.qnameExpressions) == 1 and not fObj.conceptQnames: self.xf = "{}concept-name {{{}}};".format(pIndent, fObj.qnameExpressions[0]) else: self.xf = "{}concept-name".format(pIndent) for qn in fObj.conceptQnames: self.xmlns[qn.prefix] = qn.namespaceURI self.xf = "{} {}".format(pIndent, qn) for qnExpr in fObj.qnameExpressions: self.xf = "{} {}".format(pIndent, qnExpr) self.xf = "{} ;".format(pIndent) elif isinstance(fObj, ModelConceptPeriodType): self.xf = "{}concept-period {};".format(pIndent, fObj.periodType) elif isinstance(fObj, ModelConceptBalance): self.xf = "{}concept-balance {};".format(pIndent, fObj.balance) elif isinstance(fObj, (ModelConceptDataType, ModelConceptSubstitutionGroup)): self.xf = "{}{} {} {};".format(pIndent, kebabCase(fObj.localName), "strict" if fObj.strict == "true" else "non-strict", fObj.filterQname if fObj.filterQname else "{{{}}}".format(fObj.qnameExpression)) elif isinstance(fObj, ModelExplicitDimension): members = [] for memberElt in XmlUtil.children(fObj, XbrlConst.df, "member"): members.append("member") member = XmlUtil.childText(memberElt, XbrlConst.df, "qname") if member: member = str(member) # qname, must coerce to string else: member = XmlUtil.childText(memberElt, XbrlConst.df, "qnameExpression") if member: member = "{{{}}}".format(member) else: member = "$" + XmlUtil.childText(memberElt, XbrlConst.df, "variable") members.append(member) linkrole = XmlUtil.childText(memberElt, XbrlConst.df, "linkrole") if linkrole: members.append("linkrole") members.append("\"{}\"".format(linkrole)) arcrole = XmlUtil.childText(memberElt, XbrlConst.df, "arcrole") if arcrole: members.append("arcrole") members.append("\"{}\"".format(arcrole)) axis = XmlUtil.childText(memberElt, XbrlConst.df, "axis") if axis: members.append("axis") members.append(axis) self.xf = "{}explicit-dimension {}{};".format(pIndent, fObj.dimQname or "{{{}}}".format(fObj.dimQnameExpression) if fObj.dimQnameExpression else "", " ".join(members)) elif isinstance(fObj, ModelTypedDimension): # this is a ModelTestFilter not same as genera/unit/period self.xf = "{}typed-dimension {}{};".format(pIndent, fObj.dimQname or "{{{}}}".format(fObj.dimQnameExpression) if fObj.dimQnameExpression else "", " {{{}}}".format(fObj.test) if fObj.test else "") elif isinstance(fObj, ModelTestFilter): self.xf = "{}{} {{{}}};".format(pIndent, "general" if isinstance(fObj, ModelGeneral) else "unit-general-measures" if isinstance(fObj, ModelGeneralMeasures) else "period" if isinstance(fObj, ModelPeriod) else "entity-identifier" if isinstance(fObj, ModelIdentifier) else None, fObj.test) elif isinstance(fObj, ModelDateTimeFilter): self.xf = "{}{} {{{}}}{};".format(pIndent, kebabCase(fObj.localName), fObj.date, " {{{}}}".format(fObj.time) if fObj.time else "") elif isinstance(fObj, ModelInstantDuration): self.xf = "{}instant-duration {} {};".format(pIndent, fObj.boundary, fObj.variable) elif isinstance(fObj, ModelSingleMeasure): self.xf = "{}unit {} {};".format(pIndent, fObj.boundary, fObj.variable) elif isinstance(fObj, ModelEntitySpecificIdentifier): self.xf = "{}entity scheme {{{}}} value {{{}}};".format(pIndent, fObj.scheme, fObj.value) elif isinstance(fObj, ModelEntityScheme): self.xf = "{}entity-scheme {{{}}};".format(pIndent, fObj.scheme) elif isinstance(fObj, ModelEntityRegexpScheme): self.xf = "{}entity-scheme-pattern \"{}\";".format(pIndent, fObj.pattern) elif isinstance(fObj, ModelEntityRegexpIdentifier): self.xf = "{}entity-identifier-pattern \"{}\";".format(pIndent, fObj.pattern) elif isinstance(fObj, ModelMatchFilter): self.xf = "{}{} ${} {}{};".format(pIndent, kebabCase(fObj.localName), fObj.variable, fObj.dimension or "", " match-any" if fObj.matchAny else "") elif isinstance(fObj, ModelRelativeFilter): self.xf = "{}relative ${};".format(pIndent, fObj.variable) elif isinstance(fObj, ModelAncestorFilter): self.xf = "{}ancestor {};".format(pIndent, fObj.ancestorQname or "{{{}}}".format(fObj.qnameExpression) if fObj.qnameExpression else "") elif isinstance(fObj, ModelParentFilter): self.xf = "{}parent {};".format(pIndent, fObj.parentQname or "{{{}}}".format(fObj.qnameExpression) if fObj.qnameExpression else "") elif isinstance(fObj, ModelSiblingFilter): self.xf = "{}sibling ${};".format(pIndent, fObj.variable) elif isinstance(fObj, ModelNilFilter): self.xf = "{}nilled;".format(pIndent) elif isinstance(fObj, ModelAspectCover): aspects = [] for aspectElt in XmlUtil.children(fObj, XbrlConst.acf, "aspect"): aspects.append(XmlUtil.text(aspectElt)) for dimElt in XmlUtil.descendants(fObj, XbrlConst.acf, ("qname", "qnameExpression")): dimAspect = qname( dimElt, XmlUtil.text(dimElt) ) aspects.append("exclude-dimension" if dimElt.getparent().localName == "excludeDimension" else "dimension") if dimElt.localName == "qname": aspects.append(str(qname( dimElt, XmlUtil.text(dimElt) ))) else: aspects.append("{{{}}}".format(XmlUtil.text(dimElt))) self.xf = "{}aspect-cover {};".format(pIndent, " ".join(aspects)) elif isinstance(fObj, ModelConceptRelation): conceptRelationTerms = [] if fObj.sourceQname: conceptRelationTerms.append(fObj.sourceQname) elif fObj.variable: conceptRelationTerms.append("$" + fObj.variable) else: conceptRelationTerms.append("{{{}}}".format(fObj.sourceQnameExpression)) if fObj.linkrole: conceptRelationTerms.append("linkrole") conceptRelationTerms.append(fObj.linkrole) elif fObj.linkroleExpression: conceptRelationTerms.append("linkrole") conceptRelationTerms.append("{{{}}}".format(fObj.linkroleExpression)) if fObj.arcrole: conceptRelationTerms.append("arcrole") conceptRelationTerms.append(fObj.arcrole) elif fObj.arcroleExpression: conceptRelationTerms.append("arcrole") conceptRelationTerms.append("{{{}}}".format(fObj.arcroleExpression)) if fObj.axis: conceptRelationTerms.append("axis") conceptRelationTerms.append(fObj.axis) if fObj.generations is not None: conceptRelationTerms.append("generations {}".format(fObj.generations)) if fObj.test: conceptRelationTerms.append("test") conceptRelationTerms.append("{{{}}}".format(fObj.test)) self.xf = "{}concept-relation {};".format(pIndent, " ".join(conceptRelationTerms)) elif isinstance(fObj, (ModelAndFilter, ModelOrFilter)): self.xf = "{}{} {{".format(pIndent, "and" if isinstance(fObj, ModelAndFilter)else "or") if fObj not in visited: visited.add(fObj) for modelRel in self.modelXbrl.relationshipSet(XbrlConst.booleanFilter).fromModelObject(fObj): self.doObject(modelRel.toModelObject, modelRel, cIndent, visited) visited.remove(fObj) self.xf = "{}}};".format(pIndent) elif isinstance(fObj, ModelMessage): self.xf = "{}{}{} \"{}\";".format( pIndent, "satisfied-message" if fromRel.arcrole == XbrlConst.assertionSatisfiedMessage else "unsatisfied-message", " ({})".format(fObj.xmlLang) if fObj.xmlLang else "", fObj.text) elif isinstance(fObj, ModelCustomFunctionSignature): hasImplememntation = False if fObj not in visited: visited.add(fObj) for modelRel in self.modelXbrl.relationshipSet(XbrlConst.functionImplementation).fromModelObject(fObj): self.doObject(modelRel.toModelObject, modelRel, pIndent, visited) # note: use pIndent as parent doesn't show hasImplementation = True visited.remove(fObj) if not hasImplementation: self.xf = "{}abstract-function {}({}) as {};".format(pIndent, fObj.name, ", ".join(str(t) for t in fObj.inputTypes), fObj.outputType) elif isinstance(fObj, ModelCustomFunctionImplementation): sigObj = fromRel.fromModelObject self.xf = "{}function {}({}) as {} {{;".format(pIndent, sigObj.name, ", ".join("{} as {}".format(inputName, sigObj.inputTypes[i]) for i, inputName in enumerate(fObj.inputNames)), sigObj.outputType) for name, stepExpr in fObj.stepExpressions: if "\n" not in stepExpr: self.xf = "{}step ${} {{{}}};".format(cIndent, name, stepExpr) else: self.xf = "{}step ${} {{".format(cIndent, name) for exprLine in stepExpr.split("\n"): self.xf = "{} {}".format(cIndent, exprLine.lstrip()) self.xf = "{}}};".format(cIndent) self.xf = "{}return {{{}}};".format(cIndent, fObj.outputExpression) self.xf = "{}}};".format(pIndent) elif fObj.getparent().tag == "{http://xbrl.org/2008/formula}aspects": # aspect rules arg = "" if fObj.localName == "concept": if XmlUtil.hasChild(fObj, None, "qname"): arg += " " + XmlUtil.childText(fObj, None, "qname") elif XmlUtil.hasChild(fObj, None, "qnameExpression"): arg += " {" + XmlUtil.childText(fObj, None, "qnameExpression") + "}" elif fObj.localName == "entityIdentifier": if fObj.get("scheme"): arg += " scheme {" + fObj.get("scheme") + "}" if fObj.get("identifier"): arg += " identifier {" + fObj.get("identifier") + "}" elif fObj.localName == "period": if XmlUtil.hasChild(fObj, None, "forever"): arg += " forever" if XmlUtil.hasChild(fObj, None, "instant"): arg += " instant" attr = XmlUtil.childAttr(fObj, None, "instant", "value") if attr: arg += "{" + attr + "}" if XmlUtil.hasChild(fObj, None, "duration"): arg += " duration" attr = XmlUtil.childAttr(fObj, None, "duration", "start") if attr: arg += " start {" + attr + "}" attr = XmlUtil.childAttr(fObj, None, "duration", "end") if attr: arg += " end {" + attr + "}" elif fObj.localName == "unit": if fObj.get("augment") == "true": arg += " augment" if fObj.localName in ("explicitDimension", "typedDimension"): arg += " dimension " + fObj.get("dimension") if fObj.localName in ("concept", "entityIdentifier", "period"): arg += ";" else: arg += " {" self.xf = "{}{}{}".format(pIndent, kebabCase(fObj.localName), arg) if fObj.localName == "unit": for elt in fObj.iterchildren(): arg = "" if elt.get("source"): arg += " source " + elt.get("source") if elt.get("measure"): arg += " measure {" + elt.get("measure") + "}" self.xf = "{}{}{};".format(cIndent, kebabCase(elt.localName), arg) elif fObj.localName == "explicitDimension": for elt in fObj.iterchildren(): arg = "" if XmlUtil.hasChild(elt, None, "qname"): arg += " " + XmlUtil.childText(elt, None, "qname") elif XmlUtil.hasChild(elt, None, "qnameExpression"): arg += " {" + XmlUtil.childText(elt, None, "qnameExpression") + "}" self.xf = "{}{}{};".format(cIndent, kebabCase(elt.localName), arg) elif fObj.localName == "typedDimension": for elt in fObj.iterchildren(): arg = "" if XmlUtil.hasChild(elt, None, "xpath"): arg += " xpath {" + ModelXbrl.childText(elt, None, "xpath") + "}" elif XmlUtil.hasChild(elt, None, "value"): arg += " value " + strQoute(XmlUtil.xmlstring(XmlUtil.child(elt, None, "value"), stripXmlns=True, contentsOnly=False)) self.xf = "{}{}{};".format(cIndent, kebabCase(elt.localName), arg) if fObj.localName not in ("concept", "entityIdentifier", "period"): self.xf = "{}}};".format(pIndent)
def validateFacts(val, factsToCheck): # may be called in streaming batches or all at end (final) if not streaming modelXbrl = val.modelXbrl modelDocument = modelXbrl.modelDocument # note EBA 2.1 is in ModelDocument.py timelessDatePattern = re.compile(r"\s*([0-9]{4})-([0-9]{2})-([0-9]{2})\s*$") for cntx in modelXbrl.contexts.values(): if getattr(cntx, "_batchChecked", False): continue # prior streaming batch already checked cntx._batchChecked = True val.cntxEntities.add(cntx.entityIdentifier) dateElts = XmlUtil.descendants(cntx, XbrlConst.xbrli, ("startDate","endDate","instant")) if any(not timelessDatePattern.match(e.textValue) for e in dateElts): modelXbrl.error("EBA.2.10", _('Period dates must be whole dates without time or timezone: %(dates)s.'), modelObject=cntx, dates=", ".join(e.text for e in dateElts)) if cntx.isForeverPeriod: modelXbrl.error("EBA.2.11", _('Forever context period is not allowed.'), modelObject=cntx) elif cntx.isStartEndPeriod: modelXbrl.error("EBA.2.13", _('Start-End (flow) context period is not allowed.'), modelObject=cntx) elif cntx.isInstantPeriod: # cannot pass context object to final() below, for error logging, if streaming mode val.cntxDates[cntx.instantDatetime].add(modelXbrl if getattr(val.modelXbrl, "isStreamingMode", False) else cntx) if XmlUtil.hasChild(cntx, XbrlConst.xbrli, "segment"): modelXbrl.error("EBA.2.14", _("The segment element not allowed in context Id: %(context)s"), modelObject=cntx, context=cntx.contextID) for scenElt in XmlUtil.descendants(cntx, XbrlConst.xbrli, "scenario"): childTags = ", ".join([child.prefixedName for child in scenElt.iterchildren() if isinstance(child,ModelObject) and child.tag != "{http://xbrl.org/2006/xbrldi}explicitMember" and child.tag != "{http://xbrl.org/2006/xbrldi}typedMember"]) if len(childTags) > 0: modelXbrl.error("EBA.2.15", _("Scenario of context Id %(context)s has disallowed content: %(content)s"), modelObject=cntx, context=cntx.id, content=childTags) val.unusedCntxIDs.add(cntx.id) for unit in modelXbrl.units.values(): if getattr(unit, "_batchChecked", False): continue # prior streaming batch already checked unit._batchChecked = True val.unusedUnitIDs.add(unit.id) factsByQname = defaultdict(set) # top level for this for f in factsToCheck: factsByQname[f.qname].add(f) val.unusedCntxIDs.discard(f.contextID) val.unusedUnitIDs.discard(f.unitID) for fIndicators in factsByQname[qnFIndicators]: val.numFilingIndicatorTuples += 1 for fIndicator in fIndicators.modelTupleFacts: _value = (fIndicator.xValue or fIndicator.value) # use validated xValue if DTS else value for skipDTS if _value in val.filingIndicators: modelXbrl.error("EBA.1.6.1", _('Multiple filing indicators facts for indicator %(filingIndicator)s.'), modelObject=(fIndicator, val.filingIndicators[_value]), filingIndicator=_value) val.filingIndicators[_value] = fIndicator val.unusedCntxIDs.discard(fIndicator.contextID) otherFacts = {} # (contextHash, unitHash, xmlLangHash) : fact nilFacts = [] # removed in current draft: stringFactsWithoutXmlLang = [] nonMonetaryNonPureFacts = [] for qname, facts in factsByQname.items(): for f in facts: if modelXbrl.skipDTS: c = f.qname.localName[0] isNumeric = c in ('m', 'p', 'i') isMonetary = c == 'm' isInteger = c == 'i' isPercent = c == 'p' isString = c == 's' else: concept = f.concept if concept is not None: isNumeric = concept.isNumeric isMonetary = concept.isMonetary isInteger = concept.baseXbrliType in integerItemTypes isPercent = concept.typeQname == qnPercentItemType isString = concept.baseXbrliType in ("stringItemType", "normalizedStringItemType") else: isNumeric = isString = False # error situation k = (f.getparent().objectIndex, f.qname, f.context.contextDimAwareHash if f.context is not None else None, f.unit.hash if f.unit is not None else None, hash(f.xmlLang)) if f.qname == qnFIndicators and val.validateEIOPA: pass elif k not in otherFacts: otherFacts[k] = {f} else: matches = [o for o in otherFacts[k] if (f.getparent().objectIndex == o.getparent().objectIndex and f.qname == o.qname and f.context.isEqualTo(o.context) if f.context is not None and o.context is not None else True) and # (f.unit.isEqualTo(o.unit) if f.unit is not None and o.unit is not None else True) and (f.xmlLang == o.xmlLang)] if matches: contexts = [f.contextID] + [o.contextID for o in matches] modelXbrl.error("EBA.2.16", _('Facts are duplicates %(fact)s contexts %(contexts)s.'), modelObject=[f] + matches, fact=f.qname, contexts=', '.join(contexts)) else: otherFacts[k].add(f) if isNumeric: if f.precision: modelXbrl.error("EBA.2.17", _("Numeric fact %(fact)s of context %(contextID)s has a precision attribute '%(precision)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, precision=f.precision) if f.decimals and f.decimals != "INF": try: dec = int(f.decimals) if isMonetary: if dec < -3: modelXbrl.error("EBA.2.17", _("Monetary fact %(fact)s of context %(contextID)s has a decimal attribute < -3: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) elif isInteger: if dec != 0: modelXbrl.error("EBA.2.17", _("Integer fact %(fact)s of context %(contextID)s has a decimal attribute \u2260 0: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) elif isPercent: if dec < 4: modelXbrl.error("EBA.2.17", _("Percent fact %(fact)s of context %(contextID)s has a decimal attribute < 4: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) except ValueError: pass # should have been reported as a schema error by loader '''' (not intended by EBA 2.18, paste here is from EFM) if not f.isNil and getattr(f,"xValid", 0) == 4: try: insignificance = insignificantDigits(f.xValue, decimals=f.decimals) if insignificance: # if not None, returns (truncatedDigits, insiginficantDigits) modelXbrl.error(("EFM.6.05.37", "GFM.1.02.26"), _("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s has nonzero digits in insignificant portion %(insignificantDigits)s."), modelObject=f1, fact=f1.qname, contextID=f1.contextID, decimals=f1.decimals, value=f1.xValue, truncatedDigits=insignificance[0], insignificantDigits=insignificance[1]) except (ValueError,TypeError): modelXbrl.error(("EBA.2.18"), _("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s causes Value Error exception."), modelObject=f1, fact=f1.qname, contextID=f1.contextID, decimals=f1.decimals, value=f1.value) ''' unit = f.unit if unit is not None: if isMonetary: if unit.measures[0]: val.currenciesUsed[unit.measures[0][0]] = unit elif not unit.isSingleMeasure or unit.measures[0][0] != XbrlConst.qnXbrliPure: nonMonetaryNonPureFacts.append(f) ''' removed in current draft elif isString: if not f.xmlLang: stringFactsWithoutXmlLang.append(f) ''' if f.isNil: nilFacts.append(f) if nilFacts: modelXbrl.error("EBA.2.19", _('Nil facts MUST NOT be present in the instance: %(nilFacts)s.'), modelObject=nilFacts, nilFacts=", ".join(str(f.qname) for f in nilFacts)) ''' removed in current draft if stringFactsWithoutXmlLang: modelXbrl.error("EBA.2.20", _("String facts need to report xml:lang: '%(langLessFacts)s'"), modelObject=stringFactsWithoutXmlLang, langLessFacts=", ".join(set(str(f.qname) for f in stringFactsWithoutXmlLang))) ''' if nonMonetaryNonPureFacts: modelXbrl.error("EBA.3.2", _("Non monetary (numeric) facts MUST use the pure unit: '%(langLessFacts)s'"), modelObject=nonMonetaryNonPureFacts, langLessFacts=", ".join(set(str(f.qname) for f in nonMonetaryNonPureFacts))) val.utrValidator.validateFacts() # validate facts for UTR at logLevel WARNING unitHashes = {} for unit in modelXbrl.units.values(): h = hash(unit) if h in unitHashes and unit.isEqualTo(unitHashes[h]): modelXbrl.warning("EBA.2.22", _("Duplicate units SHOULD NOT be reported, units %(unit1)s and %(unit2)s have same measures.'"), modelObject=(unit, unitHashes[h]), unit1=unit.id, unit2=unitHashes[h].id) else: unitHashes[h] = unit for elt in modelDocument.xmlRootElement.iter(): if isinstance(elt, ModelObject): # skip comments and processing instructions val.namespacePrefixesUsed[elt.qname.namespaceURI].add(elt.qname.prefix) val.prefixesUnused.discard(elt.qname.prefix)
def stepAxis(self, op, p, sourceSequence): targetSequence = [] for node in sourceSequence: if not isinstance( node, (ModelObject, etree._ElementTree, ModelAttribute)): raise XPathException( self.progHeader, 'err:XPTY0020', _('Axis step {0} context item is not a node: {1}').format( op, node)) targetNodes = [] if isinstance(p, QNameDef): ns = p.namespaceURI localname = p.localName axis = p.axis if p.isAttribute: if isinstance(node, ModelObject): attrTag = p.localName if p.unprefixed else p.clarkNotation modelAttribute = None try: modelAttribute = node.xAttributes[attrTag] except (AttributeError, TypeError, IndexError, KeyError): # may be lax or deferred validated try: xmlValidate(node.modelXbrl, node, p) modelAttribute = node.xAttributes[attrTag] except (AttributeError, TypeError, IndexError, KeyError): pass if modelAttribute is None: value = node.get(attrTag) if value is not None: targetNodes.append( ModelAttribute(node, p.clarkNotation, UNKNOWN, value, value, value)) elif modelAttribute.xValid >= VALID: targetNodes.append(modelAttribute) elif op == '/' or op is None: if axis is None or axis == "child": if isinstance(node, (ModelObject, etree._ElementTree)): targetNodes = XmlUtil.children(node, ns, localname) elif axis == "parent": if isinstance(node, ModelAttribute): parentNode = [node.modelElement] else: parentNode = [XmlUtil.parent(node)] if (isinstance(node, ModelObject) and (not ns or ns == parentNode.namespaceURI or ns == "*") and (localname == parentNode.localName or localname == "*")): targetNodes = [parentNode] elif axis == "self": if (isinstance(node, ModelObject) and (not ns or ns == node.namespaceURI or ns == "*") and (localname == node.localName or localname == "*")): targetNodes = [node] elif axis.startswith("descendant"): if isinstance(node, (ModelObject, etree._ElementTree)): targetNodes = XmlUtil.descendants( node, ns, localname) if (axis.endswith("-or-self") and isinstance(node, ModelObject) and (not ns or ns == node.namespaceURI or ns == "*") and (localname == node.localName or localname == "*")): targetNodes.append(node) elif axis.startswith("ancestor"): if isinstance(node, ModelObject): targetNodes = [ ancestor for ancestor in XmlUtil.ancestors(node) if ((not ns or ns == ancestor.namespaceURI or ns == "*") and ( localname == ancestor.localName or localname == "*")) ] if (axis.endswith("-or-self") and isinstance(node, ModelObject) and (not ns or ns == node.namespaceURI or ns == "*") and (localname == node.localName or localname == "*")): targetNodes.insert(0, node) elif axis.endswith("-sibling"): if isinstance(node, ModelObject): targetNodes = [ sibling for sibling in node.itersiblings( preceding=axis.startswith("preceding")) if ((not ns or ns == sibling.namespaceURI or ns == "*") and ( localname == sibling.localName or localname == "*")) ] elif axis == "preceding": if isinstance(node, ModelObject): for preceding in node.getroottree().iter(): if preceding == node: break elif ((not ns or ns == preceding.namespaceURI or ns == "*") and (localname == preceding.localName or localname == "*")): targetNodes.append(preceding) elif axis == "following": if isinstance(node, ModelObject): foundNode = False for following in node.getroottree().iter(): if following == node: foundNode = True elif (foundNode and (not ns or ns == following.namespaceURI or ns == "*") and (localname == following.localName or localname == "*")): targetNodes.append(following) elif op == '//': if isinstance(node, (ModelObject, etree._ElementTree)): targetNodes = XmlUtil.descendants(node, ns, localname) elif op == '..': if isinstance(node, ModelAttribute): targetNodes = [node.modelElement] else: targetNodes = [XmlUtil.parent(node)] elif isinstance(p, OperationDef) and isinstance(p.name, QNameDef): if isinstance(node, ModelObject): if p.name.localName == "text": # note this is not string value, just child text targetNodes = [node.textValue] # todo: add element, attribute, node, etc... elif p == '*': # wildcard if op == '/' or op is None: if isinstance(node, (ModelObject, etree._ElementTree)): targetNodes = XmlUtil.children(node, '*', '*') elif op == '//': if isinstance(node, (ModelObject, etree._ElementTree)): targetNodes = XmlUtil.descendants(node, '*', '*') targetSequence.extend(targetNodes) return targetSequence
def final(val): if not (val.validateEBA or val.validateEIOPA): return modelXbrl = val.modelXbrl modelDocument = modelXbrl.modelDocument _statusMsg = _("validating {0} filing rules").format(val.disclosureSystem.name) modelXbrl.profileActivity() val.showStatus(_statusMsg) if modelDocument.type == ModelDocument.Type.INSTANCE and (val.validateEBA or val.validateEIOPA): if not modelDocument.uri.endswith(".xbrl"): modelXbrl.warning("EBA.1.1", _('XBRL instance documents SHOULD use the extension ".xbrl" but it is "%(extension)s"'), modelObject=modelDocument, xmlEncoding=os.path.splitext(modelDocument.basename)[1]) if modelDocument.documentEncoding.lower() not in ("utf-8", "utf-8-sig"): modelXbrl.error("EBA.1.4", _('XBRL instance documents MUST use "UTF-8" encoding but is "%(xmlEncoding)s"'), modelObject=modelDocument, xmlEncoding=modelDocument.documentEncoding) schemaRefElts = [] schemaRefFileNames = [] for doc, docRef in modelDocument.referencesDocument.items(): if docRef.referenceType == "href": if docRef.referringModelObject.localName == "schemaRef": schemaRefElts.append(docRef.referringModelObject) schemaRefFileNames.append(doc.basename) if not UrlUtil.isAbsolute(doc.uri): modelXbrl.error("EBA.2.2", _('The link:schemaRef element in submitted instances MUST resolve to the full published entry point URL: %(url)s.'), modelObject=docRef.referringModelObject, url=doc.uri) elif docRef.referringModelObject.localName == "linkbaseRef": modelXbrl.error("EBA.2.3", _('The link:linkbaseRef element is not allowed: %(fileName)s.'), modelObject=docRef.referringModelObject, fileName=doc.basename) if len(schemaRefFileNames) > 1: modelXbrl.error("EBA.1.5", _('XBRL instance documents MUST reference only one entry point schema but %(numEntryPoints)s were found: %(entryPointNames)s'), modelObject=modelDocument, numEntryPoints=len(schemaRefFileNames), entryPointNames=', '.join(sorted(schemaRefFileNames))) ### check entry point names appropriate for filing indicator (DPM DB?) if len(schemaRefElts) != 1: modelXbrl.error("EBA.2.3", _('Any reported XBRL instance document MUST contain only one xbrli:xbrl/link:schemaRef node, but %(entryPointCount)s.'), modelObject=schemaRefElts, entryPointCount=len(schemaRefElts)) filingIndicators = {} for fIndicator in modelXbrl.factsByQname[qnFilingIndicator]: _value = (fIndicator.xValue or fIndicator.value) # use validated xValue if DTS else value for skipDTS if _value in filingIndicators: modelXbrl.error("EBA.1.6.1", _('Multiple filing indicators facts for indicator %(filingIndicator)s.'), modelObject=(fIndicator, filingIndicators[_value]), filingIndicator=_value) filingIndicators[_value] = fIndicator if not filingIndicators: modelXbrl.error("EBA.1.6", _('Missing filing indicators. Reported XBRL instances MUST include appropriate filing indicator elements'), modelObject=modelDocument) numFilingIndicatorTuples = len(modelXbrl.factsByQname[qnFIndicators]) if numFilingIndicatorTuples > 1 and not getattr(modelXbrl, "isStreamingMode", False): modelXbrl.info("EBA.1.6.2", _('Multiple filing indicators tuples when not in streaming mode (info).'), modelObject=modelXbrl.factsByQname[qnFIndicators]) # note EBA 2.1 is in ModelDocument.py cntxIDs = set() cntxEntities = set() cntxDates = defaultdict(list) timelessDatePattern = re.compile(r"\s*([0-9]{4})-([0-9]{2})-([0-9]{2})\s*$") for cntx in modelXbrl.contexts.values(): cntxIDs.add(cntx.id) cntxEntities.add(cntx.entityIdentifier) dateElts = XmlUtil.descendants(cntx, XbrlConst.xbrli, ("startDate","endDate","instant")) if any(not timelessDatePattern.match(e.textValue) for e in dateElts): modelXbrl.error("EBA.2.10", _('Period dates must be whole dates without time or timezone: %(dates)s.'), modelObject=cntx, dates=", ".join(e.text for e in dateElts)) if cntx.isForeverPeriod: modelXbrl.error("EBA.2.11", _('Forever context period is not allowed.'), modelObject=cntx) elif cntx.isStartEndPeriod: modelXbrl.error("EBA.2.13", _('Start-End (flow) context period is not allowed.'), modelObject=cntx) elif cntx.isInstantPeriod: cntxDates[cntx.instantDatetime].append(cntx) if XmlUtil.hasChild(cntx, XbrlConst.xbrli, "segment"): modelXbrl.error("EBA.2.14", _("The segment element not allowed in context Id: %(context)s"), modelObject=cntx, context=cntx.contextID) for scenElt in XmlUtil.descendants(cntx, XbrlConst.xbrli, "scenario"): childTags = ", ".join([child.prefixedName for child in scenElt.iterchildren() if isinstance(child,ModelObject) and child.tag != "{http://xbrl.org/2006/xbrldi}explicitMember" and child.tag != "{http://xbrl.org/2006/xbrldi}typedMember"]) if len(childTags) > 0: modelXbrl.error("EBA.2.15", _("Scenario of context Id %(context)s has disallowed content: %(content)s"), modelObject=cntx, context=cntx.id, content=childTags) if len(cntxDates) > 1: modelXbrl.error("EBA.2.13", _('Contexts must have the same date: %(dates)s.'), modelObject=[_cntx for _cntxs in cntxDates.values() for _cntx in _cntxs], dates=', '.join(XmlUtil.dateunionValue(_dt, subtractOneDay=True) for _dt in cntxDates.keys())) unusedCntxIDs = cntxIDs - {fact.contextID for fact in modelXbrl.factsInInstance if fact.contextID} # skip tuples if unusedCntxIDs: modelXbrl.warning("EBA.2.7", _('Unused xbrli:context nodes SHOULD NOT be present in the instance: %(unusedContextIDs)s.'), modelObject=[modelXbrl.contexts[unusedCntxID] for unusedCntxID in unusedCntxIDs], unusedContextIDs=", ".join(sorted(unusedCntxIDs))) if len(cntxEntities) > 1: modelXbrl.warning("EBA.2.9", _('All entity identifiers and schemes must be the same, %(count)s found: %(entities)s.'), modelObject=modelDocument, count=len(cntxEntities), entities=", ".join(sorted(str(cntxEntity) for cntxEntity in cntxEntities))) otherFacts = {} # (contextHash, unitHash, xmlLangHash) : fact nilFacts = [] stringFactsWithoutXmlLang = [] nonMonetaryNonPureFacts = [] unitIDsUsed = set() currenciesUsed = {} for qname, facts in modelXbrl.factsByQname.items(): for f in facts: if modelXbrl.skipDTS: c = f.qname.localName[0] isNumeric = c in ('m', 'p', 'i') isMonetary = c == 'm' isInteger = c == 'i' isPercent = c == 'p' isString = c == 's' else: concept = f.concept if concept is not None: isNumeric = concept.isNumeric isMonetary = concept.isMonetary isInteger = concept.baseXbrliType in integerItemTypes isPercent = concept.typeQname == qnPercentItemType isString = concept.baseXbrliType in ("stringItemType", "normalizedStringItemType") else: isNumeric = isString = False # error situation k = (f.getparent().objectIndex, f.qname, f.context.contextDimAwareHash if f.context is not None else None, f.unit.hash if f.unit is not None else None, hash(f.xmlLang)) if f.qname == qnFIndicators and val.validateEIOPA: pass elif k not in otherFacts: otherFacts[k] = {f} else: matches = [o for o in otherFacts[k] if (f.getparent().objectIndex == o.getparent().objectIndex and f.qname == o.qname and f.context.isEqualTo(o.context) if f.context is not None and o.context is not None else True) and (f.unit.isEqualTo(o.unit) if f.unit is not None and o.unit is not None else True) and (f.xmlLang == o.xmlLang)] if matches: contexts = [f.contextID] + [o.contextID for o in matches] modelXbrl.error("EBA.2.16", _('Facts are duplicates %(fact)s contexts %(contexts)s.'), modelObject=[f] + matches, fact=f.qname, contexts=', '.join(contexts)) else: otherFacts[k].add(f) if isNumeric: if f.precision: modelXbrl.error("EBA.2.17", _("Numeric fact %(fact)s of context %(contextID)s has a precision attribute '%(precision)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, precision=f.precision) if f.decimals and f.decimals != "INF": try: dec = int(f.decimals) if isMonetary: if dec < -3: modelXbrl.error("EBA.2.17", _("Monetary fact %(fact)s of context %(contextID)s has a decimal attribute < -3: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) elif isInteger: if dec != 0: modelXbrl.error("EBA.2.17", _("Integer fact %(fact)s of context %(contextID)s has a decimal attribute \u2260 0: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) elif isPercent: if dec < 4: modelXbrl.error("EBA.2.17", _("Percent fact %(fact)s of context %(contextID)s has a decimal attribute < 4: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) except ValueError: pass # should have been reported as a schema error by loader '''' (not intended by EBA 2.18, paste here is from EFM) if not f.isNil and getattr(f,"xValid", 0) == 4: try: insignificance = insignificantDigits(f.xValue, decimals=f.decimals) if insignificance: # if not None, returns (truncatedDigits, insiginficantDigits) modelXbrl.error(("EFM.6.05.37", "GFM.1.02.26"), _("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s has nonzero digits in insignificant portion %(insignificantDigits)s."), modelObject=f1, fact=f1.qname, contextID=f1.contextID, decimals=f1.decimals, value=f1.xValue, truncatedDigits=insignificance[0], insignificantDigits=insignificance[1]) except (ValueError,TypeError): modelXbrl.error(("EBA.2.18"), _("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s causes Value Error exception."), modelObject=f1, fact=f1.qname, contextID=f1.contextID, decimals=f1.decimals, value=f1.value) ''' unit = f.unit if unit is not None: if isMonetary: if unit.measures[0]: currenciesUsed[unit.measures[0][0]] = unit elif not unit.isSingleMeasure or unit.measures[0][0] != XbrlConst.qnXbrliPure: nonMonetaryNonPureFacts.append(f) elif isString: if not f.xmlLang: stringFactsWithoutXmlLang.append(f) if f.unitID is not None: unitIDsUsed.add(f.unitID) if f.isNil: nilFacts.append(f) if nilFacts: modelXbrl.error("EBA.2.19", _('Nil facts MUST NOT be present in the instance: %(nilFacts)s.'), modelObject=nilFacts, nilFacts=", ".join(str(f.qname) for f in nilFacts)) if stringFactsWithoutXmlLang: modelXbrl.error("EBA.2.20", _("String facts need to report xml:lang: '%(langLessFacts)s'"), modelObject=stringFactsWithoutXmlLang, langLEssFacts=", ".join(str(f.qname) for f in stringFactsWithoutXmlLang)) if nonMonetaryNonPureFacts: modelXbrl.error("EBA.3.2", _("Non monetary (numeric) facts MUST use the pure unit: '%(langLessFacts)s'"), modelObject=nonMonetaryNonPureFacts, langLessFacts=", ".join(str(f.qname) for f in nonMonetaryNonPureFacts)) unusedUnitIDs = modelXbrl.units.keys() - unitIDsUsed if unusedUnitIDs: modelXbrl.warning("EBA.2.21", _('Unused xbrli:unit nodes SHOULD NOT be present in the instance: %(unusedUnitIDs)s.'), modelObject=[modelXbrl.units[unusedUnitID] for unusedUnitID in unusedUnitIDs], unusedUnitIDs=", ".join(sorted(unusedUnitIDs))) unitHashes = {} for unit in modelXbrl.units.values(): h = hash(unit) if h in unitHashes and unit.isEqualTo(unitHashes[h]): modelXbrl.warning("EBA.2.32", _("Duplicate units SHOULD NOT be reported, units %(unit1)s and %(unit2)s have same measures.'"), modelObject=(unit, unitHashes[h]), unit1=unit.id, unit2=unitHashes[h].id) else: unitHashes[h] = unit if len(currenciesUsed) > 1: modelXbrl.error("EBA.3.1", _("There MUST be only one currency but %(numCurrencies)s were found: %(currencies)s.'"), modelObject=currenciesUsed.values(), numCurrencies=len(currenciesUsed), currencies=", ".join(str(c) for c in currenciesUsed.keys())) namespacePrefixesUsed = defaultdict(set) prefixesUnused = set(modelDocument.xmlRootElement.keys()).copy() for elt in modelDocument.xmlRootElement.iter(): if isinstance(elt, ModelObject): # skip comments and processing instructions namespacePrefixesUsed[elt.qname.namespaceURI].add(elt.qname.prefix) prefixesUnused.discard(elt.qname.prefix) if prefixesUnused: modelXbrl.warning("EBA.3.4", _("There SHOULD be no unused prefixes but these were declared: %(unusedPrefixes)s.'"), modelObject=modelDocument, unusedPrefixes=', '.join(sorted(prefixesUnused))) for ns, prefixes in namespacePrefixesUsed.items(): nsDocs = modelXbrl.namespaceDocs.get(ns) if nsDocs: for nsDoc in nsDocs: nsDocPrefix = XmlUtil.xmlnsprefix(nsDoc.xmlRootElement, ns) if any(prefix != nsDocPrefix for prefix in prefixes if prefix is not None): modelXbrl.warning("EBA.3.5", _("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s"), modelObject=modelDocument, namespace=ns, declaredPrefix=nsDocPrefix, foundPrefixes=', '.join(sorted(prefixes - {None}))) modelXbrl.profileActivity(_statusMsg, minTimeToShow=0.0) val.showStatus(None) del val.prefixNamespace, val.namespacePrefix, val.idObjects, val.typedDomainElements
def checkDTSdocument(val, modelDocument): modelXbrl = val.modelXbrl if modelDocument.type == ModelDocument.Type.INSTANCE: if not modelDocument.uri.endswith(".xbrl"): modelXbrl.warning("EBA.1.1", _('XBRL instance documents SHOULD use the extension ".xbrl" encoding but it is "%(extension)s"'), modelObject=modelDocument, xmlEncoding=os.path.splitext(modelDocument.basename)[1]) if modelDocument.documentEncoding.lower() not in ("utf-8", "utf-8-sig"): modelXbrl.error("EBA.1.4", _('XBRL instance documents MUST use "UTF-8" encoding but is "%(xmlEncoding)s"'), modelObject=modelDocument, xmlEncoding=modelDocument.documentEncoding) schemaRefElts = [] schemaRefFileNames = [] for doc, docRef in modelDocument.referencesDocument.items(): if docRef.referenceType == "href": if docRef.referringModelObject.localName == "schemaRef": schemaRefElts.append(docRef.referringModelObject) schemaRefFileNames.append(doc.basename) if not UrlUtil.isAbsolute(doc.uri): modelXbrl.error("EBA.2.2", _('The link:schemaRef element in submitted instances MUST resolve to the full published entry point URL: %(url)s.'), modelObject=docRef.referringModelObject, url=doc.uri) elif docRef.referringModelObject.localName == "linkbaseRef": modelXbrl.error("EBA.2.3", _('The link:linkbaseRef element is not allowed: %(fileName)s.'), modelObject=docRef.referringModelObject, fileName=doc.basename) if len(schemaRefFileNames) > 1: modelXbrl.error("EBA.1.5", _('XBRL instance documents MUST reference only one entry point schema but %(numEntryPoints)s were found: %(entryPointNames)s'), modelObject=modelDocument, numEntryPoints=len(schemaRefFileNames), entryPointNames=', '.join(sorted(schemaRefFileNames))) ### check entry point names appropriate for filing indicator (DPM DB?) if len(schemaRefElts) != 1: modelXbrl.error("EBA.2.3", _('Any reported XBRL instance document MUST contain only one xbrli:xbrl/link:schemaRef node, but %(entryPointCount)s.'), modelObject=schemaRefElts, entryPointCount=len(schemaRefElts)) filingIndicators = {} for fIndicator in modelXbrl.factsByQname[qnFilingIndicator]: if fIndicator.xValue in filingIndicators: modelXbrl.error("EBA.1.6.1", _('Multiple filing indicators facts for indicator %(filingIndicator)s.'), modelObject=(fIndicator, filingIndicators[filingIndicators]), filingIndicator=fIndicator.xValue) filingIndicators[fIndicator.xValue] = fIndicator if not filingIndicators: modelXbrl.error("EBA.1.6", _('Missing filing indicators. Reported XBRL instances MUST include appropriate filing indicator elements'), modelObject=modelDocument) numFilingIndicatorTuples = len(modelXbrl.factsByQname[qnFilingIndicators]) if numFilingIndicatorTuples > 1 and not getattr(modelXbrl, "isStreamingMode", False): modelXbrl.info("EBA.1.6.2", _('Multiple filing indicators tuples when not in streaming mode (info).'), modelObject=modelXbrl.factsByQname[qnFilingIndicators]) # note EBA 2.1 is in ModelDocument.py cntxIDs = set() cntxEntities = set() timelessDatePattern = re.compile(r"\s*([0-9]{4})-([0-9]{2})-([0-9]{2})\s*$") for cntx in modelXbrl.contexts.values(): cntxIDs.add(cntx.id) cntxEntities.add(cntx.entityIdentifier) dateElts = XmlUtil.descendants(cntx, XbrlConst.xbrli, ("startDate","endDate","instant")) if any(not timelessDatePattern.match(e.textValue) for e in dateElts): modelXbrl.error("EBA.2.10", _('Period dates must be whole dates without time or timezone: %(dates)s.'), modelObject=cntx, dates=", ".join(e.text for e in dateElts)) if cntx.isForeverPeriod: modelXbrl.error("EBA.2.11", _('Forever context period is not allowed.'), modelObject=cntx) if XmlUtil.hasChild(cntx, XbrlConst.xbrli, "segment"): modelXbrl.error("EBA.2.14", _("The segment element not allowed in context Id: %(context)s"), modelObject=cntx, context=cntx.contextID) for scenElt in XmlUtil.descendants(cntx, XbrlConst.xbrli, "scenario"): childTags = ", ".join([child.prefixedName for child in scenElt.iterchildren() if isinstance(child,ModelObject) and child.tag != "{http://xbrl.org/2006/xbrldi}explicitMember" and child.tag != "{http://xbrl.org/2006/xbrldi}typedMember"]) if len(childTags) > 0: modelXbrl.error("EBA.2.15", _("Scenario of context Id %(context)s has disallowed content: %(content)s"), modelObject=cntx, context=cntx.id, content=childTags) unusedCntxIDs = cntxIDs - {fact.contextID for fact in modelXbrl.facts} if unusedCntxIDs: modelXbrl.warning("EBA.2.7", _('Unused xbrli:context nodes SHOULD NOT be present in the instance: %(unusedContextIDs)s.'), modelObject=[modelXbrl.contexts[unusedCntxID] for unusedCntxID in unusedCntxIDs], unusedContextIDs=", ".join(sorted(unusedCntxIDs))) if len(cntxEntities) > 1: modelXbrl.warning("EBA.2.9", _('All entity identifiers and schemes must be the same, %(count)s found: %(entities)s.'), modelObject=modelDocument, count=len(cntxEntities), entities=", ".join(sorted(str(cntxEntity) for cntxEntity in cntxEntities))) otherFacts = {} # (contextHash, unitHash, xmlLangHash) : fact nilFacts = [] unitIDsUsed = set() currenciesUsed = {} for qname, facts in modelXbrl.factsByQname.items(): for f in facts: concept = f.concept k = (f.context.contextDimAwareHash if f.context is not None else None, f.unit.hash if f.unit is not None else None, hash(f.xmlLang)) if k not in otherFacts: otherFacts[k] = {f} else: matches = [o for o in otherFacts[k] if (f.context.isEqualTo(o.context) if f.context is not None and o.context is not None else True) and (f.unit.isEqualTo(o.unit) if f.unit is not None and o.unit is not None else True) and (f.xmlLang == o.xmlLang)] if matches: modelXbrl.error("EBA.2.16", _('Facts are duplicates %(fact)s and %(otherFacts)s.'), modelObject=[f] + matches, fact=f.qname, otherFacts=', '.join(str(f.qname) for f in matches)) else: otherFacts[k].add(f) if concept is not None and concept.isNumeric: if f.precision: modelXbrl.error("EBA.2.17", _("Numeric fact %(fact)s of context %(contextID)s has a precision attribute '%(precision)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, precision=f.precision) '''' (not intended by EBA 2.18) if f.decimals and f.decimals != "INF" and not f.isNil and getattr(f,"xValid", 0) == 4: try: insignificance = insignificantDigits(f.xValue, decimals=f.decimals) if insignificance: # if not None, returns (truncatedDigits, insiginficantDigits) modelXbrl.error(("EFM.6.05.37", "GFM.1.02.26"), _("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s has nonzero digits in insignificant portion %(insignificantDigits)s."), modelObject=f1, fact=f1.qname, contextID=f1.contextID, decimals=f1.decimals, value=f1.xValue, truncatedDigits=insignificance[0], insignificantDigits=insignificance[1]) except (ValueError,TypeError): modelXbrl.error(("EFM.6.05.37", "GFM.1.02.26"), _("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s causes Value Error exception."), modelObject=f1, fact=f1.qname, contextID=f1.contextID, decimals=f1.decimals, value=f1.value) ''' unit = f.unit if concept.isMonetary and unit is not None and unit.measures[0]: currenciesUsed[unit.measures[0][0]] = unit if f.unitID is not None: unitIDsUsed.add(f.unitID) if f.isNil: nilFacts.append(f) if nilFacts: modelXbrl.warning("EBA.2.19", _('Nil facts SHOULD NOT be present in the instance: %(nilFacts)s.'), modelObject=nilFacts, nilFacts=", ".join(str(f.qname) for f in nilFacts)) unusedUnitIDs = modelXbrl.units.keys() - unitIDsUsed if unusedUnitIDs: modelXbrl.warning("EBA.2.21", _('Unused xbrli:unit nodes SHOULD NOT be present in the instance: %(unusedUnitIDs)s.'), modelObject=[modelXbrl.units[unusedUnitID] for unusedUnitID in unusedUnitIDs], unusedUnitIDs=", ".join(sorted(unusedUnitIDs))) unitHashes = {} for unit in modelXbrl.units.values(): h = hash(unit) if h in unitHashes and unit.isEqualTo(unitHashes[h]): modelXbrl.error("EBA.2.32", _("Units %(unit1)s and %(unit2)s have same measures.'"), modelObject=(unit, unitHashes[h]), unit1=unit.id, unit2=unitHashes[h].id) else: unitHashes[h] = unit if len(currenciesUsed) > 1: modelXbrl.error("EBA.3.1", _("There MUST be only one currency but %(numCurrencies)s were found: %(currencies)s.'"), modelObject=currenciesUsed.values(), numCurrencies=len(currenciesUsed), currencies=", ".join(str(c) for c in currenciesUsed.keys())) namespacePrefixesUsed = defaultdict(set) prefixesUnused = set(modelDocument.xmlRootElement.keys()).copy() for elt in modelDocument.xmlRootElement.iter(): if isinstance(elt, ModelObject): # skip comments and processing instructions namespacePrefixesUsed[elt.qname.namespaceURI].add(elt.qname.prefix) prefixesUnused.discard(elt.qname.prefix) if prefixesUnused: modelXbrl.warning("EBA.3.4", _("There SHOULD be no unused prefixes but these were declared: %(unusedPrefixes)s.'"), modelObject=modelDocument, unusedPrefixes=', '.join(sorted(prefixesUnused))) for ns, prefixes in namespacePrefixesUsed.items(): nsDocs = modelXbrl.namespaceDocs.get(ns) if nsDocs: for nsDoc in nsDocs: nsDocPrefix = XmlUtil.xmlnsprefix(nsDoc.xmlRootElement, ns) if any(prefix != nsDocPrefix for prefix in prefixes if prefix is not None): modelXbrl.warning("EBA.3.5", _("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s"), modelObject=modelDocument, namespace=ns, declaredPrefix=nsDocPrefix, foundPrefixes=', '.join(sorted(prefixes - {None})))
def validateFacts(val, factsToCheck): # may be called in streaming batches or all at end (final) if not streaming modelXbrl = val.modelXbrl modelDocument = modelXbrl.modelDocument # note EBA 2.1 is in ModelDocument.py timelessDatePattern = re.compile(r"\s*([0-9]{4})-([0-9]{2})-([0-9]{2})\s*$") for cntx in modelXbrl.contexts.values(): if getattr(cntx, "_batchChecked", False): continue # prior streaming batch already checked cntx._batchChecked = True val.cntxEntities.add(cntx.entityIdentifier) dateElts = XmlUtil.descendants(cntx, XbrlConst.xbrli, ("startDate","endDate","instant")) if any(not timelessDatePattern.match(e.textValue) for e in dateElts): modelXbrl.error(("EBA.2.10","EIOPA.2.10"), _('Period dates must be whole dates without time or timezone: %(dates)s.'), modelObject=cntx, dates=", ".join(e.text for e in dateElts)) if cntx.isForeverPeriod: modelXbrl.error(("EBA.2.11","EIOPA.N.2.11"), _('Forever context period is not allowed.'), modelObject=cntx) elif cntx.isStartEndPeriod: modelXbrl.error(("EBA.2.13","EIOPA.N.2.11"), _('Start-End (flow) context period is not allowed.'), modelObject=cntx) elif cntx.isInstantPeriod: # cannot pass context object to final() below, for error logging, if streaming mode val.cntxDates[cntx.instantDatetime].add(modelXbrl if getattr(val.modelXbrl, "isStreamingMode", False) else cntx) if cntx.hasSegment: modelXbrl.error(("EBA.2.14","EIOPA.N.2.14"), _("Contexts MUST NOT contain xbrli:segment values: %(cntx)s.'"), modelObject=cntx, cntx=cntx.id) if cntx.nonDimValues("scenario"): modelXbrl.error(("EBA.2.15","EIOPA.S.2.15" if val.isEIOPAfullVersion else "EIOPA.N.2.15"), _("Contexts MUST NOT contain non-dimensional xbrli:scenario values: %(cntx)s.'"), modelObject=cntx, cntx=cntx.id, messageCodes=("EBA.2.15","EIOPA.N.2.15","EIOPA.S.2.15")) val.unusedCntxIDs.add(cntx.id) if val.isEIOPA_2_0_1 and len(cntx.id) > 128: modelXbrl.warning("EIOPA.S.2.6", _("Contexts IDs SHOULD be short: %(cntx)s.'"), modelObject=cntx, cntx=cntx.id) for unit in modelXbrl.units.values(): if getattr(unit, "_batchChecked", False): continue # prior streaming batch already checked unit._batchChecked = True val.unusedUnitIDs.add(unit.id) factsByQname = defaultdict(set) # top level for this for f in factsToCheck: factsByQname[f.qname].add(f) val.unusedCntxIDs.discard(f.contextID) val.unusedUnitIDs.discard(f.unitID) if f.objectIndex < val.firstFactObjectIndex: val.firstFactObjectIndex = f.objectIndex val.firstFact = f for fIndicators in factsByQname[qnFIndicators]: val.numFilingIndicatorTuples += 1 for fIndicator in fIndicators.modelTupleFacts: _value = (getattr(fIndicator, "xValue", None) or fIndicator.value) # use validated xValue if DTS else value for skipDTS _filed = fIndicator.get("{http://www.eurofiling.info/xbrl/ext/filing-indicators}filed", "true") in ("true", "1") if _value in val.filingIndicators: modelXbrl.error(("EBA.1.6.1", "EIOPA.1.6.1"), _('Multiple filing indicators facts for indicator %(filingIndicator)s.'), modelObject=(fIndicator, val.filingIndicators[_value]), filingIndicator=_value) if _filed and not val.filingIndicators[_value]: val.filingIndicators[_value] = _filed #set to filed if any of the multiple indicators are filed=true else: # not a duplicate filing indicator val.filingIndicators[_value] = _filed val.unusedCntxIDs.discard(fIndicator.contextID) cntx = fIndicator.context if cntx is not None and (cntx.hasSegment or cntx.hasScenario): modelXbrl.error("EIOPA.N.1.6.d" if val.isEIOPAfullVersion else "EIOPA.S.1.6.d", _('Filing indicators must not contain segment or scenario elements %(filingIndicator)s.'), modelObject=fIndicator, filingIndicator=_value) if fIndicators.objectIndex > val.firstFactObjectIndex: modelXbrl.warning("EIOPA.1.6.2", _('Filing indicators should precede first fact %(firstFact)s.'), modelObject=(fIndicators, val.firstFact), firstFact=val.firstFact.qname) if val.isEIOPAfullVersion: for fIndicator in factsByQname[qnFilingIndicator]: if fIndicator.getparent().qname == XbrlConst.qnXbrliXbrl: _isPos = fIndicator.get("{http://www.eurofiling.info/xbrl/ext/filing-indicators}filed", "true") in ("true", "1") _value = (getattr(fIndicator, "xValue", None) or fIndicator.value) # use validated xValue if DTS else value for skipDTS modelXbrl.error("EIOPA.1.6.a" if _isPos else "EIOPA.1.6.b", _('Filing indicators must be in a tuple %(filingIndicator)s.'), modelObject=fIndicator, filingIndicator=_value, messageCodes=("EIOPA.1.6.a", "EIOPA.1.6.b")) otherFacts = {} # (contextHash, unitHash, xmlLangHash) : fact nilFacts = [] stringFactsWithXmlLang = [] nonMonetaryNonPureFacts = [] for qname, facts in factsByQname.items(): for f in facts: if f.qname == qnFilingIndicator: continue # skip erroneous root-level filing indicators if modelXbrl.skipDTS: c = f.qname.localName[0] isNumeric = c in ('m', 'p', 'r', 'i') isMonetary = c == 'm' isInteger = c == 'i' isPercent = c == 'p' isString = c == 's' isEnum = c == 'e' else: concept = f.concept if concept is not None: isNumeric = concept.isNumeric isMonetary = concept.isMonetary isInteger = concept.baseXbrliType in integerItemTypes isPercent = concept.typeQname in (qnPercentItemType, qnPureItemType) isString = concept.baseXbrliType in ("stringItemType", "normalizedStringItemType") isEnum = concept.typeQname == qnEnumerationItemType else: isNumeric = isString = isEnum = False # error situation k = (f.getparent().objectIndex, f.qname, f.context.contextDimAwareHash if f.context is not None else None, f.unit.hash if f.unit is not None else None, hash(f.xmlLang)) if f.qname == qnFIndicators and val.validateEIOPA: pass elif k not in otherFacts: otherFacts[k] = {f} else: matches = [o for o in otherFacts[k] if (f.getparent().objectIndex == o.getparent().objectIndex and f.qname == o.qname and f.context.isEqualTo(o.context) if f.context is not None and o.context is not None else True) and # (f.unit.isEqualTo(o.unit) if f.unit is not None and o.unit is not None else True) and (f.xmlLang == o.xmlLang)] if matches: contexts = [f.contextID] + [o.contextID for o in matches] modelXbrl.error(("EBA.2.16", "EIOPA.S.2.16" if val.isEIOPAfullVersion else "EIOPA.S.2.16.a"), _('Facts are duplicates %(fact)s contexts %(contexts)s.'), modelObject=[f] + matches, fact=f.qname, contexts=', '.join(contexts), messageCodes=("EBA.2.16", "EIOPA.S.2.16", "EIOPA.S.2.16.a")) else: otherFacts[k].add(f) if isNumeric: if f.precision: modelXbrl.error(("EBA.2.17", "EIOPA.2.18.a"), _("Numeric fact %(fact)s of context %(contextID)s has a precision attribute '%(precision)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, precision=f.precision) if f.decimals and not f.isNil: # in XbrlDpmSqlDB for 2_0_1 if f.decimals == "INF": if not val.isEIOPAfullVersion: modelXbrl.error("EIOPA.S.2.18.f", _("Monetary fact %(fact)s of context %(contextID)s has a decimal attribute INF: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) else: try: xValue = f.xValue dec = int(f.decimals) if isMonetary: if val.isEIOPA_2_0_1: _absXvalue = abs(xValue) if str(f.qname) in s_2_18_c_a_met: dMin = 2 elif _absXvalue >= 100000000: dMin = -4 elif 100000000 > _absXvalue >= 1000000: dMin = -3 elif 1000000 > _absXvalue >= 1000: dMin = -2 else: dMin = -1 if dMin > dec: modelXbrl.error("EIOPA.S.2.18.c", _("Monetary fact %(fact)s of context %(contextID)s has a decimals attribute less than minimum %(minimumDecimals)s: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, minimumDecimals=dMin, decimals=f.decimals) elif dec < -3: modelXbrl.error(("EBA.2.18","EIOPA.S.2.18.c"), _("Monetary fact %(fact)s of context %(contextID)s has a decimals attribute < -3: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) else: # apply dynamic decimals check if -.1 < xValue < .1: dMin = 2 elif -1 < xValue < 1: dMin = 1 elif -10 < xValue < 10: dMin = 0 elif -100 < xValue < 100: dMin = -1 elif -1000 < xValue < 1000: dMin = -2 else: dMin = -3 if dMin > dec: modelXbrl.warning("EIOPA:factDecimalsWarning", _("Monetary fact %(fact)s of context %(contextID)s value %(value)s has an imprecise decimals attribute: %(decimals)s, minimum is %(mindec)s"), modelObject=f, fact=f.qname, contextID=f.contextID, value=xValue, decimals=f.decimals, mindec=dMin) elif isInteger: if dec != 0: modelXbrl.error(("EBA.2.18","EIOPA.S.2.18.d"), _("Integer fact %(fact)s of context %(contextID)s has a decimals attribute \u2260 0: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) elif isPercent: if dec < 4: modelXbrl.error(("EBA.2.18","EIOPA.S.2.18.e"), _("Percent fact %(fact)s of context %(contextID)s has a decimals attribute < 4: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) if val.isEIOPA_2_0_1 and xValue > 1: modelXbrl.warning(("EIOPA.3.2.b"), _("Percent fact %(fact)s of context %(contextID)s appears to be over 100% = 1.0: '%(value)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, value=xValue) else: if -.001 < xValue < .001: dMin = 4 elif -.01 < xValue < .01: dMin = 3 elif -.1 < xValue < .1: dMin = 2 elif -1 < xValue < 1: dMin = 1 else: dMin = 0 if dMin > dec: modelXbrl.warning("EIOPA:factDecimalsWarning", _("Numeric fact %(fact)s of context %(contextID)s value %(value)s has an imprecise decimals attribute: %(decimals)s, minimum is %(mindec)s"), modelObject=f, fact=f.qname, contextID=f.contextID, value=xValue, decimals=f.decimals, mindec=dMin) except (AttributeError, ValueError): pass # should have been reported as a schema error by loader '''' (not intended by EBA 2.18, paste here is from EFM) if not f.isNil and getattr(f,"xValid", 0) == 4: try: insignificance = insignificantDigits(f.xValue, decimals=f.decimals) if insignificance: # if not None, returns (truncatedDigits, insiginficantDigits) modelXbrl.error(("EFM.6.05.37", "GFM.1.02.26"), _("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s has nonzero digits in insignificant portion %(insignificantDigits)s."), modelObject=f1, fact=f1.qname, contextID=f1.contextID, decimals=f1.decimals, value=f1.xValue, truncatedDigits=insignificance[0], insignificantDigits=insignificance[1]) except (ValueError,TypeError): modelXbrl.error(("EBA.2.18"), _("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s causes Value Error exception."), modelObject=f1, fact=f1.qname, contextID=f1.contextID, decimals=f1.decimals, value=f1.value) ''' unit = f.unit if unit is not None: if isMonetary: if unit.measures[0]: _currencyMeasure = unit.measures[0][0] if val.isEIOPA_2_0_1 and f.context is not None: if f.context.dimMemberQname(val.qnDimAF) == val.qnCAx1 and val.qnDimOC in f.context.qnameDims: _ocCurrency = f.context.dimMemberQname(val.qnDimOC).localName if _currencyMeasure.localName != _ocCurrency: modelXbrl.error("EIOPA.3.1", _("There MUST be only one currency but metric %(metric)s reported OC dimension currency %(ocCurrency)s differs from unit currency: %(unitCurrency)s."), modelObject=f, metric=f.qname, ocCurrency=_ocCurrency, unitCurrency=_currencyMeasure.localName) else: val.currenciesUsed[_currencyMeasure] = unit else: val.currenciesUsed[_currencyMeasure] = unit elif not unit.isSingleMeasure or unit.measures[0][0] != XbrlConst.qnXbrliPure: nonMonetaryNonPureFacts.append(f) if isEnum: _eQn = getattr(f,"xValue", None) or qnameEltPfxName(f, f.value) if _eQn: prefixUsed(val, _eQn.namespaceURI, _eQn.prefix) if val.isEIOPA_2_0_1 and f.qname.localName == "ei1930": val.reportingCurrency = _eQn.localName elif isString: if f.xmlLang: # requires disclosureSystem to NOT specify default language stringFactsWithXmlLang.append(f) if f.isNil: nilFacts.append(f) if val.footnotesRelationshipSet.fromModelObject(f): modelXbrl.warning("EIOPA.S.19", _("Fact %(fact)s of context %(contextID)s has footnotes.'"), modelObject=f, fact=f.qname, contextID=f.contextID) if nilFacts: modelXbrl.error(("EBA.2.19", "EIOPA.S.2.19"), _('Nil facts MUST NOT be present in the instance: %(nilFacts)s.'), modelObject=nilFacts, nilFacts=", ".join(str(f.qname) for f in nilFacts)) if stringFactsWithXmlLang: modelXbrl.warning("EIOPA.2.20", # not reported for EBA _("String facts reporting xml:lang (not saved by T4U, not round-tripped): '%(factsWithLang)s'"), modelObject=stringFactsWithXmlLang, factsWithLang=", ".join(set(str(f.qname) for f in stringFactsWithXmlLang))) if nonMonetaryNonPureFacts: modelXbrl.error(("EBA.3.2","EIOPA.3.2.a"), _("Non monetary (numeric) facts MUST use the pure unit: '%(langLessFacts)s'"), modelObject=nonMonetaryNonPureFacts, langLessFacts=", ".join(set(str(f.qname) for f in nonMonetaryNonPureFacts))) val.utrValidator.validateFacts() # validate facts for UTR at logLevel WARNING unitHashes = {} for unit in modelXbrl.units.values(): h = unit.hash if h in unitHashes and unit.isEqualTo(unitHashes[h]): modelXbrl.warning("EBA.2.21", _("Duplicate units SHOULD NOT be reported, units %(unit1)s and %(unit2)s have same measures.'"), modelObject=(unit, unitHashes[h]), unit1=unit.id, unit2=unitHashes[h].id) if not getattr(modelXbrl, "isStreamingMode", False): modelXbrl.error("EIOPA.2.21", _("Duplicate units MUST NOT be reported, units %(unit1)s and %(unit2)s have same measures.'"), modelObject=(unit, unitHashes[h]), unit1=unit.id, unit2=unitHashes[h].id) else: unitHashes[h] = unit for _measures in unit.measures: for _measure in _measures: prefixUsed(val, _measure.namespaceURI, _measure.prefix) del unitHashes cntxHashes = {} for cntx in modelXbrl.contexts.values(): h = cntx.contextDimAwareHash if h in cntxHashes and cntx.isEqualTo(cntxHashes[h]): if not getattr(modelXbrl, "isStreamingMode", False): modelXbrl.log("WARNING" if val.isEIOPAfullVersion else "ERROR", "EIOPA.S.2.7.b", _("Duplicate contexts MUST NOT be reported, contexts %(cntx1)s and %(cntx2)s are equivalent.'"), modelObject=(cntx, cntxHashes[h]), cntx1=cntx.id, cntx2=cntxHashes[h].id) else: cntxHashes[h] = cntx for _dim in cntx.qnameDims.values(): _dimQn = _dim.dimensionQname prefixUsed(val, _dimQn.namespaceURI, _dimQn.prefix) if _dim.isExplicit: _memQn = _dim.memberQname else: _memQn = _dim.typedMember.qname if _memQn: prefixUsed(val, _memQn.namespaceURI, _memQn.prefix) for elt in modelDocument.xmlRootElement.iter(): if isinstance(elt, ModelObject): # skip comments and processing instructions prefixUsed(val, elt.qname.namespaceURI, elt.qname.prefix) for attrTag in elt.keys(): if attrTag.startswith("{"): _prefix, _NS, _localName = XmlUtil.clarkNotationToPrefixNsLocalname(elt, attrTag, isAttribute=True) if _prefix: prefixUsed(val, _NS, _prefix) elif val.isEIOPA_2_0_1: if elt.tag in ("{http://www.w3.org/2001/XMLSchema}documentation", "{http://www.w3.org/2001/XMLSchema}annotation"): modelXbrl.error("EIOPA.2.5", _("xs:documentation element found, all relevant business data MUST only be contained in contexts, units, schemaRef and facts."), modelObject=modelDocument) elif isinstance(elt, etree._Comment): modelXbrl.error("EIOPA.2.5", _("XML comment found, all relevant business data MUST only be contained in contexts, units, schemaRef and facts: %(comment)s"), modelObject=modelDocument, comment=elt.text)
def doObject(self, fObj, fromRel, pIndent, visited): if fObj is None: return cIndent = pIndent + " " if isinstance(fObj, ModelAssertionSet): self.xf = "{}assertion-set {} {{".format(pIndent, self.objectId(fObj, "assertionSet")) for modelRel in self.modelXbrl.relationshipSet(XbrlConst.assertionSet).fromModelObject(fObj): self.doObject(modelRel.toModelObject, modelRel, cIndent, visited) self.xf = "{}}};".format(pIndent) elif isinstance(fObj, (ModelValueAssertion, ModelExistenceAssertion, ModelFormula)): varSetType = "formula" if isinstance(fObj, ModelFormula) else "assertion" self.xf = "{}{} {} {{".format(pIndent, varSetType, self.objectId(fObj, varSetType)) for arcrole in (XbrlConst.elementLabel, XbrlConst.assertionSatisfiedMessage, XbrlConst.assertionUnsatisfiedMessage): for modelRel in self.modelXbrl.relationshipSet(arcrole).fromModelObject(fObj): self.doObject(modelRel.toModelObject, modelRel, cIndent, visited) if fObj.aspectModel == "non-dimensional": self.xf = "{}aspect-model-non-dimensional;".format(cIndent) if fObj.implicitFiltering == "false": self.xf = "{}no-implicit-filtering;".format(cIndent) if isinstance(fObj, ModelFormula): for attr in ("decimals", "precision", "value"): if fObj.get(attr): self.xf = "{}{} {{{}}};".format(cIndent, attr, fObj.get(attr)) if fObj.get("source"): self.xf = "{}source {};".format(cIndent, fObj.get("source")) for aspectsElt in XmlUtil.children(fObj, XbrlConst.formula, "aspects"): self.xf = "{}aspect-rules{} {{".format(cIndent, "source {}".format(aspectsElt.get("source")) if aspectsElt.get("source") else "") for ruleElt in XmlUtil.children(aspectsElt, XbrlConst.formula, "*"): self.doObject(ruleElt, None, cIndent + " ", visited) self.xf = "{}}};".format(cIndent) for arcrole in (XbrlConst.variableSetFilter, XbrlConst.variableSet, XbrlConst.variableSetPrecondition): for modelRel in self.modelXbrl.relationshipSet(arcrole).fromModelObject(fObj): self.doObject(modelRel.toModelObject, modelRel, cIndent, visited) if isinstance(fObj, ModelValueAssertion): self.xf = "{}test {{{}}};".format(cIndent, fObj.viewExpression) elif isinstance(fObj, ModelExistenceAssertion): self.xf = "{}evaluation-count {{{}}};".format(cIndent, fObj.viewExpression or ". gt 0") self.xf = "{}}};".format(pIndent) elif isinstance(fObj, ModelConsistencyAssertion): self.xf = "{}consistency-assertion {} {{".format(pIndent, self.objectId(fObj, "consistencyAssertion")) for arcrole in (XbrlConst.elementLabel, XbrlConst.assertionSatisfiedMessage, XbrlConst.assertionUnsatisfiedMessage): for modelRel in self.modelXbrl.relationshipSet(arcrole).fromModelObject(fObj): self.doObject(modelRel.toModelObject, modelRel, cIndent, visited) if fObj.isStrict: self.xf = "{}strict;".format(cIndent) if fObj.get("proportionalAcceptanceRadius"): self.xf = "{}proportional-acceptance-radius {{{}}};".format(cIndent, fObj.get("proportionalAcceptanceRadius")) if fObj.get("absoluteAcceptanceRadius"): self.xf = "{}absolute-acceptance-radius {{{}}};".format(cIndent, fObj.get("absoluteAcceptanceRadius")) for modelRel in self.modelXbrl.relationshipSet(XbrlConst.consistencyAssertionFormula).fromModelObject(fObj): self.doObject(modelRel.toModelObject, modelRel, cIndent, visited) self.xf = "{}}};".format(pIndent) elif isinstance(fObj, ModelFactVariable) and fromRel is not None: self.xf = "{}variable ${} {{".format(pIndent, fromRel.variableQname) if fromRel.variableQname.prefix: self.xmlns[fromRel.variableQname.prefix] = fromRel.variableQname.namespaceURI if fObj.bindAsSequence == "true": self.xf = "{}bind-as-sequence".format(cIndent) if fObj.nils == "true": self.xf = "{}nils".format(cIndent) if fObj.matches == "true": self.xf = "{}matches".format(cIndent) if fObj.fallbackValue: self.xf = "{}fallback {{{}}}".format(cIndent, fObj.fallbackValue) for modelRel in self.modelXbrl.relationshipSet(XbrlConst.variableFilter).fromModelObject(fObj): self.doObject(modelRel.toModelObject, modelRel, cIndent, visited) self.xf = "{}}};".format(pIndent) elif isinstance(fObj, ModelGeneralVariable) and fromRel is not None: self.xf = "{}variable ${} {{".format(pIndent, fromRel.variableQname) if fromRel.variableQname.prefix: self.xmlns[fromRel.variableQname.prefix] = fromRel.variableQname.namespaceURI if fObj.bindAsSequence: self.xf = "{}bind-as-sequence".format(cIndent) self.xf = "{}select {{{}}}".format(cIndent, fObj.select) elif isinstance(fObj, ModelParameter): if fromRel is not None: # parameter is referenced by a different QName on arc if fromRel.variableQname.prefix: self.xmlns[fromRel.variableQname.prefix] = fromRel.variableQname.namespaceURI self.xf = "{}parameter ${} references ${};".format(pIndent, fromRel.variableQname, fObj.parameterQname) else: # root level parameter if fObj.parameterQname.prefix: self.xmlns[fObj.parameterQname.prefix] = fObj.parameterQname.namespaceURI self.xf = "{}parameter {} {{".format(pIndent, fObj.parameterQname) if fObj.isRequired: self.xf = "{}required".format(cIndent) self.xf = "{} select {{{}}}".format(cIndent, fObj.select) if fObj.asType: self.xf = "{} as {{{}}}".format(cIndent, fObj.asType) if fObj.asType.prefix: self.xmlns[fObj.asType.prefix] = fObj.asType.namespaceURI self.xf = "{}}};".format(pIndent) elif isinstance(fObj, ModelFilter): if fromRel.isComplemented: self.xf = "{}complemented".format(pIndent) if not fromRel.isCovered and fromRel.localName == "variableFilterArc": self.xf = "{}non-covering".format(pIndent) if isinstance(fObj, ModelConceptName): if len(fObj.conceptQnames) == 1 and not fObj.qnameExpressions: qn = next(iter(fObj.conceptQnames)) self.xmlns[qn.prefix] = qn.namespaceURI self.xf = "{}concept-name {};".format(pIndent, qn) elif len(fObj.qnameExpressions) == 1 and not fObj.conceptQnames: self.xf = "{}concept-name {{{}}};".format(pIndent, fObj.qnameExpressions[0]) else: self.xf = "{}concept-name".format(pIndent) for qn in fObj.conceptQnames: self.xmlns[qn.prefix] = qn.namespaceURI self.xf = "{} {}".format(pIndent, qn) for qnExpr in fObj.qnameExpressions: self.xf = "{} {}".format(pIndent, qnExpr) self.xf = "{} ;".format(pIndent) elif isinstance(fObj, ModelConceptPeriodType): self.xf = "{}concept-period {};".format(pIndent, fObj.periodType) elif isinstance(fObj, ModelConceptBalance): self.xf = "{}concept-balance {};".format(pIndent, fObj.balance) elif isinstance(fObj, (ModelConceptDataType, ModelConceptSubstitutionGroup)): self.xf = "{}{} {} {};".format(pIndent, kebabCase(fObj.localName), "strict" if fObj.strict == "true" else "non-strict", fObj.filterQname if fObj.filterQname else "{{{}}}".format(fObj.qnameExpression)) elif isinstance(fObj, ModelExplicitDimension): members = [] for memberElt in XmlUtil.children(fObj, XbrlConst.df, "member"): members.append("member") member = XmlUtil.childText(memberElt, XbrlConst.df, "qname") if member: member = str(member) # qname, must coerce to string else: member = XmlUtil.childText(memberElt, XbrlConst.df, "qnameExpression") if member: member = "{{{}}}".format(member) else: member = "$" + XmlUtil.childText(memberElt, XbrlConst.df, "variable") members.append(member) linkrole = XmlUtil.childText(memberElt, XbrlConst.df, "linkrole") if linkrole: members.append("linkrole") members.append("\"{}\"".format(linkrole)) arcrole = XmlUtil.childText(memberElt, XbrlConst.df, "arcrole") if arcrole: members.append("arcrole") members.append("\"{}\"".format(arcrole)) axis = XmlUtil.childText(memberElt, XbrlConst.df, "axis") if axis: members.append("axis") members.append(axis) self.xf = "{}explicit-dimension {}{};".format(pIndent, fObj.dimQname or ("{{{}}}".format(fObj.dimQnameExpression) if fObj.dimQnameExpression else ""), " ".join(members)) elif isinstance(fObj, ModelTypedDimension): # this is a ModelTestFilter not same as genera/unit/period self.xf = "{}typed-dimension {}{};".format(pIndent, fObj.dimQname or ("{{{}}}".format(fObj.dimQnameExpression) if fObj.dimQnameExpression else ""), " {{{}}}".format(fObj.test) if fObj.test else "") elif isinstance(fObj, ModelTestFilter): self.xf = "{}{} {{{}}};".format(pIndent, "general" if isinstance(fObj, ModelGeneral) else "unit-general-measures" if isinstance(fObj, ModelGeneralMeasures) else "period" if isinstance(fObj, ModelPeriod) else "entity-identifier" if isinstance(fObj, ModelIdentifier) else None, fObj.test) elif isinstance(fObj, ModelDateTimeFilter): self.xf = "{}{} {{{}}}{};".format(pIndent, kebabCase(fObj.localName), fObj.date, " {{{}}}".format(fObj.time) if fObj.time else "") elif isinstance(fObj, ModelInstantDuration): self.xf = "{}instant-duration {} {};".format(pIndent, fObj.boundary, fObj.variable) elif isinstance(fObj, ModelSingleMeasure): self.xf = "{}unit-single-measure {};".format(pIndent, fObj.measureQname or ("{{{}}}".format(fObj.qnameExpression) if fObj.qnameExpression else "")) elif isinstance(fObj, ModelEntitySpecificIdentifier): self.xf = "{}entity scheme {{{}}} value {{{}}};".format(pIndent, fObj.scheme, fObj.value) elif isinstance(fObj, ModelEntityScheme): self.xf = "{}entity-scheme {{{}}};".format(pIndent, fObj.scheme) elif isinstance(fObj, ModelEntityRegexpScheme): self.xf = "{}entity-scheme-pattern \"{}\";".format(pIndent, fObj.pattern) elif isinstance(fObj, ModelEntityRegexpIdentifier): self.xf = "{}entity-identifier-pattern \"{}\";".format(pIndent, fObj.pattern) elif isinstance(fObj, ModelMatchFilter): self.xf = "{}{} ${} {}{};".format(pIndent, kebabCase(fObj.localName), fObj.variable, " dimension {}".format(fObj.dimension) if fObj.get("dimension") else "", " match-any" if fObj.matchAny else "") elif isinstance(fObj, ModelRelativeFilter): self.xf = "{}relative ${};".format(pIndent, fObj.variable) elif isinstance(fObj, ModelAncestorFilter): self.xf = "{}ancestor {};".format(pIndent, fObj.ancestorQname or ("{{{}}}".format(fObj.qnameExpression) if fObj.qnameExpression else "")) elif isinstance(fObj, ModelParentFilter): self.xf = "{}parent {};".format(pIndent, fObj.parentQname or ("{{{}}}".format(fObj.qnameExpression) if fObj.qnameExpression else "")) elif isinstance(fObj, ModelSiblingFilter): self.xf = "{}sibling ${};".format(pIndent, fObj.variable) elif isinstance(fObj, ModelNilFilter): self.xf = "{}nilled;".format(pIndent) elif isinstance(fObj, ModelAspectCover): aspects = [] for aspectElt in XmlUtil.children(fObj, XbrlConst.acf, "aspect"): aspects.append(XmlUtil.text(aspectElt)) for dimElt in XmlUtil.descendants(fObj, XbrlConst.acf, ("qname", "qnameExpression")): dimAspect = qname( dimElt, XmlUtil.text(dimElt) ) aspects.append("exclude-dimension" if dimElt.getparent().localName == "excludeDimension" else "dimension") if dimElt.localName == "qname": aspects.append(str(qname( dimElt, XmlUtil.text(dimElt) ))) else: aspects.append("{{{}}}".format(XmlUtil.text(dimElt))) self.xf = "{}aspect-cover {};".format(pIndent, " ".join(aspects)) elif isinstance(fObj, ModelConceptRelation): conceptRelationTerms = [] if fObj.sourceQname: conceptRelationTerms.append(fObj.sourceQname) elif fObj.variable: conceptRelationTerms.append("$" + fObj.variable) else: conceptRelationTerms.append("{{{}}}".format(fObj.sourceQnameExpression)) if fObj.linkrole: conceptRelationTerms.append("linkrole") conceptRelationTerms.append(fObj.linkrole) elif fObj.linkroleExpression: conceptRelationTerms.append("linkrole") conceptRelationTerms.append("{{{}}}".format(fObj.linkroleExpression)) if fObj.arcrole: conceptRelationTerms.append("arcrole") conceptRelationTerms.append(fObj.arcrole) elif fObj.arcroleExpression: conceptRelationTerms.append("arcrole") conceptRelationTerms.append("{{{}}}".format(fObj.arcroleExpression)) if fObj.axis: conceptRelationTerms.append("axis") conceptRelationTerms.append(fObj.axis) if fObj.generations is not None: conceptRelationTerms.append("generations {}".format(fObj.generations)) if fObj.test: conceptRelationTerms.append("test") conceptRelationTerms.append("{{{}}}".format(fObj.test)) self.xf = "{}concept-relation {};".format(pIndent, " ".join(conceptRelationTerms)) elif isinstance(fObj, (ModelAndFilter, ModelOrFilter)): self.xf = "{}{} {{".format(pIndent, "and" if isinstance(fObj, ModelAndFilter)else "or") if fObj not in visited: visited.add(fObj) for modelRel in self.modelXbrl.relationshipSet(XbrlConst.booleanFilter).fromModelObject(fObj): self.doObject(modelRel.toModelObject, modelRel, cIndent, visited) visited.remove(fObj) self.xf = "{}}};".format(pIndent) elif isinstance(fObj, ModelMessage): self.xf = "{}{}{} \"{}\";".format( pIndent, "satisfied-message" if fromRel.arcrole == XbrlConst.assertionSatisfiedMessage else "unsatisfied-message", " ({})".format(fObj.xmlLang) if fObj.xmlLang else "", fObj.text.replace('"', '""')) elif isinstance(fObj, ModelCustomFunctionSignature): hasImplememntation = False if fObj not in visited: visited.add(fObj) for modelRel in self.modelXbrl.relationshipSet(XbrlConst.functionImplementation).fromModelObject(fObj): self.doObject(modelRel.toModelObject, modelRel, pIndent, visited) # note: use pIndent as parent doesn't show hasImplementation = True visited.remove(fObj) if not hasImplementation: self.xmlns[fObj.functionQname.prefix] = fObj.functionQname.namespaceURI self.xf = "{}abstract-function {}({}) as {};".format(pIndent, fObj.name, ", ".join(str(t) for t in fObj.inputTypes), fObj.outputType) elif isinstance(fObj, ModelCustomFunctionImplementation): sigObj = fromRel.fromModelObject self.xmlns[sigObj.functionQname.prefix] = sigObj.functionQname.namespaceURI self.xf = "{}function {}({}) as {} {{".format(pIndent, sigObj.name, ", ".join("{} as {}".format(inputName, sigObj.inputTypes[i]) for i, inputName in enumerate(fObj.inputNames)), sigObj.outputType) for name, stepExpr in fObj.stepExpressions: if "\n" not in stepExpr: self.xf = "{}step ${} {{{}}};".format(cIndent, name, stepExpr) else: self.xf = "{}step ${} {{".format(cIndent, name) for exprLine in stepExpr.split("\n"): self.xf = "{} {}".format(cIndent, exprLine.lstrip()) self.xf = "{}}};".format(cIndent) self.xf = "{}return {{{}}};".format(cIndent, fObj.outputExpression) self.xf = "{}}};".format(pIndent) elif fObj.getparent().tag == "{http://xbrl.org/2008/formula}aspects": # aspect rules arg = "" if fObj.localName == "concept": if XmlUtil.hasChild(fObj, None, "qname"): arg += " " + XmlUtil.childText(fObj, None, "qname") elif XmlUtil.hasChild(fObj, None, "qnameExpression"): arg += " {" + XmlUtil.childText(fObj, None, "qnameExpression") + "}" elif fObj.localName == "entityIdentifier": if fObj.get("scheme"): arg += " scheme {" + fObj.get("scheme") + "}" if fObj.get("identifier"): arg += " identifier {" + fObj.get("identifier") + "}" elif fObj.localName == "period": if XmlUtil.hasChild(fObj, None, "forever"): arg += " forever" if XmlUtil.hasChild(fObj, None, "instant"): arg += " instant" attr = XmlUtil.childAttr(fObj, None, "instant", "value") if attr: arg += "{" + attr + "}" if XmlUtil.hasChild(fObj, None, "duration"): arg += " duration" attr = XmlUtil.childAttr(fObj, None, "duration", "start") if attr: arg += " start {" + attr + "}" attr = XmlUtil.childAttr(fObj, None, "duration", "end") if attr: arg += " end {" + attr + "}" elif fObj.localName == "unit": if fObj.get("augment") == "true": arg += " augment" if fObj.localName in ("explicitDimension", "typedDimension"): arg += " dimension " + fObj.get("dimension") if fObj.localName in ("concept", "entityIdentifier", "period"): arg += ";" else: arg += " {" self.xf = "{}{}{}".format(pIndent, kebabCase(fObj.localName), arg) if fObj.localName == "unit": for elt in fObj.iterchildren(): arg = "" if elt.get("source"): arg += " source " + elt.get("source") if elt.get("measure"): arg += " measure {" + elt.get("measure") + "}" self.xf = "{}{}{};".format(cIndent, kebabCase(elt.localName), arg) elif fObj.localName == "explicitDimension": for elt in fObj.iterchildren(): arg = "" if XmlUtil.hasChild(elt, None, "qname"): arg += " " + XmlUtil.childText(elt, None, "qname") elif XmlUtil.hasChild(elt, None, "qnameExpression"): arg += " {" + XmlUtil.childText(elt, None, "qnameExpression") + "}" self.xf = "{}{}{};".format(cIndent, kebabCase(elt.localName), arg) elif fObj.localName == "typedDimension": for elt in fObj.iterchildren(): arg = "" if XmlUtil.hasChild(elt, None, "xpath"): arg += " xpath {" + ModelXbrl.childText(elt, None, "xpath") + "}" elif XmlUtil.hasChild(elt, None, "value"): arg += " value " + strQoute(XmlUtil.xmlstring(XmlUtil.child(elt, None, "value"), stripXmlns=True, contentsOnly=False)) self.xf = "{}{}{};".format(cIndent, kebabCase(elt.localName), arg) if fObj.localName not in ("concept", "entityIdentifier", "period"): self.xf = "{}}};".format(pIndent) # check for prefixes in AST of programs of fObj if hasattr(fObj, "compile") and type(fObj.compile).__name__ == "method": fObj.compile() for _prog, _ast in fObj.__dict__.items(): if _prog.endswith("Prog") and isinstance(_ast, list): XPathParser.prefixDeclarations(_ast, self.xmlns, fObj)
def validateFacts(val, factsToCheck): # may be called in streaming batches or all at end (final) if not streaming modelXbrl = val.modelXbrl modelDocument = modelXbrl.modelDocument # note EBA 2.1 is in ModelDocument.py timelessDatePattern = re.compile(r"\s*([0-9]{4})-([0-9]{2})-([0-9]{2})\s*$") for cntx in modelXbrl.contexts.values(): if getattr(cntx, "_batchChecked", False): continue # prior streaming batch already checked cntx._batchChecked = True val.cntxEntities.add(cntx.entityIdentifier) dateElts = XmlUtil.descendants(cntx, XbrlConst.xbrli, ("startDate","endDate","instant")) if any(not timelessDatePattern.match(e.textValue) for e in dateElts): modelXbrl.error(("EBA.2.10","EIOPA.2.10"), _('Period dates must be whole dates without time or timezone: %(dates)s.'), modelObject=cntx, dates=", ".join(e.text for e in dateElts)) if cntx.isForeverPeriod: modelXbrl.error(("EBA.2.11","EIOPA.N.2.11"), _('Forever context period is not allowed.'), modelObject=cntx) elif cntx.isStartEndPeriod: modelXbrl.error(("EBA.2.13","EIOPA.N.2.11"), _('Start-End (flow) context period is not allowed.'), modelObject=cntx) elif cntx.isInstantPeriod: # cannot pass context object to final() below, for error logging, if streaming mode val.cntxDates[cntx.instantDatetime].add(modelXbrl if getattr(val.modelXbrl, "isStreamingMode", False) else cntx) if cntx.hasSegment: modelXbrl.error(("EBA.2.14","EIOPA.N.2.14"), _("Contexts MUST NOT contain xbrli:segment values: %(cntx)s.'"), modelObject=cntx, cntx=cntx.id) if cntx.nonDimValues("scenario"): modelXbrl.error(("EBA.2.15","EIOPA.S.2.15" if val.isEIOPAfullVersion else "EIOPA.N.2.15"), _("Contexts MUST NOT contain non-dimensional xbrli:scenario values: %(cntx)s.'"), modelObject=cntx, cntx=cntx.id, messageCodes=("EBA.2.15","EIOPA.N.2.15","EIOPA.S.2.15")) val.unusedCntxIDs.add(cntx.id) for unit in modelXbrl.units.values(): if getattr(unit, "_batchChecked", False): continue # prior streaming batch already checked unit._batchChecked = True val.unusedUnitIDs.add(unit.id) factsByQname = defaultdict(set) # top level for this for f in factsToCheck: factsByQname[f.qname].add(f) val.unusedCntxIDs.discard(f.contextID) val.unusedUnitIDs.discard(f.unitID) if f.objectIndex < val.firstFactObjectIndex: val.firstFactObjectIndex = f.objectIndex val.firstFact = f for fIndicators in factsByQname[qnFIndicators]: val.numFilingIndicatorTuples += 1 for fIndicator in fIndicators.modelTupleFacts: _value = (getattr(fIndicator, "xValue", None) or fIndicator.value) # use validated xValue if DTS else value for skipDTS if _value in val.filingIndicators: modelXbrl.error(("EBA.1.6.1", "EIOPA.1.6.1"), _('Multiple filing indicators facts for indicator %(filingIndicator)s.'), modelObject=(fIndicator, val.filingIndicators[_value]), filingIndicator=_value) val.filingIndicators[_value] = fIndicator.get("{http://www.eurofiling.info/xbrl/ext/filing-indicators}filed", "true") in ("true", "1") val.unusedCntxIDs.discard(fIndicator.contextID) cntx = fIndicator.context if cntx is not None and (cntx.hasSegment or cntx.hasScenario): modelXbrl.error("EIOPA.N.1.6.d" if val.isEIOPAfullVersion else "EIOPA.S.1.6.d", _('Filing indicators must not contain segment or scenario elements %(filingIndicator)s.'), modelObject=fIndicator, filingIndicator=_value) if fIndicators.objectIndex > val.firstFactObjectIndex: modelXbrl.warning("EIOPA.1.6.2", _('Filing indicators should precede first fact %(firstFact)s.'), modelObject=(fIndicators, val.firstFact), firstFact=val.firstFact.qname) if val.isEIOPAfullVersion: for fIndicator in factsByQname[qnFilingIndicator]: if fIndicator.getparent().qname == XbrlConst.qnXbrliXbrl: _isPos = fIndicator.get("{http://www.eurofiling.info/xbrl/ext/filing-indicators}filed", "true") in ("true", "1") _value = (getattr(fIndicator, "xValue", None) or fIndicator.value) # use validated xValue if DTS else value for skipDTS modelXbrl.error("EIOPA.1.6.a" if _isPos else "EIOPA.1.6.b", _('Filing indicators must be in a tuple %(filingIndicator)s.'), modelObject=fIndicator, filingIndicator=_value, messageCodes=("EIOPA.1.6.a", "EIOPA.1.6.b")) otherFacts = {} # (contextHash, unitHash, xmlLangHash) : fact nilFacts = [] # removed in current draft: stringFactsWithoutXmlLang = [] nonMonetaryNonPureFacts = [] for qname, facts in factsByQname.items(): for f in facts: if modelXbrl.skipDTS: c = f.qname.localName[0] isNumeric = c in ('m', 'p', 'r', 'i') isMonetary = c == 'm' isInteger = c == 'i' isPercent = c == 'p' isString = c == 's' isEnum = c == 'e' else: concept = f.concept if concept is not None: isNumeric = concept.isNumeric isMonetary = concept.isMonetary isInteger = concept.baseXbrliType in integerItemTypes isPercent = concept.typeQname in (qnPercentItemType, qnPureItemType) isString = concept.baseXbrliType in ("stringItemType", "normalizedStringItemType") isEnum = concept.typeQname == qnEnumerationItemType else: isNumeric = isString = isEnum = False # error situation k = (f.getparent().objectIndex, f.qname, f.context.contextDimAwareHash if f.context is not None else None, f.unit.hash if f.unit is not None else None, hash(f.xmlLang)) if f.qname == qnFIndicators and val.validateEIOPA: pass elif k not in otherFacts: otherFacts[k] = {f} else: matches = [o for o in otherFacts[k] if (f.getparent().objectIndex == o.getparent().objectIndex and f.qname == o.qname and f.context.isEqualTo(o.context) if f.context is not None and o.context is not None else True) and # (f.unit.isEqualTo(o.unit) if f.unit is not None and o.unit is not None else True) and (f.xmlLang == o.xmlLang)] if matches: contexts = [f.contextID] + [o.contextID for o in matches] modelXbrl.error(("EBA.2.16", "EIOPA.S.2.16" if val.isEIOPAfullVersion else "EIOPA.S.2.16.a"), _('Facts are duplicates %(fact)s contexts %(contexts)s.'), modelObject=[f] + matches, fact=f.qname, contexts=', '.join(contexts), messageCodes=("EBA.2.16", "EIOPA.S.2.16", "EIOPA.S.2.16.a")) else: otherFacts[k].add(f) if isNumeric: if f.precision: modelXbrl.error(("EBA.2.17", "EIOPA.2.18.a"), _("Numeric fact %(fact)s of context %(contextID)s has a precision attribute '%(precision)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, precision=f.precision) if f.decimals and not f.isNil: if f.decimals == "INF": if not val.isEIOPAfullVersion: modelXbrl.error("EIOPA.S.2.18.f", _("Monetary fact %(fact)s of context %(contextID)s has a decimal attribute INF: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) else: try: xValue = f.xValue dec = int(f.decimals) if isMonetary: if dec < -3: modelXbrl.error(("EBA.2.18","EIOPA.S.2.18.c"), _("Monetary fact %(fact)s of context %(contextID)s has a decimals attribute < -3: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) else: # apply dynamic decimals check if -.1 < xValue < .1: dMin = 2 elif -1 < xValue < 1: dMin = 1 elif -10 < xValue < 10: dMin = 0 elif -100 < xValue < 100: dMin = -1 elif -1000 < xValue < 1000: dMin = -2 else: dMin = -3 if dMin > dec: modelXbrl.warning("EIOPA:factDecimalsWarning", _("Monetary fact %(fact)s of context %(contextID)s value %(value)s has an imprecise decimals attribute: %(decimals)s, minimum is %(mindec)s"), modelObject=f, fact=f.qname, contextID=f.contextID, value=xValue, decimals=f.decimals, mindec=dMin) elif isInteger: if dec != 0: modelXbrl.error(("EBA.2.18","EIOPA.S.2.18.d"), _("Integer fact %(fact)s of context %(contextID)s has a decimals attribute \u2260 0: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) elif isPercent: if dec < 4: modelXbrl.error(("EBA.2.18","EIOPA.S.2.18.e"), _("Percent fact %(fact)s of context %(contextID)s has a decimals attribute < 4: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) else: if -.001 < xValue < .001: dMin = 4 elif -.01 < xValue < .01: dMin = 3 elif -.1 < xValue < .1: dMin = 2 elif -1 < xValue < 1: dMin = 1 else: dMin = 0 if dMin > dec: modelXbrl.warning("EIOPA:factDecimalsWarning", _("Numeric fact %(fact)s of context %(contextID)s value %(value)s has an imprecise decimals attribute: %(decimals)s, minimum is %(mindec)s"), modelObject=f, fact=f.qname, contextID=f.contextID, value=xValue, decimals=f.decimals, mindec=dMin) except (AttributeError, ValueError): pass # should have been reported as a schema error by loader '''' (not intended by EBA 2.18, paste here is from EFM) if not f.isNil and getattr(f,"xValid", 0) == 4: try: insignificance = insignificantDigits(f.xValue, decimals=f.decimals) if insignificance: # if not None, returns (truncatedDigits, insiginficantDigits) modelXbrl.error(("EFM.6.05.37", "GFM.1.02.26"), _("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s has nonzero digits in insignificant portion %(insignificantDigits)s."), modelObject=f1, fact=f1.qname, contextID=f1.contextID, decimals=f1.decimals, value=f1.xValue, truncatedDigits=insignificance[0], insignificantDigits=insignificance[1]) except (ValueError,TypeError): modelXbrl.error(("EBA.2.18"), _("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s causes Value Error exception."), modelObject=f1, fact=f1.qname, contextID=f1.contextID, decimals=f1.decimals, value=f1.value) ''' unit = f.unit if unit is not None: if isMonetary: if unit.measures[0]: val.currenciesUsed[unit.measures[0][0]] = unit elif not unit.isSingleMeasure or unit.measures[0][0] != XbrlConst.qnXbrliPure: nonMonetaryNonPureFacts.append(f) if isEnum: _eQn = getattr(f,"xValue", None) or qnameEltPfxName(f, f.value) if _eQn: val.namespacePrefixesUsed[_eQn.namespaceURI].add(_eQn.prefix) val.prefixesUnused.discard(_eQn.prefix) ''' removed in current draft elif isString: if not f.xmlLang: stringFactsWithoutXmlLang.append(f) ''' if f.isNil: nilFacts.append(f) if val.footnotesRelationshipSet.fromModelObject(f): modelXbrl.warning("EIOPA.S.19", _("Fact %(fact)s of context %(contextID)s has footnotes.'"), modelObject=f, fact=f.qname, contextID=f.contextID) if nilFacts: modelXbrl.error(("EBA.2.19", "EIOPA.S.2.19"), _('Nil facts MUST NOT be present in the instance: %(nilFacts)s.'), modelObject=nilFacts, nilFacts=", ".join(str(f.qname) for f in nilFacts)) ''' removed in current draft if stringFactsWithoutXmlLang: modelXbrl.error("EBA.2.20", _("String facts need to report xml:lang: '%(langLessFacts)s'"), modelObject=stringFactsWithoutXmlLang, langLessFacts=", ".join(set(str(f.qname) for f in stringFactsWithoutXmlLang))) ''' if nonMonetaryNonPureFacts: modelXbrl.error(("EBA.3.2","EIOPA.3.2.a"), _("Non monetary (numeric) facts MUST use the pure unit: '%(langLessFacts)s'"), modelObject=nonMonetaryNonPureFacts, langLessFacts=", ".join(set(str(f.qname) for f in nonMonetaryNonPureFacts))) val.utrValidator.validateFacts() # validate facts for UTR at logLevel WARNING unitHashes = {} for unit in modelXbrl.units.values(): h = unit.hash if h in unitHashes and unit.isEqualTo(unitHashes[h]): modelXbrl.warning("EBA.2.21", _("Duplicate units SHOULD NOT be reported, units %(unit1)s and %(unit2)s have same measures.'"), modelObject=(unit, unitHashes[h]), unit1=unit.id, unit2=unitHashes[h].id) if not getattr(modelXbrl, "isStreamingMode", False): modelXbrl.error("EIOPA.2.21", _("Duplicate units MUST NOT be reported, units %(unit1)s and %(unit2)s have same measures.'"), modelObject=(unit, unitHashes[h]), unit1=unit.id, unit2=unitHashes[h].id) else: unitHashes[h] = unit for _measures in unit.measures: for _measure in _measures: val.namespacePrefixesUsed[_measure.namespaceURI].add(_measure.prefix) val.prefixesUnused.discard(_measure.prefix) del unitHashes cntxHashes = {} for cntx in modelXbrl.contexts.values(): h = cntx.contextDimAwareHash if h in cntxHashes and cntx.isEqualTo(cntxHashes[h]): if not getattr(modelXbrl, "isStreamingMode", False): modelXbrl.log("WARNING" if val.isEIOPAfullVersion else "ERROR", "EIOPA.S.2.7.b", _("Duplicate contexts MUST NOT be reported, contexts %(cntx1)s and %(cntx2)s are equivalent.'"), modelObject=(cntx, cntxHashes[h]), cntx1=cntx.id, cntx2=cntxHashes[h].id) else: cntxHashes[h] = cntx for _dim in cntx.qnameDims.values(): _dimQn = _dim.dimensionQname val.namespacePrefixesUsed[_dimQn.namespaceURI].add(_dimQn.prefix) val.prefixesUnused.discard(_dimQn.prefix) if _dim.isExplicit: _memQn = _dim.memberQname else: _memQn = _dim.typedMember.qname if _memQn: val.namespacePrefixesUsed[_memQn.namespaceURI].add(_memQn.prefix) val.prefixesUnused.discard(_memQn.prefix) for elt in modelDocument.xmlRootElement.iter(): if isinstance(elt, ModelObject): # skip comments and processing instructions val.namespacePrefixesUsed[elt.qname.namespaceURI].add(elt.qname.prefix) val.prefixesUnused.discard(elt.qname.prefix) for attrTag in elt.keys(): if attrTag.startswith("{"): _prefix, _NS, _localName = XmlUtil.clarkNotationToPrefixNsLocalname(elt, attrTag, isAttribute=True) if _prefix: val.namespacePrefixesUsed[_NS].add(_prefix) val.prefixesUnused.discard(_prefix)