def validateFacts(val, factsToCheck): # may be called in streaming batches or all at end (final) if not streaming modelXbrl = val.modelXbrl modelDocument = modelXbrl.modelDocument # note EBA 2.1 is in ModelDocument.py timelessDatePattern = re.compile(r"\s*([0-9]{4})-([0-9]{2})-([0-9]{2})\s*$") for cntx in modelXbrl.contexts.values(): if getattr(cntx, "_batchChecked", False): continue # prior streaming batch already checked cntx._batchChecked = True val.cntxEntities.add(cntx.entityIdentifier) dateElts = XmlUtil.descendants(cntx, XbrlConst.xbrli, ("startDate","endDate","instant")) if any(not timelessDatePattern.match(e.textValue) for e in dateElts): modelXbrl.error(("EBA.2.10","EIOPA.2.10"), _('Period dates must be whole dates without time or timezone: %(dates)s.'), modelObject=cntx, dates=", ".join(e.text for e in dateElts)) if cntx.isForeverPeriod: modelXbrl.error(("EBA.2.11","EIOPA.N.2.11"), _('Forever context period is not allowed.'), modelObject=cntx) elif cntx.isStartEndPeriod: modelXbrl.error(("EBA.2.13","EIOPA.N.2.11"), _('Start-End (flow) context period is not allowed.'), modelObject=cntx) elif cntx.isInstantPeriod: # cannot pass context object to final() below, for error logging, if streaming mode val.cntxDates[cntx.instantDatetime].add(modelXbrl if getattr(val.modelXbrl, "isStreamingMode", False) else cntx) if cntx.hasSegment: modelXbrl.error(("EBA.2.14","EIOPA.N.2.14"), _("Contexts MUST NOT contain xbrli:segment values: %(cntx)s.'"), modelObject=cntx, cntx=cntx.id) if cntx.nonDimValues("scenario"): modelXbrl.error(("EBA.2.15","EIOPA.S.2.15" if val.isEIOPAfullVersion else "EIOPA.N.2.15"), _("Contexts MUST NOT contain non-dimensional xbrli:scenario values: %(cntx)s.'"), modelObject=cntx, cntx=cntx.id, messageCodes=("EBA.2.15","EIOPA.N.2.15","EIOPA.S.2.15")) val.unusedCntxIDs.add(cntx.id) if val.isEIOPA_2_0_1 and len(cntx.id) > 128: modelXbrl.warning("EIOPA.S.2.6", _("Contexts IDs SHOULD be short: %(cntx)s.'"), modelObject=cntx, cntx=cntx.id) for unit in modelXbrl.units.values(): if getattr(unit, "_batchChecked", False): continue # prior streaming batch already checked unit._batchChecked = True val.unusedUnitIDs.add(unit.id) factsByQname = defaultdict(set) # top level for this for f in factsToCheck: factsByQname[f.qname].add(f) val.unusedCntxIDs.discard(f.contextID) val.unusedUnitIDs.discard(f.unitID) if f.objectIndex < val.firstFactObjectIndex: val.firstFactObjectIndex = f.objectIndex val.firstFact = f for fIndicators in factsByQname[qnFIndicators]: val.numFilingIndicatorTuples += 1 for fIndicator in fIndicators.modelTupleFacts: _value = (getattr(fIndicator, "xValue", None) or fIndicator.value) # use validated xValue if DTS else value for skipDTS _filed = fIndicator.get("{http://www.eurofiling.info/xbrl/ext/filing-indicators}filed", "true") in ("true", "1") if _value in val.filingIndicators: modelXbrl.error(("EBA.1.6.1", "EIOPA.1.6.1"), _('Multiple filing indicators facts for indicator %(filingIndicator)s.'), modelObject=(fIndicator, val.filingIndicators[_value]), filingIndicator=_value) if _filed and not val.filingIndicators[_value]: val.filingIndicators[_value] = _filed #set to filed if any of the multiple indicators are filed=true else: # not a duplicate filing indicator val.filingIndicators[_value] = _filed val.unusedCntxIDs.discard(fIndicator.contextID) cntx = fIndicator.context if cntx is not None and (cntx.hasSegment or cntx.hasScenario): modelXbrl.error("EIOPA.N.1.6.d" if val.isEIOPAfullVersion else "EIOPA.S.1.6.d", _('Filing indicators must not contain segment or scenario elements %(filingIndicator)s.'), modelObject=fIndicator, filingIndicator=_value) if fIndicators.objectIndex > val.firstFactObjectIndex: modelXbrl.warning("EIOPA.1.6.2", _('Filing indicators should precede first fact %(firstFact)s.'), modelObject=(fIndicators, val.firstFact), firstFact=val.firstFact.qname) if val.isEIOPAfullVersion: for fIndicator in factsByQname[qnFilingIndicator]: if fIndicator.getparent().qname == XbrlConst.qnXbrliXbrl: _isPos = fIndicator.get("{http://www.eurofiling.info/xbrl/ext/filing-indicators}filed", "true") in ("true", "1") _value = (getattr(fIndicator, "xValue", None) or fIndicator.value) # use validated xValue if DTS else value for skipDTS modelXbrl.error("EIOPA.1.6.a" if _isPos else "EIOPA.1.6.b", _('Filing indicators must be in a tuple %(filingIndicator)s.'), modelObject=fIndicator, filingIndicator=_value, messageCodes=("EIOPA.1.6.a", "EIOPA.1.6.b")) otherFacts = {} # (contextHash, unitHash, xmlLangHash) : fact nilFacts = [] stringFactsWithXmlLang = [] nonMonetaryNonPureFacts = [] for qname, facts in factsByQname.items(): for f in facts: if f.qname == qnFilingIndicator: continue # skip erroneous root-level filing indicators if modelXbrl.skipDTS: c = f.qname.localName[0] isNumeric = c in ('m', 'p', 'r', 'i') isMonetary = c == 'm' isInteger = c == 'i' isPercent = c == 'p' isString = c == 's' isEnum = c == 'e' else: concept = f.concept if concept is not None: isNumeric = concept.isNumeric isMonetary = concept.isMonetary isInteger = concept.baseXbrliType in integerItemTypes isPercent = concept.typeQname in (qnPercentItemType, qnPureItemType) isString = concept.baseXbrliType in ("stringItemType", "normalizedStringItemType") isEnum = concept.typeQname == qnEnumerationItemType else: isNumeric = isString = isEnum = False # error situation k = (f.getparent().objectIndex, f.qname, f.context.contextDimAwareHash if f.context is not None else None, f.unit.hash if f.unit is not None else None, hash(f.xmlLang)) if f.qname == qnFIndicators and val.validateEIOPA: pass elif k not in otherFacts: otherFacts[k] = {f} else: matches = [o for o in otherFacts[k] if (f.getparent().objectIndex == o.getparent().objectIndex and f.qname == o.qname and f.context.isEqualTo(o.context) if f.context is not None and o.context is not None else True) and # (f.unit.isEqualTo(o.unit) if f.unit is not None and o.unit is not None else True) and (f.xmlLang == o.xmlLang)] if matches: contexts = [f.contextID] + [o.contextID for o in matches] modelXbrl.error(("EBA.2.16", "EIOPA.S.2.16" if val.isEIOPAfullVersion else "EIOPA.S.2.16.a"), _('Facts are duplicates %(fact)s contexts %(contexts)s.'), modelObject=[f] + matches, fact=f.qname, contexts=', '.join(contexts), messageCodes=("EBA.2.16", "EIOPA.S.2.16", "EIOPA.S.2.16.a")) else: otherFacts[k].add(f) if isNumeric: if f.precision: modelXbrl.error(("EBA.2.17", "EIOPA.2.18.a"), _("Numeric fact %(fact)s of context %(contextID)s has a precision attribute '%(precision)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, precision=f.precision) if f.decimals and not f.isNil: # in XbrlDpmSqlDB for 2_0_1 if f.decimals == "INF": if not val.isEIOPAfullVersion: modelXbrl.error("EIOPA.S.2.18.f", _("Monetary fact %(fact)s of context %(contextID)s has a decimal attribute INF: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) else: try: xValue = f.xValue dec = int(f.decimals) if isMonetary: if val.isEIOPA_2_0_1: _absXvalue = abs(xValue) if str(f.qname) in s_2_18_c_a_met: dMin = 2 elif _absXvalue >= 100000000: dMin = -4 elif 100000000 > _absXvalue >= 1000000: dMin = -3 elif 1000000 > _absXvalue >= 1000: dMin = -2 else: dMin = -1 if dMin > dec: modelXbrl.error("EIOPA.S.2.18.c", _("Monetary fact %(fact)s of context %(contextID)s has a decimals attribute less than minimum %(minimumDecimals)s: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, minimumDecimals=dMin, decimals=f.decimals) elif dec < -3: modelXbrl.error(("EBA.2.18","EIOPA.S.2.18.c"), _("Monetary fact %(fact)s of context %(contextID)s has a decimals attribute < -3: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) else: # apply dynamic decimals check if -.1 < xValue < .1: dMin = 2 elif -1 < xValue < 1: dMin = 1 elif -10 < xValue < 10: dMin = 0 elif -100 < xValue < 100: dMin = -1 elif -1000 < xValue < 1000: dMin = -2 else: dMin = -3 if dMin > dec: modelXbrl.warning("EIOPA:factDecimalsWarning", _("Monetary fact %(fact)s of context %(contextID)s value %(value)s has an imprecise decimals attribute: %(decimals)s, minimum is %(mindec)s"), modelObject=f, fact=f.qname, contextID=f.contextID, value=xValue, decimals=f.decimals, mindec=dMin) elif isInteger: if dec != 0: modelXbrl.error(("EBA.2.18","EIOPA.S.2.18.d"), _("Integer fact %(fact)s of context %(contextID)s has a decimals attribute \u2260 0: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) elif isPercent: if dec < 4: modelXbrl.error(("EBA.2.18","EIOPA.S.2.18.e"), _("Percent fact %(fact)s of context %(contextID)s has a decimals attribute < 4: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) if val.isEIOPA_2_0_1 and xValue > 1: modelXbrl.warning(("EIOPA.3.2.b"), _("Percent fact %(fact)s of context %(contextID)s appears to be over 100% = 1.0: '%(value)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, value=xValue) else: if -.001 < xValue < .001: dMin = 4 elif -.01 < xValue < .01: dMin = 3 elif -.1 < xValue < .1: dMin = 2 elif -1 < xValue < 1: dMin = 1 else: dMin = 0 if dMin > dec: modelXbrl.warning("EIOPA:factDecimalsWarning", _("Numeric fact %(fact)s of context %(contextID)s value %(value)s has an imprecise decimals attribute: %(decimals)s, minimum is %(mindec)s"), modelObject=f, fact=f.qname, contextID=f.contextID, value=xValue, decimals=f.decimals, mindec=dMin) except (AttributeError, ValueError): pass # should have been reported as a schema error by loader '''' (not intended by EBA 2.18, paste here is from EFM) if not f.isNil and getattr(f,"xValid", 0) == 4: try: insignificance = insignificantDigits(f.xValue, decimals=f.decimals) if insignificance: # if not None, returns (truncatedDigits, insiginficantDigits) modelXbrl.error(("EFM.6.05.37", "GFM.1.02.26"), _("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s has nonzero digits in insignificant portion %(insignificantDigits)s."), modelObject=f1, fact=f1.qname, contextID=f1.contextID, decimals=f1.decimals, value=f1.xValue, truncatedDigits=insignificance[0], insignificantDigits=insignificance[1]) except (ValueError,TypeError): modelXbrl.error(("EBA.2.18"), _("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s causes Value Error exception."), modelObject=f1, fact=f1.qname, contextID=f1.contextID, decimals=f1.decimals, value=f1.value) ''' unit = f.unit if unit is not None: if isMonetary: if unit.measures[0]: _currencyMeasure = unit.measures[0][0] if val.isEIOPA_2_0_1 and f.context is not None: if f.context.dimMemberQname(val.qnDimAF) == val.qnCAx1 and val.qnDimOC in f.context.qnameDims: _ocCurrency = f.context.dimMemberQname(val.qnDimOC).localName if _currencyMeasure.localName != _ocCurrency: modelXbrl.error("EIOPA.3.1", _("There MUST be only one currency but metric %(metric)s reported OC dimension currency %(ocCurrency)s differs from unit currency: %(unitCurrency)s."), modelObject=f, metric=f.qname, ocCurrency=_ocCurrency, unitCurrency=_currencyMeasure.localName) else: val.currenciesUsed[_currencyMeasure] = unit else: val.currenciesUsed[_currencyMeasure] = unit elif not unit.isSingleMeasure or unit.measures[0][0] != XbrlConst.qnXbrliPure: nonMonetaryNonPureFacts.append(f) if isEnum: _eQn = getattr(f,"xValue", None) or qnameEltPfxName(f, f.value) if _eQn: prefixUsed(val, _eQn.namespaceURI, _eQn.prefix) if val.isEIOPA_2_0_1 and f.qname.localName == "ei1930": val.reportingCurrency = _eQn.localName elif isString: if f.xmlLang: # requires disclosureSystem to NOT specify default language stringFactsWithXmlLang.append(f) if f.isNil: nilFacts.append(f) if val.footnotesRelationshipSet.fromModelObject(f): modelXbrl.warning("EIOPA.S.19", _("Fact %(fact)s of context %(contextID)s has footnotes.'"), modelObject=f, fact=f.qname, contextID=f.contextID) if nilFacts: modelXbrl.error(("EBA.2.19", "EIOPA.S.2.19"), _('Nil facts MUST NOT be present in the instance: %(nilFacts)s.'), modelObject=nilFacts, nilFacts=", ".join(str(f.qname) for f in nilFacts)) if stringFactsWithXmlLang: modelXbrl.warning("EIOPA.2.20", # not reported for EBA _("String facts reporting xml:lang (not saved by T4U, not round-tripped): '%(factsWithLang)s'"), modelObject=stringFactsWithXmlLang, factsWithLang=", ".join(set(str(f.qname) for f in stringFactsWithXmlLang))) if nonMonetaryNonPureFacts: modelXbrl.error(("EBA.3.2","EIOPA.3.2.a"), _("Non monetary (numeric) facts MUST use the pure unit: '%(langLessFacts)s'"), modelObject=nonMonetaryNonPureFacts, langLessFacts=", ".join(set(str(f.qname) for f in nonMonetaryNonPureFacts))) val.utrValidator.validateFacts() # validate facts for UTR at logLevel WARNING unitHashes = {} for unit in modelXbrl.units.values(): h = unit.hash if h in unitHashes and unit.isEqualTo(unitHashes[h]): modelXbrl.warning("EBA.2.21", _("Duplicate units SHOULD NOT be reported, units %(unit1)s and %(unit2)s have same measures.'"), modelObject=(unit, unitHashes[h]), unit1=unit.id, unit2=unitHashes[h].id) if not getattr(modelXbrl, "isStreamingMode", False): modelXbrl.error("EIOPA.2.21", _("Duplicate units MUST NOT be reported, units %(unit1)s and %(unit2)s have same measures.'"), modelObject=(unit, unitHashes[h]), unit1=unit.id, unit2=unitHashes[h].id) else: unitHashes[h] = unit for _measures in unit.measures: for _measure in _measures: prefixUsed(val, _measure.namespaceURI, _measure.prefix) del unitHashes cntxHashes = {} for cntx in modelXbrl.contexts.values(): h = cntx.contextDimAwareHash if h in cntxHashes and cntx.isEqualTo(cntxHashes[h]): if not getattr(modelXbrl, "isStreamingMode", False): modelXbrl.log("WARNING" if val.isEIOPAfullVersion else "ERROR", "EIOPA.S.2.7.b", _("Duplicate contexts MUST NOT be reported, contexts %(cntx1)s and %(cntx2)s are equivalent.'"), modelObject=(cntx, cntxHashes[h]), cntx1=cntx.id, cntx2=cntxHashes[h].id) else: cntxHashes[h] = cntx for _dim in cntx.qnameDims.values(): _dimQn = _dim.dimensionQname prefixUsed(val, _dimQn.namespaceURI, _dimQn.prefix) if _dim.isExplicit: _memQn = _dim.memberQname else: _memQn = _dim.typedMember.qname if _memQn: prefixUsed(val, _memQn.namespaceURI, _memQn.prefix) for elt in modelDocument.xmlRootElement.iter(): if isinstance(elt, ModelObject): # skip comments and processing instructions prefixUsed(val, elt.qname.namespaceURI, elt.qname.prefix) for attrTag in elt.keys(): if attrTag.startswith("{"): _prefix, _NS, _localName = XmlUtil.clarkNotationToPrefixNsLocalname(elt, attrTag, isAttribute=True) if _prefix: prefixUsed(val, _NS, _prefix) elif val.isEIOPA_2_0_1: if elt.tag in ("{http://www.w3.org/2001/XMLSchema}documentation", "{http://www.w3.org/2001/XMLSchema}annotation"): modelXbrl.error("EIOPA.2.5", _("xs:documentation element found, all relevant business data MUST only be contained in contexts, units, schemaRef and facts."), modelObject=modelDocument) elif isinstance(elt, etree._Comment): modelXbrl.error("EIOPA.2.5", _("XML comment found, all relevant business data MUST only be contained in contexts, units, schemaRef and facts: %(comment)s"), modelObject=modelDocument, comment=elt.text)
def validateFacts(val, factsToCheck): # may be called in streaming batches or all at end (final) if not streaming modelXbrl = val.modelXbrl modelDocument = modelXbrl.modelDocument # note EBA 2.1 is in ModelDocument.py timelessDatePattern = re.compile(r"\s*([0-9]{4})-([0-9]{2})-([0-9]{2})\s*$") for cntx in modelXbrl.contexts.values(): if getattr(cntx, "_batchChecked", False): continue # prior streaming batch already checked cntx._batchChecked = True val.cntxEntities.add(cntx.entityIdentifier) dateElts = XmlUtil.descendants(cntx, XbrlConst.xbrli, ("startDate","endDate","instant")) if any(not timelessDatePattern.match(e.textValue) for e in dateElts): modelXbrl.error(("EBA.2.10","EIOPA.2.10"), _('Period dates must be whole dates without time or timezone: %(dates)s.'), modelObject=cntx, dates=", ".join(e.text for e in dateElts)) if cntx.isForeverPeriod: modelXbrl.error(("EBA.2.11","EIOPA.N.2.11"), _('Forever context period is not allowed.'), modelObject=cntx) elif cntx.isStartEndPeriod: modelXbrl.error(("EBA.2.13","EIOPA.N.2.11"), _('Start-End (flow) context period is not allowed.'), modelObject=cntx) elif cntx.isInstantPeriod: # cannot pass context object to final() below, for error logging, if streaming mode val.cntxDates[cntx.instantDatetime].add(modelXbrl if getattr(val.modelXbrl, "isStreamingMode", False) else cntx) if cntx.hasSegment: modelXbrl.error(("EBA.2.14","EIOPA.N.2.14"), _("Contexts MUST NOT contain xbrli:segment values: %(cntx)s.'"), modelObject=cntx, cntx=cntx.id) if cntx.nonDimValues("scenario"): modelXbrl.error(("EBA.2.15","EIOPA.S.2.15" if val.isEIOPAfullVersion else "EIOPA.N.2.15"), _("Contexts MUST NOT contain non-dimensional xbrli:scenario values: %(cntx)s.'"), modelObject=cntx, cntx=cntx.id, messageCodes=("EBA.2.15","EIOPA.N.2.15","EIOPA.S.2.15")) val.unusedCntxIDs.add(cntx.id) for unit in modelXbrl.units.values(): if getattr(unit, "_batchChecked", False): continue # prior streaming batch already checked unit._batchChecked = True val.unusedUnitIDs.add(unit.id) factsByQname = defaultdict(set) # top level for this for f in factsToCheck: factsByQname[f.qname].add(f) val.unusedCntxIDs.discard(f.contextID) val.unusedUnitIDs.discard(f.unitID) if f.objectIndex < val.firstFactObjectIndex: val.firstFactObjectIndex = f.objectIndex val.firstFact = f for fIndicators in factsByQname[qnFIndicators]: val.numFilingIndicatorTuples += 1 for fIndicator in fIndicators.modelTupleFacts: _value = (getattr(fIndicator, "xValue", None) or fIndicator.value) # use validated xValue if DTS else value for skipDTS if _value in val.filingIndicators: modelXbrl.error(("EBA.1.6.1", "EIOPA.1.6.1"), _('Multiple filing indicators facts for indicator %(filingIndicator)s.'), modelObject=(fIndicator, val.filingIndicators[_value]), filingIndicator=_value) val.filingIndicators[_value] = fIndicator.get("{http://www.eurofiling.info/xbrl/ext/filing-indicators}filed", "true") in ("true", "1") val.unusedCntxIDs.discard(fIndicator.contextID) cntx = fIndicator.context if cntx is not None and (cntx.hasSegment or cntx.hasScenario): modelXbrl.error("EIOPA.N.1.6.d" if val.isEIOPAfullVersion else "EIOPA.S.1.6.d", _('Filing indicators must not contain segment or scenario elements %(filingIndicator)s.'), modelObject=fIndicator, filingIndicator=_value) if fIndicators.objectIndex > val.firstFactObjectIndex: modelXbrl.warning("EIOPA.1.6.2", _('Filing indicators should precede first fact %(firstFact)s.'), modelObject=(fIndicators, val.firstFact), firstFact=val.firstFact.qname) if val.isEIOPAfullVersion: for fIndicator in factsByQname[qnFilingIndicator]: if fIndicator.getparent().qname == XbrlConst.qnXbrliXbrl: _isPos = fIndicator.get("{http://www.eurofiling.info/xbrl/ext/filing-indicators}filed", "true") in ("true", "1") _value = (getattr(fIndicator, "xValue", None) or fIndicator.value) # use validated xValue if DTS else value for skipDTS modelXbrl.error("EIOPA.1.6.a" if _isPos else "EIOPA.1.6.b", _('Filing indicators must be in a tuple %(filingIndicator)s.'), modelObject=fIndicator, filingIndicator=_value, messageCodes=("EIOPA.1.6.a", "EIOPA.1.6.b")) otherFacts = {} # (contextHash, unitHash, xmlLangHash) : fact nilFacts = [] # removed in current draft: stringFactsWithoutXmlLang = [] nonMonetaryNonPureFacts = [] for qname, facts in factsByQname.items(): for f in facts: if modelXbrl.skipDTS: c = f.qname.localName[0] isNumeric = c in ('m', 'p', 'r', 'i') isMonetary = c == 'm' isInteger = c == 'i' isPercent = c == 'p' isString = c == 's' isEnum = c == 'e' else: concept = f.concept if concept is not None: isNumeric = concept.isNumeric isMonetary = concept.isMonetary isInteger = concept.baseXbrliType in integerItemTypes isPercent = concept.typeQname in (qnPercentItemType, qnPureItemType) isString = concept.baseXbrliType in ("stringItemType", "normalizedStringItemType") isEnum = concept.typeQname == qnEnumerationItemType else: isNumeric = isString = isEnum = False # error situation k = (f.getparent().objectIndex, f.qname, f.context.contextDimAwareHash if f.context is not None else None, f.unit.hash if f.unit is not None else None, hash(f.xmlLang)) if f.qname == qnFIndicators and val.validateEIOPA: pass elif k not in otherFacts: otherFacts[k] = {f} else: matches = [o for o in otherFacts[k] if (f.getparent().objectIndex == o.getparent().objectIndex and f.qname == o.qname and f.context.isEqualTo(o.context) if f.context is not None and o.context is not None else True) and # (f.unit.isEqualTo(o.unit) if f.unit is not None and o.unit is not None else True) and (f.xmlLang == o.xmlLang)] if matches: contexts = [f.contextID] + [o.contextID for o in matches] modelXbrl.error(("EBA.2.16", "EIOPA.S.2.16" if val.isEIOPAfullVersion else "EIOPA.S.2.16.a"), _('Facts are duplicates %(fact)s contexts %(contexts)s.'), modelObject=[f] + matches, fact=f.qname, contexts=', '.join(contexts), messageCodes=("EBA.2.16", "EIOPA.S.2.16", "EIOPA.S.2.16.a")) else: otherFacts[k].add(f) if isNumeric: if f.precision: modelXbrl.error(("EBA.2.17", "EIOPA.2.18.a"), _("Numeric fact %(fact)s of context %(contextID)s has a precision attribute '%(precision)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, precision=f.precision) if f.decimals and not f.isNil: if f.decimals == "INF": if not val.isEIOPAfullVersion: modelXbrl.error("EIOPA.S.2.18.f", _("Monetary fact %(fact)s of context %(contextID)s has a decimal attribute INF: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) else: try: xValue = f.xValue dec = int(f.decimals) if isMonetary: if dec < -3: modelXbrl.error(("EBA.2.18","EIOPA.S.2.18.c"), _("Monetary fact %(fact)s of context %(contextID)s has a decimals attribute < -3: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) else: # apply dynamic decimals check if -.1 < xValue < .1: dMin = 2 elif -1 < xValue < 1: dMin = 1 elif -10 < xValue < 10: dMin = 0 elif -100 < xValue < 100: dMin = -1 elif -1000 < xValue < 1000: dMin = -2 else: dMin = -3 if dMin > dec: modelXbrl.warning("EIOPA:factDecimalsWarning", _("Monetary fact %(fact)s of context %(contextID)s value %(value)s has an imprecise decimals attribute: %(decimals)s, minimum is %(mindec)s"), modelObject=f, fact=f.qname, contextID=f.contextID, value=xValue, decimals=f.decimals, mindec=dMin) elif isInteger: if dec != 0: modelXbrl.error(("EBA.2.18","EIOPA.S.2.18.d"), _("Integer fact %(fact)s of context %(contextID)s has a decimals attribute \u2260 0: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) elif isPercent: if dec < 4: modelXbrl.error(("EBA.2.18","EIOPA.S.2.18.e"), _("Percent fact %(fact)s of context %(contextID)s has a decimals attribute < 4: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) else: if -.001 < xValue < .001: dMin = 4 elif -.01 < xValue < .01: dMin = 3 elif -.1 < xValue < .1: dMin = 2 elif -1 < xValue < 1: dMin = 1 else: dMin = 0 if dMin > dec: modelXbrl.warning("EIOPA:factDecimalsWarning", _("Numeric fact %(fact)s of context %(contextID)s value %(value)s has an imprecise decimals attribute: %(decimals)s, minimum is %(mindec)s"), modelObject=f, fact=f.qname, contextID=f.contextID, value=xValue, decimals=f.decimals, mindec=dMin) except (AttributeError, ValueError): pass # should have been reported as a schema error by loader '''' (not intended by EBA 2.18, paste here is from EFM) if not f.isNil and getattr(f,"xValid", 0) == 4: try: insignificance = insignificantDigits(f.xValue, decimals=f.decimals) if insignificance: # if not None, returns (truncatedDigits, insiginficantDigits) modelXbrl.error(("EFM.6.05.37", "GFM.1.02.26"), _("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s has nonzero digits in insignificant portion %(insignificantDigits)s."), modelObject=f1, fact=f1.qname, contextID=f1.contextID, decimals=f1.decimals, value=f1.xValue, truncatedDigits=insignificance[0], insignificantDigits=insignificance[1]) except (ValueError,TypeError): modelXbrl.error(("EBA.2.18"), _("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s causes Value Error exception."), modelObject=f1, fact=f1.qname, contextID=f1.contextID, decimals=f1.decimals, value=f1.value) ''' unit = f.unit if unit is not None: if isMonetary: if unit.measures[0]: val.currenciesUsed[unit.measures[0][0]] = unit elif not unit.isSingleMeasure or unit.measures[0][0] != XbrlConst.qnXbrliPure: nonMonetaryNonPureFacts.append(f) if isEnum: _eQn = getattr(f,"xValue", None) or qnameEltPfxName(f, f.value) if _eQn: val.namespacePrefixesUsed[_eQn.namespaceURI].add(_eQn.prefix) val.prefixesUnused.discard(_eQn.prefix) ''' removed in current draft elif isString: if not f.xmlLang: stringFactsWithoutXmlLang.append(f) ''' if f.isNil: nilFacts.append(f) if val.footnotesRelationshipSet.fromModelObject(f): modelXbrl.warning("EIOPA.S.19", _("Fact %(fact)s of context %(contextID)s has footnotes.'"), modelObject=f, fact=f.qname, contextID=f.contextID) if nilFacts: modelXbrl.error(("EBA.2.19", "EIOPA.S.2.19"), _('Nil facts MUST NOT be present in the instance: %(nilFacts)s.'), modelObject=nilFacts, nilFacts=", ".join(str(f.qname) for f in nilFacts)) ''' removed in current draft if stringFactsWithoutXmlLang: modelXbrl.error("EBA.2.20", _("String facts need to report xml:lang: '%(langLessFacts)s'"), modelObject=stringFactsWithoutXmlLang, langLessFacts=", ".join(set(str(f.qname) for f in stringFactsWithoutXmlLang))) ''' if nonMonetaryNonPureFacts: modelXbrl.error(("EBA.3.2","EIOPA.3.2.a"), _("Non monetary (numeric) facts MUST use the pure unit: '%(langLessFacts)s'"), modelObject=nonMonetaryNonPureFacts, langLessFacts=", ".join(set(str(f.qname) for f in nonMonetaryNonPureFacts))) val.utrValidator.validateFacts() # validate facts for UTR at logLevel WARNING unitHashes = {} for unit in modelXbrl.units.values(): h = unit.hash if h in unitHashes and unit.isEqualTo(unitHashes[h]): modelXbrl.warning("EBA.2.21", _("Duplicate units SHOULD NOT be reported, units %(unit1)s and %(unit2)s have same measures.'"), modelObject=(unit, unitHashes[h]), unit1=unit.id, unit2=unitHashes[h].id) if not getattr(modelXbrl, "isStreamingMode", False): modelXbrl.error("EIOPA.2.21", _("Duplicate units MUST NOT be reported, units %(unit1)s and %(unit2)s have same measures.'"), modelObject=(unit, unitHashes[h]), unit1=unit.id, unit2=unitHashes[h].id) else: unitHashes[h] = unit for _measures in unit.measures: for _measure in _measures: val.namespacePrefixesUsed[_measure.namespaceURI].add(_measure.prefix) val.prefixesUnused.discard(_measure.prefix) del unitHashes cntxHashes = {} for cntx in modelXbrl.contexts.values(): h = cntx.contextDimAwareHash if h in cntxHashes and cntx.isEqualTo(cntxHashes[h]): if not getattr(modelXbrl, "isStreamingMode", False): modelXbrl.log("WARNING" if val.isEIOPAfullVersion else "ERROR", "EIOPA.S.2.7.b", _("Duplicate contexts MUST NOT be reported, contexts %(cntx1)s and %(cntx2)s are equivalent.'"), modelObject=(cntx, cntxHashes[h]), cntx1=cntx.id, cntx2=cntxHashes[h].id) else: cntxHashes[h] = cntx for _dim in cntx.qnameDims.values(): _dimQn = _dim.dimensionQname val.namespacePrefixesUsed[_dimQn.namespaceURI].add(_dimQn.prefix) val.prefixesUnused.discard(_dimQn.prefix) if _dim.isExplicit: _memQn = _dim.memberQname else: _memQn = _dim.typedMember.qname if _memQn: val.namespacePrefixesUsed[_memQn.namespaceURI].add(_memQn.prefix) val.prefixesUnused.discard(_memQn.prefix) for elt in modelDocument.xmlRootElement.iter(): if isinstance(elt, ModelObject): # skip comments and processing instructions val.namespacePrefixesUsed[elt.qname.namespaceURI].add(elt.qname.prefix) val.prefixesUnused.discard(elt.qname.prefix) for attrTag in elt.keys(): if attrTag.startswith("{"): _prefix, _NS, _localName = XmlUtil.clarkNotationToPrefixNsLocalname(elt, attrTag, isAttribute=True) if _prefix: val.namespacePrefixesUsed[_NS].add(_prefix) val.prefixesUnused.discard(_prefix)
def validateFacts(val, factsToCheck): # may be called in streaming batches or all at end (final) if not streaming modelXbrl = val.modelXbrl modelDocument = modelXbrl.modelDocument isStreamingMode = getattr(modelXbrl, "isStreamingMode", False) # note EBA 2.1 is in ModelDocument.py timelessDatePattern = re.compile(r"\s*([0-9]{4})-([0-9]{2})-([0-9]{2})\s*$") for cntx in modelXbrl.contexts.values(): if cntx is not None: if getattr(cntx, "_batchChecked", False) and isStreamingMode: continue # prior streaming batch already checked cntx._batchChecked = True val.cntxEntities.add(cntx.entityIdentifier) dateElts = XmlUtil.descendants(cntx, XbrlConst.xbrli, ("startDate","endDate","instant")) if any(not timelessDatePattern.match(e.textValue) for e in dateElts): modelXbrl.error(("EBA.2.10","EIOPA.2.10"), _('Period dates must be whole dates without time or timezone: %(dates)s.'), modelObject=cntx, dates=", ".join(e.text for e in dateElts)) if cntx.isForeverPeriod: modelXbrl.error(("EBA.2.11","EIOPA.N.2.11"), _('Forever context period is not allowed.'), modelObject=cntx) elif cntx.isStartEndPeriod: modelXbrl.error(("EBA.2.13","EIOPA.N.2.11"), _('Start-End (flow) context period is not allowed.'), modelObject=cntx) elif cntx.isInstantPeriod: # cannot pass context object to final() below, for error logging, if streaming mode val.cntxDates[cntx.instantDatetime].add(modelXbrl if getattr(val.modelXbrl, "isStreamingMode", False) else cntx) if cntx.hasSegment: modelXbrl.error(("EBA.2.14","EIOPA.N.2.14"), _("Contexts MUST NOT contain xbrli:segment values: %(cntx)s.'"), modelObject=cntx, cntx=cntx.id) if cntx.nonDimValues("scenario"): modelXbrl.error(("EBA.2.15","EIOPA.N.2.15"), _("Contexts MUST NOT contain non-dimensional xbrli:scenario values: %(cntx)s.'"), modelObject=cntx, cntx=cntx.id) val.unusedCntxIDs.add(cntx.id) for unit in modelXbrl.units.values(): if unit is None or (getattr(unit, "_batchChecked", False) and isStreamingMode): continue # prior streaming batch already checked unit._batchChecked = True val.unusedUnitIDs.add(unit.id) factsByQname = defaultdict(set) # top level for this for f in factsToCheck: factsByQname[f.qname].add(f) val.unusedCntxIDs.discard(f.contextID) val.unusedUnitIDs.discard(f.unitID) if f.objectIndex < val.firstFactObjectIndex: val.firstFactObjectIndex = f.objectIndex val.firstFact = f for fIndicators in factsByQname[qnFIndicators]: val.numFilingIndicatorTuples += 1 for fIndicator in fIndicators.modelTupleFacts: _value = (getattr(fIndicator, "xValue", None) or fIndicator.value) # use validated xValue if DTS else value for skipDTS if _value in val.filingIndicators: modelXbrl.error(("EBA.1.6.1", "EIOPA.1.6.1"), _('Multiple filing indicators facts for indicator %(filingIndicator)s.'), modelObject=(fIndicator, val.filingIndicators[_value]), filingIndicator=_value) val.filingIndicators[_value] = fIndicator.get("{http://www.eurofiling.info/xbrl/ext/filing-indicators}filed", "true") in ("true", "1") val.unusedCntxIDs.discard(fIndicator.contextID) cntx = fIndicator.context if cntx is not None and (cntx.hasSegment or cntx.hasScenario): modelXbrl.error("EIOPA.S.1.6.d", _('Filing indicators must not contain segment or scenario elements %(filingIndicator)s.'), modelObject=fIndicator, filingIndicator=_value) if fIndicators.objectIndex > val.firstFactObjectIndex: modelXbrl.warning("EIOPA.1.6.2", _('Filing indicators should precede first fact %(firstFact)s.'), modelObject=(fIndicators, val.firstFact), firstFact=val.firstFact.qname) otherFacts = {} # (contextHash, unitHash, xmlLangHash) : fact nilFacts = [] # removed in current draft: stringFactsWithoutXmlLang = [] nonMonetaryNonPureFacts = [] for qname, facts in factsByQname.items(): for f in facts: if modelXbrl.skipDTS: c = f.qname.localName[0] isNumeric = c in ('m', 'p', 'i') isMonetary = c == 'm' isInteger = c == 'i' isPercent = c == 'p' isString = c == 's' isEnum = c == 'e' else: concept = f.concept if concept is not None: isNumeric = concept.isNumeric isMonetary = concept.isMonetary isInteger = concept.baseXbrliType in integerItemTypes isPercent = concept.typeQname in (qnPercentItemType, qnPureItemType) isString = concept.baseXbrliType in ("stringItemType", "normalizedStringItemType") isEnum = concept.typeQname == qnEnumerationItemType else: isNumeric = isString = isEnum = False # error situation k = (f.getparent().objectIndex, f.qname, f.context.contextDimAwareHash if f.context is not None else None, f.unit.hash if f.unit is not None else None, hash(f.xmlLang)) if f.qname == qnFIndicators and val.validateEIOPA: pass elif k not in otherFacts: otherFacts[k] = {f} else: matches = [o for o in otherFacts[k] if (f.getparent().objectIndex == o.getparent().objectIndex and f.qname == o.qname and f.context.isEqualTo(o.context) if f.context is not None and o.context is not None else True) and # (f.unit.isEqualTo(o.unit) if f.unit is not None and o.unit is not None else True) and (f.xmlLang == o.xmlLang)] if matches: contexts = [f.contextID] + [o.contextID for o in matches] modelXbrl.error(("EBA.2.16", "EIOPA.S.2.16.a"), _('Facts are duplicates %(fact)s contexts %(contexts)s.'), modelObject=[f] + matches, fact=f.qname, contexts=', '.join(contexts)) else: otherFacts[k].add(f) if isNumeric: if f.precision: modelXbrl.error(("EBA.2.17", "EIOPA.2.18.a"), _("Numeric fact %(fact)s of context %(contextID)s has a precision attribute '%(precision)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, precision=f.precision) if f.decimals and not f.isNil: if f.decimals == "INF": modelXbrl.error("EIOPA.S.2.18.f", _("Monetary fact %(fact)s of context %(contextID)s has a decimal attribute INF: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) else: try: xValue = f.xValue dec = int(f.decimals) # will only work if getattr(f,"xValid", XmlValidate.UNVALIDATED) >= XmlValidate.VALID: if isMonetary: if dec < -3: modelXbrl.error(("EBA.2.18","EIOPA.S.2.18.c"), _("Monetary fact %(fact)s of context %(contextID)s has a decimals attribute < -3: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) else: # apply dynamic decimals check if -.1 < xValue < .1: dMin = 2 elif -1 < xValue < 1: dMin = 1 elif -10 < xValue < 10: dMin = 0 elif -100 < xValue < 100: dMin = -1 elif -1000 < xValue < 1000: dMin = -2 else: dMin = -3 if dMin > dec: modelXbrl.warning("EIOPA:factDecimalsWarning", _("Monetary fact %(fact)s of context %(contextID)s value %(value)s has an imprecise decimals attribute: %(decimals)s, minimum is %(mindec)s"), modelObject=f, fact=f.qname, contextID=f.contextID, value=xValue, decimals=f.decimals, mindec=dMin) elif isInteger: if dec != 0: modelXbrl.error(("EBA.2.18","EIOPA.S.2.18.d"), _("Integer fact %(fact)s of context %(contextID)s has a decimals attribute \u2260 0: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) elif isPercent: if dec < 4: modelXbrl.error(("EBA.2.18","EIOPA.S.2.18.e"), _("Percent fact %(fact)s of context %(contextID)s has a decimals attribute < 4: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) else: if -.001 < xValue < .001: dMin = 4 elif -.01 < xValue < .01: dMin = 3 elif -.1 < xValue < .1: dMin = 2 elif -1 < xValue < 1: dMin = 1 else: dMin = 0 if dMin > dec: modelXbrl.warning("EIOPA:factDecimalsWarning", _("Numeric fact %(fact)s of context %(contextID)s value %(value)s has an imprecise decimals attribute: %(decimals)s, minimum is %(mindec)s"), modelObject=f, fact=f.qname, contextID=f.contextID, value=xValue, decimals=f.decimals, mindec=dMin) except (AttributeError, ValueError, TypeError): pass # should have been reported as a schema error by loader '''' (not intended by EBA 2.18, paste here is from EFM) if not f.isNil and getattr(f,"xValid", 0) == 4: try: insignificance = insignificantDigits(f.xValue, decimals=f.decimals) if insignificance: # if not None, returns (truncatedDigits, insiginficantDigits) modelXbrl.error(("EFM.6.05.37", "GFM.1.02.26"), _("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s has nonzero digits in insignificant portion %(insignificantDigits)s."), modelObject=f1, fact=f1.qname, contextID=f1.contextID, decimals=f1.decimals, value=f1.xValue, truncatedDigits=insignificance[0], insignificantDigits=insignificance[1]) except (ValueError,TypeError): modelXbrl.error(("EBA.2.18"), _("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s causes Value Error exception."), modelObject=f1, fact=f1.qname, contextID=f1.contextID, decimals=f1.decimals, value=f1.value) ''' unit = f.unit if unit is not None: if isMonetary: if unit.measures[0]: val.currenciesUsed[unit.measures[0][0]] = unit elif not unit.isSingleMeasure or unit.measures[0][0] != XbrlConst.qnXbrliPure: nonMonetaryNonPureFacts.append(f) if isEnum: _eQn = getattr(f,"xValue", None) or qnameEltPfxName(f, f.value) if _eQn: val.namespacePrefixesUsed[_eQn.namespaceURI].add(_eQn.prefix) val.prefixesUnused.discard(_eQn.prefix) ''' removed in current draft elif isString: if not f.xmlLang: stringFactsWithoutXmlLang.append(f) ''' if f.isNil: nilFacts.append(f) if val.footnotesRelationshipSet.fromModelObject(f): modelXbrl.warning("EIOPA.S.19", _("Fact %(fact)s of context %(contextID)s has footnotes.'"), modelObject=f, fact=f.qname, contextID=f.contextID) if nilFacts: nilFactNames = [str(f.qname) for f in nilFacts] nilFactNames = sorted(nilFactNames) modelXbrl.error(("EBA.2.19", "EIOPA.S.2.19"), _('Nil facts MUST NOT be present in the instance: %(nilFacts)s.'), modelObject=nilFacts, nilFacts=", ".join(fname for fname in nilFactNames)) ''' removed in current draft if stringFactsWithoutXmlLang: modelXbrl.error("EBA.2.20", _("String facts need to report xml:lang: '%(langLessFacts)s'"), modelObject=stringFactsWithoutXmlLang, langLessFacts=", ".join(set(str(f.qname) for f in stringFactsWithoutXmlLang))) ''' if nonMonetaryNonPureFacts: modelXbrl.error(("EBA.3.2","EIOPA.3.2.a"), _("Non monetary (numeric) facts MUST use the pure unit: '%(langLessFacts)s'"), modelObject=nonMonetaryNonPureFacts, langLessFacts=", ".join(set(str(f.qname) for f in nonMonetaryNonPureFacts))) val.utrValidator.validateFacts() # validate facts for UTR at logLevel WARNING unitHashes = {} for unit in modelXbrl.units.values(): if unit is not None: h = unit.hash if h in unitHashes and unit.isEqualTo(unitHashes[h]): modelXbrl.warning("EBA.2.21", _("Duplicate units SHOULD NOT be reported, units %(unit1)s and %(unit2)s have same measures.'"), modelObject=(unit, unitHashes[h]), unit1=unit.id, unit2=unitHashes[h].id) if not getattr(modelXbrl, "isStreamingMode", False): modelXbrl.error("EIOPA.2.21", _("Duplicate units MUST NOT be reported, units %(unit1)s and %(unit2)s have same measures.'"), modelObject=(unit, unitHashes[h]), unit1=unit.id, unit2=unitHashes[h].id) else: unitHashes[h] = unit for _measures in unit.measures: for _measure in _measures: val.namespacePrefixesUsed[_measure.namespaceURI].add(_measure.prefix) val.prefixesUnused.discard(_measure.prefix) del unitHashes cntxHashes = {} for cntx in modelXbrl.contexts.values(): if cntx is not None: h = cntx.contextDimAwareHash if h in cntxHashes and cntx.isEqualTo(cntxHashes[h]): if not getattr(modelXbrl, "isStreamingMode", False): modelXbrl.error("EIOPA.S.2.7.b", _("Duplicate contexts MUST NOT be reported, contexts %(cntx1)s and %(cntx2)s are equivalent.'"), modelObject=(cntx, cntxHashes[h]), cntx1=cntx.id, cntx2=cntxHashes[h].id) else: cntxHashes[h] = cntx for _dim in cntx.qnameDims.values(): _dimQn = _dim.dimensionQname val.namespacePrefixesUsed[_dimQn.namespaceURI].add(_dimQn.prefix) val.prefixesUnused.discard(_dimQn.prefix) if _dim.isExplicit: _memQn = _dim.memberQname else: _memQn = _dim.typedMember.qname if _memQn: val.namespacePrefixesUsed[_memQn.namespaceURI].add(_memQn.prefix) val.prefixesUnused.discard(_memQn.prefix) for elt in modelDocument.xmlRootElement.iter(): if isinstance(elt, ModelObject): # skip comments and processing instructions val.namespacePrefixesUsed[elt.qname.namespaceURI].add(elt.qname.prefix) val.prefixesUnused.discard(elt.qname.prefix) for attrTag in elt.keys(): if attrTag.startswith("{"): _prefix, _NS, _localName = XmlUtil.clarkNotationToPrefixNsLocalname(elt, attrTag, isAttribute=True) if _prefix: val.namespacePrefixesUsed[_NS].add(_prefix) val.prefixesUnused.discard(_prefix)