def isVEqualTo(self, other): if self.isTuple or other.isTuple: return False if self.isNil: return other.isNil if other.isNil: return False if not self.context.isEqualTo(other.context): return False if self.concept.isNumeric: if other.concept.isNumeric: if not self.unit.isEqualTo(other.unit): return False if self.modelXbrl.modelManager.validateInferDecimals: d = min((inferredDecimals(self), inferredDecimals(other))); p = None else: d = None; p = min((inferredPrecision(self), inferredPrecision(other))) return roundValue(float(self.value),precision=p,decimals=d) == roundValue(float(other.value),precision=p,decimals=d) else: return False selfValue = self.value otherValue = other.value if isinstance(selfValue,str) and isinstance(otherValue,str): return selfValue.strip() == otherValue.strip() else: return selfValue == otherValue
def _assets_eq_liability_equity(modelXbrl): assets_concept = modelXbrl.nameConcepts[_ASSETS_CONCEPT][ 0] if modelXbrl.nameConcepts[_ASSETS_CONCEPT] else None liability_equity_concept = modelXbrl.nameConcepts[_LIABILITIES_CONCEPT][ 0] if modelXbrl.nameConcepts[_LIABILITIES_CONCEPT] else None if assets_concept is not None and liability_equity_concept is not None: assets_facts = modelXbrl.factsByQname[assets_concept.qname] liability_equity_facts = modelXbrl.factsByQname[ liability_equity_concept.qname] fact_dict = {} fact_dict[_ASSETS_CONCEPT] = assets_facts fact_dict[_LIABILITIES_CONCEPT] = liability_equity_facts fact_groups = facts.prepare_facts_for_calculation(fact_dict) for fact_group in fact_groups: fact_assets = fact_group[_ASSETS_CONCEPT] fact_liabilities = fact_group[_LIABILITIES_CONCEPT] if fact_assets.context is not None and fact_assets.context.instantDatetime is not None: dec_assets = inferredDecimals(fact_assets) dec_liabilities = inferredDecimals(fact_liabilities) min_dec = min(dec_assets, dec_liabilities) if roundValue(fact_assets.xValue, decimals=min_dec) != roundValue( fact_liabilities.xValue, decimals=min_dec): yield fact_assets, fact_liabilities
def isVEqualTo(self, other, deemP0Equal=False): """(bool) -- v-equality of two facts Note that facts may be in different instances """ if self.isTuple or other.isTuple: return False if self.isNil: return other.isNil if other.isNil: return False if not self.context.isEqualTo(other.context): return False if self.concept.isNumeric: if other.concept.isNumeric: if not self.unit.isEqualTo(other.unit): return False if self.modelXbrl.modelManager.validateInferDecimals: d = min((inferredDecimals(self), inferredDecimals(other))); p = None if isnan(d) and deemP0Equal: return True else: d = None; p = min((inferredPrecision(self), inferredPrecision(other))) if p == 0 and deemP0Equal: return True return roundValue(self.value,precision=p,decimals=d) == roundValue(other.value,precision=p,decimals=d) else: return False selfValue = self.value otherValue = other.value if isinstance(selfValue,str) and isinstance(otherValue,str): return selfValue.strip() == otherValue.strip() else: return selfValue == otherValue
def addMostAccurateFactToBinding(f): if f.xValid >= VALID and (nils or not f.isNil) and f.context is not None: binding = bindings[f.context.contextDimAwareHash, f.unit.hash if f.unit is not None else None] ln = f.qname.localName if ln not in binding or inferredDecimals(f) > inferredDecimals( binding[ln]): binding[ln] = f
def generateInstanceInfoset(dts, instanceInfosetFile): if dts.fileSource.isArchive: return import os, io from arelle import XmlUtil, XbrlConst from arelle.ValidateXbrlCalcs import inferredPrecision, inferredDecimals XmlUtil.setXmlns(dts.modelDocument, u"ptv", u"http://www.xbrl.org/2003/ptv") numFacts = 0 for fact in dts.facts: try: if fact.concept.periodType: fact.set(u"{http://www.xbrl.org/2003/ptv}periodType", fact.concept.periodType) if fact.concept.balance: fact.set(u"{http://www.xbrl.org/2003/ptv}balance", fact.concept.balance) if fact.isNumeric and not fact.isNil: fact.set(u"{http://www.xbrl.org/2003/ptv}decimals", unicode(inferredDecimals(fact))) fact.set(u"{http://www.xbrl.org/2003/ptv}precision", unicode(inferredPrecision(fact))) numFacts += 1 except Exception, err: dts.error(u"saveInfoset.exception", _(u"Facts exception %(fact)s %(value)s %(error)s."), modelObject=fact, fact=fact.qname, value=fact.effectiveValue, error = err)
def generateInfoset(dts, infosetFile): if dts.fileSource.isArchive: return import os, io from arelle import XmlUtil, XbrlConst from arelle.ValidateXbrlCalcs import inferredPrecision, inferredDecimals XmlUtil.setXmlns(dts.modelDocument, "ptv", "http://www.xbrl.org/2003/ptv") numFacts = 0 for fact in dts.facts: try: if fact.concept.periodType: fact.set("{http://www.xbrl.org/2003/ptv}periodType", fact.concept.periodType) if fact.concept.balance: fact.set("{http://www.xbrl.org/2003/ptv}balance", fact.concept.balance) if fact.isNumeric and not fact.isNil: fact.set("{http://www.xbrl.org/2003/ptv}decimals", str(inferredDecimals(fact))) fact.set("{http://www.xbrl.org/2003/ptv}precision", str(inferredPrecision(fact))) numFacts += 1 except Exception as err: dts.error("saveInfoset.exception", _("Facts exception %(fact)s %(value)s %(error)s."), modelObject=fact, fact=fact.qname, value=fact.effectiveValue, error = err) fh = open(infosetFile, "w", encoding="utf-8") XmlUtil.writexml(fh, dts.modelDocument.xmlDocument, encoding="utf-8") fh.close() dts.info("info:saveInfoset", _("Infoset of %(entryFile)s has %(numberOfFacts)s facts in infoset file %(infosetOutputFile)s."), modelObject=dts, entryFile=dts.uri, numberOfFacts=numFacts, infosetOutputFile=infosetFile)
def generateInstanceInfoset(dts, instanceInfosetFile): if dts.fileSource.isArchive: return import os, io from arelle import XmlUtil, XbrlConst from arelle.ValidateXbrlCalcs import inferredPrecision, inferredDecimals XmlUtil.setXmlns(dts.modelDocument, "ptv", "http://www.xbrl.org/2003/ptv") numFacts = 0 for fact in dts.facts: try: if fact.concept.periodType: fact.set("{http://www.xbrl.org/2003/ptv}periodType", fact.concept.periodType) if fact.concept.balance: fact.set("{http://www.xbrl.org/2003/ptv}balance", fact.concept.balance) if fact.isNumeric and not fact.isNil: fact.set("{http://www.xbrl.org/2003/ptv}decimals", str(inferredDecimals(fact))) fact.set("{http://www.xbrl.org/2003/ptv}precision", str(inferredPrecision(fact))) numFacts += 1 except Exception as err: dts.error("saveInfoset.exception", _("Facts exception %(fact)s %(value)s %(error)s."), modelObject=fact, fact=fact.qname, value=fact.effectiveValue, error = err) fh = open(instanceInfosetFile, "w", encoding="utf-8") XmlUtil.writexml(fh, dts.modelDocument.xmlDocument, encoding="utf-8") fh.close() dts.info("info:saveInstanceInfoset", _("Instance infoset of %(entryFile)s has %(numberOfFacts)s facts in infoset file %(infosetOutputFile)s."), modelObject=dts, entryFile=dts.uri, numberOfFacts=numFacts, infosetOutputFile=instanceInfosetFile)
def _assets_eq_liability_equity(model_xbrl): """ Yields fact assets and fact liabilities as long as it is able to :param model_xbrl: modelXbrl to check name concepts of :type model_xbrl: :class:'~arelle.ModelXbrl.ModelXbrl' :return: yields fact assets and fact liabilities that should throw errors :rtype: tuple """ assets_concept = ( model_xbrl.nameConcepts[_ASSETS_CONCEPT][0] if model_xbrl.nameConcepts[_ASSETS_CONCEPT] else None ) liability_equity_concept = ( model_xbrl.nameConcepts[_LIABILITIES_CONCEPT][0] if model_xbrl.nameConcepts[_LIABILITIES_CONCEPT] else None ) if assets_concept is not None and liability_equity_concept is not None: assets_facts = model_xbrl.factsByQname[assets_concept.qname] liability_equity_facts = ( model_xbrl.factsByQname[liability_equity_concept.qname] ) fact_dict = dict() fact_dict[_ASSETS_CONCEPT] = assets_facts fact_dict[_LIABILITIES_CONCEPT] = liability_equity_facts fact_groups = facts.prepare_facts_for_calculation(fact_dict) for fact_group in fact_groups: fact_assets = fact_group[_ASSETS_CONCEPT] fact_liabilities = fact_group[_LIABILITIES_CONCEPT] if ((fact_assets.context is not None and fact_assets.context.instantDatetime is not None)): dec_assets = inferredDecimals(fact_assets) dec_liabilities = inferredDecimals(fact_liabilities) min_dec = min(dec_assets, dec_liabilities) if _values_unequal( fact_assets.xValue, fact_liabilities.xValue, min_dec ): yield fact_assets, fact_liabilities
def _assets_eq_liability_equity(modelXbrl): assets_concept = modelXbrl.nameConcepts[_ASSETS_CONCEPT][0] if modelXbrl.nameConcepts[_ASSETS_CONCEPT] else None liability_equity_concept = modelXbrl.nameConcepts[_LIABILITIES_CONCEPT][0] if modelXbrl.nameConcepts[_LIABILITIES_CONCEPT] else None if assets_concept is not None and liability_equity_concept is not None: assets_facts = modelXbrl.factsByQname[assets_concept.qname] liability_equity_facts = modelXbrl.factsByQname[liability_equity_concept.qname] fact_dict = {} fact_dict[_ASSETS_CONCEPT] = assets_facts fact_dict[_LIABILITIES_CONCEPT] = liability_equity_facts fact_groups = facts.prepare_facts_for_calculation(fact_dict) for fact_group in fact_groups: fact_assets = fact_group[_ASSETS_CONCEPT] fact_liabilities = fact_group[_LIABILITIES_CONCEPT] if fact_assets.context is not None and fact_assets.context.instantDatetime is not None: dec_assets = inferredDecimals(fact_assets) dec_liabilities = inferredDecimals(fact_liabilities) min_dec = min(dec_assets, dec_liabilities) if roundValue(fact_assets.xValue, decimals=min_dec) != roundValue(fact_liabilities.xValue, decimals=min_dec): yield fact_assets, fact_liabilities
def _assets_eq_liability_equity(model_xbrl): """ Yields fact assets and fact liabilities as long as it is able to :param model_xbrl: modelXbrl to check name concepts of :type model_xbrl: :class:'~arelle.ModelXbrl.ModelXbrl' :return: yields fact assets and fact liabilities that should throw errors :rtype: tuple """ assets_concept = (model_xbrl.nameConcepts[_ASSETS_CONCEPT][0] if model_xbrl.nameConcepts[_ASSETS_CONCEPT] else None) liability_equity_concept = ( model_xbrl.nameConcepts[_LIABILITIES_CONCEPT][0] if model_xbrl.nameConcepts[_LIABILITIES_CONCEPT] else None) if assets_concept is not None and liability_equity_concept is not None: assets_facts = model_xbrl.factsByQname[assets_concept.qname] liability_equity_facts = ( model_xbrl.factsByQname[liability_equity_concept.qname]) fact_dict = dict() fact_dict[_ASSETS_CONCEPT] = assets_facts fact_dict[_LIABILITIES_CONCEPT] = liability_equity_facts fact_groups = facts.prepare_facts_for_calculation(fact_dict) for fact_group in fact_groups: fact_assets = fact_group[_ASSETS_CONCEPT] fact_liabilities = fact_group[_LIABILITIES_CONCEPT] if ((fact_assets.context is not None and fact_assets.context.instantDatetime is not None)): dec_assets = inferredDecimals(fact_assets) dec_liabilities = inferredDecimals(fact_liabilities) min_dec = min(dec_assets, dec_liabilities) if _values_unequal(fact_assets.xValue, fact_liabilities.xValue, min_dec): yield fact_assets, fact_liabilities
def isVEqualTo(self, other, deemP0Equal=False): # facts may be in different instances if self.isTuple or other.isTuple: return False if self.isNil: return other.isNil if other.isNil: return False if not self.context.isEqualTo(other.context): return False if self.concept.isNumeric: if other.concept.isNumeric: if not self.unit.isEqualTo(other.unit): return False if self.modelXbrl.modelManager.validateInferDecimals: d = min((inferredDecimals(self), inferredDecimals(other))) p = None else: d = None p = min( (inferredPrecision(self), inferredPrecision(other))) if p == 0 and deemP0Equal: return True return roundValue(self.value, precision=p, decimals=d) == roundValue(other.value, precision=p, decimals=d) else: return False selfValue = self.value otherValue = other.value if isinstance(selfValue, str) and isinstance(otherValue, str): return selfValue.strip() == otherValue.strip() else: return selfValue == otherValue
def factAspects(fact): aspects = {qnOimConceptAspect: oimValue(fact.qname)} if hasId and fact.id: aspects[qnOimIdAspect] = fact.id if hasLocation: aspects[qnOimLocationAspect] = elementChildSequence(fact) concept = fact.concept if concept is not None: if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang: aspects[qnOimLangAspect] = fact.xmlLang aspects[qnOimTypeAspect] = concept.baseXbrliType if fact.isItem: aspects[qnOimValueAspect] = (NILVALUE if fact.isNil else oimValue( fact.xValue, inferredDecimals(fact))) cntx = fact.context if cntx is not None: if cntx.entityIdentifierElement is not None: aspects[qnOimEntityAspect] = oimValue( qname(*cntx.entityIdentifier)) if cntx.period is not None: aspects[qnOimPeriodAspect] = oimPeriodValue(cntx) for dim in cntx.qnameDims.values(): aspects[dim.dimensionQname] = (oimValue(dim.memberQname) if dim.isExplicit else dim.typedMember.stringValue) unit = fact.unit if unit is not None: _mMul, _mDiv = unit.measures if isJSON: aspects[ qnOimUnitAspect] = ( # use tuple instead of list for hashability tuple( oimValue(m) for m in sorted(_mMul, key=lambda m: str(m))), tuple( oimValue(m) for m in sorted(_mDiv, key=lambda m: str(m)))) else: # CSV if _mMul: aspects[qnOimUnitMulAspect] = ",".join( oimValue(m) for m in sorted(_mMul, key=lambda m: str(m))) if _mDiv: aspects[qnOimUnitDivAspect] = ",".join( oimValue(m) for m in sorted(_mDiv, key=lambda m: str(m))) return aspects
def infer_precision_decimals(xc, p, args, attrName): if len(args) != 1: raise XPathContext.FunctionNumArgs() if len(args[0]) != 1: raise XPathContext.FunctionArgType(1,"xbrl:item") modelItem = xc.modelItem(args[0][0]) if modelItem: modelConcept = modelItem.concept if modelConcept.isNumeric: if modelConcept.isFraction: return 'INF' from arelle.ValidateXbrlCalcs import (inferredDecimals,inferredPrecision) p = inferredPrecision(modelItem) if attrName == "precision" else inferredDecimals(modelItem) if isinf(p): return 'INF' if isnan(p): raise XPathContext.XPathException(p, 'xfie:ItemIsNotNumeric', _('Argument 1 {0} is not inferrable.').format(attrName)) return p raise XPathContext.XPathException(p, 'xfie:ItemIsNotNumeric', _('Argument 1 is not reported with {0}.').format(attrName))
def factAspects(fact): aspects = {qnOimConceptAspect: oimValue(fact.qname)} if hasId and fact.id: aspects[qnOimIdAspect] = fact.id if hasLocation: aspects[qnOimLocationAspect] = elementChildSequence(fact) concept = fact.concept if concept is not None: if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang: aspects[qnOimLangAspect] = fact.xmlLang aspects[qnOimTypeAspect] = concept.baseXbrliType if fact.isItem: aspects[qnOimValueAspect] = (NILVALUE if fact.isNil else oimValue(fact.xValue, inferredDecimals(fact))) cntx = fact.context if cntx is not None: if cntx.entityIdentifierElement is not None: aspects[qnOimEntityAspect] = oimValue(qname(*cntx.entityIdentifier)) if cntx.period is not None: aspects[qnOimPeriodAspect] = oimPeriodValue(cntx) for dim in cntx.qnameDims.values(): aspects[dim.dimensionQname] = (oimValue(dim.memberQname) if dim.isExplicit else dim.typedMember.stringValue) unit = fact.unit if unit is not None: _mMul, _mDiv = unit.measures if isJSON: aspects[qnOimUnitAspect] = ( # use tuple instead of list for hashability tuple(oimValue(m) for m in sorted(_mMul, key=lambda m: str(m))), tuple(oimValue(m) for m in sorted(_mDiv, key=lambda m: str(m)))) else: # CSV if _mMul: aspects[qnOimUnitMulAspect] = ",".join(oimValue(m) for m in sorted(_mMul, key=lambda m: str(m))) if _mDiv: aspects[qnOimUnitDivAspect] = ",".join(oimValue(m) for m in sorted(_mDiv, key=lambda m: str(m))) return aspects
def infer_precision_decimals(xc, p, args, attrName): if len(args) != 1: raise XPathContext.FunctionNumArgs() if len(args[0]) != 1: raise XPathContext.FunctionArgType(1, "xbrl:item") modelItem = xc.modelItem(args[0][0]) if modelItem: modelConcept = modelItem.concept if modelConcept.isNumeric: if modelConcept.isFraction: return 'INF' from arelle.ValidateXbrlCalcs import (inferredDecimals, inferredPrecision) p = inferredPrecision( modelItem) if attrName == "precision" else inferredDecimals( modelItem) if isinf(p): return 'INF' if isnan(p): raise XPathContext.XPathException( p, 'xfie:ItemIsNotNumeric', _('Argument 1 {0} is not inferrable.').format(attrName)) return p raise XPathContext.XPathException( p, 'xfie:ItemIsNotNumeric', _('Argument 1 is not reported with {0}.').format(attrName))
def sectTreeRel(self, parentConcept, n, sectCalc2RelSet, inferredParentValues, visited, dimQN=None): childRels = sectCalc2RelSet.fromModelObject(parentConcept) if childRels: visited.add(parentConcept) inferredChildValues = {} # setup summation bind keys for child objects sumParentBindKeys = self.sumConceptBindKeys[parentConcept] boundSumKeys = set( ) # these are contributing fact keys, parent may be inferred boundSums = defaultdict(intervalZero) boundSummationItems = defaultdict(list) boundPerKeys = set() boundPers = defaultdict(intervalZero) boundDurationItems = defaultdict(list) boundAggKeys = set() boundAggs = defaultdict(intervalZero) boundAggItems = defaultdict(list) boundAggConcepts = defaultdict(set) for rel in childRels: childConcept = rel.toModelObject if childConcept not in visited: if rel.arcrole == summationItem: if not self.sumInit: self.sumBindFacts() boundSumKeys |= self.sumConceptBindKeys[childConcept] elif rel.arcrole == balanceChanges: if not self.perInit: self.perBindFacts() boundPerKeys |= self.perConceptBindKeys[ childConcept] # these are only duration items elif rel.arcrole == domainMember: boundAggKeys |= self.aggConceptBindKeys[dimQN] domQN = parentConcept.qname elif rel.arcrole == aggregationDomain: # this is in visited dimQN = rel.arcElement.prefixedNameQname( rel.get("dimension")) if dimQN not in self.aggDimInit: self.aggBindFacts( dimQN) # bind each referenced dimension's contexts # depth-first descent calc tree and process item after descent for rel in childRels: childConcept = rel.toModelObject if childConcept not in visited: # depth-first descent self.sectTreeRel(childConcept, n + 1, sectCalc2RelSet, inferredChildValues, visited, dimQN) # post-descent summation (allows use of inferred value) if rel.arcrole == summationItem: weight = rel.weightDecimal for sumKey in boundSumKeys: cntx, unit = sumKey factKey = (childConcept, cntx, unit) if factKey in self.sumBoundFacts: for f in self.sumBoundFacts[factKey]: addInterval(boundSums, sumKey, intervalValue(f), weight) boundSummationItems[sumKey].append(f) elif factKey in inferredChildValues: addInterval(boundSums, sumKey, inferredChildValues[factKey], weight) elif factKey in inferredParentValues: addInterval(boundSums, sumKey, inferredParentValues[factKey], weight) elif rel.arcrole == balanceChanges: weight = rel.weightDecimal for perKey in boundPerKeys: hCntx, unit, start, end = perKey factKey = (childConcept, hCntx, unit, start, end) if factKey in self.perBoundFacts: for f in self.perBoundFacts[factKey]: addInterval(boundPers, perKey, intervalValue(f), weight) boundDurationItems[perKey].append(f) elif factKey in inferredChildValues: addInterval(boundPers, perKey, inferredChildValues[factKey], weight) elif factKey in inferredParentValues: addInterval(boundPers, perKey, inferredParentValues[factKey], weight) elif rel.arcrole == domainMember: memQN = childConcept.qname for aggKey in boundAggKeys: hCntx, unit = aggKey dimMemKey = (hCntx, unit, dimQN, memQN) if dimMemKey in self.aggBoundFacts: for f in self.aggBoundFacts[dimMemKey]: a, b = intervalValue(f) factDomKey = (f.concept, hCntx, unit, dimQN, domQN) addInterval(boundAggs, factDomKey, intervalValue(f)) boundAggItems[aggKey].append(f) boundAggConcepts[aggKey].add(f.concept) elif rel.arcrole == aggregationDomain: # this is in visited childRelSet = self.modelXbrl.relationshipSet( domainMember, rel.get("targetRole")) self.sectTreeRel(childConcept, n + 1, childRelSet, inferredParentValues, {None}, dimQN) # infer global to section # process child items bound to this calc subtree for sumKey in boundSumKeys: cntx, unit = sumKey factKey = (parentConcept, cntx, unit) ia, ib = boundSums[sumKey] if factKey in self.sumBoundFacts: for f in self.sumBoundFacts[factKey]: d = inferredDecimals(f) sa, sb = intervalValue(f, d) if ((ia is NIL) ^ (sa is NIL)) or ((ia is not NIL) and (sb < ia or sa > ib)): self.modelXbrl.log( 'INCONSISTENCY', "calc2e:summationInconsistency", _("Summation inconsistent from %(concept)s in section %(section)s reported sum %(reportedSum)s, computed sum %(computedSum)s context %(contextID)s unit %(unitID)s unreported contributing items %(unreportedContributors)s" ), modelObject=boundSummationItems[sumKey], concept=parentConcept.qname, section=self.section, reportedSum=self.formatInterval(sa, sb, d), computedSum=self.formatInterval(ia, ib, d), contextID=f.context.id, unitID=f.unit.id, unreportedContributors=", ".join( str( c.qname ) # list the missing/unreported contributors in relationship order for r in childRels for c in (r.toModelObject, ) if r.arcrole == summationItem and c is not None and (c, cntx, unit) not in self.sumBoundFacts) or "none") elif inferredParentValues is not None: # value was inferred, return to parent level inferredParentValues[factKey] = (ia, ib) for perKey in boundPerKeys: hCntx, unit, start, end = perKey ia, ib = boundPers[perKey] endBalA = endBalB = ZERO endFactKey = (parentConcept, hCntx, unit, None, end) if endFactKey in self.perBoundFacts: for f in self.perBoundFacts[endFactKey]: if f.isNil: endBalA = endBalB = NIL d = 0 break d = inferredDecimals(f) a, b = intervalValue(f, d) endBalA += a endBalB += b foundStartingFact = (endBalA is NIL) while not foundStartingFact: startFactKey = (parentConcept, hCntx, unit, None, start) if startFactKey in self.perBoundFacts: for f in self.perBoundFacts[startFactKey]: if f.isNil: endBalA = endBalB = NIL foundStartingFact = True break a, b = intervalValue(f) endBalA -= a endBalB -= b foundStartingFact = True break if not foundStartingFact: # infer backing up one period _nomPer = nominalPeriod(end - start) foundEarlierAdjacentPeriodStart = False for _start in self.durationPeriodStarts.get( _nomPer, ()): if nominalPeriod( start - _start ) == _nomPer: # it's preceding period end = start start = _start perKey = hCntx, unit, start, end if perKey in boundPerKeys: chngs = boundPers[perKey] ia += chngs[0] ib += chngs[1] foundEarlierAdjacentPeriodStart = True break if not foundEarlierAdjacentPeriodStart: break if ((ia is NIL) ^ (endBalA is NIL)) or ((ia is not NIL) and (endBalB < ia or endBalA > ib)): self.modelXbrl.log( 'INCONSISTENCY', "calc2e:balanceInconsistency", _("Balance inconsistent from %(concept)s in section %(section)s reported sum %(reportedSum)s, computed sum %(computedSum)s context %(contextID)s unit %(unitID)s unreported contributing items %(unreportedContributors)s" ), modelObject=boundDurationItems[perKey], concept=parentConcept.qname, section=self.section, reportedSum=self.formatInterval( endBalA, endBalB, d), computedSum=self.formatInterval(ia, ib, d), contextID=f.context.id, unitID=f.unit.id, unreportedContributors=", ".join( str( c.qname ) # list the missing/unreported contributors in relationship order for r in childRels for c in (r.toModelObject, ) if r.arcrole == balanceChanges and c is not None and (c, hCntx, unit, start, end) not in self.perBoundFacts) or "none") for aggKey in boundAggKeys: hCntx, unit = aggKey for concept in sorted( boundAggConcepts[aggKey], key=lambda c: c.objectIndex): # repeatable errors factDomKey = (concept, hCntx, unit, dimQN, domQN) ia, ib = boundAggs[factDomKey] if factDomKey in self.aggBoundConceptFacts: for f in self.aggBoundConceptFacts[factDomKey]: d = inferredDecimals(f) sa, sb = intervalValue(f, d) if ((ia is NIL) ^ (sa is NIL)) or ((ia is not NIL) and (sb < ia or sa > ib)): self.modelXbrl.log( 'INCONSISTENCY', "calc2e:aggregationInconsistency", _("Aggregation inconsistent for %(concept)s, domain %(domain)s in section %(section)s reported sum %(reportedSum)s, computed sum %(computedSum)s context %(contextID)s unit %(unitID)s unreported contributing members %(unreportedContributors)s" ), modelObject=boundAggItems[factDomKey], concept=concept.qname, domain=parentConcept.qname, section=self.section, reportedSum=self.formatInterval(sa, sb, d), computedSum=self.formatInterval(ia, ib, d), contextID=f.context.id, unitID=f.unit.id, unreportedContributors=", ".join( str( c.qname ) # list the missing/unreported contributors in relationship order for r in childRels for c in (r.toModelObject, ) if r.arcrole == domainMember and c is not None and (concept, hCntx, unit, dimQN, c.qname ) not in self.aggBoundConceptFacts) or "none") elif inferredParentValues is not None: # value was inferred, return to parent level # allow to be retrieved by factDomKey inferredParentValues[factDomKey] = (ia, ib) if self.modelXbrl.qnameDimensionDefaults.get( dimQN) == domQN: cntxKey = (hCntx, dimQN, domQN) if cntxKey in self.eqCntx: cntx = self.eqCntx[cntxKey] else: cntx = self.aggTotalContext( hCntx, dimQN, domQN) self.eqCntx[cntxKey] = cntx if cntx is not None: # allow to be retrieved by fact line item context key self.eqCntx[(hCntx, dimQN, domQN)] = cntx inferredParentValues[(concept, cntx, unit)] = (ia, ib) visited.remove(parentConcept)
def validateXbrlFinally(val, *args, **kwargs): if not (val.validateESMAplugin): return _xhtmlNs = "{{{}}}".format(xhtml) _xhtmlNsLen = len(_xhtmlNs) modelXbrl = val.modelXbrl modelDocument = modelXbrl.modelDocument _statusMsg = _("validating {0} filing rules").format(val.disclosureSystem.name) modelXbrl.profileActivity() modelXbrl.modelManager.showStatus(_statusMsg) reportXmlLang = None firstRootmostXmlLangDepth = 9999999 if modelDocument.type == ModelDocument.Type.INSTANCE: modelXbrl.error("esma:instanceShallBeInlineXBRL", _("RTS on ESEF requires inline XBRL instances."), modelObject=modelXbrl) checkFilingDimensions(val) # sets up val.primaryItems and val.domainMembers val.hasExtensionSchema = val.hasExtensionPre = val.hasExtensionCal = val.hasExtensionDef = val.hasExtensionLbl = False checkFilingDTS(val, modelXbrl.modelDocument, []) modelXbrl.profileActivity("... filer DTS checks", minTimeToShow=1.0) if not (val.hasExtensionSchema and val.hasExtensionPre and val.hasExtensionCal and val.hasExtensionDef and val.hasExtensionLbl): missingFiles = [] if not val.hasExtensionSchema: missingFiles.append("schema file") if not val.hasExtensionPre: missingFiles.append("presentation linkbase") if not val.hasExtensionCal: missingFiles.append("calculation linkbase") if not val.hasExtensionDef: missingFiles.append("definition linkbase") if not val.hasExtensionLbl: missingFiles.append("label linkbase") modelXbrl.warning("esma:3.1.1.extensionTaxonomyWrongFilesStructure", _("Extension taxonomies MUST consist of at least a schema file and presentation, calculation, definition and label linkbases" ": missing %(missingFiles)s"), modelObject=modelXbrl, missingFiles=", ".join(missingFiles)) if modelDocument.type in (ModelDocument.Type.INLINEXBRL, ModelDocument.Type.INLINEXBRLDOCUMENTSET, ModelDocument.Type.INSTANCE): footnotesRelationshipSet = modelXbrl.relationshipSet("XBRL-footnotes") orphanedFootnotes = set() noLangFootnotes = set() footnoteRoleErrors = set() transformRegistryErrors = set() def checkFootnote(elt, text): if text: # non-empty footnote must be linked to a fact if not empty if not any(isinstance(rel.fromModelObject, ModelFact) for rel in footnotesRelationshipSet.toModelObject(elt)): orphanedFootnotes.add(elt) if not elt.xmlLang: noLangFootnotes.add(elt) if elt.role != XbrlConst.footnote or not all( rel.arcrole == XbrlConst.factFootnote and rel.linkrole == XbrlConst.defaultLinkRole for rel in footnotesRelationshipSet.toModelObject(elt)): footnoteRoleErrors.add(elt) # check file name of each inline document (which might be below a top-level IXDS) for doc in modelXbrl.urlDocs.values(): if doc.type == ModelDocument.Type.INLINEXBRL: _baseName, _baseExt = os.path.splitext(doc.basename) if _baseExt not in (".xhtml",): modelXbrl.warning("esma:TBD.fileNameExtension", _("FileName should have the extension .xhtml: %(fileName)s"), modelObject=doc, fileName=doc.basename) if modelDocument.type in (ModelDocument.Type.INLINEXBRL, ModelDocument.Type.INLINEXBRLDOCUMENTSET): ixNStag = modelXbrl.modelDocument.ixNStag ixTags = set(ixNStag + ln for ln in ("nonNumeric", "nonFraction", "references", "relationship")) ixTextTags = set(ixNStag + ln for ln in ("nonFraction", "continuation", "footnote")) ixExcludeTag = ixNStag + "exclude" ixTupleTag = ixNStag + "tuple" ixFractionTag = ixNStag + "fraction" hiddenEltIds = {} presentedHiddenEltIds = defaultdict(list) eligibleForTransformHiddenFacts = [] requiredToDisplayFacts = [] requiredToDisplayFactIds = {} firstIxdsDoc = True for ixdsHtmlRootElt in modelXbrl.ixdsHtmlElements: # ix root elements for all ix docs in IXDS for elt, depth in etreeIterWithDepth(ixdsHtmlRootElt): eltTag = elt.tag if isinstance(elt, ModelObject) and elt.namespaceURI == xhtml: eltTag = elt.localName if firstIxdsDoc and (not reportXmlLang or depth < firstRootmostXmlLangDepth): xmlLang = elt.get("{http://www.w3.org/XML/1998/namespace}lang") if xmlLang: reportXmlLang = xmlLang firstRootmostXmlLangDepth = depth elif isinstance(elt, (_ElementTree, _Comment, _ProcessingInstruction)): continue # comment or other non-parsed element else: eltTag = elt.tag if eltTag.startswith(_xhtmlNs): eltTag = eltTag[_xhtmlNsLen:] if ((eltTag in ("object", "script")) or (eltTag == "a" and "javascript:" in elt.get("href","")) or (eltTag == "img" and "javascript:" in elt.get("src",""))): modelXbrl.error("esma.2.5.1.executableCodePresent", _("Inline XBRL documents MUST NOT contain executable code: %(element)s"), modelObject=elt, element=eltTag) elif eltTag == "img": src = elt.get("src","").strip() hasParentIxTextTag = False # check if image is in an ix text-bearing element _ancestorElt = elt while (_ancestorElt is not None): if _ancestorElt.tag == ixExcludeTag: # excluded from any parent text-bearing ix element break if _ancestorElt.tag in ixTextTags: hasParentIxTextTag = True break _ancestorElt = _ancestorElt.getparent() if scheme(href) in ("http", "https", "ftp"): modelXbrl.error("esma.3.5.1.inlinXbrlContainsExternalReferences", _("Inline XBRL instance documents MUST NOT contain any reference pointing to resources outside the reporting package: %(element)s"), modelObject=elt, element=eltTag) if not src.startswith("data:image"): if hasParentIxTextTag: modelXbrl.error("esma.2.5.1.imageInIXbrlElementNotEmbedded", _("Images appearing within an inline XBRL element MUST be embedded regardless of their size."), modelObject=elt) else: # presume it to be an image file, check image contents try: base = elt.modelDocument.baseForElement(elt) normalizedUri = elt.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(graphicFile, base) if not elt.modelXbrl.fileSource.isInArchive(normalizedUri): normalizedUri = elt.modelXbrl.modelManager.cntlr.webCache.getfilename(normalizedUri) imglen = 0 with elt.modelXbrl.fileSource.file(normalizedUri,binary=True)[0] as fh: imglen += len(fh.read()) if imglen < browserMaxBase64ImageLength: modelXbrl.error("esma.2.5.1.embeddedImageNotUsingBase64Encoding", _("Images MUST be included in the XHTML document as a base64 encoded string unless their size exceeds support of browsers."), modelObject=elt) except IOError as err: modelXbrl.error("esma.2.5.1.imageFileCannotBeLoaded", _("Image file which isn't openable '%(src)s', error: %(error)s"), modelObject=elt, src=src, error=err) elif not any(src.startswith(m) for m in allowedImgMimeTypes): modelXbrl.error("esma.2.5.1.embeddedImageNotUsingBase64Encoding", _("Images MUST be included in the XHTML document as a base64 encoded string, encoding disallowed: %(src)s."), modelObject=elt, src=attrValue[:128]) elif eltTag == "a": href = elt.get("href","").strip() if scheme(href) in ("http", "https", "ftp"): modelXbrl.error("esma.3.5.1.inlinXbrlContainsExternalReferences", _("Inline XBRL instance documents MUST NOT contain any reference pointing to resources outside the reporting package: %(element)s"), modelObject=elt, element=eltTag) elif eltTag == "base" or elt.tag == "{http://www.w3.org/XML/1998/namespace}base": modelXbrl.error("esma.2.4.2.htmlOrXmlBaseUsed", _("The HTML <base> elements and xml:base attributes MUST NOT be used in the Inline XBRL document."), modelObject=elt, element=eltTag) if eltTag in ixTags and elt.get("target"): modelXbrl.error("esma.2.5.3.targetAttributeUsed", _("Target attribute MUST not be used: element %(localName)s, target attribute %(target)s."), modelObject=elt, localName=elt.elementQname, target=elt.get("target")) if eltTag == ixTupleTag: modelXbrl.error("esma.2.4.1.tupleElementUsed", _("The ix:tuple element MUST not be used in the Inline XBRL document."), modelObject=elt) if eltTag == ixFractionTag: modelXbrl.error("esma.2.4.1.fractionElementUsed", _("The ix:fraction element MUST not be used in the Inline XBRL document."), modelObject=elt) if elt.get("{http://www.w3.org/XML/1998/namespace}base") is not None: modelXbrl.error("esma.2.4.1.xmlBaseUsed", _("xml:base attributes MUST NOT be used in the Inline XBRL document: element %(localName)s, base attribute %(base)s."), modelObject=elt, localName=elt.elementQname, base=elt.get("{http://www.w3.org/XML/1998/namespace}base")) if isinstance(elt, ModelInlineFootnote): checkFootnote(elt, elt.value) elif isinstance(elt, ModelResource) and elt.qname == XbrlConst.qnLinkFootnote: checkFootnote(elt, elt.value) elif isinstance(elt, ModelInlineFact): if elt.format is not None and elt.format.namespaceURI != 'http://www.xbrl.org/inlineXBRL/transformation/2015-02-26': transformRegistryErrors.add(elt) for ixHiddenElt in ixdsHtmlRootElt.iterdescendants(tag=ixNStag + "hidden"): for tag in (ixNStag + "nonNumeric", ixNStag+"nonFraction"): for ixElt in ixHiddenElt.iterdescendants(tag=tag): if (getattr(ixElt, "xValid", 0) >= VALID # may not be validated ): # add future "and" conditions on elements which can be in hidden if (ixElt.concept.baseXsdType not in untransformableTypes and not ixElt.isNil): eligibleForTransformHiddenFacts.append(ixElt) elif ixElt.id is None: requiredToDisplayFacts.append(ixElt) if ixElt.id: hiddenEltIds[ixElt.id] = ixElt firstIxdsDoc = False if eligibleForTransformHiddenFacts: modelXbrl.warning("esma.2.4.1.transformableElementIncludedInHiddenSection", _("The ix:hidden section of Inline XBRL document MUST not include elements eligible for transformation. " "%(countEligible)s fact(s) were eligible for transformation: %(elements)s"), modelObject=eligibleForTransformHiddenFacts, countEligible=len(eligibleForTransformHiddenFacts), elements=", ".join(sorted(set(str(f.qname) for f in eligibleForTransformHiddenFacts)))) for ixdsHtmlRootElt in modelXbrl.ixdsHtmlElements: for ixElt in ixdsHtmlRootElt.getroottree().iterfind("//{http://www.w3.org/1999/xhtml}*[@style]"): hiddenFactRefMatch = styleIxHiddenPattern.match(ixElt.get("style","")) if hiddenFactRefMatch: hiddenFactRef = hiddenFactRefMatch.group(2) if hiddenFactRef not in hiddenEltIds: modelXbrl.error("esma.2.4.1.esefIxHiddenStyleNotLinkingFactInHiddenSection", _("\"-esef-ix-hidden\" style identifies @id, %(id)s of a fact that is not in ix:hidden section."), modelObject=ixElt, id=hiddenFactRef) else: presentedHiddenEltIds[hiddenFactRef].append(ixElt) for hiddenEltId, ixElt in hiddenEltIds.items(): if (hiddenEltId not in presentedHiddenEltIds and getattr(ixElt, "xValid", 0) >= VALID and # may not be validated (ixElt.concept.baseXsdType in untransformableTypes or ixElt.isNil)): requiredToDisplayFacts.append(ixElt) if requiredToDisplayFacts: modelXbrl.warning("esma.2.4.1.factInHiddenSectionNotInReport", _("The ix:hidden section contains %(countUnreferenced)s fact(s) whose @id is not applied on any \"-esef-ix- hidden\" style: %(elements)s"), modelObject=requiredToDisplayFacts, countUnreferenced=len(requiredToDisplayFacts), elements=", ".join(sorted(set(str(f.qname) for f in requiredToDisplayFacts)))) del eligibleForTransformHiddenFacts, hiddenEltIds, presentedHiddenEltIds, requiredToDisplayFacts elif modelDocument.type == ModelDocument.Type.INSTANCE: for elt in modelDocument.xmlRootElement.iter(): if elt.qname == XbrlConst.qnLinkFootnote: # for now assume no private elements extend link:footnote checkFootnote(elt, elt.stringValue) contextsWithDisallowedOCEs = [] contextsWithDisallowedOCEcontent = [] contextsWithPeriodTime = [] contextsWithPeriodTimeZone = [] contextIdentifiers = defaultdict(list) nonStandardTypedDimensions = defaultdict(set) for context in modelXbrl.contexts.values(): if XmlUtil.hasChild(context, XbrlConst.xbrli, "segment"): contextsWithDisallowedOCEs.append(context) for segScenElt in context.iterdescendants("{http://www.xbrl.org/2003/instance}scenario"): if isinstance(segScenElt,ModelObject): if any(True for child in segScenElt.iterchildren() if isinstance(child,ModelObject) and child.tag not in ("{http://xbrl.org/2006/xbrldi}explicitMember", "{http://xbrl.org/2006/xbrldi}typedMember")): contextsWithDisallowedOCEcontent.append(context) # check periods here contextIdentifiers[context.entityIdentifier].append(context) if contextsWithDisallowedOCEs: modelXbrl.error("esma.2.1.3.segmentUsed", _("xbrli:segment container MUST NOT be used in contexts: %(contextIds)s"), modelObject=contextsWithDisallowedOCEs, contextIds=", ".join(c.id for c in contextsWithDisallowedOCEs)) if contextsWithDisallowedOCEcontent: modelXbrl.error("esma.2.1.3.scenarioContainsNonDimensionalContent", _("xbrli:scenario in contexts MUST NOT contain any other content than defined in XBRL Dimensions specification: %(contextIds)s"), modelObject=contextsWithDisallowedOCEcontent, contextIds=", ".join(c.id for c in contextsWithDisallowedOCEcontent)) if len(contextIdentifiers) > 1: modelXbrl.error("esma.2.1.4.multipleIdentifiers", _("All entity identifiers in contexts MUST have identical content: %(contextIdentifiers)s"), modelObject=modelXbrl, contextIds=", ".join(i[1] for i in contextIdentifiers)) for (contextScheme, contextIdentifier), contextElts in contextIdentifiers.items(): if contextScheme != "http://standards.iso.org/iso/17442": modelXbrl.warning("esma.2.1.1.nonLEIContextScheme", _("The scheme attribute of the xbrli:identifier element should have \"http://standards.iso.org/iso/17442\" as its content: %(scheme)s"), modelObject=contextElts, scheme=contextScheme) else: leiValidity = LeiUtil.checkLei(contextIdentifier) if leiValidity == LeiUtil.LEI_INVALID_LEXICAL: modelXbrl.warning("esma.2.1.1.invalidIdentifierFormat", _("The LEI context idenntifier has an invalid format: %(identifier)s"), modelObject=contextElts, identifier=contextIdentifier) elif leiValidity == LeiUtil.LEI_INVALID_CHECKSUM: modelXbrl.warning("esma.2.1.1.invalidIdentifier", _("The LEI context idenntifier has checksum error: %(identifier)s"), modelObject=contextElts, identifier=contextIdentifier) if contextsWithPeriodTime: modelXbrl.warning("esma.2.1.2.periodWithTimeContent", _("Context period startDate, endDate and instant elements should be in whole days without time: %(contextIds)s"), modelObject=contextsWithPeriodTime, contextIds=", ".join(c.id for c in contextsWithPeriodTime)) if contextsWithPeriodTimeZone: modelXbrl.warning("esma.2.1.2.periodWithTimeZone", _("Context period startDate, endDate and instant elements should be in whole days without a timezone: %(contextIds)s"), modelObject=contextsWithPeriodTimeZone, contextIds=", ".join(c.id for c in contextsWithPeriodTimeZone)) # identify unique contexts and units mapContext = {} mapUnit = {} uniqueContextHashes = {} for context in modelXbrl.contexts.values(): h = context.contextDimAwareHash if h in uniqueContextHashes: if context.isEqualTo(uniqueContextHashes[h]): mapContext[context] = uniqueContextHashes[h] else: uniqueContextHashes[h] = context del uniqueContextHashes uniqueUnitHashes = {} for unit in modelXbrl.units.values(): h = unit.hash if h in uniqueUnitHashes: if unit.isEqualTo(uniqueUnitHashes[h]): mapUnit[unit] = uniqueUnitHashes[h] else: uniqueUnitHashes[h] = unit del uniqueUnitHashes reportedMandatory = set() precisionFacts = set() numFactsByConceptContextUnit = defaultdict(list) textFactsByConceptContext = defaultdict(list) footnotesRelationshipSet = modelXbrl.relationshipSet(XbrlConst.factFootnote, XbrlConst.defaultLinkRole) noLangFacts = [] textFactsMissingReportLang = [] conceptsUsed = set() for qn, facts in modelXbrl.factsByQname.items(): if qn in mandatory: reportedMandatory.add(qn) for f in facts: if f.precision is not None: precisionFacts.add(f) if f.isNumeric: numFactsByConceptContextUnit[(f.qname, mapContext.get(f.context,f.context), mapUnit.get(f.unit, f.unit))].append(f) elif f.concept is not None and f.concept.type is not None: if f.concept.type.isOimTextFactType: if not f.xmlLang: noLangFacts.append(f) elif f.context is not None: textFactsByConceptContext[(f.qname, mapContext.get(f.context,f.context))].append(f) conceptsUsed.add(f.concept) if f.context is not None: for dim in f.context.qnameDims.values(): conceptsUsed.add(dim.dimension) if dim.isExplicit: conceptsUsed.add(dim.member) elif dim.isTyped: conceptsUsed.add(dim.typedMember) if noLangFacts: modelXbrl.error("esma.2.5.2.undefinedLanguageForTextFact", _("Each tagged text fact MUST have the 'xml:lang' attribute assigned or inherited."), modelObject=noLangFacts) # missing report lang text facts for fList in textFactsByConceptContext.values(): if not any(f.xmlLang == reportXmlLang for f in fList): modelXbrl.error("esma.2.5.2.taggedTextFactOnlyInLanguagesOtherThanLanguageOfAReport", _("Each tagged text fact MUST have the 'xml:lang' provided in at least the language of the report: %(element)s"), modelObject=fList, element=fList[0].qname) # 2.2.4 test for fList in numFactsByConceptContextUnit.values(): if len(fList) > 1: f0 = fList[0] if any(f.isNil for f in fList): _inConsistent = not all(f.isNil for f in fList) elif all(inferredDecimals(f) == inferredDecimals(f0) for f in fList[1:]): # same decimals v0 = rangeValue(f0.value) _inConsistent = not all(rangeValue(f.value) == v0 for f in fList[1:]) else: # not all have same decimals aMax, bMin = rangeValue(f0.value, inferredDecimals(f0)) for f in fList[1:]: a, b = rangeValue(f.value, inferredDecimals(f)) if a > aMax: aMax = a if b < bMin: bMin = b _inConsistent = (bMin < aMax) if _inConsistent: modelXbrl.error(("esma:2.2.4.inconsistentDuplicateNumericFactInInlineXbrlDocument"), "Inconsistent duplicate numeric facts MUST NOT appear in the content of an inline XBRL document. %(fact)s that was used more than once in contexts equivalent to %(contextID)s: values %(values)s. ", modelObject=fList, fact=f0.qname, contextID=f0.contextID, values=", ".join(strTruncate(f.value, 128) for f in fList)) if precisionFacts: modelXbrl.warning("esma:2.2.1.precisionAttributeUsed", _("The accuracy of numeric facts SHOULD be defined with the 'decimals' attribute rather than the 'precision' attribute: %(elements)s."), modelObject=precisionFacts, elements=", ".join(sorted(str(qn) for qn in precisionFacts))) missingElements = (mandatory - reportedMandatory) if missingElements: modelXbrl.error("esma:???.missingRequiredElements", _("Required elements missing from document: %(elements)s."), modelObject=modelXbrl, elements=", ".join(sorted(str(qn) for qn in missingElements))) if transformRegistryErrors: modelXbrl.warning("esma:2.2.3.transformRegistry", _("ESMA recommends applying the latest available version of the Transformation Rules Registry marked with 'Recommendation' status for these elements: %(elements)s."), modelObject=transformRegistryErrors, elements=", ".join(sorted(str(fact.qname) for fact in transformRegistryErrors))) if orphanedFootnotes: modelXbrl.error("esma.2.3.1.unusedFootnote", _("Non-empty footnotes must be connected to fact(s)."), modelObject=orphanedFootnotes) if noLangFootnotes: modelXbrl.error("esma.2.3.2.undefinedLanguageForFootnote", _("Each footnote MUST have the 'xml:lang' attribute whose value corresponds to the language of the text in the content of the respective footnote."), modelObject=noLangFootnotes) if footnoteRoleErrors: modelXbrl.error("esma.2.3.2.nonStandardRoleForFootnote", _("The xlink:role attribute of a link:footnote and link:footnoteLink element as well as xlink:arcrole attribute of a link:footnoteArc MUST be defined in the XBRL Specification 2.1."), modelObject=footnoteRoleErrors) nonStdFootnoteElts = list() for modelLink in modelXbrl.baseSets[("XBRL-footnotes",None,None,None)]: for elt in ixdsHtmlRootElt.iter(): if isinstance(elt, (_ElementTree, _Comment, _ProcessingInstruction)): continue # comment or other non-parsed element if elt.namespaceURI != link or elt.localName not in ("loc", "link", "footnoteArc"): nonStdFootnoteElts.append(elt) if nonStdFootnoteElts: modelXbrl.error("esma.2.3.2.nonStandardElementInFootnote", _("A link:footnoteLink element MUST have no children other than link:loc, link:footnote, and link:footnoteArc."), modelObject=nonStdFootnoteElts) for qn in modelXbrl.qnameDimensionDefaults.values(): conceptsUsed.add(modelXbrl.qnameConcepts.get(qn)) # unused elements in linkbases for arcroles, err in (((parentChild,), "elementsNotUsedForTaggingAppliedInPresentationLinkbase"), ((summationItem,), "elementsNotUsedForTaggingAppliedInCalculationLinkbase"), ((dimensionDomain,domainMember), "elementsNotUsedForTaggingAppliedInDefinitionLinkbase")): lbElts = set() for arcrole in arcroles: for rel in modelXbrl.relationshipSet(arcrole).modelRelationships: fr = rel.fromModelObject to = rel.toModelObject if arcrole in (parentChild, summationItem): if fr is not None and not fr.isAbstract: lbElts.add(fr) if to is not None and not to.isAbstract: lbElts.add(to) elif arcrole == dimensionDomain: if fr is not None: # dimension, always abstract lbElts.add(fr) if to is not None and rel.isUsable: lbElts.add(to) elif arcrole == domainMember: if to is not None and rel.isUsable: lbElts.add(to) unreportedLbElts = lbElts - conceptsUsed if unreportedLbElts: modelXbrl.error("esma.3.2.6." + err, _("All usable concepts in extension taxonomy relationships MUST be applied by tagged facts: %(elements)s."), modelObject=unreportedLbElts, elements=", ".join(sorted((str(c.qname) for c in unreportedLbElts)))) modelXbrl.profileActivity(_statusMsg, minTimeToShow=0.0) modelXbrl.modelManager.showStatus(None)
def factAspects(fact): oimFact = OrderedDict() aspects = OrderedDict() if hasId and fact.id: if fact.isTuple: oimFact["tupleId"] = fact.id else: oimFact["id"] = fact.id elif (fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))): oimFact["id"] = "f{}".format(fact.objectIndex) parent = fact.getparent() concept = fact.concept aspects[str(qnOimConceptAspect)] = oimValue(concept.qname) _csvType = "Value" if not fact.isTuple: if concept is not None: _baseXsdType = concept.baseXsdType if _baseXsdType == "XBRLI_DATEUNION": if getattr(fact.xValue, "dateOnly", False): _baseXsdType = "date" else: _baseXsdType = "dateTime" _csvType = baseTypes.get(_baseXsdType,_baseXsdType) + "Value" if concept.baseXbrliType in ("stringItemType", "normalizedStringItemType") and fact.xmlLang: aspects[str(qnOimLangAspect)] = fact.xmlLang if fact.isItem: if fact.isNil: _value = None else: _inferredDecimals = inferredDecimals(fact) _value = oimValue(fact.xValue, _inferredDecimals) oimFact["value"] = _value if fact.concept is not None and fact.concept.isNumeric: _numValue = fact.xValue if isinstance(_numValue, Decimal) and not isinf(_numValue) and not isnan(_numValue): if _numValue == _numValue.to_integral(): _numValue = int(_numValue) else: _numValue = float(_numValue) if not fact.isNil: if not isinf(_inferredDecimals): # accuracy omitted if infinite oimFact["decimals"] = _inferredDecimals oimFact["dimensions"] = aspects cntx = fact.context if cntx is not None: if cntx.entityIdentifierElement is not None: aspects[str(qnOimEntityAspect)] = oimValue(qname(*cntx.entityIdentifier)) if cntx.period is not None: aspects.update(oimPeriodValue(cntx)) for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]): if dim.isExplicit: dimVal = oimValue(dim.memberQname) else: # typed if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1"): dimVal = None else: dimVal = dim.typedMember.stringValue aspects[str(dim.dimensionQname)] = dimVal unit = fact.unit if unit is not None: _mMul, _mDiv = unit.measures _sMul = '*'.join(oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m))) if _mDiv: _sDiv = '*'.join(oimValue(m) for m in sorted(_mDiv, key=lambda m: oimValue(m))) if len(_mDiv) > 1: if len(_mMul) > 1: _sUnit = "({})/({})".format(_sMul,_sDiv) else: _sUnit = "{}/({})".format(_sMul,_sDiv) else: if len(_mMul) > 1: _sUnit = "({})/{}".format(_sMul,_sDiv) else: _sUnit = "{}/{}".format(_sMul,_sDiv) else: _sUnit = _sMul aspects[str(qnOimUnitAspect)] = _sUnit # Tuples removed from xBRL-JSON #if parent.qname != XbrlConst.qnXbrliXbrl: # aspects[str(qnOimTupleParentAspect)] = parent.id if parent.id else "f{}".format(parent.objectIndex) # aspects[str(qnOimTupleOrderAspect)] = elementIndex(fact) if isJSON: factFootnotes(fact, oimFact) return oimFact
def factAspects(fact): aspects = OrderedDict() if hasId and fact.id: aspects["id"] = fact.id elif (fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSV and footnotesRelationshipSet.fromModelObject(fact))): aspects["id"] = "f{}".format(fact.objectIndex) parent = fact.getparent() concept = fact.concept if not fact.isTuple: if concept is not None: _baseXsdType = concept.baseXsdType if _baseXsdType == "XBRLI_DATEUNION": if getattr(fact.xValue, "dateOnly", False): _baseXsdType = "date" else: _baseXsdType = "dateTime" aspects["baseType"] = "xs:{}".format(_baseXsdType) if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang: aspects[qnOimLangAspect] = fact.xmlLang aspects[qnOimTypeAspect] = concept.baseXbrliType if fact.isItem: if fact.isNil: _value = None _strValue = "nil" else: _inferredDecimals = inferredDecimals(fact) _value = oimValue(fact.xValue, _inferredDecimals) _strValue = str(_value) if not isCSV: aspects["value"] = _strValue if fact.concept is not None and fact.concept.isNumeric: _numValue = fact.xValue if isinstance(_numValue, Decimal) and not isinf(_numValue) and not isnan(_numValue): if _numValue == _numValue.to_integral(): _numValue = int(_numValue) else: _numValue = float(_numValue) aspects["numericValue"] = _numValue if not fact.isNil: if isinf(_inferredDecimals): if isJSON: _accuracy = "infinity" elif isCSV: _accuracy = "INF" else: _accuracy = _inferredDecimals aspects["accuracy"] = _inferredDecimals elif isinstance(_value, bool): aspects["booleanValue"] = _value elif isCSV: aspects["stringValue"] = _strValue aspects[qnOimConceptAspect] = oimValue(fact.qname) cntx = fact.context if cntx is not None: if cntx.entityIdentifierElement is not None: aspects[qnOimEntityAspect] = oimValue(qname(*cntx.entityIdentifier)) if cntx.period is not None: if isJSON: aspects[qnOimPeriodAspect] = oimPeriodValue(cntx) elif isCSV: _periodValue = oimPeriodValue(cntx).split("/") + ["", ""] # default blank if no value aspects[qnOimPeriodStartAspect] = _periodValue[0] aspects[qnOimPeriodDurationAspect] = _periodValue[1] for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]): aspects[dim.dimensionQname] = (oimValue(dim.memberQname) if dim.isExplicit else None if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1") else dim.typedMember.stringValue) unit = fact.unit if unit is not None: _mMul, _mDiv = unit.measures if isJSON: aspects[qnOimUnitAspect] = OrderedDict( # use tuple instead of list for hashability (("numerators", tuple(oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m)))),) ) if _mDiv: aspects[qnOimUnitAspect]["denominators"] = tuple(oimValue(m) for m in sorted(_mDiv, key=lambda m: oimValue(m))) else: # CSV if _mMul: aspects[qnOimUnitNumeratorsAspect] = " ".join(oimValue(m) for m in sorted(_mMul, key=lambda m: str(m))) if _mDiv: aspects[qnOimUnitDenominatorsAspect] = " ".join(oimValue(m) for m in sorted(_mDiv, key=lambda m: str(m))) if parent.qname != XbrlConst.qnXbrliXbrl: aspects[qnOimTupleParentAspect] = parent.id if parent.id else "f{}".format(parent.objectIndex) aspects[qnOimTupleOrderAspect] = elementIndex(fact) if isJSON: _footnotes = factFootnotes(fact) if _footnotes: aspects["footnotes"] = _footnotes return aspects
def validate(self): modelXbrl = self.modelXbrl if not modelXbrl.contexts or not modelXbrl.facts: return # skip if no contexts or facts if not self.val.validateInferDecimals: # infering precision is now contrary to XBRL REC section 5.2.5.2 modelXbrl.error("calc2e:inferringPrecision", "Calc2 requires inferring decimals.") return startedAt = time.time() # check balance attributes and weights, same as XBRL 2.1 for rel in modelXbrl.relationshipSet(calc2Arcroles).modelRelationships: weight = rel.weight fromConcept = rel.fromModelObject toConcept = rel.toModelObject if fromConcept is not None and toConcept is not None: if rel.arcrole == aggregationDomain: rel.dimension = rel.arcElement.prefixedNameQname( rel.get("dimension")) if rel.dimension is None or not modelXbrl.qnameConcepts[ rel.dimension].isDimensionItem: modelXbrl.error( "calc2e:invalidAggregationDimension", _("Aggregation-domain relationship has invalid dimension %(dimension)s in link role %(linkrole)s" ), modelObject=rel, dimension=rel.get("dimension"), linkrole=ELR) elif fromConcept != toConcept or not fromConcept.isDomainMember: modelXbrl.error( "calc2e:invalidAggregationDomain", _("Calculation relationship has invalid domain %(domain)s in link role %(linkrole)s" ), modelObject=rel, domain=fromConcept, linkrole=ELR) continue if rel.arcrole == balanceChanges: if fromConcept.periodType != "instant" or toConcept.periodType != "duration": modelXbrl.error( "calc2e:invalidBalanceChangesPeriodType", _("Balance-changes relationship must have instant source concept and duration target concept in link role %(linkrole)s" ), modelObject=rel, linkrole=ELR) if weight not in (1, -1): modelXbrl.error( "calc2e:invalidWeight", _("Calculation relationship has invalid weight from %(source)s to %(target)s in link role %(linkrole)s" ), modelObject=rel, source=fromConcept.qname, target=toConcept.qname, linkrole=ELR) fromBalance = fromConcept.balance toBalance = toConcept.balance if fromBalance and toBalance: if (fromBalance == toBalance and weight < 0) or \ (fromBalance != toBalance and weight > 0): modelXbrl.error( "calc2e:balanceCalcWeightIllegal" + ("Negative" if weight < 0 else "Positive"), _("Calculation relationship has illegal weight %(weight)s from %(source)s, %(sourceBalance)s, to %(target)s, %(targetBalance)s, in link role %(linkrole)s (per 5.1.1.2 Table 6)" ), modelObject=rel, weight=weight, source=fromConcept.qname, target=toConcept.qname, linkrole=rel.linkrole, sourceBalance=fromBalance, targetBalance=toBalance, messageCodes=( "calc2e:balanceCalcWeightIllegalNegative", "calc2:balanceCalcWeightIllegalPositive")) if not fromConcept.isNumeric or not toConcept.isNumeric: modelXbrl.error( "calc2e:nonNumericCalc", _("Calculation relationship has illegal concept from %(source)s%(sourceNumericDecorator)s to %(target)s%(targetNumericDecorator)s in link role %(linkrole)s" ), modelObject=rel, source=fromConcept.qname, target=toConcept.qname, linkrole=rel.linkrole, sourceNumericDecorator="" if fromConcept.isNumeric else _(" (non-numeric)"), targetNumericDecorator="" if toConcept.isNumeric else _(" (non-numeric)")) # identify equal contexts uniqueCntxHashes = {} self.modelXbrl.profileActivity() for cntx in modelXbrl.contexts.values(): h = hash((cntx.periodHash, cntx.entityIdentifierHash, cntx.dimsHash)) # OIM-compatible hash if h in uniqueCntxHashes: if cntx.isEqualTo(uniqueCntxHashes[h]): self.eqCntx[cntx] = uniqueCntxHashes[h] else: uniqueCntxHashes[h] = cntx del uniqueCntxHashes self.modelXbrl.profileActivity("... identify aspect equal contexts", minTimeToShow=1.0) # identify equal units uniqueUnitHashes = {} for unit in self.modelXbrl.units.values(): h = unit.hash if h in uniqueUnitHashes: if unit.isEqualTo(uniqueUnitHashes[h]): self.eqUnit[unit] = uniqueUnitHashes[h] else: uniqueUnitHashes[h] = unit del uniqueUnitHashes self.modelXbrl.profileActivity("... identify equal units", minTimeToShow=1.0) sectObjs = sorted( set(rel.fromModelObject # only have numerics with context and unit for rel in modelXbrl.relationshipSet( sectionFact).modelRelationships if rel.fromModelObject is not None and rel.fromModelObject.concept is not None), key=lambda s: (s.concept.label(), s.objectIndex) ) # sort into document order for consistent error messages if not sectObjs: self.modelXbrl.error( "calc2e:noSections", "Instance contains no sections, nothing to validate.", modelObject=modelXbrl) # check by section factByConceptCntxUnit = OrderedDefaultDict( list) # sort into document order for consistent error messages self.sectionFacts = [] for sectObj in sectObjs: #print ("section {}".format(sectObj.concept.label())) self.section = sectObj.concept.label() sectLinkRoles = tuple( sectObj.concept.get(calc2linkroles, "").split()) factByConceptCntxUnit.clear() for f in sorted( ( rel. toModelObject # sort into document order for consistent error messages for rel in modelXbrl.relationshipSet( sectionFact, sectLinkRoles).fromModelObject(sectObj) if rel.toModelObject is not None and # numeric facts with context and unit rel.fromModelObject is not None and rel.toModelObject. concept is not None and rel.toModelObject.context is not None and rel.toModelObject.unit is not None), key=lambda f: f.objectIndex): factByConceptCntxUnit[f.qname, self.eqCntx.get(f.context, f.context), self.eqUnit.get(f.unit, f.unit)].append( f) for fList in factByConceptCntxUnit.values(): f0 = fList[0] if len(fList) == 1: self.sectionFacts.append(f0) else: if any(f.isNil for f in fList): _inConsistent = not all(f.isNil for f in fList) if _inConsistent: # pick a nil fact for f0 for calc validation for f in fList: if f.isNil: f0 = f break elif all( inferredDecimals(f) == inferredDecimals(f0) for f in fList[1:]): # same decimals v0 = intervalValue(f0) _inConsistent = not all( intervalValue(f) == v0 for f in fList[1:]) else: # not all have same decimals d0 = inferredDecimals(f0) aMax, bMin = intervalValue(f0, d0) for f in fList[1:]: df = inferredDecimals(f0) a, b = intervalValue(f, df) if a > aMax: aMax = a if b < bMin: bMin = b if df > d0: # take most accurate fact in section f0 = f d0 = df _inConsistent = (bMin < aMax) if _inConsistent: modelXbrl.error( "calc2e:inconsistentDuplicateInSection", "Section %(section)s contained %(fact)s inconsistent in contexts equivalent to %(contextID)s: values %(values)s", modelObject=fList, section=sectObj.concept.label(), fact=f0.qname, contextID=f0.contextID, values=", ".join( strTruncate(f.value, 128) for f in fList)) self.sectionFacts.append(f0) # sectionFacts now in document order and deduplicated #print("section {} facts {}".format(sectObj.concept.label(), ", ".join(str(f.qname)+"="+f.value for f in self.sectionFacts))) # depth-first calc tree sectCalc2RelSet = modelXbrl.relationshipSet( calc2Arcroles, sectLinkRoles) # indexers for section based on calc2 arcrole self.sumInit = False self.sumConceptBindKeys.clear() self.sumBoundFacts.clear() self.perInit = False self.perConceptBindKeys.clear() self.perBoundFacts.clear() self.durationPeriodStarts.clear() self.aggDimInit = set() self.aggConceptBindKeys.clear() self.aggBoundFacts.clear() self.aggBoundConceptFacts.clear() self.aggDimInit.clear() inferredValues = {} for rootConcept in sorted( sectCalc2RelSet.rootConcepts, key=lambda r: sectCalc2RelSet.fromModelObject(r)[0].order): self.sectTreeRel(rootConcept, 1, sectCalc2RelSet, inferredValues, {rootConcept, None})
def validateXbrlFinally(val, *args, **kwargs): if not (val.validateROSplugin): return _xhtmlNs = "{{{}}}".format(xhtml) _xhtmlNsLen = len(_xhtmlNs) modelXbrl = val.modelXbrl modelDocument = modelXbrl.modelDocument if not modelDocument: return # never loaded properly _statusMsg = _("validating {0} filing rules").format( val.disclosureSystem.name) modelXbrl.profileActivity() modelXbrl.modelManager.showStatus(_statusMsg) if modelDocument.type == ModelDocument.Type.INSTANCE: modelXbrl.error("ROS:instanceMustBeInlineXBRL", _("ROS expects inline XBRL instances."), modelObject=modelXbrl) if modelDocument.type in (ModelDocument.Type.INLINEXBRL, ModelDocument.Type.INLINEXBRLDOCUMENTSET): for ixdsHtmlRootElt in modelXbrl.ixdsHtmlElements: # ix root elements for all ix docs in IXDS ixNStag = ixdsHtmlRootElt.modelDocument.ixNStag ixTags = set(ixNStag + ln for ln in ("nonNumeric", "nonFraction", "references", "relationship")) transformRegistryErrors = set() ixTargets = set() for ixdsHtmlRootElt in modelXbrl.ixdsHtmlElements: for elt in ixdsHtmlRootElt.iter(): if isinstance( elt, (_ElementTree, _Comment, _ProcessingInstruction)): continue # comment or other non-parsed element if isinstance(elt, ModelInlineFact): if elt.format is not None and elt.format.namespaceURI not in TRnamespaces: transformRegistryErrors.add(elt) if elt.get("escape") in ("true", "1"): modelXbrl.error( "ROS.escapedHTML", _("Escaped (x)html fact content is not supported: %(element)s" ), modelObject=elt, element=eltTag) eltTag = elt.tag if eltTag in ixTags: ixTargets.add(elt.get("target")) else: if eltTag.startswith(_xhtmlNs): eltTag = eltTag[_xhtmlNsLen:] if eltTag == "link" and elt.get("type") == "text/css": modelXbrl.error( "ROS.externalCssStyle", _("CSS must be embedded in the inline XBRL document: %(element)s" ), modelObject=elt, element=eltTag) elif ((eltTag in ("object", "script")) or (eltTag == "a" and "javascript:" in elt.get("href", "")) or (eltTag == "img" and "javascript:" in elt.get("src", ""))): modelXbrl.error( "ROS.embeddedCode", _("Inline XBRL documents MUST NOT contain embedded code: %(element)s" ), modelObject=elt, element=eltTag) elif eltTag == "img": src = elt.get("src", "").strip() if not src.startswith("data:image"): modelXbrl.warning( "ROS.embeddedCode", _("Images should be inlined as a base64-encoded string: %(element)s" ), modelObject=elt, element=eltTag) if len(ixTargets) > 1: modelXbrl.error( "ROS:singleOutputDocument", _("Multiple target instance documents are not supported: %(targets)s." ), modelObject=modelXbrl, targets=", ".join((t or "(default)") for t in ixTargets)) filingTypes = set() unexpectedTaxonomyReferences = set() numIxDocs = 0 for doc in modelXbrl.urlDocs.values(): if doc.type == ModelDocument.Type.INLINEXBRL: # base file extension _baseName, _baseExt = os.path.splitext(doc.basename) if _baseExt not in (".xhtml", ".html", "htm", "ixbrl", "xml", "xhtml"): modelXbrl.error( "ROS.fileNameExtension", _("The list of acceptable file extensions for upload is: html, htm, ixbrl, xml, xhtml: %(fileName)s" ), modelObject=doc, fileName=doc.basename) # document encoding if doc.documentEncoding.lower() != "utf-8": modelXbrl.error( "ROS.documentEncoding", _("iXBRL documents submitted to Revenue should be UTF-8 encoded: %(encoding)s" ), modelObject=doc, encoding=doc.documentEncoding) # identify type of filing for referencedDoc in doc.referencesDocument.keys(): if referencedDoc.type == ModelDocument.Type.SCHEMA: if referencedDoc.uri in taxonomyReferences: filingTypes.add(referencedDoc.uri) else: unexpectedTaxonomyReferences.add(referencedDoc.uri) # count of inline docs in IXDS numIxDocs += 1 if len(filingTypes) != 1: modelXbrl.error( "ROS:multipleFilingTypes", _("Multiple filing types detected: %(filingTypes)s."), modelObject=modelXbrl, filingTypes=", ".join(sorted(filingTypes))) if unexpectedTaxonomyReferences: modelXbrl.error( "ROS:unexpectedTaxonomyReferences", _("Referenced schema(s) does not map to a taxonomy supported by Revenue (schemaRef): %(unexpectedReferences)s." ), modelObject=modelXbrl, unexpectedReferences=", ".join( sorted(unexpectedTaxonomyReferences))) # single document IXDS if numIxDocs > 1: modelXbrl.warning( "ROS:multipleInlineDocuments", _("A single inline document should be submitted but %(numberDocs)s were found." ), modelObject=modelXbrl, numberDocs=numIxDocs) # build namespace maps nsMap = {} for prefix in ("ie-common", "bus", "uk-bus", "ie-dpl", "core"): if prefix in modelXbrl.prefixedNamespaces: nsMap[prefix] = modelXbrl.prefixedNamespaces[prefix] # build mandatory table by ns qname in use mandatory = set() for prefix in mandatoryElements: if prefix in nsMap: ns = nsMap[prefix] for localName in mandatoryElements[prefix]: mandatory.add(qname(ns, prefix + ":" + localName)) equivalentManatoryQNames = [[ qname(q, nsMap) for q in equivElts ] for equivElts in equivalentMandatoryElements] # document creator requirement if "bus" in nsMap: if qname("bus:NameProductionSoftware", nsMap) not in modelXbrl.factsByQname or qname( "bus:VersionProductionSoftware", nsMap) not in modelXbrl.factsByQname: modelXbrl.warning( "ROS:documentCreatorProductInformation", _("Please use the NameProductionSoftware tag to identify the software package and the VersionProductionSoftware tag to identify the version of the software package." ), modelObject=modelXbrl) elif "uk-bus" in nsMap: if qname("uk-bus:NameAuthor", nsMap) not in modelXbrl.factsByQname or qname( "uk-bus:DescriptionOrTitleAuthor", nsMap) not in modelXbrl.factsByQname: modelXbrl.warning( "ROS:documentCreatorProductInformation", _("Revenue request that vendor, product and version information is embedded in the generated inline XBRL document using a single XBRLDocumentAuthorGrouping tuple. The NameAuthor tag should be used to identify the name and version of the software package." ), modelObject=modelXbrl) schemeEntityIds = set() mapContext = {} # identify unique contexts and units mapUnit = {} uniqueContextHashes = {} hasCRO = False unsupportedSchemeContexts = [] mismatchIdentifierContexts = [] for context in modelXbrl.contexts.values(): schemeEntityIds.add(context.entityIdentifier) scheme, entityId = context.entityIdentifier if scheme not in schemePatterns: unsupportedSchemeContexts.append(context) elif not schemePatterns[scheme].match(entityId): mismatchIdentifierContexts.append(context) if scheme == "http://www.cro.ie/": hasCRO = True h = context.contextDimAwareHash if h in uniqueContextHashes: if context.isEqualTo(uniqueContextHashes[h]): mapContext[context] = uniqueContextHashes[h] else: uniqueContextHashes[h] = context del uniqueContextHashes if len(schemeEntityIds) > 1: modelXbrl.error( "ROS:differentContextEntityIdentifiers", _("Context entity identifier not all the same: %(schemeEntityIds)s." ), modelObject=modelXbrl, schemeEntityIds=", ".join( sorted(str(s) for s in schemeEntityIds))) if unsupportedSchemeContexts: modelXbrl.error( "ROS:unsupportedContextEntityIdentifierScheme", _("Context identifier scheme(s) is not supported: %(schemes)s." ), modelObject=unsupportedSchemeContexts, schemes=", ".join( sorted( set(c.entityIdentifier[0] for c in unsupportedSchemeContexts)))) if mismatchIdentifierContexts: modelXbrl.error( "ROS:invalidContextEntityIdentifier", _("Context entity identifier(s) lexically invalid: %(identifiers)s." ), modelObject=mismatchIdentifierContexts, identifiers=", ".join( sorted( set(c.entityIdentifier[1] for c in mismatchIdentifierContexts)))) uniqueUnitHashes = {} for unit in modelXbrl.units.values(): h = unit.hash if h in uniqueUnitHashes: if unit.isEqualTo(uniqueUnitHashes[h]): mapUnit[unit] = uniqueUnitHashes[h] else: uniqueUnitHashes[h] = unit del uniqueUnitHashes if hasCRO and "ie-common" in nsMap: mandatory.add( qname("ie-common:CompaniesRegistrationOfficeNumber", nsMap)) reportedMandatory = set() numFactsByConceptContextUnit = defaultdict(list) for qn, facts in modelXbrl.factsByQname.items(): if qn in mandatory: reportedMandatory.add(qn) for f in facts: if f.isNumeric and f.parentElement.qname == qnXbrliXbrl: numFactsByConceptContextUnit[(f.qname, mapContext.get( f.context, f.context), mapUnit.get( f.unit, f.unit))].append(f) missingElements = ( mandatory - reportedMandatory ) # | (reportedFootnoteIfNil - reportedFootnoteIfNil) for qnames in equivalentManatoryQNames: # remove missing elements for which an or-match was reported if any(qn in modelXbrl.factsByQname for qn in qnames): for qn in qnames: missingElements.discard(qn) if missingElements: modelXbrl.error( "ROS:missingRequiredElements", _("Required elements missing from document: %(elements)s."), modelObject=modelXbrl, elements=", ".join(sorted(str(qn) for qn in missingElements))) for fList in numFactsByConceptContextUnit.values(): if len(fList) > 1: f0 = fList[0] if any(f.isNil for f in fList): _inConsistent = not all(f.isNil for f in fList) elif all( inferredDecimals(f) == inferredDecimals(f0) for f in fList[1:]): # same decimals v0 = rangeValue(f0.value) _inConsistent = not all( rangeValue(f.value) == v0 for f in fList[1:]) else: # not all have same decimals aMax, bMin = rangeValue(f0.value, inferredDecimals(f0)) for f in fList[1:]: a, b = rangeValue(f.value, inferredDecimals(f)) if a > aMax: aMax = a if b < bMin: bMin = b _inConsistent = (bMin < aMax) if _inConsistent: modelXbrl.error( ("ROS.inconsistentDuplicateFacts"), "Inconsistent duplicate numeric facts: %(fact)s were used more than once in contexts equivalent to %(contextID)s: values %(values)s. ", modelObject=fList, fact=f0.qname, contextID=f0.contextID, values=", ".join( strTruncate(f.value, 128) for f in fList)) modelXbrl.profileActivity(_statusMsg, minTimeToShow=0.0) modelXbrl.modelManager.showStatus(None)
def _decimals(node, sphinxContext, args): fact = factArg(node, sphinxContext, args, 0) return inferredDecimals(fact)
def factAspects(fact): oimFact = OrderedDict() aspects = OrderedDict() if hasId and fact.id: if fact.isTuple: oimFact["tupleId"] = fact.id else: oimFact["id"] = fact.id elif (fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))): oimFact["id"] = "f{}".format(fact.objectIndex) parent = fact.getparent() concept = fact.concept aspects[str(qnOimConceptAspect)] = oimValue(concept.qname) _csvType = "Value" if not fact.isTuple: if concept is not None: _baseXsdType = concept.baseXsdType if _baseXsdType == "XBRLI_DATEUNION": if getattr(fact.xValue, "dateOnly", False): _baseXsdType = "date" else: _baseXsdType = "dateTime" _csvType = baseTypes.get(_baseXsdType,_baseXsdType) + "Value" if concept.baseXbrliType in ("stringItemType", "normalizedStringItemType") and fact.xmlLang: aspects[str(qnOimLangAspect)] = fact.xmlLang if fact.isItem: if fact.isNil: _value = None else: _inferredDecimals = inferredDecimals(fact) _value = oimValue(fact.xValue, _inferredDecimals) oimFact["value"] = _value if fact.concept is not None and fact.concept.isNumeric: _numValue = fact.xValue if isinstance(_numValue, Decimal) and not isinf(_numValue) and not isnan(_numValue): if _numValue == _numValue.to_integral(): _numValue = int(_numValue) else: _numValue = float(_numValue) if not fact.isNil: if isinf(_inferredDecimals): if isJSON: _accuracy = "infinity" elif isCSVorXL: _accuracy = "INF" else: _accuracy = _inferredDecimals oimFact["accuracy"] = _accuracy oimFact["aspects"] = aspects cntx = fact.context if cntx is not None: if cntx.entityIdentifierElement is not None: aspects[str(qnOimEntityAspect)] = oimValue(qname(*cntx.entityIdentifier)) if cntx.period is not None: aspects.update(oimPeriodValue(cntx)) for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]): if dim.isExplicit: dimVal = oimValue(dim.memberQname) else: # typed if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1"): dimVal = None else: dimVal = dim.typedMember.stringValue aspects[str(dim.dimensionQname)] = dimVal unit = fact.unit if unit is not None: _mMul, _mDiv = unit.measures _sMul = '*'.join(oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m))) if _mDiv: _sDiv = '*'.join(oimValue(m) for m in sorted(_mDiv, key=lambda m: oimValue(m))) if len(_mDiv) > 1: if len(_mMul) > 1: _sUnit = "({})/({})".format(_sMul,_sDiv) else: _sUnit = "{}/({})".format(_sMul,_sDiv) else: if len(_mMul) > 1: _sUnit = "({})/{}".format(_sMul,_sDiv) else: _sUnit = "{}/{}".format(_sMul,_sDiv) else: _sUnit = _sMul aspects[str(qnOimUnitAspect)] = _sUnit # Tuples removed from xBRL-JSON #if parent.qname != XbrlConst.qnXbrliXbrl: # aspects[str(qnOimTupleParentAspect)] = parent.id if parent.id else "f{}".format(parent.objectIndex) # aspects[str(qnOimTupleOrderAspect)] = elementIndex(fact) if isJSON: factFootnotes(fact, oimFact) return oimFact
def validate(val, modelXbrl, infosetModelXbrl): infoset = infosetModelXbrl.modelDocument if infoset.type == Type.INSTANCE: # compare facts (assumed out of order) infosetFacts = defaultdict(list) for fact in infosetModelXbrl.facts: infosetFacts[fact.qname].append(fact) if len(modelXbrl.factsInInstance) != len(infosetModelXbrl.factsInInstance): modelXbrl.error("arelle:infosetTest", _("Fact counts mismatch, testcase instance %(foundFactCount)s, infoset instance %(expectedFactCount)s"), modelObject=(modelXbrl.modelDocument, infosetModelXbrl.modelDocument), foundFactCount=len(modelXbrl.factsInInstance), expectedFactCount=len(infosetModelXbrl.factsInInstance)) else: for i, instFact in enumerate(modelXbrl.facts): infosetFact = None for fact in infosetFacts[instFact.qname]: if fact.isTuple and fact.isDuplicateOf(instFact, deemP0Equal=True): infosetFact = fact break elif fact.isItem and fact.isVEqualTo(instFact, deemP0Equal=True): infosetFact = fact break if infosetFact is None: # takes precision/decimals into account if fact is not None: fact.isVEqualTo(instFact, deemP0Equal=True) modelXbrl.error("arelle:infosetTest", _("Fact %(factNumber)s mismatch %(concept)s"), modelObject=instFact, factNumber=(i+1), concept=instFact.qname) else: ptvPeriodType = infosetFact.get("{http://www.xbrl.org/2003/ptv}periodType") ptvBalance = infosetFact.get("{http://www.xbrl.org/2003/ptv}balance") ptvDecimals = infosetFact.get("{http://www.xbrl.org/2003/ptv}decimals") ptvPrecision = infosetFact.get("{http://www.xbrl.org/2003/ptv}precision") if ptvPeriodType and ptvPeriodType != instFact.concept.periodType: modelXbrl.error("arelle:infosetTest", _("Fact %(factNumber)s periodType mismatch %(concept)s expected %(expectedPeriodType)s found %(foundPeriodType)s"), modelObject=(instFact, infosetFact), factNumber=(i+1), concept=instFact.qname, expectedPeriodType=ptvPeriodType, foundPeriodType=instFact.concept.periodType) if ptvBalance and ptvBalance != instFact.concept.balance: modelXbrl.error("arelle:infosetTest", _("Fact %(factNumber)s balance mismatch %(concept)s expected %(expectedBalance)s found %(foundBalance)s"), modelObject=(instFact, infosetFact), factNumber=(i+1), concept=instFact.qname, expectedBalance=ptvBalance, foundBalance=instFact.concept.balance) if ptvDecimals and ptvDecimals != str(inferredDecimals(fact)): modelXbrl.error("arelle:infosetTest", _("Fact %(factNumber)s inferred decimals mismatch %(concept)s expected %(expectedDecimals)s found %(inferredDecimals)s"), modelObject=(instFact, infosetFact), factNumber=(i+1), concept=instFact.qname, expectedDecimals=ptvDecimals, inferredDecimals=str(inferredDecimals(fact))) if ptvPrecision and ptvPrecision != str(inferredPrecision(fact)): modelXbrl.error("arelle:infosetTest", _("Fact %(factNumber)s inferred precision mismatch %(concept)s expected %(expectedPrecision)s found %(inferredPrecision)s"), modelObject=(instFact, infosetFact), factNumber=(i+1), concept=instFact.qname, expectedPrecisions=ptvPrecision, inferredPrecision=str(inferredPrecision(fact))) elif infoset.type == Type.ARCSINFOSET: # compare arcs for arcElt in XmlUtil.children(infoset.xmlRootElement, "http://www.xbrl.org/2003/ptv", "arc"): linkType = arcElt.get("linkType") arcRole = arcElt.get("arcRole") extRole = arcElt.get("extRole") fromObj = resolvePath(modelXbrl, arcElt.get("fromPath")) if fromObj is None: modelXbrl.error("arelle:infosetTest", _("Arc fromPath not found: %(fromPath)s"), modelObject=arcElt, fromPath=arcElt.get("fromPath")) continue if linkType in ("label", "reference"): labelLang = arcElt.get("labelLang") resRole = arcElt.get("resRole") if linkType == "label": expectedLabel = XmlUtil.text(arcElt) foundLabel = fromObj.label(preferredLabel=resRole,fallbackToQname=False,lang=None,strip=True,linkrole=extRole) if foundLabel != expectedLabel: modelXbrl.error("arelle:infosetTest", _("Label expected='%(expectedLabel)s', found='%(foundLabel)s'"), modelObject=arcElt, expectedLabel=expectedLabel, foundLabel=foundLabel) continue elif linkType == "reference": expectedRef = XmlUtil.innerText(arcElt) referenceFound = False for refrel in modelXbrl.relationshipSet(XbrlConst.conceptReference,extRole).fromModelObject(fromObj): ref = refrel.toModelObject if resRole == ref.role: foundRef = XmlUtil.innerText(ref) if foundRef != expectedRef: modelXbrl.error("arelle:infosetTest", _("Reference inner text expected='%(expectedRef)s, found='%(foundRef)s'"), modelObject=arcElt, expectedRef=expectedRef, foundRef=foundRef) referenceFound = True break if referenceFound: continue modelXbrl.error("arelle:infosetTest", _("%(linkType)s not found containing '%(text)s' linkRole %(linkRole)s"), modelObject=arcElt, linkType=linkType.title(), text=XmlUtil.innerText(arcElt), linkRole=extRole) else: toObj = resolvePath(modelXbrl, arcElt.get("toPath")) if toObj is None: modelXbrl.error("arelle:infosetTest", _("Arc toPath not found: %(toPath)s"), modelObject=arcElt, toPath=arcElt.get("toPath")) continue weight = arcElt.get("weight") if weight is not None: weight = float(weight) order = arcElt.get("order") if order is not None: order = float(order) preferredLabel = arcElt.get("preferredLabel") found = False for rel in modelXbrl.relationshipSet(arcRole, extRole).fromModelObject(fromObj): if (rel.toModelObject == toObj and (weight is None or rel.weight == weight) and (order is None or rel.order == order)): found = True if not found: modelXbrl.error("arelle:infosetTest", _("Arc not found: from %(fromPath)s, to %(toPath)s, role %(arcRole)s, linkRole $(extRole)s"), modelObject=arcElt, fromPath=arcElt.get("fromPath"), toPath=arcElt.get("toPath"), arcRole=arcRole, linkRole=extRole) continue # validate dimensions of each fact factElts = XmlUtil.children(modelXbrl.modelDocument.xmlRootElement, None, "*") for itemElt in XmlUtil.children(infoset.xmlRootElement, None, "item"): try: qnElt = XmlUtil.child(itemElt,None,"qnElement") factQname = qname(qnElt, XmlUtil.text(qnElt)) sPointer = int(XmlUtil.child(itemElt,None,"sPointer").text) factElt = factElts[sPointer - 1] # 1-based xpath indexing if factElt.qname != factQname: modelXbrl.error("arelle:infosetTest", _("Fact %(sPointer)s mismatch Qname, expected %(qnElt)s, observed %(factQname)s"), modelObject=itemElt, sPointer=sPointer, qnElt=factQname, factQname=factElt.qname) elif not factElt.isItem or factElt.context is None: modelXbrl.error("arelle:infosetTest", _("Fact %(sPointer)s has no context: %(qnElt)s"), modelObject=(itemElt,factElt), sPointer=sPointer, qnElt=factQname) else: context = factElt.context memberElts = XmlUtil.children(itemElt,None,"member") numNonDefaults = 0 for memberElt in memberElts: dimElt = XmlUtil.child(memberElt, None, "qnDimension") qnDim = qname(dimElt, XmlUtil.text(dimElt)) isDefault = XmlUtil.text(XmlUtil.child(memberElt, None, "bDefaulted")) == "true" if not isDefault: numNonDefaults += 1 if not ((qnDim in context.qnameDims and not isDefault) or (qnDim in factElt.modelXbrl.qnameDimensionDefaults and isDefault)): modelXbrl.error("arelle:infosetTest", _("Fact %(sPointer)s (qnElt)s dimension mismatch %(qnDim)s"), modelObject=(itemElt, factElt, context), sPointer=sPointer, qnElt=factQname, qnDim=qnDim) if numNonDefaults != len(context.qnameDims): modelXbrl.error("arelle:infosetTest", _("Fact %(sPointer)s (qnElt)s dimensions count mismatch"), modelObject=(itemElt, factElt, context), sPointer=sPointer, qnElt=factQname) except (IndexError, ValueError, AttributeError) as err: modelXbrl.error("arelle:infosetTest", _("Invalid entity fact dimensions infoset sPointer: %(test)s, error details: %(error)s"), modelObject=itemElt, test=XmlUtil.innerTextList(itemElt), error=str(err))
def validateXbrlFinally(val, *args, **kwargs): if not (val.validateHMRCplugin) or not val.txmyType: return modelXbrl = val.modelXbrl modelDocument = modelXbrl.modelDocument _statusMsg = _("validating {0} filing rules").format( val.disclosureSystem.name) modelXbrl.profileActivity() modelXbrl.modelManager.showStatus(_statusMsg) if modelDocument.type in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL): labelHasNegativeTermPattern = re.compile(r".*[(].*\w.*[)].*") companyReferenceNumberContexts = defaultdict(list) for c1 in modelXbrl.contexts.values(): scheme, identifier = c1.entityIdentifier if scheme == "http://www.companieshouse.gov.uk/": companyReferenceNumberContexts[identifier].append(c1.id) uniqueFacts = {} # key = (qname, context hash, unit hash, lang) mandatoryFacts = {} mandatoryGDV = defaultdict(set) factForConceptContextUnitHash = defaultdict(list) hasCompaniesHouseContext = any( cntx.entityIdentifier[0] == "http://www.companieshouse.gov.uk/" for cntx in val.modelXbrl.contexts.values()) contextsUsed = set(f.context for f in modelXbrl.factsInInstance if f.context is not None) contextsWithScenario = [] for cntx in contextsUsed: for dim in cntx.qnameDims.values(): if dim.isExplicit: _memName = dim.memberQname.localName m = memNameNumPattern.match(_memName) if m: l = m.group(1) n = int(m.group(2)) else: l = _memName n = None for _gdvType in (val.txmyType, "business"): gdv = genericDimensionValidation.get( _gdvType, EMPTYDICT).get(l) if gdv: # take first match break if (gdv and (n is None or (isinstance(gdv[0], int) and isinstance(gdv[1], int) and n >= gdv[0] and n <= gdv[1]))): gdvFacts = [f for f in gdv if isinstance(f, str)] if len(gdvFacts) == 1: mandatoryGDV[gdvFacts[0]].add( GDV(gdvFacts[0], None, _memName)) elif len(gdvFacts) == 2: mandatoryGDV[gdvFacts[0]].add( GDV(gdvFacts[0], gdvFacts[1], _memName)) mandatoryGDV[gdvFacts[1]].add( GDV(gdvFacts[1], gdvFacts[0], _memName)) if XmlUtil.hasChild(cntx, xbrli, "scenario"): contextsWithScenario.append(cntx) if contextsWithScenario: modelXbrl.error( "FRC.TG.3.6.1", _("Context(s) %(identifiers)s is reported with scenario element." ), modelObject=contextsWithScenario, identifiers=", ".join(c.id for c in contextsWithScenario)) del contextsWithScenario def checkFacts(facts): for f in facts: cntx = f.context unit = f.unit if ( f.isNil or getattr(f, "xValid", 0) >= 4 ) and cntx is not None and f.concept is not None and f.concept.type is not None: factNamespaceURI = f.qname.namespaceURI factLocalName = f.qname.localName if factLocalName in mandatoryItems[val.txmyType]: mandatoryFacts[factLocalName] = f if factLocalName == "UKCompaniesHouseRegisteredNumber" and val.isAccounts: if hasCompaniesHouseContext: mandatoryFacts[factLocalName] = f for _cntx in contextsUsed: _scheme, _identifier = _cntx.entityIdentifier if _scheme == "http://www.companieshouse.gov.uk/" and f.xValue != _identifier: modelXbrl.error( "JFCVC.3316", _("Context entity identifier %(identifier)s does not match Company Reference Number (UKCompaniesHouseRegisteredNumber) Location: Accounts (context id %(id)s)" ), modelObject=(f, _cntx), identifier=_identifier, id=_cntx.id) if f.parentElement.qname == qnXbrliXbrl: # JFCVC v4.0 - only check non-tuple facts factForConceptContextUnitHash[ f.conceptContextUnitHash].append(f) if f.isNumeric: if f.precision: modelXbrl.error( "HMRC.5.4", _("Numeric fact %(fact)s of context %(contextID)s has a precision attribute '%(precision)s'" ), modelObject=f, fact=f.qname, contextID=f.contextID, precision=f.precision) try: # only process validated facts if not f.isNil: if f.xValue < 0: label = f.concept.label(lang="en") if not labelHasNegativeTermPattern.match( label): modelXbrl.error( "HMRC.5.3", _("Numeric fact %(fact)s of context %(contextID)s has a negative value '%(value)s' but label does not have a bracketed negative term (using parentheses): %(label)s" ), modelObject=f, fact=f.qname, contextID=f.contextID, value=f.value, label=label) # 6.5.37 test (insignificant digits due to rounding) if f.decimals and f.decimals != "INF": try: insignificance = insignificantDigits( f.xValue, decimals=f.decimals) if insignificance: # if not None, returns (truncatedDigits, insiginficantDigits) modelXbrl.error( "HMRC.SG.4.5", _("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s has nonzero digits in insignificant portion %(insignificantDigits)s." ), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals, value=f.xValue, truncatedDigits=insignificance[ 0], insignificantDigits= insignificance[1]) except (ValueError, TypeError): modelXbrl.error( "HMRC.SG.4.5", _("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s causes Value Error exception." ), modelObject=f1, fact=f1.qname, contextID=f1.contextID, decimals=f1.decimals, value=f1.value) except AttributeError: pass # if not validated it should have failed with a schema error # check GDV if f.qname.localName in mandatoryGDV: _gdvReqList = mandatoryGDV[factLocalName] _gdvReqRemovals = [] for _gdvReq in _gdvReqList: if any(_gdvReq.memLocalName == dim.memberQname.localName for dim in cntx.qnameDims.values() if dim.isExplicit): _gdvReqRemovals.append(_gdvReq) if _gdvReq.altFact in mandatoryGDV: _gdvAltList = mandatoryGDV[_gdvReq.altFact] _gdvAltRemovals = [] for _gdvAlt in _gdvAltList: if any(_gdvAlt.memLocalName == dim.memberQname.localName for dim in cntx.qnameDims.values() if dim.isExplicit): _gdvAltRemovals.append(_gdvAlt) for _gdvAlt in _gdvAltRemovals: _gdvAltList.remove(_gdvAlt) if _gdvReqRemovals and not f.xValue: # fact was a mandatory name or description modelXbrl.error( "JFCVC.3315", _("Generic dimension members associated name/description has no text: %(fact)s" ), modelObject=f, fact=f.qname) for _gdvReq in _gdvReqRemovals: _gdvReqList.remove(_gdvReq) if f.modelTupleFacts: checkFacts(f.modelTupleFacts) checkFacts(modelXbrl.facts) if val.isAccounts: _missingItems = mandatoryItems[ val.txmyType] - mandatoryFacts.keys() if hasCompaniesHouseContext and "UKCompaniesHouseRegisteredNumber" not in mandatoryFacts: _missingItems.add("UKCompaniesHouseRegisteredNumber") if _missingItems: modelXbrl.error("JFCVC.3312", _("Facts are MANDATORY: %(missingItems)s"), modelObject=modelXbrl, missingItems=", ".join(sorted(_missingItems))) ''' removed with JFCVC v4.0 2020-06-09 f = mandatoryFacts.get("StartDateForPeriodCoveredByReport") if f is not None and (f.isNil or f.xValue < _6_APR_2008): modelXbrl.error("JFCVC.3313", _("Period Start Date (StartDateForPeriodCoveredByReport) must be 6 April 2008 or later, but is %(value)s"), modelObject=f, value=f.value) ''' memLocalNamesMissing = set( "{}({})".format(_gdvRec.memLocalName, _gdvRec.factNames) for _gdv in mandatoryGDV.values() for _gdvRec in _gdv) if memLocalNamesMissing: modelXbrl.error( "JFCVC.3315", _("Generic dimension members have no associated name or description item, member names (name or description item): %(memberNames)s" ), modelObject=modelXbrl, memberNames=", ".join(sorted(memLocalNamesMissing))) aspectEqualFacts = defaultdict( dict) # dict [(qname,lang)] of dict(cntx,unit) of [fact, fact] for hashEquivalentFacts in factForConceptContextUnitHash.values(): if len(hashEquivalentFacts) > 1: for f in hashEquivalentFacts: # check for hash collision by value checks on context and unit if getattr(f, "xValid", 0) >= 4: cuDict = aspectEqualFacts[( f.qname, (f.xmlLang or "").lower() if f.concept.type.isWgnStringFactType else None)] _matched = False for (_cntx, _unit), fList in cuDict.items(): if (((_cntx is None and f.context is None) or (f.context is not None and f.context.isEqualTo(_cntx))) and ((_unit is None and f.unit is None) or (f.unit is not None and f.unit.isEqualTo(_unit)))): _matched = True fList.append(f) break if not _matched: cuDict[(f.context, f.unit)] = [f] decVals = {} for cuDict in aspectEqualFacts.values(): # dups by qname, lang for fList in cuDict.values( ): # dups by equal-context equal-unit if len(fList) > 1: f0 = fList[0] if f0.concept.isNumeric: if any(f.isNil for f in fList): _inConsistent = not all(f.isNil for f in fList) else: # not all have same decimals _d = inferredDecimals(f0) _v = f0.xValue _inConsistent = isnan( _v ) # NaN is incomparable, always makes dups inconsistent decVals[_d] = _v aMax, bMin = rangeValue(_v, _d) for f in fList[1:]: _d = inferredDecimals(f) _v = f.xValue if isnan(_v): _inConsistent = True break if _d in decVals: _inConsistent |= _v != decVals[_d] else: decVals[_d] = _v a, b = rangeValue(_v, _d) if a > aMax: aMax = a if b < bMin: bMin = b if not _inConsistent: _inConsistent = (bMin < aMax) decVals.clear() if _inConsistent: modelXbrl.error( "JFCVC.3314", "Inconsistent duplicate fact values %(fact)s: %(values)s.", modelObject=fList, fact=f0.qname, contextID=f0.contextID, values=", ".join(f.value for f in fList)) aspectEqualFacts.clear() del factForConceptContextUnitHash, aspectEqualFacts if modelXbrl.modelDocument.type == ModelDocument.Type.INLINEXBRL: rootElt = modelXbrl.modelDocument.xmlRootElement if rootElt.tag in ("html", "xhtml") or not rootElt.tag.startswith( "{http://www.w3.org/1999/xhtml}"): modelXbrl.error( "HMRC.SG.3.3", _("InlineXBRL root element <%(element)s> MUST be html and have the xhtml namespace." ), modelObject=rootElt, element=rootElt.tag) for elt in rootElt.iterdescendants( tag="{http://www.w3.org/1999/xhtml}script"): modelXbrl.error("HMRC.SG.3.3", _("Script element is disallowed."), modelObject=elt) for tag, localName, attr in (("{http://www.w3.org/1999/xhtml}a", "a", "href"), ("{http://www.w3.org/1999/xhtml}img", "img", "src")): for elt in rootElt.iterdescendants(tag=tag): attrValue = (elt.get(attr) or "").strip() if "javascript:" in attrValue: modelXbrl.error( "HMRC.SG.3.3", _("Element %(localName)s javascript %(javascript)s is disallowed." ), modelObject=elt, localName=localName, javascript=attrValue[:64]) if localName == "img" and not any( attrValue.startswith(m) for m in allowedImgMimeTypes): modelXbrl.error( "HMRC.SG.3.8", _("Image scope must be base-64 encoded string (starting with data:image/*;base64), *=gif, jpeg or png. src disallowed: %(src)s." ), modelObject=elt, src=attrValue[:128]) for elt in rootElt.iterdescendants( tag="{http://www.w3.org/1999/xhtml}style"): if elt.text and styleImgUrlPattern.match(elt.text): modelXbrl.error( "HMRC.SG.3.8", _("Style element has disallowed image reference: %(styleImage)s." ), modelObject=elt, styleImage=styleImgUrlPattern.match(elt.text).group(1)) for elt in rootElt.xpath( "//xhtml:*[@style]", namespaces={"xhtml": "http://www.w3.org/1999/xhtml"}): for match in styleImgUrlPattern.findall(elt.get("style")): modelXbrl.error( "HMRC.SG.3.8", _("Element %(elt)s style attribute has disallowed image reference: %(styleImage)s." ), modelObject=elt, elt=elt.tag.rpartition("}")[2], styleImage=match) modelXbrl.profileActivity(_statusMsg, minTimeToShow=0.0) modelXbrl.modelManager.showStatus(None)
def showInfo(cntlr, options, modelXbrl, _entrypoint, *args, **kwargs): for url, doc in sorted(modelXbrl.urlDocs.items(), key=lambda i: i[0]): if not any(url.startswith(w) for w in ("https://xbrl.sec.gov/", "http://xbrl.sec.gov/", "http://xbrl.fasb.org/", "http://www.xbrl.org/", "http://xbrl.ifrs.org/", "http://www.esma.europa.eu/")): cntlr.addToLog("File {} size {:,}".format(doc.basename, os.path.getsize(doc.filepath)), messageCode="info", level=logging.DEBUG) cntlr.addToLog("Heap memory before loading {:,}".format(memoryAtStartup), messageCode="info", level=logging.DEBUG) cntlr.addToLog("Heap memory after loading {:,}".format(cntlr.memoryUsed), messageCode="info", level=logging.DEBUG) cntlr.addToLog("Time to load {:.2f} seconds".format(time.time() - timeAtStart), messageCode="info", level=logging.DEBUG) isInlineXbrl = modelXbrl.modelDocument.type in (ModelDocument.Type.INLINEXBRL, ModelDocument.Type.INLINEXBRLDOCUMENTSET) if isInlineXbrl: instanceType = "inline XBRL, number of documents {}".format(len(modelXbrl.ixdsHtmlElements)) else: instanceType = "xBRL-XML" cntlr.addToLog("Instance type {}".format(instanceType), messageCode="info", level=logging.DEBUG) numContexts = len(modelXbrl.contexts) numLongContexts = 0 bytesSaveable = 0 frequencyOfDims = {} sumNumDims = 0 distinctDurations = set() distinctInstants = set() shortContextIdLen = int(math.log10(numContexts or 1)) + 2 # if no contexts, use 1 for log function to work xbrlQnameCount = 0 xbrlQnameLengths = 0 for c in modelXbrl.contexts.values(): sumNumDims += len(c.qnameDims) for d in c.qnameDims.values(): dimQname = str(d.dimensionQname) frequencyOfDims[dimQname] = frequencyOfDims.get(dimQname,0) + 1 xbrlQnameCount += 1 xbrlQnameLengths += len(d.dimensionQname.localName) if c.isInstantPeriod: distinctInstants.add(c.instantDatetime) elif c.isStartEndPeriod: distinctDurations.add((c.startDatetime, c.endDatetime)) if len(c.id) > shortContextIdLen: bytesSaveable += len(c.id) - shortContextIdLen cntlr.addToLog("Number of contexts {:,}".format(numContexts), messageCode="info", level=logging.DEBUG) cntlr.addToLog("Number of distinct durations {:,}".format(len(distinctDurations)), messageCode="info", level=logging.DEBUG) cntlr.addToLog("Number of distinct instants {:,}".format(len(distinctInstants)), messageCode="info", level=logging.DEBUG) cntlr.addToLog("Avg number dimensions per contexts {:,.2f}".format(sumNumDims/numContexts if numContexts else 0), messageCode="info", level=logging.DEBUG) mostPopularDims = sorted(frequencyOfDims.items(), key=lambda i:"{:0>9},{}".format(999999999-i[1],i[0])) for dimName, count in mostPopularDims[0:3]: cntlr.addToLog("Dimension {} used in {:,} contexts".format(dimName, count), messageCode="info", level=logging.DEBUG) numFacts = 0 numTableTextBlockFacts = 0 lenTableTextBlockFacts = 0 numTextBlockFacts = 0 lenTextBlockFacts = 0 distinctElementsInFacts = set() factsPerContext = {} factForConceptContextUnitHash = defaultdict(list) for f in modelXbrl.factsInInstance: context = f.context concept = f.concept distinctElementsInFacts.add(f.qname) numFacts += 1 if f.qname.localName.endswith("TableTextBlock"): numTableTextBlockFacts += 1 lenTableTextBlockFacts += len(f.xValue) elif f.qname.localName.endswith("TextBlock"): numTextBlockFacts += 1 lenTextBlockFacts += len(f.xValue) if context is not None and concept is not None: factsPerContext[context.id] = factsPerContext.get(context.id,0) + 1 factForConceptContextUnitHash[f.conceptContextUnitHash].append(f) bytesSaveable += len(context.id) - shortContextIdLen mostPopularContexts = sorted(factsPerContext.items(), key=lambda i:"{:0>9},{}".format(999999999-i[1],i[0])) cntlr.addToLog("Number of facts {:,}".format(numFacts), messageCode="info", level=logging.DEBUG) cntlr.addToLog("Number of TableTextBlock facts {:,} avg len {:,.0f}".format(numTableTextBlockFacts, lenTableTextBlockFacts/numTableTextBlockFacts if numTableTextBlockFacts else 0), messageCode="info", level=logging.DEBUG) cntlr.addToLog("Number of TextBlock facts {:,} avg len {:,.0f}".format(numTextBlockFacts, lenTextBlockFacts/numTableTextBlockFacts if numTableTextBlockFacts else 0), messageCode="info", level=logging.DEBUG) cntlr.addToLog("Max number facts per context {:,}".format(mostPopularContexts[0][1] if mostPopularContexts else 0), messageCode="info", level=logging.DEBUG) cntlr.addToLog("Avg number facts per context {:,.2f}".format(sum([v for v in factsPerContext.values()])/numContexts if numContexts else 0), messageCode="info", level=logging.DEBUG) cntlr.addToLog("Distinct elements in facts {:,}".format(len(distinctElementsInFacts)), messageCode="info", level=logging.DEBUG) cntlr.addToLog("Number of bytes saveable context id of {} length is {:,}".format(shortContextIdLen, bytesSaveable), messageCode="info", level=logging.DEBUG) aspectEqualFacts = defaultdict(list) numConsistentDupFacts = numInConsistentDupFacts = 0 for hashEquivalentFacts in factForConceptContextUnitHash.values(): if len(hashEquivalentFacts) > 1: for f in hashEquivalentFacts: aspectEqualFacts[(f.qname,f.contextID,f.unitID, f.xmlLang if f.concept.type.isWgnStringFactType else None)].append(f) for fList in aspectEqualFacts.values(): f0 = fList[0] if f0.concept.isNumeric: if any(f.isNil for f in fList): _inConsistent = not all(f.isNil for f in fList) elif all(inferredDecimals(f) == inferredDecimals(f0) for f in fList[1:]): # same decimals v0 = rangeValue(f0.value) _inConsistent = not all(rangeValue(f.value) == v0 for f in fList[1:]) else: # not all have same decimals aMax, bMin = rangeValue(f0.value, inferredDecimals(f0)) for f in fList[1:]: a, b = rangeValue(f.value, inferredDecimals(f)) if a > aMax: aMax = a if b < bMin: bMin = b _inConsistent = (bMin < aMax) else: _inConsistent = any(not f.isVEqualTo(f0) for f in fList[1:]) if _inConsistent: numInConsistentDupFacts += 1 else: numConsistentDupFacts += 1 aspectEqualFacts.clear() cntlr.addToLog("Number of duplicate facts consistent {:,} inconsistent {:,}".format(numConsistentDupFacts, numInConsistentDupFacts), messageCode="info", level=logging.DEBUG) styleAttrCounts = {} totalStyleLen = 0 continuationElements = {} ixNsPrefix = "{http://www.xbrl.org/2013/inlineXBRL}" for ixdsHtmlRootElt in getattr(modelXbrl, "ixdsHtmlElements", ()): # ix root elements if inline for ixElt in ixdsHtmlRootElt.iterdescendants(): style = ixElt.get("style") ixEltTag = str(ixElt.tag) if style: styleAttrCounts[style] = styleAttrCounts.get(style,0) + 1 if styleIxHiddenPattern.match(style) is None: totalStyleLen += len(style) if ixEltTag == "{http://www.xbrl.org/2013/inlineXBRL}continuation" and ixElt.id: continuationElements[ixElt.id] = ixElt if ixEltTag.startswith(ixNsPrefix): localName = ixEltTag[len(ixNsPrefix):] if localName == "continuation" and ixElt.id: continuationElements[ixElt.id] = ixElt elif localName in ("nonFraction", "nonNumeric", "fraction"): xbrlQnameCount += 1 xbrlQnameLengths += len(ixElt.qname.localName) elif isinstance(ixElt, ModelFact): xbrlQnameCount += 2 xbrlQnameLengths += len(ixElt.qname.localName) def locateContinuation(element, chain=None): contAt = element.get("continuedAt") if contAt: if contAt in continuationElements: if chain is None: chain = [element] contElt = continuationElements[contAt] if contElt not in chain: chain.append(contElt) element._continuationElement = contElt return locateContinuation(contElt, chain) elif chain: # end of chain return len(chain) numContinuations = 0 maxLenLen = 0 maxLenHops = 0 maxHops = 0 maxHopsLen = 0 for f in modelXbrl.factsInInstance: if f.get("continuedAt"): numContinuations += 1 _len = len(f.xValue) _hops = locateContinuation(f) if _hops > maxHops: maxHops = _hops maxHopsLen = _len if _len > maxLenLen: maxLenLen = _len maxLenHops = _hops cntlr.addToLog("Number of continuation facts {:,}".format(numContinuations), messageCode="info", level=logging.DEBUG) cntlr.addToLog("Longest continuation fact {:,} number of hops {:,}".format(maxLenLen, maxLenHops), messageCode="info", level=logging.DEBUG) cntlr.addToLog("Most continuation hops {:,} fact len {:,}".format(maxHops, maxHopsLen), messageCode="info", level=logging.DEBUG) numDupStyles = sum(1 for n in styleAttrCounts.values() if n > 1) bytesSaveableByCss = sum(len(s)*(n-1) for s,n in styleAttrCounts.items() if n > 1) cntlr.addToLog("Number of duplicate styles {:,}, bytes saveable by CSS {:,}, len of all non-ix-hidden @styles {:,}".format(numDupStyles, bytesSaveableByCss, totalStyleLen), messageCode="info", level=logging.DEBUG) cntlr.addToLog("Number of XBRL QNames {:,}, bytes saveable by EBA-style element names {:,}".format(xbrlQnameCount, xbrlQnameLengths - (5*xbrlQnameCount)), messageCode="info", level=logging.DEBUG)
def createViewer(self, scriptUrl="js/dist/ixbrlviewer.js"): """ Create an iXBRL file with XBRL data as a JSON blob, and script tags added """ dts = self.dts idGen = 0 self.roleMap.getPrefix(XbrlConst.standardLabel,"std") self.roleMap.getPrefix(XbrlConst.documentationLabel,"doc") self.roleMap.getPrefix(XbrlConst.summationItem,"calc") self.roleMap.getPrefix(XbrlConst.parentChild,"pres") for f in dts.facts: if f.id is None: f.set("id","ixv-%d" % (idGen)) idGen += 1 conceptName = self.nsmap.qname(f.qname) scheme, ident = f.context.entityIdentifier aspects = { "c": conceptName, "e": self.nsmap.qname(QName(self.nsmap.getPrefix(scheme,"e"), scheme, ident)), } factData = { "v": f.value, "a": aspects, } if f.format is not None: factData["f"] = str(f.format) if f.isNumeric: if f.unit: # XXX does not support complex units unit = self.nsmap.qname(f.unit.measures[0][0]) aspects["u"] = unit d = inferredDecimals(f) if d != float("INF") and not math.isnan(d): factData["d"] = d for d, v in f.context.qnameDims.items(): if v.memberQname is None: # Typed dimension, not yet supported. continue aspects[self.nsmap.qname(v.dimensionQname)] = self.nsmap.qname(v.memberQname) self.addConcept(v.dimension) self.addConcept(v.member) # XXX does not support forever periods if f.context.isInstantPeriod: aspects["p"] = self.dateFormat(f.context.instantDatetime.isoformat()) elif f.context.isStartEndPeriod: aspects["p"] = "%s/%s" % ( self.dateFormat(f.context.startDatetime.isoformat()), self.dateFormat(f.context.endDatetime.isoformat()) ) self.taxonomyData["facts"][f.id] = factData self.addConcept(f.concept) self.taxonomyData["prefixes"] = self.nsmap.prefixmap self.taxonomyData["roles"] = self.roleMap.prefixmap self.taxonomyData["rels"] = self.getRelationnShips() taxonomyDataJSON = self.escapeJSONForScriptTag(json.dumps(self.taxonomyData, indent=1, allow_nan=False)) dts.info("viewer:info", "Creating iXBRL viewer") for child in dts.modelDocument.xmlDocument.getroot(): if child.tag == '{http://www.w3.org/1999/xhtml}body': child.append(etree.Comment("BEGIN IXBRL VIEWER EXTENSIONS")) e = etree.fromstring("<script xmlns='http://www.w3.org/1999/xhtml' src='%s' />" % scriptUrl) e.text = '' child.append(e) # Putting this in the header can interfere with character set # auto detection e = etree.fromstring("<script xmlns='http://www.w3.org/1999/xhtml' id='taxonomy-data' type='application/json'></script>") e.text = taxonomyDataJSON child.append(e) child.append(etree.Comment("END IXBRL VIEWER EXTENSIONS")) break return dts.modelDocument.xmlDocument
def createViewer(self, scriptUrl="js/dist/ixbrlviewer.js"): """ Create an iXBRL file with XBRL data as a JSON blob, and script tags added """ dts = self.dts iv = iXBRLViewer(dts) idGen = 0 self.roleMap.getPrefix(XbrlConst.standardLabel, "std") self.roleMap.getPrefix(XbrlConst.documentationLabel, "doc") self.roleMap.getPrefix(XbrlConst.summationItem, "calc") self.roleMap.getPrefix(XbrlConst.parentChild, "pres") for f in dts.facts: if f.id is None: f.set("id", "ixv-%d" % (idGen)) idGen += 1 conceptName = self.nsmap.qname(f.qname) scheme, ident = f.context.entityIdentifier aspects = { "c": conceptName, "e": self.nsmap.qname( QName(self.nsmap.getPrefix(scheme, "e"), scheme, ident)), } factData = { "v": f.value, "a": aspects, } if f.format is not None: factData["f"] = str(f.format) if f.isNumeric: if f.unit is not None: # XXX does not support complex units unit = self.nsmap.qname(f.unit.measures[0][0]) aspects["u"] = unit else: # The presence of the unit aspect is used by the viewer to # identify numeric facts. If the fact has no unit (invalid # XBRL, but we want to support it for draft documents), # include the unit aspect with a null value. aspects["u"] = None d = inferredDecimals(f) if d != float("INF") and not math.isnan(d): factData["d"] = d for d, v in f.context.qnameDims.items(): if v.memberQname is None: # Typed dimension, not yet supported. continue aspects[self.nsmap.qname(v.dimensionQname)] = self.nsmap.qname( v.memberQname) self.addConcept(v.dimension) self.addConcept(v.member) if f.context.isForeverPeriod: aspects["p"] = "f" elif f.context.isInstantPeriod: aspects["p"] = self.dateFormat( f.context.instantDatetime.isoformat()) elif f.context.isStartEndPeriod: aspects["p"] = "%s/%s" % ( self.dateFormat(f.context.startDatetime.isoformat()), self.dateFormat(f.context.endDatetime.isoformat())) self.taxonomyData["facts"][f.id] = factData self.addConcept(f.concept) self.taxonomyData["prefixes"] = self.nsmap.prefixmap self.taxonomyData["roles"] = self.roleMap.prefixmap self.taxonomyData["rels"] = self.getRelationships() dts.info("viewer:info", "Creating iXBRL viewer") if dts.modelDocument.type == Type.INLINEXBRLDOCUMENTSET: # Sort by object index to preserve order in which files were specified. docSet = sorted(dts.modelDocument.referencesDocument.keys(), key=lambda x: x.objectIndex) docSetFiles = list( map(lambda x: os.path.basename(x.filepath), docSet)) self.taxonomyData["docSetFiles"] = docSetFiles for n in range(0, len(docSet)): iv.addFile( iXBRLViewerFile(docSetFiles[n], docSet[n].xmlDocument)) xmlDocument = docSet[0].xmlDocument else: xmlDocument = dts.modelDocument.xmlDocument filename = os.path.basename(dts.modelDocument.filepath) iv.addFile(iXBRLViewerFile(filename, xmlDocument)) taxonomyDataJSON = self.escapeJSONForScriptTag( json.dumps(self.taxonomyData, indent=1, allow_nan=False)) for child in xmlDocument.getroot(): if child.tag == '{http://www.w3.org/1999/xhtml}body': child.append(etree.Comment("BEGIN IXBRL VIEWER EXTENSIONS")) e = etree.fromstring( "<script xmlns='http://www.w3.org/1999/xhtml' src='%s' type='text/javascript' />" % scriptUrl) # Don't self close e.text = '' child.append(e) # Putting this in the header can interfere with character set # auto detection e = etree.fromstring( "<script xmlns='http://www.w3.org/1999/xhtml' type='application/x.ixbrl-viewer+json'></script>" ) e.text = taxonomyDataJSON child.append(e) child.append(etree.Comment("END IXBRL VIEWER EXTENSIONS")) break return iv
def factAspects(fact): aspects = OrderedDict() if hasId and fact.id: aspects["id"] = fact.id elif (fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))): aspects["id"] = "f{}".format(fact.objectIndex) parent = fact.getparent() concept = fact.concept _csvType = "Value" if not fact.isTuple: if concept is not None: _baseXsdType = concept.baseXsdType if _baseXsdType == "XBRLI_DATEUNION": if getattr(fact.xValue, "dateOnly", False): _baseXsdType = "date" else: _baseXsdType = "dateTime" aspects["baseType"] = "xs:{}".format(_baseXsdType) _csvType = baseTypes.get(_baseXsdType,_baseXsdType) + "Value" if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang: aspects[qnOimLangAspect] = fact.xmlLang if fact.isItem: if fact.isNil: _value = None _strValue = "nil" else: _inferredDecimals = inferredDecimals(fact) _value = oimValue(fact.xValue, _inferredDecimals) _strValue = str(_value) if not isCSVorXL: aspects["value"] = _strValue if fact.concept is not None and fact.concept.isNumeric: _numValue = fact.xValue if isinstance(_numValue, Decimal) and not isinf(_numValue) and not isnan(_numValue): if _numValue == _numValue.to_integral(): _numValue = int(_numValue) else: _numValue = float(_numValue) if isCSVorXL: aspects[_csvType] = _numValue else: aspects["numericValue"] = _numValue if not fact.isNil: if isinf(_inferredDecimals): if isJSON: _accuracy = "infinity" elif isCSVorXL: _accuracy = "INF" else: _accuracy = _inferredDecimals aspects["accuracy"] = _accuracy elif isinstance(_value, bool): aspects["booleanValue"] = _value elif isCSVorXL: aspects[_csvType] = _strValue aspects[qnOimConceptAspect] = oimValue(fact.qname) cntx = fact.context if cntx is not None: if cntx.entityIdentifierElement is not None: aspects[qnOimEntityAspect] = oimValue(qname(*cntx.entityIdentifier)) if cntx.period is not None: if isJSON: aspects[qnOimPeriodAspect] = oimPeriodValue(cntx) elif isCSVorXL: _periodValue = oimPeriodValue(cntx) aspects[qnOimPeriodStartAspect] = _periodValue["start"] aspects[qnOimPeriodEndAspect] = _periodValue["end"] for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]): aspects[dim.dimensionQname] = (oimValue(dim.memberQname) if dim.isExplicit else None if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1") else dim.typedMember.stringValue) unit = fact.unit if unit is not None: _mMul, _mDiv = unit.measures _sMul = '*'.join(oimValue(m) for m in sorted(_mMul, key=lambda m: str(m))) if _mDiv: _sDiv = '*'.join(oimValue(m) for m in sorted(_mDiv, key=lambda m: str(m))) if len(mDiv) > 1: if len(mMul) > 1: _sUnit = "({})/({})".format(_sMul,_sDiv) else: _sUnit = "{}/({})".format(_sMul,_sDiv) else: if len(mMul) > 1: _sUnit = "({})/{}".format(_sMul,_sDiv) else: _sUnit = "{}/{}".format(_sMul,_sDiv) else: _sUnit = _sMul aspects[qnOimUnitAspect] = _sUnit if parent.qname != XbrlConst.qnXbrliXbrl: aspects[qnOimTupleParentAspect] = parent.id if parent.id else "f{}".format(parent.objectIndex) aspects[qnOimTupleOrderAspect] = elementIndex(fact) if isJSON: _footnotes = factFootnotes(fact) if _footnotes: aspects["footnotes"] = _footnotes return aspects
def intervalValue(fact, dec=None): # value in decimals if fact.isNil: return NILinterval if dec is None: dec = inferredDecimals(fact) return rangeValue(fact.value, dec)
def factAspects(fact): aspects = OrderedDict() if hasId and fact.id: aspects["id"] = fact.id elif fact.isTuple or footnotesRelationshipSet.toModelObject(fact): aspects["id"] = "f{}".format(fact.objectIndex) parent = fact.getparent() concept = fact.concept if not fact.isTuple: if concept is not None: _baseXsdType = concept.baseXsdType if _baseXsdType == "XBRLI_DATEUNION": if getattr(fact.xValue, "dateOnly", False): _baseXsdType = "date" else: _baseXsdType = "dateTime" aspects["baseType"] = "xs:{}".format(_baseXsdType) if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang: aspects[qnOimLangAspect] = fact.xmlLang aspects[qnOimTypeAspect] = concept.baseXbrliType if fact.isItem: if fact.isNil: _value = None _strValue = "nil" else: _inferredDecimals = inferredDecimals(fact) _value = oimValue(fact.xValue, _inferredDecimals) _strValue = str(_value) aspects["value"] = _strValue if fact.concept is not None and fact.concept.isNumeric: _numValue = fact.xValue if isinstance(_numValue, Decimal) and not isinf(_numValue) and not isnan(_numValue): if _numValue == _numValue.to_integral(): _numValue = int(_numValue) else: _numValue = float(_numValue) aspects["numericValue"] = _numValue if not fact.isNil: aspects["accuracy"] = "infinity" if isinf(_inferredDecimals) else _inferredDecimals elif isinstance(_value, bool): aspects["booleanValue"] = _value aspects[qnOimConceptAspect] = oimValue(fact.qname) cntx = fact.context if cntx is not None: if cntx.entityIdentifierElement is not None: aspects[qnOimEntityAspect] = oimValue(qname(*cntx.entityIdentifier)) if cntx.period is not None: aspects[qnOimPeriodAspect] = oimPeriodValue(cntx) for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]): aspects[dim.dimensionQname] = (oimValue(dim.memberQname) if dim.isExplicit else None if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1") else dim.typedMember.stringValue) unit = fact.unit if unit is not None: _mMul, _mDiv = unit.measures if isJSON: aspects[qnOimUnitAspect] = { # use tuple instead of list for hashability "numerators": tuple(oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m))) } if _mDiv: aspects[qnOimUnitAspect]["denominators"] = tuple(oimValue(m) for m in sorted(_mDiv, key=lambda m: oimValue(m))) else: # CSV if _mMul: aspects[qnOimUnitMulAspect] = ",".join(oimValue(m) for m in sorted(_mMul, key=lambda m: q(m))) if _mDiv: aspects[qnOimUnitDivAspect] = ",".join(oimValue(m) for m in sorted(_mDiv, key=lambda m: str(m))) if parent.qname != XbrlConst.qnXbrliXbrl: aspects[qnOimTupleParentAspect] = parent.id if parent.id else "f{}".format(parent.objectIndex) aspects[qnOimTupleOrderAspect] = elementIndex(fact) footnotes = [] for footnoteRel in footnotesRelationshipSet.fromModelObject(fact): footnote = {"group": footnoteRel.arcrole} footnotes.append(footnote) toObj = footnoteRel.toModelObject if isinstance(toObj, ModelFact): footnote["factRef"] = toObj.id if toObj.id else "f{}".format(toObj.objectIndex) else: footnote["footnoteType"] = toObj.role footnote["footnote"] = xmlstring(toObj, stripXmlns=True, contentsOnly=True, includeText=True) if toObj.xmlLang: footnote["language"] = toObj.xmlLang if footnotes: aspects["footnotes"] = footnotes return aspects
def showInfo(cntlr, options, modelXbrl, _entrypoint, *args, **kwargs): for url, doc in sorted(modelXbrl.urlDocs.items(), key=lambda i: i[0]): if not any(url.startswith(w) for w in ("https://xbrl.sec.gov/", "http://xbrl.sec.gov/", "http://xbrl.fasb.org/", "http://www.xbrl.org/", "http://xbrl.ifrs.org/", "http://www.esma.europa.eu/")): if os.path.exists(doc.filepath): # skip if in an archive or stream cntlr.addToLog("File {} size {:,}".format(doc.basename, os.path.getsize(doc.filepath)), messageCode="info", level=logging.DEBUG) cntlr.addToLog("Heap memory before loading {:,}".format(memoryAtStartup), messageCode="info", level=logging.DEBUG) cntlr.addToLog("Heap memory after loading {:,}".format(cntlr.memoryUsed), messageCode="info", level=logging.DEBUG) cntlr.addToLog("Time to load {:.2f} seconds".format(time.time() - timeAtStart), messageCode="info", level=logging.DEBUG) isInlineXbrl = modelXbrl.modelDocument.type in (ModelDocument.Type.INLINEXBRL, ModelDocument.Type.INLINEXBRLDOCUMENTSET) if isInlineXbrl: instanceType = "inline XBRL, number of documents {}".format(len(modelXbrl.ixdsHtmlElements)) else: instanceType = "xBRL-XML" cntlr.addToLog("Instance type {}".format(instanceType), messageCode="info", level=logging.DEBUG) numContexts = len(modelXbrl.contexts) numLongContexts = 0 bytesSaveableInline = 0 bytesSaveableInlineWithCsv = 0 frequencyOfDims = {} sumNumDims = 0 distinctDurations = set() distinctInstants = set() shortContextIdLen = int(math.log10(numContexts or 1)) + 2 # if no contexts, use 1 for log function to work xbrlQnameCountInline = 0 xbrlQnameCountInlineWithCsv = 0 xbrlQnameLengthsInline = 0 xbrlQnameLengthsInlineWithCsv = 0 for c in modelXbrl.contexts.values(): sumNumDims += len(c.qnameDims) for d in c.qnameDims.values(): dimQname = str(d.dimensionQname) frequencyOfDims[dimQname] = frequencyOfDims.get(dimQname,0) + 1 xbrlQnameCountInline += 1 xbrlQnameCountInlineWithCsv += 1 xbrlQnameLengthsInline += len(d.dimensionQname.localName) xbrlQnameLengthsInlineWithCsv += len(d.dimensionQname.localName) if c.isInstantPeriod: distinctInstants.add(c.instantDatetime) elif c.isStartEndPeriod: distinctDurations.add((c.startDatetime, c.endDatetime)) if len(c.id) > shortContextIdLen: bytesSaveableInline += len(c.id) - shortContextIdLen bytesSaveableInlineWithCsv += len(c.id) - shortContextIdLen cntlr.addToLog("Number of contexts {:,}".format(numContexts), messageCode="info", level=logging.DEBUG) cntlr.addToLog("Number of distinct durations {:,}".format(len(distinctDurations)), messageCode="info", level=logging.DEBUG) cntlr.addToLog("Number of distinct instants {:,}".format(len(distinctInstants)), messageCode="info", level=logging.DEBUG) cntlr.addToLog("Avg number dimensions per contexts {:,.2f}".format(sumNumDims/numContexts if numContexts else 0), messageCode="info", level=logging.DEBUG) mostPopularDims = sorted(frequencyOfDims.items(), key=lambda i:"{:0>9},{}".format(999999999-i[1],i[0])) for dimName, count in mostPopularDims[0:3]: cntlr.addToLog("Dimension {} used in {:,} contexts".format(dimName, count), messageCode="info", level=logging.DEBUG) # analyze for tables which could be composed from CSV data tblFacts = defaultdict(set) tblNestedTables = defaultdict(set) factSize = {} for f in modelXbrl.factsInInstance: for tdElt in ancestors(f, xhtml, "td"): factSize[f] = len(xmlstring(tdElt,stripXmlns=True)) break childTblElt = None for tblElt in ancestors(f, xhtml, "table"): tblFacts[tblElt].add(f) if childTblElt: tblNestedTables[tblElt].add(childTblElt) # find tables containing only numeric facts def tblNestedFactCount(tbl): c = len(tblFacts.get(tbl, ())) for nestedTbl in tblNestedTables.get(tbl,()): c += tblNestedFactCount(nestedTbl) return c factsInInstance = len(modelXbrl.factsInInstance) factsInTables = len(set.union(*(fset for fset in tblFacts.values()))) cntlr.addToLog("Facts in instance: {:,}, facts in tables: {:,}".format(factsInInstance,factsInTables), messageCode="info", level=logging.DEBUG) numTblsEligible = 0 numFactsEligible = 0 bytesCsvSavings = 0 factsEligibleForCsv = set() tablesWithEligibleFacts = set() if tblFacts and factSize: # find eligible tables, have facts and not nested tables with other facts for tbl, facts in tblFacts.items(): if len(facts) == tblNestedFactCount(tbl): s = sum(factSize.get(f,0) for f in facts) - sum(len(str(f.value)) for f in facts) if s > 10000: numTblsEligible += 1 bytesCsvSavings += s numFactsEligible += len(facts) factsEligibleForCsv |= facts tablesWithEligibleFacts.add(tbl) numFacts = 0 numTableTextBlockFacts = 0 lenTableTextBlockFacts = 0 numTextBlockFacts = 0 lenTextBlockFacts = 0 distinctElementsInFacts = set() factsPerContext = {} factForConceptContextUnitHash = defaultdict(list) for f in modelXbrl.factsInInstance: context = f.context concept = f.concept distinctElementsInFacts.add(f.qname) numFacts += 1 if f.qname.localName.endswith("TableTextBlock"): numTableTextBlockFacts += 1 lenTableTextBlockFacts += len(f.xValue) elif f.qname.localName.endswith("TextBlock"): numTextBlockFacts += 1 lenTextBlockFacts += len(f.xValue) if context is not None and concept is not None: factsPerContext[context.id] = factsPerContext.get(context.id,0) + 1 factForConceptContextUnitHash[f.conceptContextUnitHash].append(f) bytesSaveableInline += len(context.id) - shortContextIdLen if f not in factsEligibleForCsv: bytesSaveableInlineWithCsv += len(context.id) - shortContextIdLen if numTblsEligible: cntlr.addToLog("Tables eligible for facts in CSV: {:,}, facts eligible for CSV: {:,}, bytes saveable by facts in CSV {:,}".format(numTblsEligible, numFactsEligible, bytesCsvSavings), messageCode="info", level=logging.DEBUG) else: cntlr.addToLog("No tables eligible for facts in CSV", messageCode="info", level=logging.DEBUG) mostPopularContexts = sorted(factsPerContext.items(), key=lambda i:"{:0>9},{}".format(999999999-i[1],i[0])) cntlr.addToLog("Number of facts {:,}".format(numFacts), messageCode="info", level=logging.DEBUG) cntlr.addToLog("Number of TableTextBlock facts {:,} avg len {:,.0f}".format(numTableTextBlockFacts, lenTableTextBlockFacts/numTableTextBlockFacts if numTableTextBlockFacts else 0), messageCode="info", level=logging.DEBUG) cntlr.addToLog("Number of TextBlock facts {:,} avg len {:,.0f}".format(numTextBlockFacts, lenTextBlockFacts/numTableTextBlockFacts if numTableTextBlockFacts else 0), messageCode="info", level=logging.DEBUG) cntlr.addToLog("Max number facts per context {:,}".format(mostPopularContexts[0][1] if mostPopularContexts else 0), messageCode="info", level=logging.DEBUG) cntlr.addToLog("Avg number facts per context {:,.2f}".format(sum([v for v in factsPerContext.values()])/numContexts if numContexts else 0), messageCode="info", level=logging.DEBUG) cntlr.addToLog("Distinct elements in facts {:,}".format(len(distinctElementsInFacts)), messageCode="info", level=logging.DEBUG) cntlr.addToLog("Number of bytes saveable context id of {} length is {:,}".format(shortContextIdLen, bytesSaveableInline), messageCode="info", level=logging.DEBUG) cntlr.addToLog("Excepting facts eligible for CSV, number of bytes saveable context id of {} length is {:,}".format(shortContextIdLen, bytesSaveableInlineWithCsv), messageCode="info", level=logging.DEBUG) aspectEqualFacts = defaultdict(list) decVals = {} numConsistentDupFacts = numInConsistentDupFacts = 0 for hashEquivalentFacts in factForConceptContextUnitHash.values(): if len(hashEquivalentFacts) > 1: for f in hashEquivalentFacts: aspectEqualFacts[(f.qname,f.contextID,f.unitID, f.xmlLang.lower() if f.concept.type.isWgnStringFactType else None)].append(f) for fList in aspectEqualFacts.values(): f0 = fList[0] if f0.concept.isNumeric: if any(f.isNil for f in fList): _inConsistent = not all(f.isNil for f in fList) else: # not all have same decimals _d = inferredDecimals(f0) _v = f0.xValue _inConsistent = isnan(_v) # NaN is incomparable, always makes dups inconsistent decVals[_d] = _v aMax, bMin = rangeValue(_v, _d) for f in fList[1:]: _d = inferredDecimals(f) _v = f.xValue if isnan(_v): _inConsistent = True break if _d in decVals: _inConsistent |= _v != decVals[_d] else: decVals[_d] = _v a, b = rangeValue(_v, _d) if a > aMax: aMax = a if b < bMin: bMin = b if not _inConsistent: _inConsistent = (bMin < aMax) decVals.clear() else: _inConsistent = any(not f.isVEqualTo(f0) for f in fList[1:]) if _inConsistent: numInConsistentDupFacts += 1 else: numConsistentDupFacts += 1 aspectEqualFacts.clear() cntlr.addToLog("Number of duplicate facts consistent {:,} inconsistent {:,}".format(numConsistentDupFacts, numInConsistentDupFacts), messageCode="info", level=logging.DEBUG) styleAttrCountsInline = {} styleAttrCountsInlineWithCsv = {} totalStyleLenInline = 0 totalStyleLenInlineWithCsv = 0 continuationElements = {} ixNsPrefix = "{http://www.xbrl.org/2013/inlineXBRL}" for ixdsHtmlRootElt in getattr(modelXbrl, "ixdsHtmlElements", ()): # ix root elements if inline for ixElt in ixdsHtmlRootElt.iterdescendants(): inEligibleTableForCsv = any(p in tablesWithEligibleFacts for p in ixElt.iterancestors("{http://www.w3.org/1999/xhtml}table")) style = ixElt.get("style") ixEltTag = str(ixElt.tag) if style: styleAttrCountsInline[style] = styleAttrCountsInline.get(style,0) + 1 if not inEligibleTableForCsv: styleAttrCountsInlineWithCsv[style] = styleAttrCountsInlineWithCsv.get(style,0) + 1 if styleIxHiddenPattern.match(style) is None: totalStyleLenInline += len(style) if not inEligibleTableForCsv: totalStyleLenInlineWithCsv += len(style) if ixEltTag == "{http://www.xbrl.org/2013/inlineXBRL}continuation" and ixElt.id: continuationElements[ixElt.id] = ixElt if ixEltTag.startswith(ixNsPrefix): localName = ixEltTag[len(ixNsPrefix):] if localName == "continuation" and ixElt.id: continuationElements[ixElt.id] = ixElt elif localName in ("nonFraction", "nonNumeric", "fraction"): xbrlQnameCountInline += 1 xbrlQnameLengthsInline += len(ixElt.qname.localName) if not inEligibleTableForCsv: xbrlQnameCountInlineWithCsv += 1 xbrlQnameLengthsInlineWithCsv += len(ixElt.qname.localName) elif isinstance(ixElt, ModelFact): xbrlQnameCountInline += 2 xbrlQnameLengthsInline += len(ixElt.qname.localName) if not inEligibleTableForCsv: xbrlQnameCountInlineWithCsv += 2 xbrlQnameLengthsInlineWithCsv += len(ixElt.qname.localName) def locateContinuation(element, chain=None): contAt = element.get("continuedAt") if contAt: if contAt in continuationElements: if chain is None: chain = [element] contElt = continuationElements[contAt] if contElt not in chain: chain.append(contElt) element._continuationElement = contElt return locateContinuation(contElt, chain) elif chain: # end of chain return len(chain) numContinuations = 0 maxLenLen = 0 maxLenHops = 0 maxHops = 0 maxHopsLen = 0 for f in modelXbrl.factsInInstance: if f.get("continuedAt"): numContinuations += 1 _len = len(f.xValue) _hops = locateContinuation(f) if _hops > maxHops: maxHops = _hops maxHopsLen = _len if _len > maxLenLen: maxLenLen = _len maxLenHops = _hops cntlr.addToLog("Number of continuation facts {:,}".format(numContinuations), messageCode="info", level=logging.DEBUG) cntlr.addToLog("Longest continuation fact {:,} number of hops {:,}".format(maxLenLen, maxLenHops), messageCode="info", level=logging.DEBUG) cntlr.addToLog("Most continuation hops {:,} fact len {:,}".format(maxHops, maxHopsLen), messageCode="info", level=logging.DEBUG) numDupStyles = sum(1 for n in styleAttrCountsInline.values() if n > 1) bytesSaveableByCssInline = sum(len(s)*(n-1) for s,n in styleAttrCountsInline.items() if n > 1) cntlr.addToLog("Number of duplicate styles {:,}, bytes saveable by CSS {:,}, len of all non-ix-hidden @styles {:,}".format(numDupStyles, bytesSaveableByCssInline, totalStyleLenInline), messageCode="info", level=logging.DEBUG) cntlr.addToLog("Number of XBRL QNames {:,}, bytes saveable by EBA-style element names {:,}".format(xbrlQnameCountInline, xbrlQnameLengthsInline - (5*xbrlQnameCountInline)), messageCode="info", level=logging.DEBUG) numDupStyles = sum(1 for n in styleAttrCountsInlineWithCsv.values() if n > 1) bytesSaveableByCssInlineWithCsv = sum(len(s)*(n-1) for s,n in styleAttrCountsInlineWithCsv.items() if n > 1) cntlr.addToLog("Excepting facts eligible for CSV, number of duplicate styles {:,}, bytes saveable by CSS {:,}, len of all non-ix-hidden @styles {:,}".format(numDupStyles, bytesSaveableByCssInlineWithCsv, totalStyleLenInlineWithCsv), messageCode="info", level=logging.DEBUG) cntlr.addToLog("Excepting facts eligible for CSV, number of XBRL QNames {:,}, bytes saveable by EBA-style element names {:,}".format(xbrlQnameCountInlineWithCsv, xbrlQnameLengthsInlineWithCsv - (5*xbrlQnameCountInlineWithCsv)), messageCode="info", level=logging.DEBUG)
def leastDecimals(binding, localNames): nonNilFacts = [binding[ln] for ln in localNames if not binding[ln].isNil] if nonNilFacts: return min((inferredDecimals(f) for f in nonNilFacts)) return floatINF
def sectTreeRel(self, parentConcept, n, sectCalc2RelSet, inferredParentValues, visited, dimQN=None): childRels = sectCalc2RelSet.fromModelObject(parentConcept) if childRels: visited.add(parentConcept) inferredChildValues = {} # setup summation bind keys for child objects sumParentBindKeys = self.sumConceptBindKeys[parentConcept] boundSumKeys = set() # these are contributing fact keys, parent may be inferred boundSums = defaultdict(intervalZero) boundSummationItems = defaultdict(list) boundPerKeys = set() boundPers = defaultdict(intervalZero) boundDurationItems = defaultdict(list) boundAggKeys = set() boundAggs = defaultdict(intervalZero) boundAggItems = defaultdict(list) boundAggConcepts = defaultdict(set) for rel in childRels: childConcept = rel.toModelObject if childConcept not in visited: if rel.arcrole == summationItem: if not self.sumInit: self.sumBindFacts() boundSumKeys |= self.sumConceptBindKeys[childConcept] elif rel.arcrole == balanceChanges: if not self.perInit: self.perBindFacts() boundPerKeys |= self.perConceptBindKeys[childConcept] # these are only duration items elif rel.arcrole == domainMember: boundAggKeys |= self.aggConceptBindKeys[dimQN] domQN = parentConcept.qname elif rel.arcrole == aggregationDomain: # this is in visited dimQN = rel.arcElement.prefixedNameQname(rel.get("dimension")) if dimQN not in self.aggDimInit: self.aggBindFacts(dimQN) # bind each referenced dimension's contexts # depth-first descent calc tree and process item after descent for rel in childRels: childConcept = rel.toModelObject if childConcept not in visited: # depth-first descent self.sectTreeRel(childConcept, n+1, sectCalc2RelSet, inferredChildValues, visited, dimQN) # post-descent summation (allows use of inferred value) if rel.arcrole == summationItem: weight = rel.weightDecimal for sumKey in boundSumKeys: cntx, unit = sumKey factKey = (childConcept, cntx, unit) if factKey in self.sumBoundFacts: for f in self.sumBoundFacts[factKey]: addInterval(boundSums, sumKey, intervalValue(f), weight) boundSummationItems[sumKey].append(f) elif factKey in inferredChildValues: addInterval(boundSums, sumKey, inferredChildValues[factKey], weight) elif factKey in inferredParentValues: addInterval(boundSums, sumKey, inferredParentValues[factKey], weight) elif rel.arcrole == balanceChanges: weight = rel.weightDecimal for perKey in boundPerKeys: hCntx, unit, start, end = perKey factKey = (childConcept, hCntx, unit, start, end) if factKey in self.perBoundFacts: for f in self.perBoundFacts[factKey]: addInterval(boundPers, perKey, intervalValue(f), weight) boundDurationItems[perKey].append(f) elif factKey in inferredChildValues: addInterval(boundPers, perKey, inferredChildValues[factKey], weight) elif factKey in inferredParentValues: addInterval(boundPers, perKey, inferredParentValues[factKey], weight) elif rel.arcrole == domainMember: memQN = childConcept.qname for aggKey in boundAggKeys: hCntx, unit = aggKey dimMemKey = (hCntx, unit, dimQN, memQN) if dimMemKey in self.aggBoundFacts: for f in self.aggBoundFacts[dimMemKey]: a, b = intervalValue(f) factDomKey = (f.concept, hCntx, unit, dimQN, domQN) addInterval(boundAggs, factDomKey, intervalValue(f)) boundAggItems[aggKey].append(f) boundAggConcepts[aggKey].add(f.concept) elif rel.arcrole == aggregationDomain: # this is in visited childRelSet = self.modelXbrl.relationshipSet(domainMember,rel.get("targetRole")) self.sectTreeRel(childConcept, n+1, childRelSet, inferredParentValues, {None}, dimQN) # infer global to section # process child items bound to this calc subtree for sumKey in boundSumKeys: cntx, unit = sumKey factKey = (parentConcept, cntx, unit) ia, ib = boundSums[sumKey] if factKey in self.sumBoundFacts: for f in self.sumBoundFacts[factKey]: d = inferredDecimals(f) sa, sb = intervalValue(f, d) if ((ia is NIL) ^ (sa is NIL)) or ((ia is not NIL) and (sb < ia or sa > ib)): self.modelXbrl.log('INCONSISTENCY', "calc2e:summationInconsistency", _("Summation inconsistent from %(concept)s in section %(section)s reported sum %(reportedSum)s, computed sum %(computedSum)s context %(contextID)s unit %(unitID)s unreported contributing items %(unreportedContributors)s"), modelObject=boundSummationItems[sumKey], concept=parentConcept.qname, section=self.section, reportedSum=self.formatInterval(sa, sb, d), computedSum=self.formatInterval(ia, ib, d), contextID=f.context.id, unitID=f.unit.id, unreportedContributors=", ".join(str(c.qname) # list the missing/unreported contributors in relationship order for r in childRels for c in (r.toModelObject,) if r.arcrole == summationItem and c is not None and (c, cntx, unit) not in self.sumBoundFacts) or "none") elif inferredParentValues is not None: # value was inferred, return to parent level inferredParentValues[factKey] = (ia, ib) for perKey in boundPerKeys: hCntx, unit, start, end = perKey ia, ib = boundPers[perKey] endBalA = endBalB = ZERO endFactKey = (parentConcept, hCntx, unit, None, end) if endFactKey in self.perBoundFacts: for f in self.perBoundFacts[endFactKey]: if f.isNil: endBalA = endBalB = NIL d = 0 break d = inferredDecimals(f) a, b = intervalValue(f,d) endBalA += a endBalB += b foundStartingFact = (endBalA is NIL) while not foundStartingFact: startFactKey = (parentConcept, hCntx, unit, None, start) if startFactKey in self.perBoundFacts: for f in self.perBoundFacts[startFactKey]: if f.isNil: endBalA = endBalB = NIL foundStartingFact = True break a, b = intervalValue(f) endBalA -= a endBalB -= b foundStartingFact = True break if not foundStartingFact: # infer backing up one period _nomPer = nominalPeriod(end - start) foundEarlierAdjacentPeriodStart = False for _start in self.durationPeriodStarts.get(_nomPer, ()): if nominalPeriod(start - _start) == _nomPer: # it's preceding period end = start start = _start perKey = hCntx, unit, start, end if perKey in boundPerKeys: chngs = boundPers[perKey] ia += chngs[0] ib += chngs[1] foundEarlierAdjacentPeriodStart = True break if not foundEarlierAdjacentPeriodStart: break if ((ia is NIL) ^ (endBalA is NIL)) or ((ia is not NIL) and (endBalB < ia or endBalA > ib)): self.modelXbrl.log('INCONSISTENCY', "calc2e:balanceInconsistency", _("Balance inconsistent from %(concept)s in section %(section)s reported sum %(reportedSum)s, computed sum %(computedSum)s context %(contextID)s unit %(unitID)s unreported contributing items %(unreportedContributors)s"), modelObject=boundDurationItems[perKey], concept=parentConcept.qname, section=self.section, reportedSum=self.formatInterval(endBalA, endBalB, d), computedSum=self.formatInterval(ia, ib, d), contextID=f.context.id, unitID=f.unit.id, unreportedContributors=", ".join(str(c.qname) # list the missing/unreported contributors in relationship order for r in childRels for c in (r.toModelObject,) if r.arcrole == balanceChanges and c is not None and (c, hCntx, unit, start, end) not in self.perBoundFacts) or "none") for aggKey in boundAggKeys: hCntx, unit = aggKey for concept in sorted(boundAggConcepts[aggKey], key=lambda c:c.objectIndex): # repeatable errors factDomKey = (concept, hCntx, unit, dimQN, domQN) ia, ib = boundAggs[factDomKey] if factDomKey in self.aggBoundConceptFacts: for f in self.aggBoundConceptFacts[factDomKey]: d = inferredDecimals(f) sa, sb = intervalValue(f, d) if ((ia is NIL) ^ (sa is NIL)) or ((ia is not NIL) and (sb < ia or sa > ib)): self.modelXbrl.log('INCONSISTENCY', "calc2e:aggregationInconsistency", _("Aggregation inconsistent for %(concept)s, domain %(domain)s in section %(section)s reported sum %(reportedSum)s, computed sum %(computedSum)s context %(contextID)s unit %(unitID)s unreported contributing members %(unreportedContributors)s"), modelObject=boundAggItems[factDomKey], concept=concept.qname, domain=parentConcept.qname, section=self.section, reportedSum=self.formatInterval(sa, sb, d), computedSum=self.formatInterval(ia, ib, d), contextID=f.context.id, unitID=f.unit.id, unreportedContributors=", ".join(str(c.qname) # list the missing/unreported contributors in relationship order for r in childRels for c in (r.toModelObject,) if r.arcrole == domainMember and c is not None and (concept, hCntx, unit, dimQN, c.qname) not in self.aggBoundConceptFacts) or "none") elif inferredParentValues is not None: # value was inferred, return to parent level # allow to be retrieved by factDomKey inferredParentValues[factDomKey] = (ia, ib) if self.modelXbrl.qnameDimensionDefaults.get(dimQN) == domQN: cntxKey = (hCntx, dimQN, domQN) if cntxKey in self.eqCntx: cntx = self.eqCntx[cntxKey] else: cntx = self.aggTotalContext(hCntx, dimQN, domQN) self.eqCntx[cntxKey] = cntx if cntx is not None: # allow to be retrieved by fact line item context key self.eqCntx[(hCntx, dimQN, domQN)] = cntx inferredParentValues[(concept, cntx, unit)] = (ia, ib) visited.remove(parentConcept)
def factAspects(fact): aspects = OrderedDict() if hasId and fact.id: aspects["id"] = fact.id elif (fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))): aspects["id"] = "f{}".format(fact.objectIndex) parent = fact.getparent() concept = fact.concept _csvType = "Value" if not fact.isTuple: if concept is not None: _baseXsdType = concept.baseXsdType if _baseXsdType == "XBRLI_DATEUNION": if getattr(fact.xValue, "dateOnly", False): _baseXsdType = "date" else: _baseXsdType = "dateTime" aspects["baseType"] = "xsd:{}".format(_baseXsdType) _csvType = baseTypes.get(_baseXsdType, _baseXsdType) + "Value" if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang: aspects[qnOimLangAspect] = fact.xmlLang if fact.isItem: if fact.isNil: _value = None _strValue = "nil" else: _inferredDecimals = inferredDecimals(fact) _value = oimValue(fact.xValue, _inferredDecimals) _strValue = str(_value) if not isCSVorXL: aspects["value"] = _strValue if fact.concept is not None and fact.concept.isNumeric: _numValue = fact.xValue if isinstance(_numValue, Decimal) and not isinf( _numValue) and not isnan(_numValue): if _numValue == _numValue.to_integral(): _numValue = int(_numValue) else: _numValue = float(_numValue) if isCSVorXL: aspects[_csvType] = _numValue else: aspects["numericValue"] = _numValue if not fact.isNil: if isinf(_inferredDecimals): if isJSON: _accuracy = "infinity" elif isCSVorXL: _accuracy = "INF" else: _accuracy = _inferredDecimals aspects["accuracy"] = _accuracy elif isinstance(_value, bool): aspects["booleanValue"] = _value elif isCSVorXL: aspects[_csvType] = _strValue aspects[qnOimConceptAspect] = oimValue(fact.qname) cntx = fact.context if cntx is not None: if cntx.entityIdentifierElement is not None: aspects[qnOimEntityAspect] = oimValue( qname(*cntx.entityIdentifier)) if cntx.period is not None: if isJSON: aspects[qnOimPeriodAspect] = oimPeriodValue(cntx) elif isCSVorXL: _periodValue = oimPeriodValue(cntx) aspects[qnOimPeriodStartAspect] = _periodValue["start"] aspects[qnOimPeriodEndAspect] = _periodValue["end"] for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]): aspects[dim.dimensionQname] = ( oimValue(dim.memberQname) if dim.isExplicit else None if dim.typedMember.get( "{http://www.w3.org/2001/XMLSchema-instance}nil") in ( "true", "1") else dim.typedMember.stringValue) unit = fact.unit if unit is not None: _mMul, _mDiv = unit.measures _sMul = '*'.join( oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m))) if _mDiv: _sDiv = '*'.join( oimValue(m) for m in sorted(_mDiv, key=lambda m: oimValue(m))) if len(_mDiv) > 1: if len(_mMul) > 1: _sUnit = "({})/({})".format(_sMul, _sDiv) else: _sUnit = "{}/({})".format(_sMul, _sDiv) else: if len(_mMul) > 1: _sUnit = "({})/{}".format(_sMul, _sDiv) else: _sUnit = "{}/{}".format(_sMul, _sDiv) else: _sUnit = _sMul aspects[qnOimUnitAspect] = _sUnit if parent.qname != XbrlConst.qnXbrliXbrl: aspects[ qnOimTupleParentAspect] = parent.id if parent.id else "f{}".format( parent.objectIndex) aspects[qnOimTupleOrderAspect] = elementIndex(fact) if isJSON: _footnotes = factFootnotes(fact) if _footnotes: aspects["footnotes"] = _footnotes return aspects
def intervalValue(fact, dec=None): # value in decimals if fact.isNil: return NILinterval if dec is None: dec = inferredDecimals(fact) return rangeValue(fact.value, dec)
def factAspects(fact): oimFact = OrderedDict() aspects = OrderedDict() if isCSVorXL: oimFact["id"] = fact.id or "f{}".format(fact.objectIndex) parent = fact.getparent() concept = fact.concept aspects[str(qnOimConceptAspect)] = oimValue(concept.qname) if concept is not None: if concept.type.isOimTextFactType and fact.xmlLang: aspects[str(qnOimLangAspect)] = fact.xmlLang if fact.isItem: if fact.isNil: _value = None else: _inferredDecimals = inferredDecimals(fact) _value = oimValue(fact.xValue, _inferredDecimals) oimFact["value"] = _value if fact.concept is not None and fact.concept.isNumeric: _numValue = fact.xValue if isinstance(_numValue, Decimal) and not isinf( _numValue) and not isnan(_numValue): if _numValue == _numValue.to_integral(): _numValue = int(_numValue) else: _numValue = float(_numValue) if not fact.isNil: if not isinf( _inferredDecimals): # accuracy omitted if infinite oimFact["decimals"] = _inferredDecimals oimFact["dimensions"] = aspects cntx = fact.context if cntx is not None: if cntx.entityIdentifierElement is not None and cntx.entityIdentifier != ENTITY_NA_QNAME: aspects[str(qnOimEntityAspect)] = oimValue( qname(*cntx.entityIdentifier)) if cntx.period is not None and not cntx.isForeverPeriod: aspects.update(oimPeriodValue(cntx)) for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]): if dim.isExplicit: dimVal = oimValue(dim.memberQname) else: # typed if dim.typedMember.get( "{http://www.w3.org/2001/XMLSchema-instance}nil" ) in ("true", "1"): dimVal = None else: dimVal = dim.typedMember.stringValue aspects[str(dim.dimensionQname)] = dimVal unit = fact.unit if unit is not None: _mMul, _mDiv = unit.measures _sMul = '*'.join( oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m))) if _mDiv: _sDiv = '*'.join( oimValue(m) for m in sorted(_mDiv, key=lambda m: oimValue(m))) if len(_mDiv) > 1: if len(_mMul) > 1: _sUnit = "({})/({})".format(_sMul, _sDiv) else: _sUnit = "{}/({})".format(_sMul, _sDiv) else: if len(_mMul) > 1: _sUnit = "({})/{}".format(_sMul, _sDiv) else: _sUnit = "{}/{}".format(_sMul, _sDiv) else: _sUnit = _sMul if _sUnit != "xbrli:pure": aspects[str(qnOimUnitAspect)] = _sUnit # Tuples removed from xBRL-JSON #if parent.qname != XbrlConst.qnXbrliXbrl: # aspects[str(qnOimTupleParentAspect)] = parent.id if parent.id else "f{}".format(parent.objectIndex) # aspects[str(qnOimTupleOrderAspect)] = elementIndex(fact) if isJSON: factFootnotes(fact, oimFact=oimFact) return oimFact
def validate(self): modelXbrl = self.modelXbrl if not modelXbrl.contexts or not modelXbrl.facts: return # skip if no contexts or facts if not self.val.validateInferDecimals: # infering precision is now contrary to XBRL REC section 5.2.5.2 modelXbrl.error("calc2e:inferringPrecision","Calc2 requires inferring decimals.") return startedAt = time.time() # check balance attributes and weights, same as XBRL 2.1 for rel in modelXbrl.relationshipSet(calc2Arcroles).modelRelationships: weight = rel.weight fromConcept = rel.fromModelObject toConcept = rel.toModelObject if fromConcept is not None and toConcept is not None: if rel.arcrole == aggregationDomain: rel.dimension = rel.arcElement.prefixedNameQname(rel.get("dimension")) if rel.dimension is None or not modelXbrl.qnameConcepts[rel.dimension].isDimensionItem: modelXbrl.error("calc2e:invalidAggregationDimension", _("Aggregation-domain relationship has invalid dimension %(dimension)s in link role %(linkrole)s"), modelObject=rel, dimension=rel.get("dimension"), linkrole=ELR) elif fromConcept != toConcept or not fromConcept.isDomainMember: modelXbrl.error("calc2e:invalidAggregationDomain", _("Calculation relationship has invalid domain %(domain)s in link role %(linkrole)s"), modelObject=rel, domain=fromConcept, linkrole=ELR) continue if rel.arcrole == balanceChanges: if fromConcept.periodType != "instant" or toConcept.periodType != "duration": modelXbrl.error("calc2e:invalidBalanceChangesPeriodType", _("Balance-changes relationship must have instant source concept and duration target concept in link role %(linkrole)s"), modelObject=rel, linkrole=ELR) if weight not in (1, -1): modelXbrl.error("calc2e:invalidWeight", _("Calculation relationship has invalid weight from %(source)s to %(target)s in link role %(linkrole)s"), modelObject=rel, source=fromConcept.qname, target=toConcept.qname, linkrole=ELR) fromBalance = fromConcept.balance toBalance = toConcept.balance if fromBalance and toBalance: if (fromBalance == toBalance and weight < 0) or \ (fromBalance != toBalance and weight > 0): modelXbrl.error("calc2e:balanceCalcWeightIllegal" + ("Negative" if weight < 0 else "Positive"), _("Calculation relationship has illegal weight %(weight)s from %(source)s, %(sourceBalance)s, to %(target)s, %(targetBalance)s, in link role %(linkrole)s (per 5.1.1.2 Table 6)"), modelObject=rel, weight=weight, source=fromConcept.qname, target=toConcept.qname, linkrole=rel.linkrole, sourceBalance=fromBalance, targetBalance=toBalance, messageCodes=("calc2e:balanceCalcWeightIllegalNegative", "calc2:balanceCalcWeightIllegalPositive")) if not fromConcept.isNumeric or not toConcept.isNumeric: modelXbrl.error("calc2e:nonNumericCalc", _("Calculation relationship has illegal concept from %(source)s%(sourceNumericDecorator)s to %(target)s%(targetNumericDecorator)s in link role %(linkrole)s"), modelObject=rel, source=fromConcept.qname, target=toConcept.qname, linkrole=rel.linkrole, sourceNumericDecorator="" if fromConcept.isNumeric else _(" (non-numeric)"), targetNumericDecorator="" if toConcept.isNumeric else _(" (non-numeric)")) # identify equal contexts uniqueCntxHashes = {} self.modelXbrl.profileActivity() for cntx in modelXbrl.contexts.values(): h = hash( (cntx.periodHash, cntx.entityIdentifierHash, cntx.dimsHash) ) # OIM-compatible hash if h in uniqueCntxHashes: if cntx.isEqualTo(uniqueCntxHashes[h]): self.eqCntx[cntx] = uniqueCntxHashes[h] else: uniqueCntxHashes[h] = cntx del uniqueCntxHashes self.modelXbrl.profileActivity("... identify aspect equal contexts", minTimeToShow=1.0) # identify equal units uniqueUnitHashes = {} for unit in self.modelXbrl.units.values(): h = unit.hash if h in uniqueUnitHashes: if unit.isEqualTo(uniqueUnitHashes[h]): self.eqUnit[unit] = uniqueUnitHashes[h] else: uniqueUnitHashes[h] = unit del uniqueUnitHashes self.modelXbrl.profileActivity("... identify equal units", minTimeToShow=1.0) sectObjs = sorted(set(rel.fromModelObject # only have numerics with context and unit for rel in modelXbrl.relationshipSet(sectionFact).modelRelationships if rel.fromModelObject is not None and rel.fromModelObject.concept is not None), key=lambda s: (s.concept.label(), s.objectIndex)) # sort into document order for consistent error messages if not sectObjs: self.modelXbrl.error("calc2e:noSections", "Instance contains no sections, nothing to validate.", modelObject=modelXbrl) # check by section factByConceptCntxUnit = OrderedDefaultDict(list) # sort into document order for consistent error messages self.sectionFacts = [] for sectObj in sectObjs: #print ("section {}".format(sectObj.concept.label())) self.section = sectObj.concept.label() sectLinkRoles = tuple(sectObj.concept.get(calc2linkroles,"").split()) factByConceptCntxUnit.clear() for f in sorted((rel.toModelObject # sort into document order for consistent error messages for rel in modelXbrl.relationshipSet(sectionFact,sectLinkRoles).fromModelObject(sectObj) if rel.toModelObject is not None and # numeric facts with context and unit rel.fromModelObject is not None and rel.toModelObject.concept is not None and rel.toModelObject.context is not None and rel.toModelObject.unit is not None), key=lambda f: f.objectIndex): factByConceptCntxUnit[f.qname, self.eqCntx.get(f.context,f.context), self.eqUnit.get(f.unit,f.unit)].append(f) for fList in factByConceptCntxUnit.values(): f0 = fList[0] if len(fList) == 1: self.sectionFacts.append(f0) else: if any(f.isNil for f in fList): _inConsistent = not all(f.isNil for f in fList) if _inConsistent: # pick a nil fact for f0 for calc validation for f in fList: if f.isNil: f0 = f break elif all(inferredDecimals(f) == inferredDecimals(f0) for f in fList[1:]): # same decimals v0 = intervalValue(f0) _inConsistent = not all(intervalValue(f) == v0 for f in fList[1:]) else: # not all have same decimals d0 = inferredDecimals(f0) aMax, bMin = intervalValue(f0, d0) for f in fList[1:]: df = inferredDecimals(f0) a, b = intervalValue(f, df) if a > aMax: aMax = a if b < bMin: bMin = b if df > d0: # take most accurate fact in section f0 = f d0 = df _inConsistent = (bMin < aMax) if _inConsistent: modelXbrl.error("calc2e:inconsistentDuplicateInSection", "Section %(section)s contained %(fact)s inconsistent in contexts equivalent to %(contextID)s: values %(values)s", modelObject=fList, section=sectObj.concept.label(), fact=f0.qname, contextID=f0.contextID, values=", ".join(strTruncate(f.value, 128) for f in fList)) self.sectionFacts.append(f0) # sectionFacts now in document order and deduplicated #print("section {} facts {}".format(sectObj.concept.label(), ", ".join(str(f.qname)+"="+f.value for f in self.sectionFacts))) # depth-first calc tree sectCalc2RelSet = modelXbrl.relationshipSet(calc2Arcroles, sectLinkRoles) # indexers for section based on calc2 arcrole self.sumInit = False self.sumConceptBindKeys.clear() self.sumBoundFacts.clear() self.perInit = False self.perConceptBindKeys.clear() self.perBoundFacts.clear() self.durationPeriodStarts.clear() self.aggDimInit = set() self.aggConceptBindKeys.clear() self.aggBoundFacts.clear() self.aggBoundConceptFacts.clear() self.aggDimInit.clear() inferredValues = {} for rootConcept in sorted(sectCalc2RelSet.rootConcepts, key=lambda r: sectCalc2RelSet.fromModelObject(r)[0].order): self.sectTreeRel(rootConcept, 1, sectCalc2RelSet, inferredValues, {rootConcept, None})
def _decimals(node, sphinxContext, args): fact = factArg(node, sphinxContext, args, 0) return inferredDecimals(fact)
def validateXbrlFinally(val, *args, **kwargs): if not (val.validateESEFplugin): return _xhtmlNs = "{{{}}}".format(xhtml) _xhtmlNsLen = len(_xhtmlNs) modelXbrl = val.modelXbrl modelDocument = modelXbrl.modelDocument _statusMsg = _("validating {0} filing rules").format( val.disclosureSystem.name) modelXbrl.profileActivity() modelXbrl.modelManager.showStatus(_statusMsg) reportXmlLang = None firstRootmostXmlLangDepth = 9999999 _ifrsNs = None for targetNs in modelXbrl.namespaceDocs.keys(): if ifrsNsPattern.match(targetNs): _ifrsNs = targetNs if not _ifrsNs: modelXbrl.error("ESEF.RTS.ifrsRequired", _("RTS on ESEF requires IFRS taxonomy."), modelObject=modelXbrl) return esefPrimaryStatementPlaceholders = set( qname(_ifrsNs, n) for n in esefPrimaryStatementPlaceholderNames) esefMandatoryElements2020 = set( qname(_ifrsNs, n) for n in esefMandatoryElementNames2020) if modelDocument.type == ModelDocument.Type.INSTANCE: modelXbrl.error("ESEF.I.1.instanceShallBeInlineXBRL", _("RTS on ESEF requires inline XBRL instances."), modelObject=modelXbrl) checkFilingDimensions( val) # sets up val.primaryItems and val.domainMembers val.hasExtensionSchema = val.hasExtensionPre = val.hasExtensionCal = val.hasExtensionDef = val.hasExtensionLbl = False checkFilingDTS(val, modelXbrl.modelDocument, []) modelXbrl.profileActivity("... filer DTS checks", minTimeToShow=1.0) if not (val.hasExtensionSchema and val.hasExtensionPre and val.hasExtensionCal and val.hasExtensionDef and val.hasExtensionLbl): missingFiles = [] if not val.hasExtensionSchema: missingFiles.append("schema file") if not val.hasExtensionPre: missingFiles.append("presentation linkbase") if not val.hasExtensionCal: missingFiles.append("calculation linkbase") if not val.hasExtensionDef: missingFiles.append("definition linkbase") if not val.hasExtensionLbl: missingFiles.append("label linkbase") modelXbrl.warning( "ESEF.3.1.1.extensionTaxonomyWrongFilesStructure", _("Extension taxonomies MUST consist of at least a schema file and presentation, calculation, definition and label linkbases" ": missing %(missingFiles)s"), modelObject=modelXbrl, missingFiles=", ".join(missingFiles)) #if modelDocument.type == ModelDocument.Type.INLINEXBRLDOCUMENTSET: # # reports only under reports, none elsewhere # modelXbrl.fileSource.dir if modelDocument.type in (ModelDocument.Type.INLINEXBRL, ModelDocument.Type.INLINEXBRLDOCUMENTSET, ModelDocument.Type.INSTANCE): footnotesRelationshipSet = modelXbrl.relationshipSet("XBRL-footnotes") orphanedFootnotes = set() noLangFootnotes = set() factLangFootnotes = defaultdict(set) footnoteRoleErrors = set() transformRegistryErrors = set() def checkFootnote(elt, text): if text: # non-empty footnote must be linked to a fact if not empty if not any( isinstance(rel.fromModelObject, ModelFact) for rel in footnotesRelationshipSet.toModelObject(elt)): orphanedFootnotes.add(elt) lang = elt.xmlLang if not lang: noLangFootnotes.add(elt) else: for rel in footnotesRelationshipSet.toModelObject(elt): if rel.fromModelObject is not None: factLangFootnotes[rel.fromModelObject].add(lang) if elt.role != XbrlConst.footnote or not all( rel.arcrole == XbrlConst.factFootnote and rel.linkrole == XbrlConst.defaultLinkRole for rel in footnotesRelationshipSet.toModelObject(elt)): footnoteRoleErrors.add(elt) # check file name of each inline document (which might be below a top-level IXDS) for doc in modelXbrl.urlDocs.values(): if doc.type == ModelDocument.Type.INLINEXBRL: _baseName, _baseExt = os.path.splitext(doc.basename) if _baseExt not in (".xhtml", ".html"): modelXbrl.warning( "ESEF.RTS.Art.3.fileNameExtension", _("FileName SHOULD have the extension .xhtml or .html: %(fileName)s" ), modelObject=doc, fileName=doc.basename) docinfo = doc.xmlRootElement.getroottree().docinfo if " html" in docinfo.doctype: modelXbrl.warning( "ESEF.RTS.Art.3.htmlDoctype", _("Doctype SHOULD NOT be html: %(fileName)s"), modelObject=doc, fileName=doc.basename) if modelDocument.type in (ModelDocument.Type.INLINEXBRL, ModelDocument.Type.INLINEXBRLDOCUMENTSET): hiddenEltIds = {} presentedHiddenEltIds = defaultdict(list) eligibleForTransformHiddenFacts = [] requiredToDisplayFacts = [] requiredToDisplayFactIds = {} firstIxdsDoc = True for ixdsHtmlRootElt in modelXbrl.ixdsHtmlElements: # ix root elements for all ix docs in IXDS ixNStag = ixdsHtmlRootElt.modelDocument.ixNStag ixTags = set(ixNStag + ln for ln in ("nonNumeric", "nonFraction", "references", "relationship")) ixTextTags = set(ixNStag + ln for ln in ("nonFraction", "continuation", "footnote")) ixExcludeTag = ixNStag + "exclude" ixTupleTag = ixNStag + "tuple" ixFractionTag = ixNStag + "fraction" for elt, depth in etreeIterWithDepth(ixdsHtmlRootElt): eltTag = elt.tag if isinstance( elt, (_ElementTree, _Comment, _ProcessingInstruction)): continue # comment or other non-parsed element else: eltTag = elt.tag if eltTag.startswith(_xhtmlNs): eltTag = eltTag[_xhtmlNsLen:] if firstIxdsDoc and (not reportXmlLang or depth < firstRootmostXmlLangDepth): xmlLang = elt.get( "{http://www.w3.org/XML/1998/namespace}lang" ) if xmlLang: reportXmlLang = xmlLang firstRootmostXmlLangDepth = depth if ((eltTag in ("object", "script")) or (eltTag == "a" and "javascript:" in elt.get("href", "")) or (eltTag == "img" and "javascript:" in elt.get("src", ""))): modelXbrl.error( "ESEF.2.5.1.executableCodePresent", _("Inline XBRL documents MUST NOT contain executable code: %(element)s" ), modelObject=elt, element=eltTag) elif eltTag == "img": src = elt.get("src", "").strip() hasParentIxTextTag = False # check if image is in an ix text-bearing element _ancestorElt = elt while (_ancestorElt is not None): if _ancestorElt.tag == ixExcludeTag: # excluded from any parent text-bearing ix element break if _ancestorElt.tag in ixTextTags: hasParentIxTextTag = True break _ancestorElt = _ancestorElt.getparent() if scheme(src) in ("http", "https", "ftp"): modelXbrl.error( "ESEF.3.5.1.inlinXbrlContainsExternalReferences", _("Inline XBRL instance documents MUST NOT contain any reference pointing to resources outside the reporting package: %(element)s" ), modelObject=elt, element=eltTag) elif not src.startswith("data:image"): if hasParentIxTextTag: modelXbrl.error( "ESEF.2.5.1.imageInIXbrlElementNotEmbedded", _("Images appearing within an inline XBRL element MUST be embedded regardless of their size." ), modelObject=elt) else: # presume it to be an image file, check image contents try: base = elt.modelDocument.baseForElement( elt) normalizedUri = elt.modelXbrl.modelManager.cntlr.webCache.normalizeUrl( src, base) if not elt.modelXbrl.fileSource.isInArchive( normalizedUri): normalizedUri = elt.modelXbrl.modelManager.cntlr.webCache.getfilename( normalizedUri) imglen = 0 with elt.modelXbrl.fileSource.file( normalizedUri, binary=True)[0] as fh: imglen += len(fh.read()) if imglen < browserMaxBase64ImageLength: modelXbrl.error( "ESEF.2.5.1.embeddedImageNotUsingBase64Encoding", _("Images MUST be included in the XHTML document as a base64 encoded string unless their size exceeds support of browsers (%(maxImageSize)s): %(file)s." ), modelObject=elt, maxImageSize= browserMaxBase64ImageLength, file=os.path.basename( normalizedUri)) except IOError as err: modelXbrl.error( "ESEF.2.5.1.imageFileCannotBeLoaded", _("Image file which isn't openable '%(src)s', error: %(error)s" ), modelObject=elt, src=src, error=err) elif not any( src.startswith(m) for m in allowedImgMimeTypes): modelXbrl.error( "ESEF.2.5.1.embeddedImageNotUsingBase64Encoding", _("Images MUST be included in the XHTML document as a base64 encoded string, encoding disallowed: %(src)s." ), modelObject=elt, src=attrValue[:128]) elif eltTag == "a": href = elt.get("href", "").strip() if scheme(href) in ("http", "https", "ftp"): modelXbrl.error( "ESEF.3.5.1.inlinXbrlContainsExternalReferences", _("Inline XBRL instance documents MUST NOT contain any reference pointing to resources outside the reporting package: %(element)s" ), modelObject=elt, element=eltTag) elif eltTag == "base" or elt.tag == "{http://www.w3.org/XML/1998/namespace}base": modelXbrl.error( "ESEF.2.4.2.htmlOrXmlBaseUsed", _("The HTML <base> elements and xml:base attributes MUST NOT be used in the Inline XBRL document." ), modelObject=elt, element=eltTag) elif eltTag == "link" and elt.get( "type") == "text/css": if len(modelXbrl.ixdsHtmlElements) > 1: f = elt.get("href") if not f or isHttpUrl(f) or os.path.isabs(f): modelXbrl.warning( "ESEF.2.5.4.externalCssReportPackage", _("The CSS file should be physically stored within the report package: %{file}s." ), modelObject=elt, file=f) else: modelXbrl.error( "ESEF.2.5.4.externalCssFileForSingleIXbrlDocument", _("Where an Inline XBRL document set contains a single document, the CSS MUST be embedded within the document." ), modelObject=elt, element=eltTag) elif eltTag == "style" and elt.get( "type") == "text/css": if len(modelXbrl.ixdsHtmlElements) > 1: modelXbrl.warning( "ESEF.2.5.4.embeddedCssForMultiHtmlIXbrlDocumentSets", _("Where an Inline XBRL document set contains multiple documents, the CSS SHOULD be defined in a separate file." ), modelObject=elt, element=eltTag) if eltTag in ixTags and elt.get("target"): modelXbrl.error( "ESEF.2.5.3.targetAttributeUsed", _("Target attribute MUST not be used: element %(localName)s, target attribute %(target)s." ), modelObject=elt, localName=elt.elementQname, target=elt.get("target")) if eltTag == ixTupleTag: modelXbrl.error( "ESEF.2.4.1.tupleElementUsed", _("The ix:tuple element MUST not be used in the Inline XBRL document: %(qname)s." ), modelObject=elt, qname=elt.qname) if eltTag == ixFractionTag: modelXbrl.error( "ESEF.2.4.1.fractionElementUsed", _("The ix:fraction element MUST not be used in the Inline XBRL document." ), modelObject=elt) if elt.get("{http://www.w3.org/XML/1998/namespace}base" ) is not None: modelXbrl.error( "ESEF.2.4.1.xmlBaseUsed", _("xml:base attributes MUST NOT be used in the Inline XBRL document: element %(localName)s, base attribute %(base)s." ), modelObject=elt, localName=elt.elementQname, base=elt.get( "{http://www.w3.org/XML/1998/namespace}base")) if isinstance(elt, ModelInlineFootnote): checkFootnote(elt, elt.value) elif isinstance( elt, ModelResource ) and elt.qname == XbrlConst.qnLinkFootnote: checkFootnote(elt, elt.value) elif isinstance(elt, ModelInlineFact): if elt.format is not None and elt.format.namespaceURI not in IXT_NAMESPACES: transformRegistryErrors.add(elt) for ixHiddenElt in ixdsHtmlRootElt.iterdescendants( tag=ixNStag + "hidden"): for tag in (ixNStag + "nonNumeric", ixNStag + "nonFraction"): for ixElt in ixHiddenElt.iterdescendants(tag=tag): if ( getattr(ixElt, "xValid", 0) >= VALID # may not be validated ): # add future "and" conditions on elements which can be in hidden if (ixElt.concept.baseXsdType not in untransformableTypes and not ixElt.isNil): eligibleForTransformHiddenFacts.append( ixElt) elif ixElt.id is None: requiredToDisplayFacts.append(ixElt) if ixElt.id: hiddenEltIds[ixElt.id] = ixElt firstIxdsDoc = False if eligibleForTransformHiddenFacts: modelXbrl.warning( "ESEF.2.4.1.transformableElementIncludedInHiddenSection", _("The ix:hidden section of Inline XBRL document MUST not include elements eligible for transformation. " "%(countEligible)s fact(s) were eligible for transformation: %(elements)s" ), modelObject=eligibleForTransformHiddenFacts, countEligible=len(eligibleForTransformHiddenFacts), elements=", ".join( sorted( set( str(f.qname) for f in eligibleForTransformHiddenFacts)))) for ixdsHtmlRootElt in modelXbrl.ixdsHtmlElements: for ixElt in ixdsHtmlRootElt.getroottree().iterfind( "//{http://www.w3.org/1999/xhtml}*[@style]"): hiddenFactRefMatch = styleIxHiddenPattern.match( ixElt.get("style", "")) if hiddenFactRefMatch: hiddenFactRef = hiddenFactRefMatch.group(2) if hiddenFactRef not in hiddenEltIds: modelXbrl.error( "ESEF.2.4.1.esefIxHiddenStyleNotLinkingFactInHiddenSection", _("\"-esef-ix-hidden\" style identifies @id, %(id)s of a fact that is not in ix:hidden section." ), modelObject=ixElt, id=hiddenFactRef) else: presentedHiddenEltIds[hiddenFactRef].append(ixElt) for hiddenEltId, ixElt in hiddenEltIds.items(): if (hiddenEltId not in presentedHiddenEltIds and getattr(ixElt, "xValid", 0) >= VALID and # may not be validated (ixElt.concept.baseXsdType in untransformableTypes or ixElt.isNil)): requiredToDisplayFacts.append(ixElt) if requiredToDisplayFacts: modelXbrl.warning( "ESEF.2.4.1.factInHiddenSectionNotInReport", _("The ix:hidden section contains %(countUnreferenced)s fact(s) whose @id is not applied on any \"-esef-ix- hidden\" style: %(elements)s" ), modelObject=requiredToDisplayFacts, countUnreferenced=len(requiredToDisplayFacts), elements=", ".join( sorted( set(str(f.qname) for f in requiredToDisplayFacts)))) del eligibleForTransformHiddenFacts, hiddenEltIds, presentedHiddenEltIds, requiredToDisplayFacts elif modelDocument.type == ModelDocument.Type.INSTANCE: for elt in modelDocument.xmlRootElement.iter(): if elt.qname == XbrlConst.qnLinkFootnote: # for now assume no private elements extend link:footnote checkFootnote(elt, elt.stringValue) contextsWithDisallowedOCEs = [] contextsWithDisallowedOCEcontent = [] contextsWithPeriodTime = [] contextsWithPeriodTimeZone = [] contextIdentifiers = defaultdict(list) nonStandardTypedDimensions = defaultdict(set) for context in modelXbrl.contexts.values(): for elt in context.iterdescendants( "{http://www.xbrl.org/2003/instance}startDate", "{http://www.xbrl.org/2003/instance}endDate", "{http://www.xbrl.org/2003/instance}instant"): m = datetimePattern.match(elt.stringValue) if m: if m.group(1): contextsWithPeriodTime.append(context) if m.group(3): contextsWithPeriodTimeZone.append(context) for elt in context.iterdescendants( "{http://www.xbrl.org/2003/instance}segment"): contextsWithDisallowedOCEs.append(context) break for elt in context.iterdescendants( "{http://www.xbrl.org/2003/instance}scenario"): if isinstance(elt, ModelObject): if any(True for child in elt.iterchildren() if isinstance(child, ModelObject) and child.tag not in ( "{http://xbrl.org/2006/xbrldi}explicitMember", "{http://xbrl.org/2006/xbrldi}typedMember")): contextsWithDisallowedOCEcontent.append(context) # check periods here contextIdentifiers[context.entityIdentifier].append(context) if contextsWithDisallowedOCEs: modelXbrl.error( "ESEF.2.1.3.segmentUsed", _("xbrli:segment container MUST NOT be used in contexts: %(contextIds)s" ), modelObject=contextsWithDisallowedOCEs, contextIds=", ".join(c.id for c in contextsWithDisallowedOCEs)) if contextsWithDisallowedOCEcontent: modelXbrl.error( "ESEF.2.1.3.scenarioContainsNonDimensionalContent", _("xbrli:scenario in contexts MUST NOT contain any other content than defined in XBRL Dimensions specification: %(contextIds)s" ), modelObject=contextsWithDisallowedOCEcontent, contextIds=", ".join( c.id for c in contextsWithDisallowedOCEcontent)) if len(contextIdentifiers) > 1: modelXbrl.error( "ESEF.2.1.4.multipleIdentifiers", _("All entity identifiers in contexts MUST have identical content: %(contextIdentifiers)s" ), modelObject=modelXbrl, contextIds=", ".join(i[1] for i in contextIdentifiers)) for (contextScheme, contextIdentifier), contextElts in contextIdentifiers.items(): if contextScheme != iso17442: modelXbrl.warning( "ESEF.2.1.1.nonLEIContextScheme", _("The scheme attribute of the xbrli:identifier element should have \"%(leiScheme)s\" as its content: %(contextScheme)s" ), modelObject=contextElts, contextScheme=contextScheme, leiScheme=iso17442) else: leiValidity = LeiUtil.checkLei(contextIdentifier) if leiValidity == LeiUtil.LEI_INVALID_LEXICAL: modelXbrl.warning( "ESEF.2.1.1.invalidIdentifierFormat", _("The LEI context identifier has an invalid format: %(identifier)s" ), modelObject=contextElts, identifier=contextIdentifier) elif leiValidity == LeiUtil.LEI_INVALID_CHECKSUM: modelXbrl.warning( "ESEF.2.1.1.invalidIdentifier", _("The LEI context identifier has checksum error: %(identifier)s" ), modelObject=contextElts, identifier=contextIdentifier) if contextsWithPeriodTime: modelXbrl.warning( "ESEF.2.1.2.periodWithTimeContent", _("Context period startDate, endDate and instant elements should be in whole days without time: %(contextIds)s" ), modelObject=contextsWithPeriodTime, contextIds=", ".join(c.id for c in contextsWithPeriodTime)) if contextsWithPeriodTimeZone: modelXbrl.warning( "ESEF.2.1.2.periodWithTimeZone", _("Context period startDate, endDate and instant elements should be in whole days without a timezone: %(contextIds)s" ), modelObject=contextsWithPeriodTimeZone, contextIds=", ".join(c.id for c in contextsWithPeriodTimeZone)) # identify unique contexts and units mapContext = {} mapUnit = {} uniqueContextHashes = {} for context in modelXbrl.contexts.values(): h = context.contextDimAwareHash if h in uniqueContextHashes: if context.isEqualTo(uniqueContextHashes[h]): mapContext[context] = uniqueContextHashes[h] else: uniqueContextHashes[h] = context del uniqueContextHashes uniqueUnitHashes = {} utrValidator = ValidateUtr(modelXbrl) utrUnitIds = set( u.unitId for unitItemType in utrValidator.utrItemTypeEntries.values() for u in unitItemType.values()) for unit in modelXbrl.units.values(): h = unit.hash if h in uniqueUnitHashes: if unit.isEqualTo(uniqueUnitHashes[h]): mapUnit[unit] = uniqueUnitHashes[h] else: uniqueUnitHashes[h] = unit # check if any custom measure is in UTR for measureTerm in unit.measures: for measure in measureTerm: ns = measure.namespaceURI if ns != XbrlConst.iso4217 and not ns.startswith( "http://www.xbrl.org/"): if measure.localName in utrUnitIds: modelXbrl.error( "ESEF.RTS.III.1.G1-7-1.customUnitInUtr", _("Custom measure SHOULD NOT duplicate a UnitID of UTR: %(measure)s" ), modelObject=unit, measure=measure) del uniqueUnitHashes reportedMandatory = set() precisionFacts = set() numFactsByConceptContextUnit = defaultdict(list) textFactsByConceptContext = defaultdict(list) footnotesRelationshipSet = modelXbrl.relationshipSet( XbrlConst.factFootnote, XbrlConst.defaultLinkRole) noLangFacts = [] textFactsMissingReportLang = [] conceptsUsed = set() for qn, facts in modelXbrl.factsByQname.items(): if qn in mandatory: reportedMandatory.add(qn) for f in facts: if f.precision is not None: precisionFacts.add(f) if f.isNumeric: numFactsByConceptContextUnit[(f.qname, mapContext.get( f.context, f.context), mapUnit.get( f.unit, f.unit))].append(f) if f.concept is not None and not f.isNil and f.xValid >= VALID and f.xValue > 1 and f.concept.type is not None and ( f.concept.type.qname == PERCENT_TYPE or f.concept.type.isDerivedFrom(PERCENT_TYPE)): modelXbrl.warning( "ESEF.2.2.2.percentGreaterThan100", _("A percent fact should have value <= 100: %(element)s in context %(context)s value %(value)s" ), modelObject=f, element=f.qname, context=f.context.id, value=f.xValue) elif f.concept is not None and f.concept.type is not None: if f.concept.type.isOimTextFactType: if not f.xmlLang: noLangFacts.append(f) elif f.context is not None: textFactsByConceptContext[( f.qname, mapContext.get(f.context, f.context))].append(f) conceptsUsed.add(f.concept) if f.context is not None: for dim in f.context.qnameDims.values(): conceptsUsed.add(dim.dimension) if dim.isExplicit: conceptsUsed.add(dim.member) elif dim.isTyped: conceptsUsed.add(dim.typedMember) if noLangFacts: modelXbrl.error( "ESEF.2.5.2.undefinedLanguageForTextFact", _("Each tagged text fact MUST have the 'xml:lang' attribute assigned or inherited." ), modelObject=noLangFacts) # missing report lang text facts if reportXmlLang: for fList in textFactsByConceptContext.values(): if not any(f.xmlLang == reportXmlLang for f in fList): modelXbrl.error( "ESEF.2.5.2.taggedTextFactOnlyInLanguagesOtherThanLanguageOfAReport", _("Each tagged text fact MUST have the 'xml:lang' provided in at least the language of the report: %(element)s" ), modelObject=fList, element=fList[0].qname) # 2.2.4 test for fList in numFactsByConceptContextUnit.values(): if len(fList) > 1: f0 = fList[0] if any(f.isNil for f in fList): _inConsistent = not all(f.isNil for f in fList) elif all( inferredDecimals(f) == inferredDecimals(f0) for f in fList[1:]): # same decimals v0 = rangeValue(f0.value) _inConsistent = not all( rangeValue(f.value) == v0 for f in fList[1:]) else: # not all have same decimals aMax, bMin = rangeValue(f0.value, inferredDecimals(f0)) for f in fList[1:]: a, b = rangeValue(f.value, inferredDecimals(f)) if a > aMax: aMax = a if b < bMin: bMin = b _inConsistent = (bMin < aMax) if _inConsistent: modelXbrl.error( ("ESEF.2.2.4.inconsistentDuplicateNumericFactInInlineXbrlDocument" ), "Inconsistent duplicate numeric facts MUST NOT appear in the content of an inline XBRL document. %(fact)s that was used more than once in contexts equivalent to %(contextID)s: values %(values)s. ", modelObject=fList, fact=f0.qname, contextID=f0.contextID, values=", ".join( strTruncate(f.value, 128) for f in fList)) if precisionFacts: modelXbrl.warning( "ESEF.2.2.1.precisionAttributeUsed", _("The accuracy of numeric facts SHOULD be defined with the 'decimals' attribute rather than the 'precision' attribute: %(elements)s." ), modelObject=precisionFacts, elements=", ".join(sorted( str(e.qname) for e in precisionFacts))) missingElements = (mandatory - reportedMandatory) if missingElements: modelXbrl.error( "ESEF.???.missingRequiredElements", _("Required elements missing from document: %(elements)s."), modelObject=modelXbrl, elements=", ".join(sorted(str(qn) for qn in missingElements))) if transformRegistryErrors: modelXbrl.warning( "ESEF.2.2.3.transformRegistry", _("ESMA recommends applying the latest available version of the Transformation Rules Registry marked with 'Recommendation' status for these elements: %(elements)s." ), modelObject=transformRegistryErrors, elements=", ".join( sorted( str(fact.qname) for fact in transformRegistryErrors))) if orphanedFootnotes: modelXbrl.error( "ESEF.2.3.1.unusedFootnote", _("Non-empty footnotes must be connected to fact(s)."), modelObject=orphanedFootnotes) if noLangFootnotes: modelXbrl.error( "ESEF.2.3.1.undefinedLanguageForFootnote", _("Each footnote MUST have the 'xml:lang' attribute whose value corresponds to the language of the text in the content of the respective footnote." ), modelObject=noLangFootnotes) nonDefLangFtFacts = set(f for f, langs in factLangFootnotes.items() if reportXmlLang not in langs) if nonDefLangFtFacts: modelXbrl.error( "ESEF.2.3.1.footnoteOnlyInLanguagesOtherThanLanguageOfAReport", _("Each fact MUST have at least one footnote with 'xml:lang' attribute whose value corresponds to the language of the text in the content of the respective footnote: %(qnames)s." ), modelObject=nonDefLangFtFacts, qnames=", ".join( sorted(str(f.qname) for f in nonDefLangFtFacts))) del nonDefLangFtFacts if footnoteRoleErrors: modelXbrl.error( "ESEF.2.3.1.nonStandardRoleForFootnote", _("The xlink:role attribute of a link:footnote and link:footnoteLink element as well as xlink:arcrole attribute of a link:footnoteArc MUST be defined in the XBRL Specification 2.1." ), modelObject=footnoteRoleErrors) nonStdFootnoteElts = list() for modelLink in modelXbrl.baseSets[("XBRL-footnotes", None, None, None)]: for elt in modelLink.iterchildren(): if isinstance( elt, (_ElementTree, _Comment, _ProcessingInstruction)): continue # comment or other non-parsed element if elt.qname not in FOOTNOTE_LINK_CHILDREN: nonStdFootnoteElts.append(elt) if nonStdFootnoteElts: modelXbrl.error( "ESEF.2.3.2.nonStandardElementInFootnote", _("A link:footnoteLink element MUST have no children other than link:loc, link:footnote, and link:footnoteArc." ), modelObject=nonStdFootnoteElts) for qn in modelXbrl.qnameDimensionDefaults.values(): conceptsUsed.add(modelXbrl.qnameConcepts.get(qn)) # unused elements in linkbases for arcroles, err in ( ((parentChild, ), "elementsNotUsedForTaggingAppliedInPresentationLinkbase"), ((summationItem, ), "elementsNotUsedForTaggingAppliedInCalculationLinkbase"), ((dimensionDomain, domainMember), "elementsNotUsedForTaggingAppliedInDefinitionLinkbase")): unreportedLbElts = set() for arcrole in arcroles: for rel in modelXbrl.relationshipSet( arcrole).modelRelationships: fr = rel.fromModelObject to = rel.toModelObject if arcrole in (parentChild, summationItem): if fr is not None and not fr.isAbstract and fr not in conceptsUsed and isExtension( val, rel): unreportedLbElts.add(fr) if to is not None and not to.isAbstract and to not in conceptsUsed and isExtension( val, rel): unreportedLbElts.add(to) elif arcrole == dimensionDomain: # dimension, always abstract if fr is not None and fr not in conceptsUsed and isExtension( val, rel): unreportedLbElts.add(fr) if to is not None and rel.isUsable and to not in conceptsUsed and isExtension( val, rel): unreportedLbElts.add(to) elif arcrole == domainMember: if to is not None and not fr.isAbstract and rel.isUsable and to not in conceptsUsed and isExtension( val, rel): unreportedLbElts.add(to) if unreportedLbElts: modelXbrl.error( "ESEF.3.4.6." + err, _("All usable concepts in extension taxonomy relationships MUST be applied by tagged facts: %(elements)s." ), modelObject=unreportedLbElts, elements=", ".join( sorted((str(c.qname) for c in unreportedLbElts)))) # 3.4.4 check for presentation preferred labels missingConceptLabels = defaultdict(set) # by role pfsConceptsRootInPreLB = set() def checkLabels(parent, relSet, labelrole, visited): if not parent.label( labelrole, lang=reportXmlLang, fallbackToQname=False): if parent.name != "NotesAccountingPoliciesAndMandatoryTags": # TEMPORARY TBD remove missingConceptLabels[labelrole].add(parent) visited.add(parent) conceptRels = defaultdict( list) # counts for concepts without preferred label role for rel in relSet.fromModelObject(parent): child = rel.toModelObject if child is not None: labelrole = rel.preferredLabel if not labelrole: conceptRels[child].append(rel) if child not in visited: checkLabels(child, relSet, labelrole, visited) for concept, rels in conceptRels.items(): if len(rels) > 1: modelXbrl.warning( "ESEF.3.4.4.missingPreferredLabelRole", _("Preferred label role SHOULD be used when concept is duplicated in same presentation tree location: %(qname)s." ), modelObject=rels + [concept], qname=concept.qname) visited.remove(parent) for ELR in modelXbrl.relationshipSet(parentChild).linkRoleUris: relSet = modelXbrl.relationshipSet(parentChild, ELR) for rootConcept in relSet.rootConcepts: checkLabels(rootConcept, relSet, None, set()) # check for PFS element which isn't an orphan if rootConcept.qname in esefPrimaryStatementPlaceholders and relSet.fromModelObject( rootConcept): pfsConceptsRootInPreLB.add(rootConcept) for labelrole, concepts in missingConceptLabels.items(): modelXbrl.warning( "ESEF.3.4.5.missingLabelForRoleInReportLanguage", _("Label for %(role)s role SHOULD be available in report language for concepts: %(qnames)s." ), modelObject=concepts, qnames=", ".join(str(c.qname) for c in concepts), role=os.path.basename(labelrole) if labelrole else "standard") if not pfsConceptsRootInPreLB: # no PFS statements were recognized modelXbrl.error( "ESEF.RTS.Annex.II.Par.1.Par.7.missingPrimaryFinancialStatement", _("A primary financial statement placeholder element MUST be a root of a presentation linkbase tree." ), modelObject=modelXbrl) # dereference del missingConceptLabels, pfsConceptsRootInPreLB # mandatory factc RTS Annex II missingMandatoryElements = esefMandatoryElements2020 - modelXbrl.factsByQname.keys( ) if missingMandatoryElements: modelXbrl.error( "ESEF.RTS.Annex.II.Par.2.missingMandatoryMarkups", _("Mandatory elements to be marked up are missing: %(qnames)s." ), modelObject=missingMandatoryElements, qnames=", ".join( sorted(str(qn) for qn in missingMandatoryElements))) # duplicated core taxonomy elements for name, concepts in modelXbrl.nameConcepts.items(): if len(concepts) > 1: i = None # ifrs Concept for c in concepts: if c.qname.namespaceURI == _ifrsNs: i = c break if i is not None: for c in concepts: if c != i and c.balance == i.balance and c.periodType == i.periodType: modelXbrl.error( "ESEF.RTS.Annex.IV.Par.4.2.extensionElementDuplicatesCoreElement", _("Extension elements must not duplicate the existing elements from the core taxonomy and be identifiable %(qname)s." ), modelObject=(c, i), qname=c.qname) modelXbrl.profileActivity(_statusMsg, minTimeToShow=0.0) modelXbrl.modelManager.showStatus(None)
def validate(val, modelXbrl, infosetModelXbrl): infoset = infosetModelXbrl.modelDocument if infoset.type == Type.INSTANCE: # compare facts (assumed out of order) infosetFacts = defaultdict(list) for fact in infosetModelXbrl.facts: infosetFacts[fact.qname].append(fact) if len(modelXbrl.factsInInstance) != len( infosetModelXbrl.factsInInstance): modelXbrl.error( "arelle:infosetTest", _("Fact counts mismatch, testcase instance %(foundFactCount)s, infoset instance %(expectedFactCount)s" ), modelObject=(modelXbrl.modelDocument, infosetModelXbrl.modelDocument), foundFactCount=len(modelXbrl.factsInInstance), expectedFactCount=len(infosetModelXbrl.factsInInstance)) else: for i, instFact in enumerate(modelXbrl.facts): infosetFact = None for fact in infosetFacts[instFact.qname]: if fact.isTuple and fact.isDuplicateOf(instFact, deemP0Equal=True): infosetFact = fact break elif fact.isItem and fact.isVEqualTo(instFact, deemP0Equal=True): infosetFact = fact break if infosetFact is None: # takes precision/decimals into account if fact is not None: fact.isVEqualTo(instFact, deemP0Equal=True) modelXbrl.error( "arelle:infosetTest", _("Fact %(factNumber)s mismatch %(concept)s"), modelObject=instFact, factNumber=(i + 1), concept=instFact.qname) else: ptvPeriodType = infosetFact.get( "{http://www.xbrl.org/2003/ptv}periodType") ptvBalance = infosetFact.get( "{http://www.xbrl.org/2003/ptv}balance") ptvDecimals = infosetFact.get( "{http://www.xbrl.org/2003/ptv}decimals") ptvPrecision = infosetFact.get( "{http://www.xbrl.org/2003/ptv}precision") if ptvPeriodType and ptvPeriodType != instFact.concept.periodType: modelXbrl.error( "arelle:infosetTest", _("Fact %(factNumber)s periodType mismatch %(concept)s expected %(expectedPeriodType)s found %(foundPeriodType)s" ), modelObject=(instFact, infosetFact), factNumber=(i + 1), concept=instFact.qname, expectedPeriodType=ptvPeriodType, foundPeriodType=instFact.concept.periodType) if ptvBalance and ptvBalance != instFact.concept.balance: modelXbrl.error( "arelle:infosetTest", _("Fact %(factNumber)s balance mismatch %(concept)s expected %(expectedBalance)s found %(foundBalance)s" ), modelObject=(instFact, infosetFact), factNumber=(i + 1), concept=instFact.qname, expectedBalance=ptvBalance, foundBalance=instFact.concept.balance) if ptvDecimals and ptvDecimals != str( inferredDecimals(fact)): modelXbrl.error( "arelle:infosetTest", _("Fact %(factNumber)s inferred decimals mismatch %(concept)s expected %(expectedDecimals)s found %(inferredDecimals)s" ), modelObject=(instFact, infosetFact), factNumber=(i + 1), concept=instFact.qname, expectedDecimals=ptvDecimals, inferredDecimals=str(inferredDecimals(fact))) if ptvPrecision and ptvPrecision != str( inferredPrecision(fact)): modelXbrl.error( "arelle:infosetTest", _("Fact %(factNumber)s inferred precision mismatch %(concept)s expected %(expectedPrecision)s found %(inferredPrecision)s" ), modelObject=(instFact, infosetFact), factNumber=(i + 1), concept=instFact.qname, expectedPrecisions=ptvPrecision, inferredPrecision=str(inferredPrecision(fact))) elif infoset.type == Type.ARCSINFOSET: # compare arcs for arcElt in XmlUtil.children(infoset.xmlRootElement, "http://www.xbrl.org/2003/ptv", "arc"): linkType = arcElt.get("linkType") arcRole = arcElt.get("arcRole") extRole = arcElt.get("extRole") fromObj = resolvePath(modelXbrl, arcElt.get("fromPath")) if fromObj is None: modelXbrl.error("arelle:infosetTest", _("Arc fromPath not found: %(fromPath)s"), modelObject=arcElt, fromPath=arcElt.get("fromPath")) continue if linkType in ("label", "reference"): labelLang = arcElt.get("labelLang") resRole = arcElt.get("resRole") if linkType == "label": expectedLabel = XmlUtil.text(arcElt) foundLabel = fromObj.label(preferredLabel=resRole, fallbackToQname=False, lang=None, strip=True, linkrole=extRole) if foundLabel != expectedLabel: modelXbrl.error( "arelle:infosetTest", _("Label expected='%(expectedLabel)s', found='%(foundLabel)s'" ), modelObject=arcElt, expectedLabel=expectedLabel, foundLabel=foundLabel) continue elif linkType == "reference": expectedRef = XmlUtil.innerText(arcElt) referenceFound = False for refrel in modelXbrl.relationshipSet( XbrlConst.conceptReference, extRole).fromModelObject(fromObj): ref = refrel.toModelObject if resRole == ref.role: foundRef = XmlUtil.innerText(ref) if foundRef != expectedRef: modelXbrl.error( "arelle:infosetTest", _("Reference inner text expected='%(expectedRef)s, found='%(foundRef)s'" ), modelObject=arcElt, expectedRef=expectedRef, foundRef=foundRef) referenceFound = True break if referenceFound: continue modelXbrl.error( "arelle:infosetTest", _("%(linkType)s not found containing '%(text)s' linkRole %(linkRole)s" ), modelObject=arcElt, linkType=linkType.title(), text=XmlUtil.innerText(arcElt), linkRole=extRole) else: toObj = resolvePath(modelXbrl, arcElt.get("toPath")) if toObj is None: modelXbrl.error("arelle:infosetTest", _("Arc toPath not found: %(toPath)s"), modelObject=arcElt, toPath=arcElt.get("toPath")) continue weight = arcElt.get("weight") if weight is not None: weight = float(weight) order = arcElt.get("order") if order is not None: order = float(order) preferredLabel = arcElt.get("preferredLabel") found = False for rel in modelXbrl.relationshipSet( arcRole, extRole).fromModelObject(fromObj): if (rel.toModelObject == toObj and (weight is None or rel.weight == weight) and (order is None or rel.order == order)): found = True if not found: modelXbrl.error( "arelle:infosetTest", _("Arc not found: from %(fromPath)s, to %(toPath)s, role %(arcRole)s, linkRole $(extRole)s" ), modelObject=arcElt, fromPath=arcElt.get("fromPath"), toPath=arcElt.get("toPath"), arcRole=arcRole, linkRole=extRole) continue # validate dimensions of each fact factElts = XmlUtil.children(modelXbrl.modelDocument.xmlRootElement, None, "*") for itemElt in XmlUtil.children(infoset.xmlRootElement, None, "item"): try: qnElt = XmlUtil.child(itemElt, None, "qnElement") factQname = qname(qnElt, XmlUtil.text(qnElt)) sPointer = int(XmlUtil.child(itemElt, None, "sPointer").text) factElt = factElts[sPointer - 1] # 1-based xpath indexing if factElt.qname != factQname: modelXbrl.error( "arelle:infosetTest", _("Fact %(sPointer)s mismatch Qname, expected %(qnElt)s, observed %(factQname)s" ), modelObject=itemElt, sPointer=sPointer, qnElt=factQname, factQname=factElt.qname) elif not factElt.isItem or factElt.context is None: modelXbrl.error( "arelle:infosetTest", _("Fact %(sPointer)s has no context: %(qnElt)s"), modelObject=(itemElt, factElt), sPointer=sPointer, qnElt=factQname) else: context = factElt.context memberElts = XmlUtil.children(itemElt, None, "member") numNonDefaults = 0 for memberElt in memberElts: dimElt = XmlUtil.child(memberElt, None, "qnDimension") qnDim = qname(dimElt, XmlUtil.text(dimElt)) isDefault = XmlUtil.text( XmlUtil.child(memberElt, None, "bDefaulted")) == "true" if not isDefault: numNonDefaults += 1 if not ( (qnDim in context.qnameDims and not isDefault) or (qnDim in factElt.modelXbrl.qnameDimensionDefaults and isDefault)): modelXbrl.error( "arelle:infosetTest", _("Fact %(sPointer)s (qnElt)s dimension mismatch %(qnDim)s" ), modelObject=(itemElt, factElt, context), sPointer=sPointer, qnElt=factQname, qnDim=qnDim) if numNonDefaults != len(context.qnameDims): modelXbrl.error( "arelle:infosetTest", _("Fact %(sPointer)s (qnElt)s dimensions count mismatch" ), modelObject=(itemElt, factElt, context), sPointer=sPointer, qnElt=factQname) except (IndexError, ValueError, AttributeError) as err: modelXbrl.error( "arelle:infosetTest", _("Invalid entity fact dimensions infoset sPointer: %(test)s, error details: %(error)s" ), modelObject=itemElt, test=XmlUtil.innerTextList(itemElt), error=str(err))
def addFact(self, f): if f.id is None: f.set("id", "ixv-%d" % (self.idGen)) self.idGen += 1 conceptName = self.nsmap.qname(f.qname) scheme, ident = f.context.entityIdentifier aspects = { "c": conceptName, "e": self.nsmap.qname( QName(self.nsmap.getPrefix(scheme, "e"), scheme, ident)), } factData = { "a": aspects, } if f.isNil: factData["v"] = None elif f.concept is not None and f.concept.isEnumeration: qnEnums = f.xValue if not isinstance(qnEnums, list): qnEnums = (qnEnums, ) factData["v"] = " ".join(self.nsmap.qname(qn) for qn in qnEnums) for qn in qnEnums: self.addConcept(self.dts.qnameConcepts.get(qn)) else: factData["v"] = f.value if f.value == INVALIDixVALUE: factData["err"] = 'INVALID_IX_VALUE' if f.format is not None: factData["f"] = str(f.format) if f.isNumeric: if f.unit is not None and len(f.unit.measures[0]): # XXX does not support complex units unit = self.nsmap.qname(f.unit.measures[0][0]) aspects["u"] = unit else: # The presence of the unit aspect is used by the viewer to # identify numeric facts. If the fact has no unit (invalid # XBRL, but we want to support it for draft documents), # include the unit aspect with a null value. aspects["u"] = None d = inferredDecimals(f) if d != float("INF") and not math.isnan(d): factData["d"] = d for d, v in f.context.qnameDims.items(): if v.memberQname is not None: aspects[self.nsmap.qname(v.dimensionQname)] = self.nsmap.qname( v.memberQname) self.addConcept(v.member) self.addConcept(v.dimension, dimensionType="e") elif v.typedMember is not None: aspects[self.nsmap.qname( v.dimensionQname)] = v.typedMember.text self.addConcept(v.dimension, dimensionType="t") if f.context.isForeverPeriod: aspects["p"] = "f" elif f.context.isInstantPeriod and f.context.instantDatetime is not None: aspects["p"] = self.dateFormat( f.context.instantDatetime.isoformat()) elif f.context.isStartEndPeriod and f.context.startDatetime is not None and f.context.endDatetime is not None: aspects["p"] = "%s/%s" % ( self.dateFormat(f.context.startDatetime.isoformat()), self.dateFormat(f.context.endDatetime.isoformat())) frels = self.footnoteRelationshipSet.fromModelObject(f) if frels: for frel in frels: if frel.toModelObject is not None: factData.setdefault("fn", []).append(frel.toModelObject.id) self.taxonomyData["facts"][f.id] = factData self.addConcept(f.concept)