def isVEqualTo(self, other):
if self.isTuple or other.isTuple:
return False
if self.isNil:
return other.isNil
if other.isNil:
return False
if not self.context.isEqualTo(other.context):
return False
if self.concept.isNumeric:
if other.concept.isNumeric:
if not self.unit.isEqualTo(other.unit):
return False
if self.modelXbrl.modelManager.validateInferDecimals:
d = min((inferredDecimals(self), inferredDecimals(other))); p = None
else:
d = None; p = min((inferredPrecision(self), inferredPrecision(other)))
return roundValue(float(self.value),precision=p,decimals=d) == roundValue(float(other.value),precision=p,decimals=d)
else:
return False
selfValue = self.value
otherValue = other.value
if isinstance(selfValue,str) and isinstance(otherValue,str):
return selfValue.strip() == otherValue.strip()
else:
return selfValue == otherValue
def _assets_eq_liability_equity(modelXbrl):
assets_concept = modelXbrl.nameConcepts[_ASSETS_CONCEPT][
0] if modelXbrl.nameConcepts[_ASSETS_CONCEPT] else None
liability_equity_concept = modelXbrl.nameConcepts[_LIABILITIES_CONCEPT][
0] if modelXbrl.nameConcepts[_LIABILITIES_CONCEPT] else None
if assets_concept is not None and liability_equity_concept is not None:
assets_facts = modelXbrl.factsByQname[assets_concept.qname]
liability_equity_facts = modelXbrl.factsByQname[
liability_equity_concept.qname]
fact_dict = {}
fact_dict[_ASSETS_CONCEPT] = assets_facts
fact_dict[_LIABILITIES_CONCEPT] = liability_equity_facts
fact_groups = facts.prepare_facts_for_calculation(fact_dict)
for fact_group in fact_groups:
fact_assets = fact_group[_ASSETS_CONCEPT]
fact_liabilities = fact_group[_LIABILITIES_CONCEPT]
if fact_assets.context is not None and fact_assets.context.instantDatetime is not None:
dec_assets = inferredDecimals(fact_assets)
dec_liabilities = inferredDecimals(fact_liabilities)
min_dec = min(dec_assets, dec_liabilities)
if roundValue(fact_assets.xValue,
decimals=min_dec) != roundValue(
fact_liabilities.xValue, decimals=min_dec):
yield fact_assets, fact_liabilities
def isVEqualTo(self, other, deemP0Equal=False):
"""(bool) -- v-equality of two facts
Note that facts may be in different instances
"""
if self.isTuple or other.isTuple:
return False
if self.isNil:
return other.isNil
if other.isNil:
return False
if not self.context.isEqualTo(other.context):
return False
if self.concept.isNumeric:
if other.concept.isNumeric:
if not self.unit.isEqualTo(other.unit):
return False
if self.modelXbrl.modelManager.validateInferDecimals:
d = min((inferredDecimals(self), inferredDecimals(other))); p = None
if isnan(d) and deemP0Equal:
return True
else:
d = None; p = min((inferredPrecision(self), inferredPrecision(other)))
if p == 0 and deemP0Equal:
return True
return roundValue(self.value,precision=p,decimals=d) == roundValue(other.value,precision=p,decimals=d)
else:
return False
selfValue = self.value
otherValue = other.value
if isinstance(selfValue,str) and isinstance(otherValue,str):
return selfValue.strip() == otherValue.strip()
else:
return selfValue == otherValue
def addMostAccurateFactToBinding(f):
if f.xValid >= VALID and (nils
or not f.isNil) and f.context is not None:
binding = bindings[f.context.contextDimAwareHash,
f.unit.hash if f.unit is not None else None]
ln = f.qname.localName
if ln not in binding or inferredDecimals(f) > inferredDecimals(
binding[ln]):
binding[ln] = f
def generateInstanceInfoset(dts, instanceInfosetFile):
if dts.fileSource.isArchive:
return
import os, io
from arelle import XmlUtil, XbrlConst
from arelle.ValidateXbrlCalcs import inferredPrecision, inferredDecimals
XmlUtil.setXmlns(dts.modelDocument, u"ptv", u"http://www.xbrl.org/2003/ptv")
numFacts = 0
for fact in dts.facts:
try:
if fact.concept.periodType:
fact.set(u"{http://www.xbrl.org/2003/ptv}periodType", fact.concept.periodType)
if fact.concept.balance:
fact.set(u"{http://www.xbrl.org/2003/ptv}balance", fact.concept.balance)
if fact.isNumeric and not fact.isNil:
fact.set(u"{http://www.xbrl.org/2003/ptv}decimals", unicode(inferredDecimals(fact)))
fact.set(u"{http://www.xbrl.org/2003/ptv}precision", unicode(inferredPrecision(fact)))
numFacts += 1
except Exception, err:
dts.error(u"saveInfoset.exception",
_(u"Facts exception %(fact)s %(value)s %(error)s."),
modelObject=fact, fact=fact.qname, value=fact.effectiveValue, error = err)
def generateInfoset(dts, infosetFile):
if dts.fileSource.isArchive:
return
import os, io
from arelle import XmlUtil, XbrlConst
from arelle.ValidateXbrlCalcs import inferredPrecision, inferredDecimals
XmlUtil.setXmlns(dts.modelDocument, "ptv", "http://www.xbrl.org/2003/ptv")
numFacts = 0
for fact in dts.facts:
try:
if fact.concept.periodType:
fact.set("{http://www.xbrl.org/2003/ptv}periodType", fact.concept.periodType)
if fact.concept.balance:
fact.set("{http://www.xbrl.org/2003/ptv}balance", fact.concept.balance)
if fact.isNumeric and not fact.isNil:
fact.set("{http://www.xbrl.org/2003/ptv}decimals", str(inferredDecimals(fact)))
fact.set("{http://www.xbrl.org/2003/ptv}precision", str(inferredPrecision(fact)))
numFacts += 1
except Exception as err:
dts.error("saveInfoset.exception",
_("Facts exception %(fact)s %(value)s %(error)s."),
modelObject=fact, fact=fact.qname, value=fact.effectiveValue, error = err)
fh = open(infosetFile, "w", encoding="utf-8")
XmlUtil.writexml(fh, dts.modelDocument.xmlDocument, encoding="utf-8")
fh.close()
dts.info("info:saveInfoset",
_("Infoset of %(entryFile)s has %(numberOfFacts)s facts in infoset file %(infosetOutputFile)s."),
modelObject=dts,
entryFile=dts.uri, numberOfFacts=numFacts, infosetOutputFile=infosetFile)
def generateInstanceInfoset(dts, instanceInfosetFile):
if dts.fileSource.isArchive:
return
import os, io
from arelle import XmlUtil, XbrlConst
from arelle.ValidateXbrlCalcs import inferredPrecision, inferredDecimals
XmlUtil.setXmlns(dts.modelDocument, "ptv", "http://www.xbrl.org/2003/ptv")
numFacts = 0
for fact in dts.facts:
try:
if fact.concept.periodType:
fact.set("{http://www.xbrl.org/2003/ptv}periodType", fact.concept.periodType)
if fact.concept.balance:
fact.set("{http://www.xbrl.org/2003/ptv}balance", fact.concept.balance)
if fact.isNumeric and not fact.isNil:
fact.set("{http://www.xbrl.org/2003/ptv}decimals", str(inferredDecimals(fact)))
fact.set("{http://www.xbrl.org/2003/ptv}precision", str(inferredPrecision(fact)))
numFacts += 1
except Exception as err:
dts.error("saveInfoset.exception",
_("Facts exception %(fact)s %(value)s %(error)s."),
modelObject=fact, fact=fact.qname, value=fact.effectiveValue, error = err)
fh = open(instanceInfosetFile, "w", encoding="utf-8")
XmlUtil.writexml(fh, dts.modelDocument.xmlDocument, encoding="utf-8")
fh.close()
dts.info("info:saveInstanceInfoset",
_("Instance infoset of %(entryFile)s has %(numberOfFacts)s facts in infoset file %(infosetOutputFile)s."),
modelObject=dts,
entryFile=dts.uri, numberOfFacts=numFacts, infosetOutputFile=instanceInfosetFile)
def _assets_eq_liability_equity(model_xbrl):
"""
Yields fact assets and fact liabilities as long as it is able to
:param model_xbrl: modelXbrl to check name concepts of
:type model_xbrl: :class:'~arelle.ModelXbrl.ModelXbrl'
:return: yields fact assets and fact liabilities that should throw errors
:rtype: tuple
"""
assets_concept = (
model_xbrl.nameConcepts[_ASSETS_CONCEPT][0]
if model_xbrl.nameConcepts[_ASSETS_CONCEPT] else None
)
liability_equity_concept = (
model_xbrl.nameConcepts[_LIABILITIES_CONCEPT][0]
if model_xbrl.nameConcepts[_LIABILITIES_CONCEPT]
else None
)
if assets_concept is not None and liability_equity_concept is not None:
assets_facts = model_xbrl.factsByQname[assets_concept.qname]
liability_equity_facts = (
model_xbrl.factsByQname[liability_equity_concept.qname]
)
fact_dict = dict()
fact_dict[_ASSETS_CONCEPT] = assets_facts
fact_dict[_LIABILITIES_CONCEPT] = liability_equity_facts
fact_groups = facts.prepare_facts_for_calculation(fact_dict)
for fact_group in fact_groups:
fact_assets = fact_group[_ASSETS_CONCEPT]
fact_liabilities = fact_group[_LIABILITIES_CONCEPT]
if ((fact_assets.context is not None and
fact_assets.context.instantDatetime is not None)):
dec_assets = inferredDecimals(fact_assets)
dec_liabilities = inferredDecimals(fact_liabilities)
min_dec = min(dec_assets, dec_liabilities)
if _values_unequal(
fact_assets.xValue, fact_liabilities.xValue, min_dec
):
yield fact_assets, fact_liabilities
def _assets_eq_liability_equity(modelXbrl):
assets_concept = modelXbrl.nameConcepts[_ASSETS_CONCEPT][0] if modelXbrl.nameConcepts[_ASSETS_CONCEPT] else None
liability_equity_concept = modelXbrl.nameConcepts[_LIABILITIES_CONCEPT][0] if modelXbrl.nameConcepts[_LIABILITIES_CONCEPT] else None
if assets_concept is not None and liability_equity_concept is not None:
assets_facts = modelXbrl.factsByQname[assets_concept.qname]
liability_equity_facts = modelXbrl.factsByQname[liability_equity_concept.qname]
fact_dict = {}
fact_dict[_ASSETS_CONCEPT] = assets_facts
fact_dict[_LIABILITIES_CONCEPT] = liability_equity_facts
fact_groups = facts.prepare_facts_for_calculation(fact_dict)
for fact_group in fact_groups:
fact_assets = fact_group[_ASSETS_CONCEPT]
fact_liabilities = fact_group[_LIABILITIES_CONCEPT]
if fact_assets.context is not None and fact_assets.context.instantDatetime is not None:
dec_assets = inferredDecimals(fact_assets)
dec_liabilities = inferredDecimals(fact_liabilities)
min_dec = min(dec_assets, dec_liabilities)
if roundValue(fact_assets.xValue, decimals=min_dec) != roundValue(fact_liabilities.xValue, decimals=min_dec):
yield fact_assets, fact_liabilities
def _assets_eq_liability_equity(model_xbrl):
"""
Yields fact assets and fact liabilities as long as it is able to
:param model_xbrl: modelXbrl to check name concepts of
:type model_xbrl: :class:'~arelle.ModelXbrl.ModelXbrl'
:return: yields fact assets and fact liabilities that should throw errors
:rtype: tuple
"""
assets_concept = (model_xbrl.nameConcepts[_ASSETS_CONCEPT][0]
if model_xbrl.nameConcepts[_ASSETS_CONCEPT] else None)
liability_equity_concept = (
model_xbrl.nameConcepts[_LIABILITIES_CONCEPT][0]
if model_xbrl.nameConcepts[_LIABILITIES_CONCEPT] else None)
if assets_concept is not None and liability_equity_concept is not None:
assets_facts = model_xbrl.factsByQname[assets_concept.qname]
liability_equity_facts = (
model_xbrl.factsByQname[liability_equity_concept.qname])
fact_dict = dict()
fact_dict[_ASSETS_CONCEPT] = assets_facts
fact_dict[_LIABILITIES_CONCEPT] = liability_equity_facts
fact_groups = facts.prepare_facts_for_calculation(fact_dict)
for fact_group in fact_groups:
fact_assets = fact_group[_ASSETS_CONCEPT]
fact_liabilities = fact_group[_LIABILITIES_CONCEPT]
if ((fact_assets.context is not None
and fact_assets.context.instantDatetime is not None)):
dec_assets = inferredDecimals(fact_assets)
dec_liabilities = inferredDecimals(fact_liabilities)
min_dec = min(dec_assets, dec_liabilities)
if _values_unequal(fact_assets.xValue, fact_liabilities.xValue,
min_dec):
yield fact_assets, fact_liabilities
def isVEqualTo(self,
other,
deemP0Equal=False): # facts may be in different instances
if self.isTuple or other.isTuple:
return False
if self.isNil:
return other.isNil
if other.isNil:
return False
if not self.context.isEqualTo(other.context):
return False
if self.concept.isNumeric:
if other.concept.isNumeric:
if not self.unit.isEqualTo(other.unit):
return False
if self.modelXbrl.modelManager.validateInferDecimals:
d = min((inferredDecimals(self), inferredDecimals(other)))
p = None
else:
d = None
p = min(
(inferredPrecision(self), inferredPrecision(other)))
if p == 0 and deemP0Equal:
return True
return roundValue(self.value, precision=p,
decimals=d) == roundValue(other.value,
precision=p,
decimals=d)
else:
return False
selfValue = self.value
otherValue = other.value
if isinstance(selfValue, str) and isinstance(otherValue, str):
return selfValue.strip() == otherValue.strip()
else:
return selfValue == otherValue
def factAspects(fact):
aspects = {qnOimConceptAspect: oimValue(fact.qname)}
if hasId and fact.id:
aspects[qnOimIdAspect] = fact.id
if hasLocation:
aspects[qnOimLocationAspect] = elementChildSequence(fact)
concept = fact.concept
if concept is not None:
if concept.baseXbrliType in ("string", "normalizedString",
"token") and fact.xmlLang:
aspects[qnOimLangAspect] = fact.xmlLang
aspects[qnOimTypeAspect] = concept.baseXbrliType
if fact.isItem:
aspects[qnOimValueAspect] = (NILVALUE if fact.isNil else oimValue(
fact.xValue, inferredDecimals(fact)))
cntx = fact.context
if cntx is not None:
if cntx.entityIdentifierElement is not None:
aspects[qnOimEntityAspect] = oimValue(
qname(*cntx.entityIdentifier))
if cntx.period is not None:
aspects[qnOimPeriodAspect] = oimPeriodValue(cntx)
for dim in cntx.qnameDims.values():
aspects[dim.dimensionQname] = (oimValue(dim.memberQname)
if dim.isExplicit else
dim.typedMember.stringValue)
unit = fact.unit
if unit is not None:
_mMul, _mDiv = unit.measures
if isJSON:
aspects[
qnOimUnitAspect] = ( # use tuple instead of list for hashability
tuple(
oimValue(m)
for m in sorted(_mMul, key=lambda m: str(m))),
tuple(
oimValue(m)
for m in sorted(_mDiv, key=lambda m: str(m))))
else: # CSV
if _mMul:
aspects[qnOimUnitMulAspect] = ",".join(
oimValue(m)
for m in sorted(_mMul, key=lambda m: str(m)))
if _mDiv:
aspects[qnOimUnitDivAspect] = ",".join(
oimValue(m)
for m in sorted(_mDiv, key=lambda m: str(m)))
return aspects
def infer_precision_decimals(xc, p, args, attrName):
if len(args) != 1: raise XPathContext.FunctionNumArgs()
if len(args[0]) != 1: raise XPathContext.FunctionArgType(1,"xbrl:item")
modelItem = xc.modelItem(args[0][0])
if modelItem:
modelConcept = modelItem.concept
if modelConcept.isNumeric:
if modelConcept.isFraction: return 'INF'
from arelle.ValidateXbrlCalcs import (inferredDecimals,inferredPrecision)
p = inferredPrecision(modelItem) if attrName == "precision" else inferredDecimals(modelItem)
if isinf(p):
return 'INF'
if isnan(p):
raise XPathContext.XPathException(p, 'xfie:ItemIsNotNumeric', _('Argument 1 {0} is not inferrable.').format(attrName))
return p
raise XPathContext.XPathException(p, 'xfie:ItemIsNotNumeric', _('Argument 1 is not reported with {0}.').format(attrName))
def factAspects(fact):
aspects = {qnOimConceptAspect: oimValue(fact.qname)}
if hasId and fact.id:
aspects[qnOimIdAspect] = fact.id
if hasLocation:
aspects[qnOimLocationAspect] = elementChildSequence(fact)
concept = fact.concept
if concept is not None:
if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang:
aspects[qnOimLangAspect] = fact.xmlLang
aspects[qnOimTypeAspect] = concept.baseXbrliType
if fact.isItem:
aspects[qnOimValueAspect] = (NILVALUE if fact.isNil else
oimValue(fact.xValue, inferredDecimals(fact)))
cntx = fact.context
if cntx is not None:
if cntx.entityIdentifierElement is not None:
aspects[qnOimEntityAspect] = oimValue(qname(*cntx.entityIdentifier))
if cntx.period is not None:
aspects[qnOimPeriodAspect] = oimPeriodValue(cntx)
for dim in cntx.qnameDims.values():
aspects[dim.dimensionQname] = (oimValue(dim.memberQname) if dim.isExplicit
else dim.typedMember.stringValue)
unit = fact.unit
if unit is not None:
_mMul, _mDiv = unit.measures
if isJSON:
aspects[qnOimUnitAspect] = ( # use tuple instead of list for hashability
tuple(oimValue(m) for m in sorted(_mMul, key=lambda m: str(m))),
tuple(oimValue(m) for m in sorted(_mDiv, key=lambda m: str(m))))
else: # CSV
if _mMul:
aspects[qnOimUnitMulAspect] = ",".join(oimValue(m)
for m in sorted(_mMul, key=lambda m: str(m)))
if _mDiv:
aspects[qnOimUnitDivAspect] = ",".join(oimValue(m)
for m in sorted(_mDiv, key=lambda m: str(m)))
return aspects
def infer_precision_decimals(xc, p, args, attrName):
if len(args) != 1: raise XPathContext.FunctionNumArgs()
if len(args[0]) != 1: raise XPathContext.FunctionArgType(1, "xbrl:item")
modelItem = xc.modelItem(args[0][0])
if modelItem:
modelConcept = modelItem.concept
if modelConcept.isNumeric:
if modelConcept.isFraction: return 'INF'
from arelle.ValidateXbrlCalcs import (inferredDecimals,
inferredPrecision)
p = inferredPrecision(
modelItem) if attrName == "precision" else inferredDecimals(
modelItem)
if isinf(p):
return 'INF'
if isnan(p):
raise XPathContext.XPathException(
p, 'xfie:ItemIsNotNumeric',
_('Argument 1 {0} is not inferrable.').format(attrName))
return p
raise XPathContext.XPathException(
p, 'xfie:ItemIsNotNumeric',
_('Argument 1 is not reported with {0}.').format(attrName))
def sectTreeRel(self,
parentConcept,
n,
sectCalc2RelSet,
inferredParentValues,
visited,
dimQN=None):
childRels = sectCalc2RelSet.fromModelObject(parentConcept)
if childRels:
visited.add(parentConcept)
inferredChildValues = {}
# setup summation bind keys for child objects
sumParentBindKeys = self.sumConceptBindKeys[parentConcept]
boundSumKeys = set(
) # these are contributing fact keys, parent may be inferred
boundSums = defaultdict(intervalZero)
boundSummationItems = defaultdict(list)
boundPerKeys = set()
boundPers = defaultdict(intervalZero)
boundDurationItems = defaultdict(list)
boundAggKeys = set()
boundAggs = defaultdict(intervalZero)
boundAggItems = defaultdict(list)
boundAggConcepts = defaultdict(set)
for rel in childRels:
childConcept = rel.toModelObject
if childConcept not in visited:
if rel.arcrole == summationItem:
if not self.sumInit:
self.sumBindFacts()
boundSumKeys |= self.sumConceptBindKeys[childConcept]
elif rel.arcrole == balanceChanges:
if not self.perInit:
self.perBindFacts()
boundPerKeys |= self.perConceptBindKeys[
childConcept] # these are only duration items
elif rel.arcrole == domainMember:
boundAggKeys |= self.aggConceptBindKeys[dimQN]
domQN = parentConcept.qname
elif rel.arcrole == aggregationDomain: # this is in visited
dimQN = rel.arcElement.prefixedNameQname(
rel.get("dimension"))
if dimQN not in self.aggDimInit:
self.aggBindFacts(
dimQN) # bind each referenced dimension's contexts
# depth-first descent calc tree and process item after descent
for rel in childRels:
childConcept = rel.toModelObject
if childConcept not in visited:
# depth-first descent
self.sectTreeRel(childConcept, n + 1, sectCalc2RelSet,
inferredChildValues, visited, dimQN)
# post-descent summation (allows use of inferred value)
if rel.arcrole == summationItem:
weight = rel.weightDecimal
for sumKey in boundSumKeys:
cntx, unit = sumKey
factKey = (childConcept, cntx, unit)
if factKey in self.sumBoundFacts:
for f in self.sumBoundFacts[factKey]:
addInterval(boundSums, sumKey,
intervalValue(f), weight)
boundSummationItems[sumKey].append(f)
elif factKey in inferredChildValues:
addInterval(boundSums, sumKey,
inferredChildValues[factKey],
weight)
elif factKey in inferredParentValues:
addInterval(boundSums, sumKey,
inferredParentValues[factKey],
weight)
elif rel.arcrole == balanceChanges:
weight = rel.weightDecimal
for perKey in boundPerKeys:
hCntx, unit, start, end = perKey
factKey = (childConcept, hCntx, unit, start, end)
if factKey in self.perBoundFacts:
for f in self.perBoundFacts[factKey]:
addInterval(boundPers, perKey,
intervalValue(f), weight)
boundDurationItems[perKey].append(f)
elif factKey in inferredChildValues:
addInterval(boundPers, perKey,
inferredChildValues[factKey],
weight)
elif factKey in inferredParentValues:
addInterval(boundPers, perKey,
inferredParentValues[factKey],
weight)
elif rel.arcrole == domainMember:
memQN = childConcept.qname
for aggKey in boundAggKeys:
hCntx, unit = aggKey
dimMemKey = (hCntx, unit, dimQN, memQN)
if dimMemKey in self.aggBoundFacts:
for f in self.aggBoundFacts[dimMemKey]:
a, b = intervalValue(f)
factDomKey = (f.concept, hCntx, unit,
dimQN, domQN)
addInterval(boundAggs, factDomKey,
intervalValue(f))
boundAggItems[aggKey].append(f)
boundAggConcepts[aggKey].add(f.concept)
elif rel.arcrole == aggregationDomain: # this is in visited
childRelSet = self.modelXbrl.relationshipSet(
domainMember, rel.get("targetRole"))
self.sectTreeRel(childConcept, n + 1, childRelSet,
inferredParentValues, {None},
dimQN) # infer global to section
# process child items bound to this calc subtree
for sumKey in boundSumKeys:
cntx, unit = sumKey
factKey = (parentConcept, cntx, unit)
ia, ib = boundSums[sumKey]
if factKey in self.sumBoundFacts:
for f in self.sumBoundFacts[factKey]:
d = inferredDecimals(f)
sa, sb = intervalValue(f, d)
if ((ia is NIL) ^
(sa is NIL)) or ((ia is not NIL) and
(sb < ia or sa > ib)):
self.modelXbrl.log(
'INCONSISTENCY',
"calc2e:summationInconsistency",
_("Summation inconsistent from %(concept)s in section %(section)s reported sum %(reportedSum)s, computed sum %(computedSum)s context %(contextID)s unit %(unitID)s unreported contributing items %(unreportedContributors)s"
),
modelObject=boundSummationItems[sumKey],
concept=parentConcept.qname,
section=self.section,
reportedSum=self.formatInterval(sa, sb, d),
computedSum=self.formatInterval(ia, ib, d),
contextID=f.context.id,
unitID=f.unit.id,
unreportedContributors=", ".join(
str(
c.qname
) # list the missing/unreported contributors in relationship order
for r in childRels
for c in (r.toModelObject, )
if r.arcrole == summationItem
and c is not None and
(c, cntx, unit) not in self.sumBoundFacts)
or "none")
elif inferredParentValues is not None: # value was inferred, return to parent level
inferredParentValues[factKey] = (ia, ib)
for perKey in boundPerKeys:
hCntx, unit, start, end = perKey
ia, ib = boundPers[perKey]
endBalA = endBalB = ZERO
endFactKey = (parentConcept, hCntx, unit, None, end)
if endFactKey in self.perBoundFacts:
for f in self.perBoundFacts[endFactKey]:
if f.isNil:
endBalA = endBalB = NIL
d = 0
break
d = inferredDecimals(f)
a, b = intervalValue(f, d)
endBalA += a
endBalB += b
foundStartingFact = (endBalA is NIL)
while not foundStartingFact:
startFactKey = (parentConcept, hCntx, unit, None,
start)
if startFactKey in self.perBoundFacts:
for f in self.perBoundFacts[startFactKey]:
if f.isNil:
endBalA = endBalB = NIL
foundStartingFact = True
break
a, b = intervalValue(f)
endBalA -= a
endBalB -= b
foundStartingFact = True
break
if not foundStartingFact:
# infer backing up one period
_nomPer = nominalPeriod(end - start)
foundEarlierAdjacentPeriodStart = False
for _start in self.durationPeriodStarts.get(
_nomPer, ()):
if nominalPeriod(
start - _start
) == _nomPer: # it's preceding period
end = start
start = _start
perKey = hCntx, unit, start, end
if perKey in boundPerKeys:
chngs = boundPers[perKey]
ia += chngs[0]
ib += chngs[1]
foundEarlierAdjacentPeriodStart = True
break
if not foundEarlierAdjacentPeriodStart:
break
if ((ia is NIL) ^
(endBalA is NIL)) or ((ia is not NIL) and
(endBalB < ia or endBalA > ib)):
self.modelXbrl.log(
'INCONSISTENCY',
"calc2e:balanceInconsistency",
_("Balance inconsistent from %(concept)s in section %(section)s reported sum %(reportedSum)s, computed sum %(computedSum)s context %(contextID)s unit %(unitID)s unreported contributing items %(unreportedContributors)s"
),
modelObject=boundDurationItems[perKey],
concept=parentConcept.qname,
section=self.section,
reportedSum=self.formatInterval(
endBalA, endBalB, d),
computedSum=self.formatInterval(ia, ib, d),
contextID=f.context.id,
unitID=f.unit.id,
unreportedContributors=", ".join(
str(
c.qname
) # list the missing/unreported contributors in relationship order
for r in childRels for c in (r.toModelObject, )
if r.arcrole == balanceChanges
and c is not None and
(c, hCntx, unit, start,
end) not in self.perBoundFacts) or "none")
for aggKey in boundAggKeys:
hCntx, unit = aggKey
for concept in sorted(
boundAggConcepts[aggKey],
key=lambda c: c.objectIndex): # repeatable errors
factDomKey = (concept, hCntx, unit, dimQN, domQN)
ia, ib = boundAggs[factDomKey]
if factDomKey in self.aggBoundConceptFacts:
for f in self.aggBoundConceptFacts[factDomKey]:
d = inferredDecimals(f)
sa, sb = intervalValue(f, d)
if ((ia is NIL) ^
(sa is NIL)) or ((ia is not NIL) and
(sb < ia or sa > ib)):
self.modelXbrl.log(
'INCONSISTENCY',
"calc2e:aggregationInconsistency",
_("Aggregation inconsistent for %(concept)s, domain %(domain)s in section %(section)s reported sum %(reportedSum)s, computed sum %(computedSum)s context %(contextID)s unit %(unitID)s unreported contributing members %(unreportedContributors)s"
),
modelObject=boundAggItems[factDomKey],
concept=concept.qname,
domain=parentConcept.qname,
section=self.section,
reportedSum=self.formatInterval(sa, sb, d),
computedSum=self.formatInterval(ia, ib, d),
contextID=f.context.id,
unitID=f.unit.id,
unreportedContributors=", ".join(
str(
c.qname
) # list the missing/unreported contributors in relationship order
for r in childRels
for c in (r.toModelObject, )
if r.arcrole == domainMember
and c is not None and
(concept, hCntx, unit, dimQN, c.qname
) not in self.aggBoundConceptFacts)
or "none")
elif inferredParentValues is not None: # value was inferred, return to parent level
# allow to be retrieved by factDomKey
inferredParentValues[factDomKey] = (ia, ib)
if self.modelXbrl.qnameDimensionDefaults.get(
dimQN) == domQN:
cntxKey = (hCntx, dimQN, domQN)
if cntxKey in self.eqCntx:
cntx = self.eqCntx[cntxKey]
else:
cntx = self.aggTotalContext(
hCntx, dimQN, domQN)
self.eqCntx[cntxKey] = cntx
if cntx is not None:
# allow to be retrieved by fact line item context key
self.eqCntx[(hCntx, dimQN, domQN)] = cntx
inferredParentValues[(concept, cntx,
unit)] = (ia, ib)
visited.remove(parentConcept)
def validateXbrlFinally(val, *args, **kwargs):
if not (val.validateESMAplugin):
return
_xhtmlNs = "{{{}}}".format(xhtml)
_xhtmlNsLen = len(_xhtmlNs)
modelXbrl = val.modelXbrl
modelDocument = modelXbrl.modelDocument
_statusMsg = _("validating {0} filing rules").format(val.disclosureSystem.name)
modelXbrl.profileActivity()
modelXbrl.modelManager.showStatus(_statusMsg)
reportXmlLang = None
firstRootmostXmlLangDepth = 9999999
if modelDocument.type == ModelDocument.Type.INSTANCE:
modelXbrl.error("esma:instanceShallBeInlineXBRL",
_("RTS on ESEF requires inline XBRL instances."),
modelObject=modelXbrl)
checkFilingDimensions(val) # sets up val.primaryItems and val.domainMembers
val.hasExtensionSchema = val.hasExtensionPre = val.hasExtensionCal = val.hasExtensionDef = val.hasExtensionLbl = False
checkFilingDTS(val, modelXbrl.modelDocument, [])
modelXbrl.profileActivity("... filer DTS checks", minTimeToShow=1.0)
if not (val.hasExtensionSchema and val.hasExtensionPre and val.hasExtensionCal and val.hasExtensionDef and val.hasExtensionLbl):
missingFiles = []
if not val.hasExtensionSchema: missingFiles.append("schema file")
if not val.hasExtensionPre: missingFiles.append("presentation linkbase")
if not val.hasExtensionCal: missingFiles.append("calculation linkbase")
if not val.hasExtensionDef: missingFiles.append("definition linkbase")
if not val.hasExtensionLbl: missingFiles.append("label linkbase")
modelXbrl.warning("esma:3.1.1.extensionTaxonomyWrongFilesStructure",
_("Extension taxonomies MUST consist of at least a schema file and presentation, calculation, definition and label linkbases"
": missing %(missingFiles)s"),
modelObject=modelXbrl, missingFiles=", ".join(missingFiles))
if modelDocument.type in (ModelDocument.Type.INLINEXBRL, ModelDocument.Type.INLINEXBRLDOCUMENTSET, ModelDocument.Type.INSTANCE):
footnotesRelationshipSet = modelXbrl.relationshipSet("XBRL-footnotes")
orphanedFootnotes = set()
noLangFootnotes = set()
footnoteRoleErrors = set()
transformRegistryErrors = set()
def checkFootnote(elt, text):
if text: # non-empty footnote must be linked to a fact if not empty
if not any(isinstance(rel.fromModelObject, ModelFact)
for rel in footnotesRelationshipSet.toModelObject(elt)):
orphanedFootnotes.add(elt)
if not elt.xmlLang:
noLangFootnotes.add(elt)
if elt.role != XbrlConst.footnote or not all(
rel.arcrole == XbrlConst.factFootnote and rel.linkrole == XbrlConst.defaultLinkRole
for rel in footnotesRelationshipSet.toModelObject(elt)):
footnoteRoleErrors.add(elt)
# check file name of each inline document (which might be below a top-level IXDS)
for doc in modelXbrl.urlDocs.values():
if doc.type == ModelDocument.Type.INLINEXBRL:
_baseName, _baseExt = os.path.splitext(doc.basename)
if _baseExt not in (".xhtml",):
modelXbrl.warning("esma:TBD.fileNameExtension",
_("FileName should have the extension .xhtml: %(fileName)s"),
modelObject=doc, fileName=doc.basename)
if modelDocument.type in (ModelDocument.Type.INLINEXBRL, ModelDocument.Type.INLINEXBRLDOCUMENTSET):
ixNStag = modelXbrl.modelDocument.ixNStag
ixTags = set(ixNStag + ln for ln in ("nonNumeric", "nonFraction", "references", "relationship"))
ixTextTags = set(ixNStag + ln for ln in ("nonFraction", "continuation", "footnote"))
ixExcludeTag = ixNStag + "exclude"
ixTupleTag = ixNStag + "tuple"
ixFractionTag = ixNStag + "fraction"
hiddenEltIds = {}
presentedHiddenEltIds = defaultdict(list)
eligibleForTransformHiddenFacts = []
requiredToDisplayFacts = []
requiredToDisplayFactIds = {}
firstIxdsDoc = True
for ixdsHtmlRootElt in modelXbrl.ixdsHtmlElements: # ix root elements for all ix docs in IXDS
for elt, depth in etreeIterWithDepth(ixdsHtmlRootElt):
eltTag = elt.tag
if isinstance(elt, ModelObject) and elt.namespaceURI == xhtml:
eltTag = elt.localName
if firstIxdsDoc and (not reportXmlLang or depth < firstRootmostXmlLangDepth):
xmlLang = elt.get("{http://www.w3.org/XML/1998/namespace}lang")
if xmlLang:
reportXmlLang = xmlLang
firstRootmostXmlLangDepth = depth
elif isinstance(elt, (_ElementTree, _Comment, _ProcessingInstruction)):
continue # comment or other non-parsed element
else:
eltTag = elt.tag
if eltTag.startswith(_xhtmlNs):
eltTag = eltTag[_xhtmlNsLen:]
if ((eltTag in ("object", "script")) or
(eltTag == "a" and "javascript:" in elt.get("href","")) or
(eltTag == "img" and "javascript:" in elt.get("src",""))):
modelXbrl.error("esma.2.5.1.executableCodePresent",
_("Inline XBRL documents MUST NOT contain executable code: %(element)s"),
modelObject=elt, element=eltTag)
elif eltTag == "img":
src = elt.get("src","").strip()
hasParentIxTextTag = False # check if image is in an ix text-bearing element
_ancestorElt = elt
while (_ancestorElt is not None):
if _ancestorElt.tag == ixExcludeTag: # excluded from any parent text-bearing ix element
break
if _ancestorElt.tag in ixTextTags:
hasParentIxTextTag = True
break
_ancestorElt = _ancestorElt.getparent()
if scheme(href) in ("http", "https", "ftp"):
modelXbrl.error("esma.3.5.1.inlinXbrlContainsExternalReferences",
_("Inline XBRL instance documents MUST NOT contain any reference pointing to resources outside the reporting package: %(element)s"),
modelObject=elt, element=eltTag)
if not src.startswith("data:image"):
if hasParentIxTextTag:
modelXbrl.error("esma.2.5.1.imageInIXbrlElementNotEmbedded",
_("Images appearing within an inline XBRL element MUST be embedded regardless of their size."),
modelObject=elt)
else:
# presume it to be an image file, check image contents
try:
base = elt.modelDocument.baseForElement(elt)
normalizedUri = elt.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(graphicFile, base)
if not elt.modelXbrl.fileSource.isInArchive(normalizedUri):
normalizedUri = elt.modelXbrl.modelManager.cntlr.webCache.getfilename(normalizedUri)
imglen = 0
with elt.modelXbrl.fileSource.file(normalizedUri,binary=True)[0] as fh:
imglen += len(fh.read())
if imglen < browserMaxBase64ImageLength:
modelXbrl.error("esma.2.5.1.embeddedImageNotUsingBase64Encoding",
_("Images MUST be included in the XHTML document as a base64 encoded string unless their size exceeds support of browsers."),
modelObject=elt)
except IOError as err:
modelXbrl.error("esma.2.5.1.imageFileCannotBeLoaded",
_("Image file which isn't openable '%(src)s', error: %(error)s"),
modelObject=elt, src=src, error=err)
elif not any(src.startswith(m) for m in allowedImgMimeTypes):
modelXbrl.error("esma.2.5.1.embeddedImageNotUsingBase64Encoding",
_("Images MUST be included in the XHTML document as a base64 encoded string, encoding disallowed: %(src)s."),
modelObject=elt, src=attrValue[:128])
elif eltTag == "a":
href = elt.get("href","").strip()
if scheme(href) in ("http", "https", "ftp"):
modelXbrl.error("esma.3.5.1.inlinXbrlContainsExternalReferences",
_("Inline XBRL instance documents MUST NOT contain any reference pointing to resources outside the reporting package: %(element)s"),
modelObject=elt, element=eltTag)
elif eltTag == "base" or elt.tag == "{http://www.w3.org/XML/1998/namespace}base":
modelXbrl.error("esma.2.4.2.htmlOrXmlBaseUsed",
_("The HTML <base> elements and xml:base attributes MUST NOT be used in the Inline XBRL document."),
modelObject=elt, element=eltTag)
if eltTag in ixTags and elt.get("target"):
modelXbrl.error("esma.2.5.3.targetAttributeUsed",
_("Target attribute MUST not be used: element %(localName)s, target attribute %(target)s."),
modelObject=elt, localName=elt.elementQname, target=elt.get("target"))
if eltTag == ixTupleTag:
modelXbrl.error("esma.2.4.1.tupleElementUsed",
_("The ix:tuple element MUST not be used in the Inline XBRL document."),
modelObject=elt)
if eltTag == ixFractionTag:
modelXbrl.error("esma.2.4.1.fractionElementUsed",
_("The ix:fraction element MUST not be used in the Inline XBRL document."),
modelObject=elt)
if elt.get("{http://www.w3.org/XML/1998/namespace}base") is not None:
modelXbrl.error("esma.2.4.1.xmlBaseUsed",
_("xml:base attributes MUST NOT be used in the Inline XBRL document: element %(localName)s, base attribute %(base)s."),
modelObject=elt, localName=elt.elementQname, base=elt.get("{http://www.w3.org/XML/1998/namespace}base"))
if isinstance(elt, ModelInlineFootnote):
checkFootnote(elt, elt.value)
elif isinstance(elt, ModelResource) and elt.qname == XbrlConst.qnLinkFootnote:
checkFootnote(elt, elt.value)
elif isinstance(elt, ModelInlineFact):
if elt.format is not None and elt.format.namespaceURI != 'http://www.xbrl.org/inlineXBRL/transformation/2015-02-26':
transformRegistryErrors.add(elt)
for ixHiddenElt in ixdsHtmlRootElt.iterdescendants(tag=ixNStag + "hidden"):
for tag in (ixNStag + "nonNumeric", ixNStag+"nonFraction"):
for ixElt in ixHiddenElt.iterdescendants(tag=tag):
if (getattr(ixElt, "xValid", 0) >= VALID # may not be validated
): # add future "and" conditions on elements which can be in hidden
if (ixElt.concept.baseXsdType not in untransformableTypes and
not ixElt.isNil):
eligibleForTransformHiddenFacts.append(ixElt)
elif ixElt.id is None:
requiredToDisplayFacts.append(ixElt)
if ixElt.id:
hiddenEltIds[ixElt.id] = ixElt
firstIxdsDoc = False
if eligibleForTransformHiddenFacts:
modelXbrl.warning("esma.2.4.1.transformableElementIncludedInHiddenSection",
_("The ix:hidden section of Inline XBRL document MUST not include elements eligible for transformation. "
"%(countEligible)s fact(s) were eligible for transformation: %(elements)s"),
modelObject=eligibleForTransformHiddenFacts,
countEligible=len(eligibleForTransformHiddenFacts),
elements=", ".join(sorted(set(str(f.qname) for f in eligibleForTransformHiddenFacts))))
for ixdsHtmlRootElt in modelXbrl.ixdsHtmlElements:
for ixElt in ixdsHtmlRootElt.getroottree().iterfind("//{http://www.w3.org/1999/xhtml}*[@style]"):
hiddenFactRefMatch = styleIxHiddenPattern.match(ixElt.get("style",""))
if hiddenFactRefMatch:
hiddenFactRef = hiddenFactRefMatch.group(2)
if hiddenFactRef not in hiddenEltIds:
modelXbrl.error("esma.2.4.1.esefIxHiddenStyleNotLinkingFactInHiddenSection",
_("\"-esef-ix-hidden\" style identifies @id, %(id)s of a fact that is not in ix:hidden section."),
modelObject=ixElt, id=hiddenFactRef)
else:
presentedHiddenEltIds[hiddenFactRef].append(ixElt)
for hiddenEltId, ixElt in hiddenEltIds.items():
if (hiddenEltId not in presentedHiddenEltIds and
getattr(ixElt, "xValid", 0) >= VALID and # may not be validated
(ixElt.concept.baseXsdType in untransformableTypes or ixElt.isNil)):
requiredToDisplayFacts.append(ixElt)
if requiredToDisplayFacts:
modelXbrl.warning("esma.2.4.1.factInHiddenSectionNotInReport",
_("The ix:hidden section contains %(countUnreferenced)s fact(s) whose @id is not applied on any \"-esef-ix- hidden\" style: %(elements)s"),
modelObject=requiredToDisplayFacts,
countUnreferenced=len(requiredToDisplayFacts),
elements=", ".join(sorted(set(str(f.qname) for f in requiredToDisplayFacts))))
del eligibleForTransformHiddenFacts, hiddenEltIds, presentedHiddenEltIds, requiredToDisplayFacts
elif modelDocument.type == ModelDocument.Type.INSTANCE:
for elt in modelDocument.xmlRootElement.iter():
if elt.qname == XbrlConst.qnLinkFootnote: # for now assume no private elements extend link:footnote
checkFootnote(elt, elt.stringValue)
contextsWithDisallowedOCEs = []
contextsWithDisallowedOCEcontent = []
contextsWithPeriodTime = []
contextsWithPeriodTimeZone = []
contextIdentifiers = defaultdict(list)
nonStandardTypedDimensions = defaultdict(set)
for context in modelXbrl.contexts.values():
if XmlUtil.hasChild(context, XbrlConst.xbrli, "segment"):
contextsWithDisallowedOCEs.append(context)
for segScenElt in context.iterdescendants("{http://www.xbrl.org/2003/instance}scenario"):
if isinstance(segScenElt,ModelObject):
if any(True for child in segScenElt.iterchildren()
if isinstance(child,ModelObject) and
child.tag not in ("{http://xbrl.org/2006/xbrldi}explicitMember",
"{http://xbrl.org/2006/xbrldi}typedMember")):
contextsWithDisallowedOCEcontent.append(context)
# check periods here
contextIdentifiers[context.entityIdentifier].append(context)
if contextsWithDisallowedOCEs:
modelXbrl.error("esma.2.1.3.segmentUsed",
_("xbrli:segment container MUST NOT be used in contexts: %(contextIds)s"),
modelObject=contextsWithDisallowedOCEs, contextIds=", ".join(c.id for c in contextsWithDisallowedOCEs))
if contextsWithDisallowedOCEcontent:
modelXbrl.error("esma.2.1.3.scenarioContainsNonDimensionalContent",
_("xbrli:scenario in contexts MUST NOT contain any other content than defined in XBRL Dimensions specification: %(contextIds)s"),
modelObject=contextsWithDisallowedOCEcontent, contextIds=", ".join(c.id for c in contextsWithDisallowedOCEcontent))
if len(contextIdentifiers) > 1:
modelXbrl.error("esma.2.1.4.multipleIdentifiers",
_("All entity identifiers in contexts MUST have identical content: %(contextIdentifiers)s"),
modelObject=modelXbrl, contextIds=", ".join(i[1] for i in contextIdentifiers))
for (contextScheme, contextIdentifier), contextElts in contextIdentifiers.items():
if contextScheme != "http://standards.iso.org/iso/17442":
modelXbrl.warning("esma.2.1.1.nonLEIContextScheme",
_("The scheme attribute of the xbrli:identifier element should have \"http://standards.iso.org/iso/17442\" as its content: %(scheme)s"),
modelObject=contextElts, scheme=contextScheme)
else:
leiValidity = LeiUtil.checkLei(contextIdentifier)
if leiValidity == LeiUtil.LEI_INVALID_LEXICAL:
modelXbrl.warning("esma.2.1.1.invalidIdentifierFormat",
_("The LEI context idenntifier has an invalid format: %(identifier)s"),
modelObject=contextElts, identifier=contextIdentifier)
elif leiValidity == LeiUtil.LEI_INVALID_CHECKSUM:
modelXbrl.warning("esma.2.1.1.invalidIdentifier",
_("The LEI context idenntifier has checksum error: %(identifier)s"),
modelObject=contextElts, identifier=contextIdentifier)
if contextsWithPeriodTime:
modelXbrl.warning("esma.2.1.2.periodWithTimeContent",
_("Context period startDate, endDate and instant elements should be in whole days without time: %(contextIds)s"),
modelObject=contextsWithPeriodTime, contextIds=", ".join(c.id for c in contextsWithPeriodTime))
if contextsWithPeriodTimeZone:
modelXbrl.warning("esma.2.1.2.periodWithTimeZone",
_("Context period startDate, endDate and instant elements should be in whole days without a timezone: %(contextIds)s"),
modelObject=contextsWithPeriodTimeZone, contextIds=", ".join(c.id for c in contextsWithPeriodTimeZone))
# identify unique contexts and units
mapContext = {}
mapUnit = {}
uniqueContextHashes = {}
for context in modelXbrl.contexts.values():
h = context.contextDimAwareHash
if h in uniqueContextHashes:
if context.isEqualTo(uniqueContextHashes[h]):
mapContext[context] = uniqueContextHashes[h]
else:
uniqueContextHashes[h] = context
del uniqueContextHashes
uniqueUnitHashes = {}
for unit in modelXbrl.units.values():
h = unit.hash
if h in uniqueUnitHashes:
if unit.isEqualTo(uniqueUnitHashes[h]):
mapUnit[unit] = uniqueUnitHashes[h]
else:
uniqueUnitHashes[h] = unit
del uniqueUnitHashes
reportedMandatory = set()
precisionFacts = set()
numFactsByConceptContextUnit = defaultdict(list)
textFactsByConceptContext = defaultdict(list)
footnotesRelationshipSet = modelXbrl.relationshipSet(XbrlConst.factFootnote, XbrlConst.defaultLinkRole)
noLangFacts = []
textFactsMissingReportLang = []
conceptsUsed = set()
for qn, facts in modelXbrl.factsByQname.items():
if qn in mandatory:
reportedMandatory.add(qn)
for f in facts:
if f.precision is not None:
precisionFacts.add(f)
if f.isNumeric:
numFactsByConceptContextUnit[(f.qname, mapContext.get(f.context,f.context), mapUnit.get(f.unit, f.unit))].append(f)
elif f.concept is not None and f.concept.type is not None:
if f.concept.type.isOimTextFactType:
if not f.xmlLang:
noLangFacts.append(f)
elif f.context is not None:
textFactsByConceptContext[(f.qname, mapContext.get(f.context,f.context))].append(f)
conceptsUsed.add(f.concept)
if f.context is not None:
for dim in f.context.qnameDims.values():
conceptsUsed.add(dim.dimension)
if dim.isExplicit:
conceptsUsed.add(dim.member)
elif dim.isTyped:
conceptsUsed.add(dim.typedMember)
if noLangFacts:
modelXbrl.error("esma.2.5.2.undefinedLanguageForTextFact",
_("Each tagged text fact MUST have the 'xml:lang' attribute assigned or inherited."),
modelObject=noLangFacts)
# missing report lang text facts
for fList in textFactsByConceptContext.values():
if not any(f.xmlLang == reportXmlLang for f in fList):
modelXbrl.error("esma.2.5.2.taggedTextFactOnlyInLanguagesOtherThanLanguageOfAReport",
_("Each tagged text fact MUST have the 'xml:lang' provided in at least the language of the report: %(element)s"),
modelObject=fList, element=fList[0].qname)
# 2.2.4 test
for fList in numFactsByConceptContextUnit.values():
if len(fList) > 1:
f0 = fList[0]
if any(f.isNil for f in fList):
_inConsistent = not all(f.isNil for f in fList)
elif all(inferredDecimals(f) == inferredDecimals(f0) for f in fList[1:]): # same decimals
v0 = rangeValue(f0.value)
_inConsistent = not all(rangeValue(f.value) == v0 for f in fList[1:])
else: # not all have same decimals
aMax, bMin = rangeValue(f0.value, inferredDecimals(f0))
for f in fList[1:]:
a, b = rangeValue(f.value, inferredDecimals(f))
if a > aMax: aMax = a
if b < bMin: bMin = b
_inConsistent = (bMin < aMax)
if _inConsistent:
modelXbrl.error(("esma:2.2.4.inconsistentDuplicateNumericFactInInlineXbrlDocument"),
"Inconsistent duplicate numeric facts MUST NOT appear in the content of an inline XBRL document. %(fact)s that was used more than once in contexts equivalent to %(contextID)s: values %(values)s. ",
modelObject=fList, fact=f0.qname, contextID=f0.contextID, values=", ".join(strTruncate(f.value, 128) for f in fList))
if precisionFacts:
modelXbrl.warning("esma:2.2.1.precisionAttributeUsed",
_("The accuracy of numeric facts SHOULD be defined with the 'decimals' attribute rather than the 'precision' attribute: %(elements)s."),
modelObject=precisionFacts, elements=", ".join(sorted(str(qn) for qn in precisionFacts)))
missingElements = (mandatory - reportedMandatory)
if missingElements:
modelXbrl.error("esma:???.missingRequiredElements",
_("Required elements missing from document: %(elements)s."),
modelObject=modelXbrl, elements=", ".join(sorted(str(qn) for qn in missingElements)))
if transformRegistryErrors:
modelXbrl.warning("esma:2.2.3.transformRegistry",
_("ESMA recommends applying the latest available version of the Transformation Rules Registry marked with 'Recommendation' status for these elements: %(elements)s."),
modelObject=transformRegistryErrors,
elements=", ".join(sorted(str(fact.qname) for fact in transformRegistryErrors)))
if orphanedFootnotes:
modelXbrl.error("esma.2.3.1.unusedFootnote",
_("Non-empty footnotes must be connected to fact(s)."),
modelObject=orphanedFootnotes)
if noLangFootnotes:
modelXbrl.error("esma.2.3.2.undefinedLanguageForFootnote",
_("Each footnote MUST have the 'xml:lang' attribute whose value corresponds to the language of the text in the content of the respective footnote."),
modelObject=noLangFootnotes)
if footnoteRoleErrors:
modelXbrl.error("esma.2.3.2.nonStandardRoleForFootnote",
_("The xlink:role attribute of a link:footnote and link:footnoteLink element as well as xlink:arcrole attribute of a link:footnoteArc MUST be defined in the XBRL Specification 2.1."),
modelObject=footnoteRoleErrors)
nonStdFootnoteElts = list()
for modelLink in modelXbrl.baseSets[("XBRL-footnotes",None,None,None)]:
for elt in ixdsHtmlRootElt.iter():
if isinstance(elt, (_ElementTree, _Comment, _ProcessingInstruction)):
continue # comment or other non-parsed element
if elt.namespaceURI != link or elt.localName not in ("loc", "link", "footnoteArc"):
nonStdFootnoteElts.append(elt)
if nonStdFootnoteElts:
modelXbrl.error("esma.2.3.2.nonStandardElementInFootnote",
_("A link:footnoteLink element MUST have no children other than link:loc, link:footnote, and link:footnoteArc."),
modelObject=nonStdFootnoteElts)
for qn in modelXbrl.qnameDimensionDefaults.values():
conceptsUsed.add(modelXbrl.qnameConcepts.get(qn))
# unused elements in linkbases
for arcroles, err in (((parentChild,), "elementsNotUsedForTaggingAppliedInPresentationLinkbase"),
((summationItem,), "elementsNotUsedForTaggingAppliedInCalculationLinkbase"),
((dimensionDomain,domainMember), "elementsNotUsedForTaggingAppliedInDefinitionLinkbase")):
lbElts = set()
for arcrole in arcroles:
for rel in modelXbrl.relationshipSet(arcrole).modelRelationships:
fr = rel.fromModelObject
to = rel.toModelObject
if arcrole in (parentChild, summationItem):
if fr is not None and not fr.isAbstract:
lbElts.add(fr)
if to is not None and not to.isAbstract:
lbElts.add(to)
elif arcrole == dimensionDomain:
if fr is not None: # dimension, always abstract
lbElts.add(fr)
if to is not None and rel.isUsable:
lbElts.add(to)
elif arcrole == domainMember:
if to is not None and rel.isUsable:
lbElts.add(to)
unreportedLbElts = lbElts - conceptsUsed
if unreportedLbElts:
modelXbrl.error("esma.3.2.6." + err,
_("All usable concepts in extension taxonomy relationships MUST be applied by tagged facts: %(elements)s."),
modelObject=unreportedLbElts, elements=", ".join(sorted((str(c.qname) for c in unreportedLbElts))))
modelXbrl.profileActivity(_statusMsg, minTimeToShow=0.0)
modelXbrl.modelManager.showStatus(None)
def factAspects(fact):
oimFact = OrderedDict()
aspects = OrderedDict()
if hasId and fact.id:
if fact.isTuple:
oimFact["tupleId"] = fact.id
else:
oimFact["id"] = fact.id
elif (fact.isTuple or
footnotesRelationshipSet.toModelObject(fact) or
(isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))):
oimFact["id"] = "f{}".format(fact.objectIndex)
parent = fact.getparent()
concept = fact.concept
aspects[str(qnOimConceptAspect)] = oimValue(concept.qname)
_csvType = "Value"
if not fact.isTuple:
if concept is not None:
_baseXsdType = concept.baseXsdType
if _baseXsdType == "XBRLI_DATEUNION":
if getattr(fact.xValue, "dateOnly", False):
_baseXsdType = "date"
else:
_baseXsdType = "dateTime"
_csvType = baseTypes.get(_baseXsdType,_baseXsdType) + "Value"
if concept.baseXbrliType in ("stringItemType", "normalizedStringItemType") and fact.xmlLang:
aspects[str(qnOimLangAspect)] = fact.xmlLang
if fact.isItem:
if fact.isNil:
_value = None
else:
_inferredDecimals = inferredDecimals(fact)
_value = oimValue(fact.xValue, _inferredDecimals)
oimFact["value"] = _value
if fact.concept is not None and fact.concept.isNumeric:
_numValue = fact.xValue
if isinstance(_numValue, Decimal) and not isinf(_numValue) and not isnan(_numValue):
if _numValue == _numValue.to_integral():
_numValue = int(_numValue)
else:
_numValue = float(_numValue)
if not fact.isNil:
if not isinf(_inferredDecimals): # accuracy omitted if infinite
oimFact["decimals"] = _inferredDecimals
oimFact["dimensions"] = aspects
cntx = fact.context
if cntx is not None:
if cntx.entityIdentifierElement is not None:
aspects[str(qnOimEntityAspect)] = oimValue(qname(*cntx.entityIdentifier))
if cntx.period is not None:
aspects.update(oimPeriodValue(cntx))
for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]):
if dim.isExplicit:
dimVal = oimValue(dim.memberQname)
else: # typed
if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1"):
dimVal = None
else:
dimVal = dim.typedMember.stringValue
aspects[str(dim.dimensionQname)] = dimVal
unit = fact.unit
if unit is not None:
_mMul, _mDiv = unit.measures
_sMul = '*'.join(oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m)))
if _mDiv:
_sDiv = '*'.join(oimValue(m) for m in sorted(_mDiv, key=lambda m: oimValue(m)))
if len(_mDiv) > 1:
if len(_mMul) > 1:
_sUnit = "({})/({})".format(_sMul,_sDiv)
else:
_sUnit = "{}/({})".format(_sMul,_sDiv)
else:
if len(_mMul) > 1:
_sUnit = "({})/{}".format(_sMul,_sDiv)
else:
_sUnit = "{}/{}".format(_sMul,_sDiv)
else:
_sUnit = _sMul
aspects[str(qnOimUnitAspect)] = _sUnit
# Tuples removed from xBRL-JSON
#if parent.qname != XbrlConst.qnXbrliXbrl:
# aspects[str(qnOimTupleParentAspect)] = parent.id if parent.id else "f{}".format(parent.objectIndex)
# aspects[str(qnOimTupleOrderAspect)] = elementIndex(fact)
if isJSON:
factFootnotes(fact, oimFact)
return oimFact
def factAspects(fact):
aspects = OrderedDict()
if hasId and fact.id:
aspects["id"] = fact.id
elif (fact.isTuple or
footnotesRelationshipSet.toModelObject(fact) or
(isCSV and footnotesRelationshipSet.fromModelObject(fact))):
aspects["id"] = "f{}".format(fact.objectIndex)
parent = fact.getparent()
concept = fact.concept
if not fact.isTuple:
if concept is not None:
_baseXsdType = concept.baseXsdType
if _baseXsdType == "XBRLI_DATEUNION":
if getattr(fact.xValue, "dateOnly", False):
_baseXsdType = "date"
else:
_baseXsdType = "dateTime"
aspects["baseType"] = "xs:{}".format(_baseXsdType)
if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang:
aspects[qnOimLangAspect] = fact.xmlLang
aspects[qnOimTypeAspect] = concept.baseXbrliType
if fact.isItem:
if fact.isNil:
_value = None
_strValue = "nil"
else:
_inferredDecimals = inferredDecimals(fact)
_value = oimValue(fact.xValue, _inferredDecimals)
_strValue = str(_value)
if not isCSV:
aspects["value"] = _strValue
if fact.concept is not None and fact.concept.isNumeric:
_numValue = fact.xValue
if isinstance(_numValue, Decimal) and not isinf(_numValue) and not isnan(_numValue):
if _numValue == _numValue.to_integral():
_numValue = int(_numValue)
else:
_numValue = float(_numValue)
aspects["numericValue"] = _numValue
if not fact.isNil:
if isinf(_inferredDecimals):
if isJSON: _accuracy = "infinity"
elif isCSV: _accuracy = "INF"
else:
_accuracy = _inferredDecimals
aspects["accuracy"] = _inferredDecimals
elif isinstance(_value, bool):
aspects["booleanValue"] = _value
elif isCSV:
aspects["stringValue"] = _strValue
aspects[qnOimConceptAspect] = oimValue(fact.qname)
cntx = fact.context
if cntx is not None:
if cntx.entityIdentifierElement is not None:
aspects[qnOimEntityAspect] = oimValue(qname(*cntx.entityIdentifier))
if cntx.period is not None:
if isJSON:
aspects[qnOimPeriodAspect] = oimPeriodValue(cntx)
elif isCSV:
_periodValue = oimPeriodValue(cntx).split("/") + ["", ""] # default blank if no value
aspects[qnOimPeriodStartAspect] = _periodValue[0]
aspects[qnOimPeriodDurationAspect] = _periodValue[1]
for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]):
aspects[dim.dimensionQname] = (oimValue(dim.memberQname) if dim.isExplicit
else None if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1")
else dim.typedMember.stringValue)
unit = fact.unit
if unit is not None:
_mMul, _mDiv = unit.measures
if isJSON:
aspects[qnOimUnitAspect] = OrderedDict( # use tuple instead of list for hashability
(("numerators", tuple(oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m)))),)
)
if _mDiv:
aspects[qnOimUnitAspect]["denominators"] = tuple(oimValue(m) for m in sorted(_mDiv, key=lambda m: oimValue(m)))
else: # CSV
if _mMul:
aspects[qnOimUnitNumeratorsAspect] = " ".join(oimValue(m)
for m in sorted(_mMul, key=lambda m: str(m)))
if _mDiv:
aspects[qnOimUnitDenominatorsAspect] = " ".join(oimValue(m)
for m in sorted(_mDiv, key=lambda m: str(m)))
if parent.qname != XbrlConst.qnXbrliXbrl:
aspects[qnOimTupleParentAspect] = parent.id if parent.id else "f{}".format(parent.objectIndex)
aspects[qnOimTupleOrderAspect] = elementIndex(fact)
if isJSON:
_footnotes = factFootnotes(fact)
if _footnotes:
aspects["footnotes"] = _footnotes
return aspects
def validate(self):
modelXbrl = self.modelXbrl
if not modelXbrl.contexts or not modelXbrl.facts:
return # skip if no contexts or facts
if not self.val.validateInferDecimals: # infering precision is now contrary to XBRL REC section 5.2.5.2
modelXbrl.error("calc2e:inferringPrecision",
"Calc2 requires inferring decimals.")
return
startedAt = time.time()
# check balance attributes and weights, same as XBRL 2.1
for rel in modelXbrl.relationshipSet(calc2Arcroles).modelRelationships:
weight = rel.weight
fromConcept = rel.fromModelObject
toConcept = rel.toModelObject
if fromConcept is not None and toConcept is not None:
if rel.arcrole == aggregationDomain:
rel.dimension = rel.arcElement.prefixedNameQname(
rel.get("dimension"))
if rel.dimension is None or not modelXbrl.qnameConcepts[
rel.dimension].isDimensionItem:
modelXbrl.error(
"calc2e:invalidAggregationDimension",
_("Aggregation-domain relationship has invalid dimension %(dimension)s in link role %(linkrole)s"
),
modelObject=rel,
dimension=rel.get("dimension"),
linkrole=ELR)
elif fromConcept != toConcept or not fromConcept.isDomainMember:
modelXbrl.error(
"calc2e:invalidAggregationDomain",
_("Calculation relationship has invalid domain %(domain)s in link role %(linkrole)s"
),
modelObject=rel,
domain=fromConcept,
linkrole=ELR)
continue
if rel.arcrole == balanceChanges:
if fromConcept.periodType != "instant" or toConcept.periodType != "duration":
modelXbrl.error(
"calc2e:invalidBalanceChangesPeriodType",
_("Balance-changes relationship must have instant source concept and duration target concept in link role %(linkrole)s"
),
modelObject=rel,
linkrole=ELR)
if weight not in (1, -1):
modelXbrl.error(
"calc2e:invalidWeight",
_("Calculation relationship has invalid weight from %(source)s to %(target)s in link role %(linkrole)s"
),
modelObject=rel,
source=fromConcept.qname,
target=toConcept.qname,
linkrole=ELR)
fromBalance = fromConcept.balance
toBalance = toConcept.balance
if fromBalance and toBalance:
if (fromBalance == toBalance and weight < 0) or \
(fromBalance != toBalance and weight > 0):
modelXbrl.error(
"calc2e:balanceCalcWeightIllegal" +
("Negative" if weight < 0 else "Positive"),
_("Calculation relationship has illegal weight %(weight)s from %(source)s, %(sourceBalance)s, to %(target)s, %(targetBalance)s, in link role %(linkrole)s (per 5.1.1.2 Table 6)"
),
modelObject=rel,
weight=weight,
source=fromConcept.qname,
target=toConcept.qname,
linkrole=rel.linkrole,
sourceBalance=fromBalance,
targetBalance=toBalance,
messageCodes=(
"calc2e:balanceCalcWeightIllegalNegative",
"calc2:balanceCalcWeightIllegalPositive"))
if not fromConcept.isNumeric or not toConcept.isNumeric:
modelXbrl.error(
"calc2e:nonNumericCalc",
_("Calculation relationship has illegal concept from %(source)s%(sourceNumericDecorator)s to %(target)s%(targetNumericDecorator)s in link role %(linkrole)s"
),
modelObject=rel,
source=fromConcept.qname,
target=toConcept.qname,
linkrole=rel.linkrole,
sourceNumericDecorator=""
if fromConcept.isNumeric else _(" (non-numeric)"),
targetNumericDecorator=""
if toConcept.isNumeric else _(" (non-numeric)"))
# identify equal contexts
uniqueCntxHashes = {}
self.modelXbrl.profileActivity()
for cntx in modelXbrl.contexts.values():
h = hash((cntx.periodHash, cntx.entityIdentifierHash,
cntx.dimsHash)) # OIM-compatible hash
if h in uniqueCntxHashes:
if cntx.isEqualTo(uniqueCntxHashes[h]):
self.eqCntx[cntx] = uniqueCntxHashes[h]
else:
uniqueCntxHashes[h] = cntx
del uniqueCntxHashes
self.modelXbrl.profileActivity("... identify aspect equal contexts",
minTimeToShow=1.0)
# identify equal units
uniqueUnitHashes = {}
for unit in self.modelXbrl.units.values():
h = unit.hash
if h in uniqueUnitHashes:
if unit.isEqualTo(uniqueUnitHashes[h]):
self.eqUnit[unit] = uniqueUnitHashes[h]
else:
uniqueUnitHashes[h] = unit
del uniqueUnitHashes
self.modelXbrl.profileActivity("... identify equal units",
minTimeToShow=1.0)
sectObjs = sorted(
set(rel.fromModelObject # only have numerics with context and unit
for rel in modelXbrl.relationshipSet(
sectionFact).modelRelationships
if rel.fromModelObject is not None
and rel.fromModelObject.concept is not None),
key=lambda s: (s.concept.label(), s.objectIndex)
) # sort into document order for consistent error messages
if not sectObjs:
self.modelXbrl.error(
"calc2e:noSections",
"Instance contains no sections, nothing to validate.",
modelObject=modelXbrl)
# check by section
factByConceptCntxUnit = OrderedDefaultDict(
list) # sort into document order for consistent error messages
self.sectionFacts = []
for sectObj in sectObjs:
#print ("section {}".format(sectObj.concept.label()))
self.section = sectObj.concept.label()
sectLinkRoles = tuple(
sectObj.concept.get(calc2linkroles, "").split())
factByConceptCntxUnit.clear()
for f in sorted(
(
rel.
toModelObject # sort into document order for consistent error messages
for rel in modelXbrl.relationshipSet(
sectionFact, sectLinkRoles).fromModelObject(sectObj)
if rel.toModelObject is not None
and # numeric facts with context and unit
rel.fromModelObject is not None and rel.toModelObject.
concept is not None and rel.toModelObject.context
is not None and rel.toModelObject.unit is not None),
key=lambda f: f.objectIndex):
factByConceptCntxUnit[f.qname,
self.eqCntx.get(f.context, f.context),
self.eqUnit.get(f.unit, f.unit)].append(
f)
for fList in factByConceptCntxUnit.values():
f0 = fList[0]
if len(fList) == 1:
self.sectionFacts.append(f0)
else:
if any(f.isNil for f in fList):
_inConsistent = not all(f.isNil for f in fList)
if _inConsistent: # pick a nil fact for f0 for calc validation
for f in fList:
if f.isNil:
f0 = f
break
elif all(
inferredDecimals(f) == inferredDecimals(f0)
for f in fList[1:]): # same decimals
v0 = intervalValue(f0)
_inConsistent = not all(
intervalValue(f) == v0 for f in fList[1:])
else: # not all have same decimals
d0 = inferredDecimals(f0)
aMax, bMin = intervalValue(f0, d0)
for f in fList[1:]:
df = inferredDecimals(f0)
a, b = intervalValue(f, df)
if a > aMax: aMax = a
if b < bMin: bMin = b
if df > d0: # take most accurate fact in section
f0 = f
d0 = df
_inConsistent = (bMin < aMax)
if _inConsistent:
modelXbrl.error(
"calc2e:inconsistentDuplicateInSection",
"Section %(section)s contained %(fact)s inconsistent in contexts equivalent to %(contextID)s: values %(values)s",
modelObject=fList,
section=sectObj.concept.label(),
fact=f0.qname,
contextID=f0.contextID,
values=", ".join(
strTruncate(f.value, 128) for f in fList))
self.sectionFacts.append(f0)
# sectionFacts now in document order and deduplicated
#print("section {} facts {}".format(sectObj.concept.label(), ", ".join(str(f.qname)+"="+f.value for f in self.sectionFacts)))
# depth-first calc tree
sectCalc2RelSet = modelXbrl.relationshipSet(
calc2Arcroles, sectLinkRoles)
# indexers for section based on calc2 arcrole
self.sumInit = False
self.sumConceptBindKeys.clear()
self.sumBoundFacts.clear()
self.perInit = False
self.perConceptBindKeys.clear()
self.perBoundFacts.clear()
self.durationPeriodStarts.clear()
self.aggDimInit = set()
self.aggConceptBindKeys.clear()
self.aggBoundFacts.clear()
self.aggBoundConceptFacts.clear()
self.aggDimInit.clear()
inferredValues = {}
for rootConcept in sorted(
sectCalc2RelSet.rootConcepts,
key=lambda r: sectCalc2RelSet.fromModelObject(r)[0].order):
self.sectTreeRel(rootConcept, 1, sectCalc2RelSet,
inferredValues, {rootConcept, None})
def validateXbrlFinally(val, *args, **kwargs):
if not (val.validateROSplugin):
return
_xhtmlNs = "{{{}}}".format(xhtml)
_xhtmlNsLen = len(_xhtmlNs)
modelXbrl = val.modelXbrl
modelDocument = modelXbrl.modelDocument
if not modelDocument:
return # never loaded properly
_statusMsg = _("validating {0} filing rules").format(
val.disclosureSystem.name)
modelXbrl.profileActivity()
modelXbrl.modelManager.showStatus(_statusMsg)
if modelDocument.type == ModelDocument.Type.INSTANCE:
modelXbrl.error("ROS:instanceMustBeInlineXBRL",
_("ROS expects inline XBRL instances."),
modelObject=modelXbrl)
if modelDocument.type in (ModelDocument.Type.INLINEXBRL,
ModelDocument.Type.INLINEXBRLDOCUMENTSET):
for ixdsHtmlRootElt in modelXbrl.ixdsHtmlElements: # ix root elements for all ix docs in IXDS
ixNStag = ixdsHtmlRootElt.modelDocument.ixNStag
ixTags = set(ixNStag + ln for ln in ("nonNumeric", "nonFraction",
"references", "relationship"))
transformRegistryErrors = set()
ixTargets = set()
for ixdsHtmlRootElt in modelXbrl.ixdsHtmlElements:
for elt in ixdsHtmlRootElt.iter():
if isinstance(
elt, (_ElementTree, _Comment, _ProcessingInstruction)):
continue # comment or other non-parsed element
if isinstance(elt, ModelInlineFact):
if elt.format is not None and elt.format.namespaceURI not in TRnamespaces:
transformRegistryErrors.add(elt)
if elt.get("escape") in ("true", "1"):
modelXbrl.error(
"ROS.escapedHTML",
_("Escaped (x)html fact content is not supported: %(element)s"
),
modelObject=elt,
element=eltTag)
eltTag = elt.tag
if eltTag in ixTags:
ixTargets.add(elt.get("target"))
else:
if eltTag.startswith(_xhtmlNs):
eltTag = eltTag[_xhtmlNsLen:]
if eltTag == "link" and elt.get("type") == "text/css":
modelXbrl.error(
"ROS.externalCssStyle",
_("CSS must be embedded in the inline XBRL document: %(element)s"
),
modelObject=elt,
element=eltTag)
elif ((eltTag in ("object", "script"))
or (eltTag == "a"
and "javascript:" in elt.get("href", ""))
or (eltTag == "img"
and "javascript:" in elt.get("src", ""))):
modelXbrl.error(
"ROS.embeddedCode",
_("Inline XBRL documents MUST NOT contain embedded code: %(element)s"
),
modelObject=elt,
element=eltTag)
elif eltTag == "img":
src = elt.get("src", "").strip()
if not src.startswith("data:image"):
modelXbrl.warning(
"ROS.embeddedCode",
_("Images should be inlined as a base64-encoded string: %(element)s"
),
modelObject=elt,
element=eltTag)
if len(ixTargets) > 1:
modelXbrl.error(
"ROS:singleOutputDocument",
_("Multiple target instance documents are not supported: %(targets)s."
),
modelObject=modelXbrl,
targets=", ".join((t or "(default)") for t in ixTargets))
filingTypes = set()
unexpectedTaxonomyReferences = set()
numIxDocs = 0
for doc in modelXbrl.urlDocs.values():
if doc.type == ModelDocument.Type.INLINEXBRL:
# base file extension
_baseName, _baseExt = os.path.splitext(doc.basename)
if _baseExt not in (".xhtml", ".html", "htm", "ixbrl", "xml",
"xhtml"):
modelXbrl.error(
"ROS.fileNameExtension",
_("The list of acceptable file extensions for upload is: html, htm, ixbrl, xml, xhtml: %(fileName)s"
),
modelObject=doc,
fileName=doc.basename)
# document encoding
if doc.documentEncoding.lower() != "utf-8":
modelXbrl.error(
"ROS.documentEncoding",
_("iXBRL documents submitted to Revenue should be UTF-8 encoded: %(encoding)s"
),
modelObject=doc,
encoding=doc.documentEncoding)
# identify type of filing
for referencedDoc in doc.referencesDocument.keys():
if referencedDoc.type == ModelDocument.Type.SCHEMA:
if referencedDoc.uri in taxonomyReferences:
filingTypes.add(referencedDoc.uri)
else:
unexpectedTaxonomyReferences.add(referencedDoc.uri)
# count of inline docs in IXDS
numIxDocs += 1
if len(filingTypes) != 1:
modelXbrl.error(
"ROS:multipleFilingTypes",
_("Multiple filing types detected: %(filingTypes)s."),
modelObject=modelXbrl,
filingTypes=", ".join(sorted(filingTypes)))
if unexpectedTaxonomyReferences:
modelXbrl.error(
"ROS:unexpectedTaxonomyReferences",
_("Referenced schema(s) does not map to a taxonomy supported by Revenue (schemaRef): %(unexpectedReferences)s."
),
modelObject=modelXbrl,
unexpectedReferences=", ".join(
sorted(unexpectedTaxonomyReferences)))
# single document IXDS
if numIxDocs > 1:
modelXbrl.warning(
"ROS:multipleInlineDocuments",
_("A single inline document should be submitted but %(numberDocs)s were found."
),
modelObject=modelXbrl,
numberDocs=numIxDocs)
# build namespace maps
nsMap = {}
for prefix in ("ie-common", "bus", "uk-bus", "ie-dpl", "core"):
if prefix in modelXbrl.prefixedNamespaces:
nsMap[prefix] = modelXbrl.prefixedNamespaces[prefix]
# build mandatory table by ns qname in use
mandatory = set()
for prefix in mandatoryElements:
if prefix in nsMap:
ns = nsMap[prefix]
for localName in mandatoryElements[prefix]:
mandatory.add(qname(ns, prefix + ":" + localName))
equivalentManatoryQNames = [[
qname(q, nsMap) for q in equivElts
] for equivElts in equivalentMandatoryElements]
# document creator requirement
if "bus" in nsMap:
if qname("bus:NameProductionSoftware",
nsMap) not in modelXbrl.factsByQname or qname(
"bus:VersionProductionSoftware",
nsMap) not in modelXbrl.factsByQname:
modelXbrl.warning(
"ROS:documentCreatorProductInformation",
_("Please use the NameProductionSoftware tag to identify the software package and the VersionProductionSoftware tag to identify the version of the software package."
),
modelObject=modelXbrl)
elif "uk-bus" in nsMap:
if qname("uk-bus:NameAuthor",
nsMap) not in modelXbrl.factsByQname or qname(
"uk-bus:DescriptionOrTitleAuthor",
nsMap) not in modelXbrl.factsByQname:
modelXbrl.warning(
"ROS:documentCreatorProductInformation",
_("Revenue request that vendor, product and version information is embedded in the generated inline XBRL document using a single XBRLDocumentAuthorGrouping tuple. The NameAuthor tag should be used to identify the name and version of the software package."
),
modelObject=modelXbrl)
schemeEntityIds = set()
mapContext = {} # identify unique contexts and units
mapUnit = {}
uniqueContextHashes = {}
hasCRO = False
unsupportedSchemeContexts = []
mismatchIdentifierContexts = []
for context in modelXbrl.contexts.values():
schemeEntityIds.add(context.entityIdentifier)
scheme, entityId = context.entityIdentifier
if scheme not in schemePatterns:
unsupportedSchemeContexts.append(context)
elif not schemePatterns[scheme].match(entityId):
mismatchIdentifierContexts.append(context)
if scheme == "http://www.cro.ie/":
hasCRO = True
h = context.contextDimAwareHash
if h in uniqueContextHashes:
if context.isEqualTo(uniqueContextHashes[h]):
mapContext[context] = uniqueContextHashes[h]
else:
uniqueContextHashes[h] = context
del uniqueContextHashes
if len(schemeEntityIds) > 1:
modelXbrl.error(
"ROS:differentContextEntityIdentifiers",
_("Context entity identifier not all the same: %(schemeEntityIds)s."
),
modelObject=modelXbrl,
schemeEntityIds=", ".join(
sorted(str(s) for s in schemeEntityIds)))
if unsupportedSchemeContexts:
modelXbrl.error(
"ROS:unsupportedContextEntityIdentifierScheme",
_("Context identifier scheme(s) is not supported: %(schemes)s."
),
modelObject=unsupportedSchemeContexts,
schemes=", ".join(
sorted(
set(c.entityIdentifier[0]
for c in unsupportedSchemeContexts))))
if mismatchIdentifierContexts:
modelXbrl.error(
"ROS:invalidContextEntityIdentifier",
_("Context entity identifier(s) lexically invalid: %(identifiers)s."
),
modelObject=mismatchIdentifierContexts,
identifiers=", ".join(
sorted(
set(c.entityIdentifier[1]
for c in mismatchIdentifierContexts))))
uniqueUnitHashes = {}
for unit in modelXbrl.units.values():
h = unit.hash
if h in uniqueUnitHashes:
if unit.isEqualTo(uniqueUnitHashes[h]):
mapUnit[unit] = uniqueUnitHashes[h]
else:
uniqueUnitHashes[h] = unit
del uniqueUnitHashes
if hasCRO and "ie-common" in nsMap:
mandatory.add(
qname("ie-common:CompaniesRegistrationOfficeNumber", nsMap))
reportedMandatory = set()
numFactsByConceptContextUnit = defaultdict(list)
for qn, facts in modelXbrl.factsByQname.items():
if qn in mandatory:
reportedMandatory.add(qn)
for f in facts:
if f.isNumeric and f.parentElement.qname == qnXbrliXbrl:
numFactsByConceptContextUnit[(f.qname,
mapContext.get(
f.context, f.context),
mapUnit.get(
f.unit,
f.unit))].append(f)
missingElements = (
mandatory - reportedMandatory
) # | (reportedFootnoteIfNil - reportedFootnoteIfNil)
for qnames in equivalentManatoryQNames: # remove missing elements for which an or-match was reported
if any(qn in modelXbrl.factsByQname for qn in qnames):
for qn in qnames:
missingElements.discard(qn)
if missingElements:
modelXbrl.error(
"ROS:missingRequiredElements",
_("Required elements missing from document: %(elements)s."),
modelObject=modelXbrl,
elements=", ".join(sorted(str(qn) for qn in missingElements)))
for fList in numFactsByConceptContextUnit.values():
if len(fList) > 1:
f0 = fList[0]
if any(f.isNil for f in fList):
_inConsistent = not all(f.isNil for f in fList)
elif all(
inferredDecimals(f) == inferredDecimals(f0)
for f in fList[1:]): # same decimals
v0 = rangeValue(f0.value)
_inConsistent = not all(
rangeValue(f.value) == v0 for f in fList[1:])
else: # not all have same decimals
aMax, bMin = rangeValue(f0.value, inferredDecimals(f0))
for f in fList[1:]:
a, b = rangeValue(f.value, inferredDecimals(f))
if a > aMax: aMax = a
if b < bMin: bMin = b
_inConsistent = (bMin < aMax)
if _inConsistent:
modelXbrl.error(
("ROS.inconsistentDuplicateFacts"),
"Inconsistent duplicate numeric facts: %(fact)s were used more than once in contexts equivalent to %(contextID)s: values %(values)s. ",
modelObject=fList,
fact=f0.qname,
contextID=f0.contextID,
values=", ".join(
strTruncate(f.value, 128) for f in fList))
modelXbrl.profileActivity(_statusMsg, minTimeToShow=0.0)
modelXbrl.modelManager.showStatus(None)
def _decimals(node, sphinxContext, args):
fact = factArg(node, sphinxContext, args, 0)
return inferredDecimals(fact)
def factAspects(fact):
oimFact = OrderedDict()
aspects = OrderedDict()
if hasId and fact.id:
if fact.isTuple:
oimFact["tupleId"] = fact.id
else:
oimFact["id"] = fact.id
elif (fact.isTuple or
footnotesRelationshipSet.toModelObject(fact) or
(isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))):
oimFact["id"] = "f{}".format(fact.objectIndex)
parent = fact.getparent()
concept = fact.concept
aspects[str(qnOimConceptAspect)] = oimValue(concept.qname)
_csvType = "Value"
if not fact.isTuple:
if concept is not None:
_baseXsdType = concept.baseXsdType
if _baseXsdType == "XBRLI_DATEUNION":
if getattr(fact.xValue, "dateOnly", False):
_baseXsdType = "date"
else:
_baseXsdType = "dateTime"
_csvType = baseTypes.get(_baseXsdType,_baseXsdType) + "Value"
if concept.baseXbrliType in ("stringItemType", "normalizedStringItemType") and fact.xmlLang:
aspects[str(qnOimLangAspect)] = fact.xmlLang
if fact.isItem:
if fact.isNil:
_value = None
else:
_inferredDecimals = inferredDecimals(fact)
_value = oimValue(fact.xValue, _inferredDecimals)
oimFact["value"] = _value
if fact.concept is not None and fact.concept.isNumeric:
_numValue = fact.xValue
if isinstance(_numValue, Decimal) and not isinf(_numValue) and not isnan(_numValue):
if _numValue == _numValue.to_integral():
_numValue = int(_numValue)
else:
_numValue = float(_numValue)
if not fact.isNil:
if isinf(_inferredDecimals):
if isJSON: _accuracy = "infinity"
elif isCSVorXL: _accuracy = "INF"
else:
_accuracy = _inferredDecimals
oimFact["accuracy"] = _accuracy
oimFact["aspects"] = aspects
cntx = fact.context
if cntx is not None:
if cntx.entityIdentifierElement is not None:
aspects[str(qnOimEntityAspect)] = oimValue(qname(*cntx.entityIdentifier))
if cntx.period is not None:
aspects.update(oimPeriodValue(cntx))
for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]):
if dim.isExplicit:
dimVal = oimValue(dim.memberQname)
else: # typed
if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1"):
dimVal = None
else:
dimVal = dim.typedMember.stringValue
aspects[str(dim.dimensionQname)] = dimVal
unit = fact.unit
if unit is not None:
_mMul, _mDiv = unit.measures
_sMul = '*'.join(oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m)))
if _mDiv:
_sDiv = '*'.join(oimValue(m) for m in sorted(_mDiv, key=lambda m: oimValue(m)))
if len(_mDiv) > 1:
if len(_mMul) > 1:
_sUnit = "({})/({})".format(_sMul,_sDiv)
else:
_sUnit = "{}/({})".format(_sMul,_sDiv)
else:
if len(_mMul) > 1:
_sUnit = "({})/{}".format(_sMul,_sDiv)
else:
_sUnit = "{}/{}".format(_sMul,_sDiv)
else:
_sUnit = _sMul
aspects[str(qnOimUnitAspect)] = _sUnit
# Tuples removed from xBRL-JSON
#if parent.qname != XbrlConst.qnXbrliXbrl:
# aspects[str(qnOimTupleParentAspect)] = parent.id if parent.id else "f{}".format(parent.objectIndex)
# aspects[str(qnOimTupleOrderAspect)] = elementIndex(fact)
if isJSON:
factFootnotes(fact, oimFact)
return oimFact
def validate(val, modelXbrl, infosetModelXbrl):
infoset = infosetModelXbrl.modelDocument
if infoset.type == Type.INSTANCE:
# compare facts (assumed out of order)
infosetFacts = defaultdict(list)
for fact in infosetModelXbrl.facts:
infosetFacts[fact.qname].append(fact)
if len(modelXbrl.factsInInstance) != len(infosetModelXbrl.factsInInstance):
modelXbrl.error("arelle:infosetTest",
_("Fact counts mismatch, testcase instance %(foundFactCount)s, infoset instance %(expectedFactCount)s"),
modelObject=(modelXbrl.modelDocument, infosetModelXbrl.modelDocument),
foundFactCount=len(modelXbrl.factsInInstance),
expectedFactCount=len(infosetModelXbrl.factsInInstance))
else:
for i, instFact in enumerate(modelXbrl.facts):
infosetFact = None
for fact in infosetFacts[instFact.qname]:
if fact.isTuple and fact.isDuplicateOf(instFact, deemP0Equal=True):
infosetFact = fact
break
elif fact.isItem and fact.isVEqualTo(instFact, deemP0Equal=True):
infosetFact = fact
break
if infosetFact is None: # takes precision/decimals into account
if fact is not None:
fact.isVEqualTo(instFact, deemP0Equal=True)
modelXbrl.error("arelle:infosetTest",
_("Fact %(factNumber)s mismatch %(concept)s"),
modelObject=instFact,
factNumber=(i+1),
concept=instFact.qname)
else:
ptvPeriodType = infosetFact.get("{http://www.xbrl.org/2003/ptv}periodType")
ptvBalance = infosetFact.get("{http://www.xbrl.org/2003/ptv}balance")
ptvDecimals = infosetFact.get("{http://www.xbrl.org/2003/ptv}decimals")
ptvPrecision = infosetFact.get("{http://www.xbrl.org/2003/ptv}precision")
if ptvPeriodType and ptvPeriodType != instFact.concept.periodType:
modelXbrl.error("arelle:infosetTest",
_("Fact %(factNumber)s periodType mismatch %(concept)s expected %(expectedPeriodType)s found %(foundPeriodType)s"),
modelObject=(instFact, infosetFact),
factNumber=(i+1),
concept=instFact.qname,
expectedPeriodType=ptvPeriodType,
foundPeriodType=instFact.concept.periodType)
if ptvBalance and ptvBalance != instFact.concept.balance:
modelXbrl.error("arelle:infosetTest",
_("Fact %(factNumber)s balance mismatch %(concept)s expected %(expectedBalance)s found %(foundBalance)s"),
modelObject=(instFact, infosetFact),
factNumber=(i+1),
concept=instFact.qname,
expectedBalance=ptvBalance,
foundBalance=instFact.concept.balance)
if ptvDecimals and ptvDecimals != str(inferredDecimals(fact)):
modelXbrl.error("arelle:infosetTest",
_("Fact %(factNumber)s inferred decimals mismatch %(concept)s expected %(expectedDecimals)s found %(inferredDecimals)s"),
modelObject=(instFact, infosetFact),
factNumber=(i+1),
concept=instFact.qname,
expectedDecimals=ptvDecimals,
inferredDecimals=str(inferredDecimals(fact)))
if ptvPrecision and ptvPrecision != str(inferredPrecision(fact)):
modelXbrl.error("arelle:infosetTest",
_("Fact %(factNumber)s inferred precision mismatch %(concept)s expected %(expectedPrecision)s found %(inferredPrecision)s"),
modelObject=(instFact, infosetFact),
factNumber=(i+1),
concept=instFact.qname,
expectedPrecisions=ptvPrecision,
inferredPrecision=str(inferredPrecision(fact)))
elif infoset.type == Type.ARCSINFOSET:
# compare arcs
for arcElt in XmlUtil.children(infoset.xmlRootElement, "http://www.xbrl.org/2003/ptv", "arc"):
linkType = arcElt.get("linkType")
arcRole = arcElt.get("arcRole")
extRole = arcElt.get("extRole")
fromObj = resolvePath(modelXbrl, arcElt.get("fromPath"))
if fromObj is None:
modelXbrl.error("arelle:infosetTest",
_("Arc fromPath not found: %(fromPath)s"),
modelObject=arcElt, fromPath=arcElt.get("fromPath"))
continue
if linkType in ("label", "reference"):
labelLang = arcElt.get("labelLang")
resRole = arcElt.get("resRole")
if linkType == "label":
expectedLabel = XmlUtil.text(arcElt)
foundLabel = fromObj.label(preferredLabel=resRole,fallbackToQname=False,lang=None,strip=True,linkrole=extRole)
if foundLabel != expectedLabel:
modelXbrl.error("arelle:infosetTest",
_("Label expected='%(expectedLabel)s', found='%(foundLabel)s'"),
modelObject=arcElt, expectedLabel=expectedLabel, foundLabel=foundLabel)
continue
elif linkType == "reference":
expectedRef = XmlUtil.innerText(arcElt)
referenceFound = False
for refrel in modelXbrl.relationshipSet(XbrlConst.conceptReference,extRole).fromModelObject(fromObj):
ref = refrel.toModelObject
if resRole == ref.role:
foundRef = XmlUtil.innerText(ref)
if foundRef != expectedRef:
modelXbrl.error("arelle:infosetTest",
_("Reference inner text expected='%(expectedRef)s, found='%(foundRef)s'"),
modelObject=arcElt, expectedRef=expectedRef, foundRef=foundRef)
referenceFound = True
break
if referenceFound:
continue
modelXbrl.error("arelle:infosetTest",
_("%(linkType)s not found containing '%(text)s' linkRole %(linkRole)s"),
modelObject=arcElt, linkType=linkType.title(), text=XmlUtil.innerText(arcElt), linkRole=extRole)
else:
toObj = resolvePath(modelXbrl, arcElt.get("toPath"))
if toObj is None:
modelXbrl.error("arelle:infosetTest",
_("Arc toPath not found: %(toPath)s"),
modelObject=arcElt, toPath=arcElt.get("toPath"))
continue
weight = arcElt.get("weight")
if weight is not None:
weight = float(weight)
order = arcElt.get("order")
if order is not None:
order = float(order)
preferredLabel = arcElt.get("preferredLabel")
found = False
for rel in modelXbrl.relationshipSet(arcRole, extRole).fromModelObject(fromObj):
if (rel.toModelObject == toObj and
(weight is None or rel.weight == weight) and
(order is None or rel.order == order)):
found = True
if not found:
modelXbrl.error("arelle:infosetTest",
_("Arc not found: from %(fromPath)s, to %(toPath)s, role %(arcRole)s, linkRole $(extRole)s"),
modelObject=arcElt, fromPath=arcElt.get("fromPath"), toPath=arcElt.get("toPath"), arcRole=arcRole, linkRole=extRole)
continue
# validate dimensions of each fact
factElts = XmlUtil.children(modelXbrl.modelDocument.xmlRootElement, None, "*")
for itemElt in XmlUtil.children(infoset.xmlRootElement, None, "item"):
try:
qnElt = XmlUtil.child(itemElt,None,"qnElement")
factQname = qname(qnElt, XmlUtil.text(qnElt))
sPointer = int(XmlUtil.child(itemElt,None,"sPointer").text)
factElt = factElts[sPointer - 1] # 1-based xpath indexing
if factElt.qname != factQname:
modelXbrl.error("arelle:infosetTest",
_("Fact %(sPointer)s mismatch Qname, expected %(qnElt)s, observed %(factQname)s"),
modelObject=itemElt, sPointer=sPointer, qnElt=factQname, factQname=factElt.qname)
elif not factElt.isItem or factElt.context is None:
modelXbrl.error("arelle:infosetTest",
_("Fact %(sPointer)s has no context: %(qnElt)s"),
modelObject=(itemElt,factElt), sPointer=sPointer, qnElt=factQname)
else:
context = factElt.context
memberElts = XmlUtil.children(itemElt,None,"member")
numNonDefaults = 0
for memberElt in memberElts:
dimElt = XmlUtil.child(memberElt, None, "qnDimension")
qnDim = qname(dimElt, XmlUtil.text(dimElt))
isDefault = XmlUtil.text(XmlUtil.child(memberElt, None, "bDefaulted")) == "true"
if not isDefault:
numNonDefaults += 1
if not ((qnDim in context.qnameDims and not isDefault) or
(qnDim in factElt.modelXbrl.qnameDimensionDefaults and isDefault)):
modelXbrl.error("arelle:infosetTest",
_("Fact %(sPointer)s (qnElt)s dimension mismatch %(qnDim)s"),
modelObject=(itemElt, factElt, context), sPointer=sPointer, qnElt=factQname, qnDim=qnDim)
if numNonDefaults != len(context.qnameDims):
modelXbrl.error("arelle:infosetTest",
_("Fact %(sPointer)s (qnElt)s dimensions count mismatch"),
modelObject=(itemElt, factElt, context), sPointer=sPointer, qnElt=factQname)
except (IndexError, ValueError, AttributeError) as err:
modelXbrl.error("arelle:infosetTest",
_("Invalid entity fact dimensions infoset sPointer: %(test)s, error details: %(error)s"),
modelObject=itemElt, test=XmlUtil.innerTextList(itemElt), error=str(err))
def validateXbrlFinally(val, *args, **kwargs):
if not (val.validateHMRCplugin) or not val.txmyType:
return
modelXbrl = val.modelXbrl
modelDocument = modelXbrl.modelDocument
_statusMsg = _("validating {0} filing rules").format(
val.disclosureSystem.name)
modelXbrl.profileActivity()
modelXbrl.modelManager.showStatus(_statusMsg)
if modelDocument.type in (ModelDocument.Type.INSTANCE,
ModelDocument.Type.INLINEXBRL):
labelHasNegativeTermPattern = re.compile(r".*[(].*\w.*[)].*")
companyReferenceNumberContexts = defaultdict(list)
for c1 in modelXbrl.contexts.values():
scheme, identifier = c1.entityIdentifier
if scheme == "http://www.companieshouse.gov.uk/":
companyReferenceNumberContexts[identifier].append(c1.id)
uniqueFacts = {} # key = (qname, context hash, unit hash, lang)
mandatoryFacts = {}
mandatoryGDV = defaultdict(set)
factForConceptContextUnitHash = defaultdict(list)
hasCompaniesHouseContext = any(
cntx.entityIdentifier[0] == "http://www.companieshouse.gov.uk/"
for cntx in val.modelXbrl.contexts.values())
contextsUsed = set(f.context for f in modelXbrl.factsInInstance
if f.context is not None)
contextsWithScenario = []
for cntx in contextsUsed:
for dim in cntx.qnameDims.values():
if dim.isExplicit:
_memName = dim.memberQname.localName
m = memNameNumPattern.match(_memName)
if m:
l = m.group(1)
n = int(m.group(2))
else:
l = _memName
n = None
for _gdvType in (val.txmyType, "business"):
gdv = genericDimensionValidation.get(
_gdvType, EMPTYDICT).get(l)
if gdv: # take first match
break
if (gdv and
(n is None or
(isinstance(gdv[0], int) and isinstance(gdv[1], int)
and n >= gdv[0] and n <= gdv[1]))):
gdvFacts = [f for f in gdv if isinstance(f, str)]
if len(gdvFacts) == 1:
mandatoryGDV[gdvFacts[0]].add(
GDV(gdvFacts[0], None, _memName))
elif len(gdvFacts) == 2:
mandatoryGDV[gdvFacts[0]].add(
GDV(gdvFacts[0], gdvFacts[1], _memName))
mandatoryGDV[gdvFacts[1]].add(
GDV(gdvFacts[1], gdvFacts[0], _memName))
if XmlUtil.hasChild(cntx, xbrli, "scenario"):
contextsWithScenario.append(cntx)
if contextsWithScenario:
modelXbrl.error(
"FRC.TG.3.6.1",
_("Context(s) %(identifiers)s is reported with scenario element."
),
modelObject=contextsWithScenario,
identifiers=", ".join(c.id for c in contextsWithScenario))
del contextsWithScenario
def checkFacts(facts):
for f in facts:
cntx = f.context
unit = f.unit
if (
f.isNil or getattr(f, "xValid", 0) >= 4
) and cntx is not None and f.concept is not None and f.concept.type is not None:
factNamespaceURI = f.qname.namespaceURI
factLocalName = f.qname.localName
if factLocalName in mandatoryItems[val.txmyType]:
mandatoryFacts[factLocalName] = f
if factLocalName == "UKCompaniesHouseRegisteredNumber" and val.isAccounts:
if hasCompaniesHouseContext:
mandatoryFacts[factLocalName] = f
for _cntx in contextsUsed:
_scheme, _identifier = _cntx.entityIdentifier
if _scheme == "http://www.companieshouse.gov.uk/" and f.xValue != _identifier:
modelXbrl.error(
"JFCVC.3316",
_("Context entity identifier %(identifier)s does not match Company Reference Number (UKCompaniesHouseRegisteredNumber) Location: Accounts (context id %(id)s)"
),
modelObject=(f, _cntx),
identifier=_identifier,
id=_cntx.id)
if f.parentElement.qname == qnXbrliXbrl: # JFCVC v4.0 - only check non-tuple facts
factForConceptContextUnitHash[
f.conceptContextUnitHash].append(f)
if f.isNumeric:
if f.precision:
modelXbrl.error(
"HMRC.5.4",
_("Numeric fact %(fact)s of context %(contextID)s has a precision attribute '%(precision)s'"
),
modelObject=f,
fact=f.qname,
contextID=f.contextID,
precision=f.precision)
try: # only process validated facts
if not f.isNil:
if f.xValue < 0:
label = f.concept.label(lang="en")
if not labelHasNegativeTermPattern.match(
label):
modelXbrl.error(
"HMRC.5.3",
_("Numeric fact %(fact)s of context %(contextID)s has a negative value '%(value)s' but label does not have a bracketed negative term (using parentheses): %(label)s"
),
modelObject=f,
fact=f.qname,
contextID=f.contextID,
value=f.value,
label=label)
# 6.5.37 test (insignificant digits due to rounding)
if f.decimals and f.decimals != "INF":
try:
insignificance = insignificantDigits(
f.xValue, decimals=f.decimals)
if insignificance: # if not None, returns (truncatedDigits, insiginficantDigits)
modelXbrl.error(
"HMRC.SG.4.5",
_("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s has nonzero digits in insignificant portion %(insignificantDigits)s."
),
modelObject=f,
fact=f.qname,
contextID=f.contextID,
decimals=f.decimals,
value=f.xValue,
truncatedDigits=insignificance[
0],
insignificantDigits=
insignificance[1])
except (ValueError, TypeError):
modelXbrl.error(
"HMRC.SG.4.5",
_("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s causes Value Error exception."
),
modelObject=f1,
fact=f1.qname,
contextID=f1.contextID,
decimals=f1.decimals,
value=f1.value)
except AttributeError:
pass # if not validated it should have failed with a schema error
# check GDV
if f.qname.localName in mandatoryGDV:
_gdvReqList = mandatoryGDV[factLocalName]
_gdvReqRemovals = []
for _gdvReq in _gdvReqList:
if any(_gdvReq.memLocalName ==
dim.memberQname.localName
for dim in cntx.qnameDims.values()
if dim.isExplicit):
_gdvReqRemovals.append(_gdvReq)
if _gdvReq.altFact in mandatoryGDV:
_gdvAltList = mandatoryGDV[_gdvReq.altFact]
_gdvAltRemovals = []
for _gdvAlt in _gdvAltList:
if any(_gdvAlt.memLocalName ==
dim.memberQname.localName for
dim in cntx.qnameDims.values()
if dim.isExplicit):
_gdvAltRemovals.append(_gdvAlt)
for _gdvAlt in _gdvAltRemovals:
_gdvAltList.remove(_gdvAlt)
if _gdvReqRemovals and not f.xValue: # fact was a mandatory name or description
modelXbrl.error(
"JFCVC.3315",
_("Generic dimension members associated name/description has no text: %(fact)s"
),
modelObject=f,
fact=f.qname)
for _gdvReq in _gdvReqRemovals:
_gdvReqList.remove(_gdvReq)
if f.modelTupleFacts:
checkFacts(f.modelTupleFacts)
checkFacts(modelXbrl.facts)
if val.isAccounts:
_missingItems = mandatoryItems[
val.txmyType] - mandatoryFacts.keys()
if hasCompaniesHouseContext and "UKCompaniesHouseRegisteredNumber" not in mandatoryFacts:
_missingItems.add("UKCompaniesHouseRegisteredNumber")
if _missingItems:
modelXbrl.error("JFCVC.3312",
_("Facts are MANDATORY: %(missingItems)s"),
modelObject=modelXbrl,
missingItems=", ".join(sorted(_missingItems)))
''' removed with JFCVC v4.0 2020-06-09
f = mandatoryFacts.get("StartDateForPeriodCoveredByReport")
if f is not None and (f.isNil or f.xValue < _6_APR_2008):
modelXbrl.error("JFCVC.3313",
_("Period Start Date (StartDateForPeriodCoveredByReport) must be 6 April 2008 or later, but is %(value)s"),
modelObject=f, value=f.value)
'''
memLocalNamesMissing = set(
"{}({})".format(_gdvRec.memLocalName, _gdvRec.factNames)
for _gdv in mandatoryGDV.values() for _gdvRec in _gdv)
if memLocalNamesMissing:
modelXbrl.error(
"JFCVC.3315",
_("Generic dimension members have no associated name or description item, member names (name or description item): %(memberNames)s"
),
modelObject=modelXbrl,
memberNames=", ".join(sorted(memLocalNamesMissing)))
aspectEqualFacts = defaultdict(
dict) # dict [(qname,lang)] of dict(cntx,unit) of [fact, fact]
for hashEquivalentFacts in factForConceptContextUnitHash.values():
if len(hashEquivalentFacts) > 1:
for f in hashEquivalentFacts: # check for hash collision by value checks on context and unit
if getattr(f, "xValid", 0) >= 4:
cuDict = aspectEqualFacts[(
f.qname, (f.xmlLang or "").lower()
if f.concept.type.isWgnStringFactType else None)]
_matched = False
for (_cntx, _unit), fList in cuDict.items():
if (((_cntx is None and f.context is None) or
(f.context is not None
and f.context.isEqualTo(_cntx)))
and ((_unit is None and f.unit is None) or
(f.unit is not None
and f.unit.isEqualTo(_unit)))):
_matched = True
fList.append(f)
break
if not _matched:
cuDict[(f.context, f.unit)] = [f]
decVals = {}
for cuDict in aspectEqualFacts.values(): # dups by qname, lang
for fList in cuDict.values(
): # dups by equal-context equal-unit
if len(fList) > 1:
f0 = fList[0]
if f0.concept.isNumeric:
if any(f.isNil for f in fList):
_inConsistent = not all(f.isNil
for f in fList)
else: # not all have same decimals
_d = inferredDecimals(f0)
_v = f0.xValue
_inConsistent = isnan(
_v
) # NaN is incomparable, always makes dups inconsistent
decVals[_d] = _v
aMax, bMin = rangeValue(_v, _d)
for f in fList[1:]:
_d = inferredDecimals(f)
_v = f.xValue
if isnan(_v):
_inConsistent = True
break
if _d in decVals:
_inConsistent |= _v != decVals[_d]
else:
decVals[_d] = _v
a, b = rangeValue(_v, _d)
if a > aMax: aMax = a
if b < bMin: bMin = b
if not _inConsistent:
_inConsistent = (bMin < aMax)
decVals.clear()
if _inConsistent:
modelXbrl.error(
"JFCVC.3314",
"Inconsistent duplicate fact values %(fact)s: %(values)s.",
modelObject=fList,
fact=f0.qname,
contextID=f0.contextID,
values=", ".join(f.value for f in fList))
aspectEqualFacts.clear()
del factForConceptContextUnitHash, aspectEqualFacts
if modelXbrl.modelDocument.type == ModelDocument.Type.INLINEXBRL:
rootElt = modelXbrl.modelDocument.xmlRootElement
if rootElt.tag in ("html", "xhtml") or not rootElt.tag.startswith(
"{http://www.w3.org/1999/xhtml}"):
modelXbrl.error(
"HMRC.SG.3.3",
_("InlineXBRL root element <%(element)s> MUST be html and have the xhtml namespace."
),
modelObject=rootElt,
element=rootElt.tag)
for elt in rootElt.iterdescendants(
tag="{http://www.w3.org/1999/xhtml}script"):
modelXbrl.error("HMRC.SG.3.3",
_("Script element is disallowed."),
modelObject=elt)
for tag, localName, attr in (("{http://www.w3.org/1999/xhtml}a", "a",
"href"),
("{http://www.w3.org/1999/xhtml}img",
"img", "src")):
for elt in rootElt.iterdescendants(tag=tag):
attrValue = (elt.get(attr) or "").strip()
if "javascript:" in attrValue:
modelXbrl.error(
"HMRC.SG.3.3",
_("Element %(localName)s javascript %(javascript)s is disallowed."
),
modelObject=elt,
localName=localName,
javascript=attrValue[:64])
if localName == "img" and not any(
attrValue.startswith(m) for m in allowedImgMimeTypes):
modelXbrl.error(
"HMRC.SG.3.8",
_("Image scope must be base-64 encoded string (starting with data:image/*;base64), *=gif, jpeg or png. src disallowed: %(src)s."
),
modelObject=elt,
src=attrValue[:128])
for elt in rootElt.iterdescendants(
tag="{http://www.w3.org/1999/xhtml}style"):
if elt.text and styleImgUrlPattern.match(elt.text):
modelXbrl.error(
"HMRC.SG.3.8",
_("Style element has disallowed image reference: %(styleImage)s."
),
modelObject=elt,
styleImage=styleImgUrlPattern.match(elt.text).group(1))
for elt in rootElt.xpath(
"//xhtml:*[@style]",
namespaces={"xhtml": "http://www.w3.org/1999/xhtml"}):
for match in styleImgUrlPattern.findall(elt.get("style")):
modelXbrl.error(
"HMRC.SG.3.8",
_("Element %(elt)s style attribute has disallowed image reference: %(styleImage)s."
),
modelObject=elt,
elt=elt.tag.rpartition("}")[2],
styleImage=match)
modelXbrl.profileActivity(_statusMsg, minTimeToShow=0.0)
modelXbrl.modelManager.showStatus(None)
def showInfo(cntlr, options, modelXbrl, _entrypoint, *args, **kwargs):
for url, doc in sorted(modelXbrl.urlDocs.items(), key=lambda i: i[0]):
if not any(url.startswith(w) for w in ("https://xbrl.sec.gov/", "http://xbrl.sec.gov/", "http://xbrl.fasb.org/", "http://www.xbrl.org/",
"http://xbrl.ifrs.org/", "http://www.esma.europa.eu/")):
cntlr.addToLog("File {} size {:,}".format(doc.basename, os.path.getsize(doc.filepath)), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Heap memory before loading {:,}".format(memoryAtStartup), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Heap memory after loading {:,}".format(cntlr.memoryUsed), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Time to load {:.2f} seconds".format(time.time() - timeAtStart), messageCode="info", level=logging.DEBUG)
isInlineXbrl = modelXbrl.modelDocument.type in (ModelDocument.Type.INLINEXBRL, ModelDocument.Type.INLINEXBRLDOCUMENTSET)
if isInlineXbrl:
instanceType = "inline XBRL, number of documents {}".format(len(modelXbrl.ixdsHtmlElements))
else:
instanceType = "xBRL-XML"
cntlr.addToLog("Instance type {}".format(instanceType), messageCode="info", level=logging.DEBUG)
numContexts = len(modelXbrl.contexts)
numLongContexts = 0
bytesSaveable = 0
frequencyOfDims = {}
sumNumDims = 0
distinctDurations = set()
distinctInstants = set()
shortContextIdLen = int(math.log10(numContexts or 1)) + 2 # if no contexts, use 1 for log function to work
xbrlQnameCount = 0
xbrlQnameLengths = 0
for c in modelXbrl.contexts.values():
sumNumDims += len(c.qnameDims)
for d in c.qnameDims.values():
dimQname = str(d.dimensionQname)
frequencyOfDims[dimQname] = frequencyOfDims.get(dimQname,0) + 1
xbrlQnameCount += 1
xbrlQnameLengths += len(d.dimensionQname.localName)
if c.isInstantPeriod:
distinctInstants.add(c.instantDatetime)
elif c.isStartEndPeriod:
distinctDurations.add((c.startDatetime, c.endDatetime))
if len(c.id) > shortContextIdLen:
bytesSaveable += len(c.id) - shortContextIdLen
cntlr.addToLog("Number of contexts {:,}".format(numContexts), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Number of distinct durations {:,}".format(len(distinctDurations)), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Number of distinct instants {:,}".format(len(distinctInstants)), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Avg number dimensions per contexts {:,.2f}".format(sumNumDims/numContexts if numContexts else 0), messageCode="info", level=logging.DEBUG)
mostPopularDims = sorted(frequencyOfDims.items(), key=lambda i:"{:0>9},{}".format(999999999-i[1],i[0]))
for dimName, count in mostPopularDims[0:3]:
cntlr.addToLog("Dimension {} used in {:,} contexts".format(dimName, count), messageCode="info", level=logging.DEBUG)
numFacts = 0
numTableTextBlockFacts = 0
lenTableTextBlockFacts = 0
numTextBlockFacts = 0
lenTextBlockFacts = 0
distinctElementsInFacts = set()
factsPerContext = {}
factForConceptContextUnitHash = defaultdict(list)
for f in modelXbrl.factsInInstance:
context = f.context
concept = f.concept
distinctElementsInFacts.add(f.qname)
numFacts += 1
if f.qname.localName.endswith("TableTextBlock"):
numTableTextBlockFacts += 1
lenTableTextBlockFacts += len(f.xValue)
elif f.qname.localName.endswith("TextBlock"):
numTextBlockFacts += 1
lenTextBlockFacts += len(f.xValue)
if context is not None and concept is not None:
factsPerContext[context.id] = factsPerContext.get(context.id,0) + 1
factForConceptContextUnitHash[f.conceptContextUnitHash].append(f)
bytesSaveable += len(context.id) - shortContextIdLen
mostPopularContexts = sorted(factsPerContext.items(), key=lambda i:"{:0>9},{}".format(999999999-i[1],i[0]))
cntlr.addToLog("Number of facts {:,}".format(numFacts), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Number of TableTextBlock facts {:,} avg len {:,.0f}".format(numTableTextBlockFacts, lenTableTextBlockFacts/numTableTextBlockFacts if numTableTextBlockFacts else 0), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Number of TextBlock facts {:,} avg len {:,.0f}".format(numTextBlockFacts, lenTextBlockFacts/numTableTextBlockFacts if numTableTextBlockFacts else 0), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Max number facts per context {:,}".format(mostPopularContexts[0][1] if mostPopularContexts else 0), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Avg number facts per context {:,.2f}".format(sum([v for v in factsPerContext.values()])/numContexts if numContexts else 0), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Distinct elements in facts {:,}".format(len(distinctElementsInFacts)), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Number of bytes saveable context id of {} length is {:,}".format(shortContextIdLen, bytesSaveable), messageCode="info", level=logging.DEBUG)
aspectEqualFacts = defaultdict(list)
numConsistentDupFacts = numInConsistentDupFacts = 0
for hashEquivalentFacts in factForConceptContextUnitHash.values():
if len(hashEquivalentFacts) > 1:
for f in hashEquivalentFacts:
aspectEqualFacts[(f.qname,f.contextID,f.unitID,
f.xmlLang if f.concept.type.isWgnStringFactType else None)].append(f)
for fList in aspectEqualFacts.values():
f0 = fList[0]
if f0.concept.isNumeric:
if any(f.isNil for f in fList):
_inConsistent = not all(f.isNil for f in fList)
elif all(inferredDecimals(f) == inferredDecimals(f0) for f in fList[1:]): # same decimals
v0 = rangeValue(f0.value)
_inConsistent = not all(rangeValue(f.value) == v0 for f in fList[1:])
else: # not all have same decimals
aMax, bMin = rangeValue(f0.value, inferredDecimals(f0))
for f in fList[1:]:
a, b = rangeValue(f.value, inferredDecimals(f))
if a > aMax: aMax = a
if b < bMin: bMin = b
_inConsistent = (bMin < aMax)
else:
_inConsistent = any(not f.isVEqualTo(f0) for f in fList[1:])
if _inConsistent:
numInConsistentDupFacts += 1
else:
numConsistentDupFacts += 1
aspectEqualFacts.clear()
cntlr.addToLog("Number of duplicate facts consistent {:,} inconsistent {:,}".format(numConsistentDupFacts, numInConsistentDupFacts), messageCode="info", level=logging.DEBUG)
styleAttrCounts = {}
totalStyleLen = 0
continuationElements = {}
ixNsPrefix = "{http://www.xbrl.org/2013/inlineXBRL}"
for ixdsHtmlRootElt in getattr(modelXbrl, "ixdsHtmlElements", ()): # ix root elements if inline
for ixElt in ixdsHtmlRootElt.iterdescendants():
style = ixElt.get("style")
ixEltTag = str(ixElt.tag)
if style:
styleAttrCounts[style] = styleAttrCounts.get(style,0) + 1
if styleIxHiddenPattern.match(style) is None:
totalStyleLen += len(style)
if ixEltTag == "{http://www.xbrl.org/2013/inlineXBRL}continuation" and ixElt.id:
continuationElements[ixElt.id] = ixElt
if ixEltTag.startswith(ixNsPrefix):
localName = ixEltTag[len(ixNsPrefix):]
if localName == "continuation" and ixElt.id:
continuationElements[ixElt.id] = ixElt
elif localName in ("nonFraction", "nonNumeric", "fraction"):
xbrlQnameCount += 1
xbrlQnameLengths += len(ixElt.qname.localName)
elif isinstance(ixElt, ModelFact):
xbrlQnameCount += 2
xbrlQnameLengths += len(ixElt.qname.localName)
def locateContinuation(element, chain=None):
contAt = element.get("continuedAt")
if contAt:
if contAt in continuationElements:
if chain is None: chain = [element]
contElt = continuationElements[contAt]
if contElt not in chain:
chain.append(contElt)
element._continuationElement = contElt
return locateContinuation(contElt, chain)
elif chain: # end of chain
return len(chain)
numContinuations = 0
maxLenLen = 0
maxLenHops = 0
maxHops = 0
maxHopsLen = 0
for f in modelXbrl.factsInInstance:
if f.get("continuedAt"):
numContinuations += 1
_len = len(f.xValue)
_hops = locateContinuation(f)
if _hops > maxHops:
maxHops = _hops
maxHopsLen = _len
if _len > maxLenLen:
maxLenLen = _len
maxLenHops = _hops
cntlr.addToLog("Number of continuation facts {:,}".format(numContinuations), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Longest continuation fact {:,} number of hops {:,}".format(maxLenLen, maxLenHops), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Most continuation hops {:,} fact len {:,}".format(maxHops, maxHopsLen), messageCode="info", level=logging.DEBUG)
numDupStyles = sum(1 for n in styleAttrCounts.values() if n > 1)
bytesSaveableByCss = sum(len(s)*(n-1) for s,n in styleAttrCounts.items() if n > 1)
cntlr.addToLog("Number of duplicate styles {:,}, bytes saveable by CSS {:,}, len of all non-ix-hidden @styles {:,}".format(numDupStyles, bytesSaveableByCss, totalStyleLen), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Number of XBRL QNames {:,}, bytes saveable by EBA-style element names {:,}".format(xbrlQnameCount, xbrlQnameLengths - (5*xbrlQnameCount)), messageCode="info", level=logging.DEBUG)
def createViewer(self, scriptUrl="js/dist/ixbrlviewer.js"):
"""
Create an iXBRL file with XBRL data as a JSON blob, and script tags added
"""
dts = self.dts
idGen = 0
self.roleMap.getPrefix(XbrlConst.standardLabel,"std")
self.roleMap.getPrefix(XbrlConst.documentationLabel,"doc")
self.roleMap.getPrefix(XbrlConst.summationItem,"calc")
self.roleMap.getPrefix(XbrlConst.parentChild,"pres")
for f in dts.facts:
if f.id is None:
f.set("id","ixv-%d" % (idGen))
idGen += 1
conceptName = self.nsmap.qname(f.qname)
scheme, ident = f.context.entityIdentifier
aspects = {
"c": conceptName,
"e": self.nsmap.qname(QName(self.nsmap.getPrefix(scheme,"e"), scheme, ident)),
}
factData = {
"v": f.value,
"a": aspects,
}
if f.format is not None:
factData["f"] = str(f.format)
if f.isNumeric:
if f.unit:
# XXX does not support complex units
unit = self.nsmap.qname(f.unit.measures[0][0])
aspects["u"] = unit
d = inferredDecimals(f)
if d != float("INF") and not math.isnan(d):
factData["d"] = d
for d, v in f.context.qnameDims.items():
if v.memberQname is None:
# Typed dimension, not yet supported.
continue
aspects[self.nsmap.qname(v.dimensionQname)] = self.nsmap.qname(v.memberQname)
self.addConcept(v.dimension)
self.addConcept(v.member)
# XXX does not support forever periods
if f.context.isInstantPeriod:
aspects["p"] = self.dateFormat(f.context.instantDatetime.isoformat())
elif f.context.isStartEndPeriod:
aspects["p"] = "%s/%s" % (
self.dateFormat(f.context.startDatetime.isoformat()),
self.dateFormat(f.context.endDatetime.isoformat())
)
self.taxonomyData["facts"][f.id] = factData
self.addConcept(f.concept)
self.taxonomyData["prefixes"] = self.nsmap.prefixmap
self.taxonomyData["roles"] = self.roleMap.prefixmap
self.taxonomyData["rels"] = self.getRelationnShips()
taxonomyDataJSON = self.escapeJSONForScriptTag(json.dumps(self.taxonomyData, indent=1, allow_nan=False))
dts.info("viewer:info", "Creating iXBRL viewer")
for child in dts.modelDocument.xmlDocument.getroot():
if child.tag == '{http://www.w3.org/1999/xhtml}body':
child.append(etree.Comment("BEGIN IXBRL VIEWER EXTENSIONS"))
e = etree.fromstring("<script xmlns='http://www.w3.org/1999/xhtml' src='%s' />" % scriptUrl)
e.text = ''
child.append(e)
# Putting this in the header can interfere with character set
# auto detection
e = etree.fromstring("<script xmlns='http://www.w3.org/1999/xhtml' id='taxonomy-data' type='application/json'></script>")
e.text = taxonomyDataJSON
child.append(e)
child.append(etree.Comment("END IXBRL VIEWER EXTENSIONS"))
break
return dts.modelDocument.xmlDocument
def createViewer(self, scriptUrl="js/dist/ixbrlviewer.js"):
"""
Create an iXBRL file with XBRL data as a JSON blob, and script tags added
"""
dts = self.dts
iv = iXBRLViewer(dts)
idGen = 0
self.roleMap.getPrefix(XbrlConst.standardLabel, "std")
self.roleMap.getPrefix(XbrlConst.documentationLabel, "doc")
self.roleMap.getPrefix(XbrlConst.summationItem, "calc")
self.roleMap.getPrefix(XbrlConst.parentChild, "pres")
for f in dts.facts:
if f.id is None:
f.set("id", "ixv-%d" % (idGen))
idGen += 1
conceptName = self.nsmap.qname(f.qname)
scheme, ident = f.context.entityIdentifier
aspects = {
"c":
conceptName,
"e":
self.nsmap.qname(
QName(self.nsmap.getPrefix(scheme, "e"), scheme, ident)),
}
factData = {
"v": f.value,
"a": aspects,
}
if f.format is not None:
factData["f"] = str(f.format)
if f.isNumeric:
if f.unit is not None:
# XXX does not support complex units
unit = self.nsmap.qname(f.unit.measures[0][0])
aspects["u"] = unit
else:
# The presence of the unit aspect is used by the viewer to
# identify numeric facts. If the fact has no unit (invalid
# XBRL, but we want to support it for draft documents),
# include the unit aspect with a null value.
aspects["u"] = None
d = inferredDecimals(f)
if d != float("INF") and not math.isnan(d):
factData["d"] = d
for d, v in f.context.qnameDims.items():
if v.memberQname is None:
# Typed dimension, not yet supported.
continue
aspects[self.nsmap.qname(v.dimensionQname)] = self.nsmap.qname(
v.memberQname)
self.addConcept(v.dimension)
self.addConcept(v.member)
if f.context.isForeverPeriod:
aspects["p"] = "f"
elif f.context.isInstantPeriod:
aspects["p"] = self.dateFormat(
f.context.instantDatetime.isoformat())
elif f.context.isStartEndPeriod:
aspects["p"] = "%s/%s" % (
self.dateFormat(f.context.startDatetime.isoformat()),
self.dateFormat(f.context.endDatetime.isoformat()))
self.taxonomyData["facts"][f.id] = factData
self.addConcept(f.concept)
self.taxonomyData["prefixes"] = self.nsmap.prefixmap
self.taxonomyData["roles"] = self.roleMap.prefixmap
self.taxonomyData["rels"] = self.getRelationships()
dts.info("viewer:info", "Creating iXBRL viewer")
if dts.modelDocument.type == Type.INLINEXBRLDOCUMENTSET:
# Sort by object index to preserve order in which files were specified.
docSet = sorted(dts.modelDocument.referencesDocument.keys(),
key=lambda x: x.objectIndex)
docSetFiles = list(
map(lambda x: os.path.basename(x.filepath), docSet))
self.taxonomyData["docSetFiles"] = docSetFiles
for n in range(0, len(docSet)):
iv.addFile(
iXBRLViewerFile(docSetFiles[n], docSet[n].xmlDocument))
xmlDocument = docSet[0].xmlDocument
else:
xmlDocument = dts.modelDocument.xmlDocument
filename = os.path.basename(dts.modelDocument.filepath)
iv.addFile(iXBRLViewerFile(filename, xmlDocument))
taxonomyDataJSON = self.escapeJSONForScriptTag(
json.dumps(self.taxonomyData, indent=1, allow_nan=False))
for child in xmlDocument.getroot():
if child.tag == '{http://www.w3.org/1999/xhtml}body':
child.append(etree.Comment("BEGIN IXBRL VIEWER EXTENSIONS"))
e = etree.fromstring(
"<script xmlns='http://www.w3.org/1999/xhtml' src='%s' type='text/javascript' />"
% scriptUrl)
# Don't self close
e.text = ''
child.append(e)
# Putting this in the header can interfere with character set
# auto detection
e = etree.fromstring(
"<script xmlns='http://www.w3.org/1999/xhtml' type='application/x.ixbrl-viewer+json'></script>"
)
e.text = taxonomyDataJSON
child.append(e)
child.append(etree.Comment("END IXBRL VIEWER EXTENSIONS"))
break
return iv
def factAspects(fact):
aspects = OrderedDict()
if hasId and fact.id:
aspects["id"] = fact.id
elif (fact.isTuple or
footnotesRelationshipSet.toModelObject(fact) or
(isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))):
aspects["id"] = "f{}".format(fact.objectIndex)
parent = fact.getparent()
concept = fact.concept
_csvType = "Value"
if not fact.isTuple:
if concept is not None:
_baseXsdType = concept.baseXsdType
if _baseXsdType == "XBRLI_DATEUNION":
if getattr(fact.xValue, "dateOnly", False):
_baseXsdType = "date"
else:
_baseXsdType = "dateTime"
aspects["baseType"] = "xs:{}".format(_baseXsdType)
_csvType = baseTypes.get(_baseXsdType,_baseXsdType) + "Value"
if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang:
aspects[qnOimLangAspect] = fact.xmlLang
if fact.isItem:
if fact.isNil:
_value = None
_strValue = "nil"
else:
_inferredDecimals = inferredDecimals(fact)
_value = oimValue(fact.xValue, _inferredDecimals)
_strValue = str(_value)
if not isCSVorXL:
aspects["value"] = _strValue
if fact.concept is not None and fact.concept.isNumeric:
_numValue = fact.xValue
if isinstance(_numValue, Decimal) and not isinf(_numValue) and not isnan(_numValue):
if _numValue == _numValue.to_integral():
_numValue = int(_numValue)
else:
_numValue = float(_numValue)
if isCSVorXL:
aspects[_csvType] = _numValue
else:
aspects["numericValue"] = _numValue
if not fact.isNil:
if isinf(_inferredDecimals):
if isJSON: _accuracy = "infinity"
elif isCSVorXL: _accuracy = "INF"
else:
_accuracy = _inferredDecimals
aspects["accuracy"] = _accuracy
elif isinstance(_value, bool):
aspects["booleanValue"] = _value
elif isCSVorXL:
aspects[_csvType] = _strValue
aspects[qnOimConceptAspect] = oimValue(fact.qname)
cntx = fact.context
if cntx is not None:
if cntx.entityIdentifierElement is not None:
aspects[qnOimEntityAspect] = oimValue(qname(*cntx.entityIdentifier))
if cntx.period is not None:
if isJSON:
aspects[qnOimPeriodAspect] = oimPeriodValue(cntx)
elif isCSVorXL:
_periodValue = oimPeriodValue(cntx)
aspects[qnOimPeriodStartAspect] = _periodValue["start"]
aspects[qnOimPeriodEndAspect] = _periodValue["end"]
for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]):
aspects[dim.dimensionQname] = (oimValue(dim.memberQname) if dim.isExplicit
else None if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1")
else dim.typedMember.stringValue)
unit = fact.unit
if unit is not None:
_mMul, _mDiv = unit.measures
_sMul = '*'.join(oimValue(m) for m in sorted(_mMul, key=lambda m: str(m)))
if _mDiv:
_sDiv = '*'.join(oimValue(m) for m in sorted(_mDiv, key=lambda m: str(m)))
if len(mDiv) > 1:
if len(mMul) > 1:
_sUnit = "({})/({})".format(_sMul,_sDiv)
else:
_sUnit = "{}/({})".format(_sMul,_sDiv)
else:
if len(mMul) > 1:
_sUnit = "({})/{}".format(_sMul,_sDiv)
else:
_sUnit = "{}/{}".format(_sMul,_sDiv)
else:
_sUnit = _sMul
aspects[qnOimUnitAspect] = _sUnit
if parent.qname != XbrlConst.qnXbrliXbrl:
aspects[qnOimTupleParentAspect] = parent.id if parent.id else "f{}".format(parent.objectIndex)
aspects[qnOimTupleOrderAspect] = elementIndex(fact)
if isJSON:
_footnotes = factFootnotes(fact)
if _footnotes:
aspects["footnotes"] = _footnotes
return aspects
def intervalValue(fact, dec=None): # value in decimals
if fact.isNil:
return NILinterval
if dec is None:
dec = inferredDecimals(fact)
return rangeValue(fact.value, dec)
def factAspects(fact):
aspects = OrderedDict()
if hasId and fact.id:
aspects["id"] = fact.id
elif fact.isTuple or footnotesRelationshipSet.toModelObject(fact):
aspects["id"] = "f{}".format(fact.objectIndex)
parent = fact.getparent()
concept = fact.concept
if not fact.isTuple:
if concept is not None:
_baseXsdType = concept.baseXsdType
if _baseXsdType == "XBRLI_DATEUNION":
if getattr(fact.xValue, "dateOnly", False):
_baseXsdType = "date"
else:
_baseXsdType = "dateTime"
aspects["baseType"] = "xs:{}".format(_baseXsdType)
if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang:
aspects[qnOimLangAspect] = fact.xmlLang
aspects[qnOimTypeAspect] = concept.baseXbrliType
if fact.isItem:
if fact.isNil:
_value = None
_strValue = "nil"
else:
_inferredDecimals = inferredDecimals(fact)
_value = oimValue(fact.xValue, _inferredDecimals)
_strValue = str(_value)
aspects["value"] = _strValue
if fact.concept is not None and fact.concept.isNumeric:
_numValue = fact.xValue
if isinstance(_numValue, Decimal) and not isinf(_numValue) and not isnan(_numValue):
if _numValue == _numValue.to_integral():
_numValue = int(_numValue)
else:
_numValue = float(_numValue)
aspects["numericValue"] = _numValue
if not fact.isNil:
aspects["accuracy"] = "infinity" if isinf(_inferredDecimals) else _inferredDecimals
elif isinstance(_value, bool):
aspects["booleanValue"] = _value
aspects[qnOimConceptAspect] = oimValue(fact.qname)
cntx = fact.context
if cntx is not None:
if cntx.entityIdentifierElement is not None:
aspects[qnOimEntityAspect] = oimValue(qname(*cntx.entityIdentifier))
if cntx.period is not None:
aspects[qnOimPeriodAspect] = oimPeriodValue(cntx)
for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]):
aspects[dim.dimensionQname] = (oimValue(dim.memberQname) if dim.isExplicit
else None if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1")
else dim.typedMember.stringValue)
unit = fact.unit
if unit is not None:
_mMul, _mDiv = unit.measures
if isJSON:
aspects[qnOimUnitAspect] = { # use tuple instead of list for hashability
"numerators": tuple(oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m)))
}
if _mDiv:
aspects[qnOimUnitAspect]["denominators"] = tuple(oimValue(m) for m in sorted(_mDiv, key=lambda m: oimValue(m)))
else: # CSV
if _mMul:
aspects[qnOimUnitMulAspect] = ",".join(oimValue(m)
for m in sorted(_mMul, key=lambda m: q(m)))
if _mDiv:
aspects[qnOimUnitDivAspect] = ",".join(oimValue(m)
for m in sorted(_mDiv, key=lambda m: str(m)))
if parent.qname != XbrlConst.qnXbrliXbrl:
aspects[qnOimTupleParentAspect] = parent.id if parent.id else "f{}".format(parent.objectIndex)
aspects[qnOimTupleOrderAspect] = elementIndex(fact)
footnotes = []
for footnoteRel in footnotesRelationshipSet.fromModelObject(fact):
footnote = {"group": footnoteRel.arcrole}
footnotes.append(footnote)
toObj = footnoteRel.toModelObject
if isinstance(toObj, ModelFact):
footnote["factRef"] = toObj.id if toObj.id else "f{}".format(toObj.objectIndex)
else:
footnote["footnoteType"] = toObj.role
footnote["footnote"] = xmlstring(toObj, stripXmlns=True, contentsOnly=True, includeText=True)
if toObj.xmlLang:
footnote["language"] = toObj.xmlLang
if footnotes:
aspects["footnotes"] = footnotes
return aspects
def showInfo(cntlr, options, modelXbrl, _entrypoint, *args, **kwargs):
for url, doc in sorted(modelXbrl.urlDocs.items(), key=lambda i: i[0]):
if not any(url.startswith(w) for w in ("https://xbrl.sec.gov/", "http://xbrl.sec.gov/", "http://xbrl.fasb.org/", "http://www.xbrl.org/",
"http://xbrl.ifrs.org/", "http://www.esma.europa.eu/")):
if os.path.exists(doc.filepath): # skip if in an archive or stream
cntlr.addToLog("File {} size {:,}".format(doc.basename, os.path.getsize(doc.filepath)), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Heap memory before loading {:,}".format(memoryAtStartup), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Heap memory after loading {:,}".format(cntlr.memoryUsed), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Time to load {:.2f} seconds".format(time.time() - timeAtStart), messageCode="info", level=logging.DEBUG)
isInlineXbrl = modelXbrl.modelDocument.type in (ModelDocument.Type.INLINEXBRL, ModelDocument.Type.INLINEXBRLDOCUMENTSET)
if isInlineXbrl:
instanceType = "inline XBRL, number of documents {}".format(len(modelXbrl.ixdsHtmlElements))
else:
instanceType = "xBRL-XML"
cntlr.addToLog("Instance type {}".format(instanceType), messageCode="info", level=logging.DEBUG)
numContexts = len(modelXbrl.contexts)
numLongContexts = 0
bytesSaveableInline = 0
bytesSaveableInlineWithCsv = 0
frequencyOfDims = {}
sumNumDims = 0
distinctDurations = set()
distinctInstants = set()
shortContextIdLen = int(math.log10(numContexts or 1)) + 2 # if no contexts, use 1 for log function to work
xbrlQnameCountInline = 0
xbrlQnameCountInlineWithCsv = 0
xbrlQnameLengthsInline = 0
xbrlQnameLengthsInlineWithCsv = 0
for c in modelXbrl.contexts.values():
sumNumDims += len(c.qnameDims)
for d in c.qnameDims.values():
dimQname = str(d.dimensionQname)
frequencyOfDims[dimQname] = frequencyOfDims.get(dimQname,0) + 1
xbrlQnameCountInline += 1
xbrlQnameCountInlineWithCsv += 1
xbrlQnameLengthsInline += len(d.dimensionQname.localName)
xbrlQnameLengthsInlineWithCsv += len(d.dimensionQname.localName)
if c.isInstantPeriod:
distinctInstants.add(c.instantDatetime)
elif c.isStartEndPeriod:
distinctDurations.add((c.startDatetime, c.endDatetime))
if len(c.id) > shortContextIdLen:
bytesSaveableInline += len(c.id) - shortContextIdLen
bytesSaveableInlineWithCsv += len(c.id) - shortContextIdLen
cntlr.addToLog("Number of contexts {:,}".format(numContexts), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Number of distinct durations {:,}".format(len(distinctDurations)), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Number of distinct instants {:,}".format(len(distinctInstants)), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Avg number dimensions per contexts {:,.2f}".format(sumNumDims/numContexts if numContexts else 0), messageCode="info", level=logging.DEBUG)
mostPopularDims = sorted(frequencyOfDims.items(), key=lambda i:"{:0>9},{}".format(999999999-i[1],i[0]))
for dimName, count in mostPopularDims[0:3]:
cntlr.addToLog("Dimension {} used in {:,} contexts".format(dimName, count), messageCode="info", level=logging.DEBUG)
# analyze for tables which could be composed from CSV data
tblFacts = defaultdict(set)
tblNestedTables = defaultdict(set)
factSize = {}
for f in modelXbrl.factsInInstance:
for tdElt in ancestors(f, xhtml, "td"):
factSize[f] = len(xmlstring(tdElt,stripXmlns=True))
break
childTblElt = None
for tblElt in ancestors(f, xhtml, "table"):
tblFacts[tblElt].add(f)
if childTblElt:
tblNestedTables[tblElt].add(childTblElt)
# find tables containing only numeric facts
def tblNestedFactCount(tbl):
c = len(tblFacts.get(tbl, ()))
for nestedTbl in tblNestedTables.get(tbl,()):
c += tblNestedFactCount(nestedTbl)
return c
factsInInstance = len(modelXbrl.factsInInstance)
factsInTables = len(set.union(*(fset for fset in tblFacts.values())))
cntlr.addToLog("Facts in instance: {:,}, facts in tables: {:,}".format(factsInInstance,factsInTables), messageCode="info", level=logging.DEBUG)
numTblsEligible = 0
numFactsEligible = 0
bytesCsvSavings = 0
factsEligibleForCsv = set()
tablesWithEligibleFacts = set()
if tblFacts and factSize:
# find eligible tables, have facts and not nested tables with other facts
for tbl, facts in tblFacts.items():
if len(facts) == tblNestedFactCount(tbl):
s = sum(factSize.get(f,0) for f in facts) - sum(len(str(f.value)) for f in facts)
if s > 10000:
numTblsEligible += 1
bytesCsvSavings += s
numFactsEligible += len(facts)
factsEligibleForCsv |= facts
tablesWithEligibleFacts.add(tbl)
numFacts = 0
numTableTextBlockFacts = 0
lenTableTextBlockFacts = 0
numTextBlockFacts = 0
lenTextBlockFacts = 0
distinctElementsInFacts = set()
factsPerContext = {}
factForConceptContextUnitHash = defaultdict(list)
for f in modelXbrl.factsInInstance:
context = f.context
concept = f.concept
distinctElementsInFacts.add(f.qname)
numFacts += 1
if f.qname.localName.endswith("TableTextBlock"):
numTableTextBlockFacts += 1
lenTableTextBlockFacts += len(f.xValue)
elif f.qname.localName.endswith("TextBlock"):
numTextBlockFacts += 1
lenTextBlockFacts += len(f.xValue)
if context is not None and concept is not None:
factsPerContext[context.id] = factsPerContext.get(context.id,0) + 1
factForConceptContextUnitHash[f.conceptContextUnitHash].append(f)
bytesSaveableInline += len(context.id) - shortContextIdLen
if f not in factsEligibleForCsv:
bytesSaveableInlineWithCsv += len(context.id) - shortContextIdLen
if numTblsEligible:
cntlr.addToLog("Tables eligible for facts in CSV: {:,}, facts eligible for CSV: {:,}, bytes saveable by facts in CSV {:,}".format(numTblsEligible, numFactsEligible, bytesCsvSavings), messageCode="info", level=logging.DEBUG)
else:
cntlr.addToLog("No tables eligible for facts in CSV", messageCode="info", level=logging.DEBUG)
mostPopularContexts = sorted(factsPerContext.items(), key=lambda i:"{:0>9},{}".format(999999999-i[1],i[0]))
cntlr.addToLog("Number of facts {:,}".format(numFacts), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Number of TableTextBlock facts {:,} avg len {:,.0f}".format(numTableTextBlockFacts, lenTableTextBlockFacts/numTableTextBlockFacts if numTableTextBlockFacts else 0), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Number of TextBlock facts {:,} avg len {:,.0f}".format(numTextBlockFacts, lenTextBlockFacts/numTableTextBlockFacts if numTableTextBlockFacts else 0), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Max number facts per context {:,}".format(mostPopularContexts[0][1] if mostPopularContexts else 0), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Avg number facts per context {:,.2f}".format(sum([v for v in factsPerContext.values()])/numContexts if numContexts else 0), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Distinct elements in facts {:,}".format(len(distinctElementsInFacts)), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Number of bytes saveable context id of {} length is {:,}".format(shortContextIdLen, bytesSaveableInline), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Excepting facts eligible for CSV, number of bytes saveable context id of {} length is {:,}".format(shortContextIdLen, bytesSaveableInlineWithCsv), messageCode="info", level=logging.DEBUG)
aspectEqualFacts = defaultdict(list)
decVals = {}
numConsistentDupFacts = numInConsistentDupFacts = 0
for hashEquivalentFacts in factForConceptContextUnitHash.values():
if len(hashEquivalentFacts) > 1:
for f in hashEquivalentFacts:
aspectEqualFacts[(f.qname,f.contextID,f.unitID,
f.xmlLang.lower() if f.concept.type.isWgnStringFactType else None)].append(f)
for fList in aspectEqualFacts.values():
f0 = fList[0]
if f0.concept.isNumeric:
if any(f.isNil for f in fList):
_inConsistent = not all(f.isNil for f in fList)
else: # not all have same decimals
_d = inferredDecimals(f0)
_v = f0.xValue
_inConsistent = isnan(_v) # NaN is incomparable, always makes dups inconsistent
decVals[_d] = _v
aMax, bMin = rangeValue(_v, _d)
for f in fList[1:]:
_d = inferredDecimals(f)
_v = f.xValue
if isnan(_v):
_inConsistent = True
break
if _d in decVals:
_inConsistent |= _v != decVals[_d]
else:
decVals[_d] = _v
a, b = rangeValue(_v, _d)
if a > aMax: aMax = a
if b < bMin: bMin = b
if not _inConsistent:
_inConsistent = (bMin < aMax)
decVals.clear()
else:
_inConsistent = any(not f.isVEqualTo(f0) for f in fList[1:])
if _inConsistent:
numInConsistentDupFacts += 1
else:
numConsistentDupFacts += 1
aspectEqualFacts.clear()
cntlr.addToLog("Number of duplicate facts consistent {:,} inconsistent {:,}".format(numConsistentDupFacts, numInConsistentDupFacts), messageCode="info", level=logging.DEBUG)
styleAttrCountsInline = {}
styleAttrCountsInlineWithCsv = {}
totalStyleLenInline = 0
totalStyleLenInlineWithCsv = 0
continuationElements = {}
ixNsPrefix = "{http://www.xbrl.org/2013/inlineXBRL}"
for ixdsHtmlRootElt in getattr(modelXbrl, "ixdsHtmlElements", ()): # ix root elements if inline
for ixElt in ixdsHtmlRootElt.iterdescendants():
inEligibleTableForCsv = any(p in tablesWithEligibleFacts for p in ixElt.iterancestors("{http://www.w3.org/1999/xhtml}table"))
style = ixElt.get("style")
ixEltTag = str(ixElt.tag)
if style:
styleAttrCountsInline[style] = styleAttrCountsInline.get(style,0) + 1
if not inEligibleTableForCsv:
styleAttrCountsInlineWithCsv[style] = styleAttrCountsInlineWithCsv.get(style,0) + 1
if styleIxHiddenPattern.match(style) is None:
totalStyleLenInline += len(style)
if not inEligibleTableForCsv:
totalStyleLenInlineWithCsv += len(style)
if ixEltTag == "{http://www.xbrl.org/2013/inlineXBRL}continuation" and ixElt.id:
continuationElements[ixElt.id] = ixElt
if ixEltTag.startswith(ixNsPrefix):
localName = ixEltTag[len(ixNsPrefix):]
if localName == "continuation" and ixElt.id:
continuationElements[ixElt.id] = ixElt
elif localName in ("nonFraction", "nonNumeric", "fraction"):
xbrlQnameCountInline += 1
xbrlQnameLengthsInline += len(ixElt.qname.localName)
if not inEligibleTableForCsv:
xbrlQnameCountInlineWithCsv += 1
xbrlQnameLengthsInlineWithCsv += len(ixElt.qname.localName)
elif isinstance(ixElt, ModelFact):
xbrlQnameCountInline += 2
xbrlQnameLengthsInline += len(ixElt.qname.localName)
if not inEligibleTableForCsv:
xbrlQnameCountInlineWithCsv += 2
xbrlQnameLengthsInlineWithCsv += len(ixElt.qname.localName)
def locateContinuation(element, chain=None):
contAt = element.get("continuedAt")
if contAt:
if contAt in continuationElements:
if chain is None: chain = [element]
contElt = continuationElements[contAt]
if contElt not in chain:
chain.append(contElt)
element._continuationElement = contElt
return locateContinuation(contElt, chain)
elif chain: # end of chain
return len(chain)
numContinuations = 0
maxLenLen = 0
maxLenHops = 0
maxHops = 0
maxHopsLen = 0
for f in modelXbrl.factsInInstance:
if f.get("continuedAt"):
numContinuations += 1
_len = len(f.xValue)
_hops = locateContinuation(f)
if _hops > maxHops:
maxHops = _hops
maxHopsLen = _len
if _len > maxLenLen:
maxLenLen = _len
maxLenHops = _hops
cntlr.addToLog("Number of continuation facts {:,}".format(numContinuations), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Longest continuation fact {:,} number of hops {:,}".format(maxLenLen, maxLenHops), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Most continuation hops {:,} fact len {:,}".format(maxHops, maxHopsLen), messageCode="info", level=logging.DEBUG)
numDupStyles = sum(1 for n in styleAttrCountsInline.values() if n > 1)
bytesSaveableByCssInline = sum(len(s)*(n-1) for s,n in styleAttrCountsInline.items() if n > 1)
cntlr.addToLog("Number of duplicate styles {:,}, bytes saveable by CSS {:,}, len of all non-ix-hidden @styles {:,}".format(numDupStyles, bytesSaveableByCssInline, totalStyleLenInline), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Number of XBRL QNames {:,}, bytes saveable by EBA-style element names {:,}".format(xbrlQnameCountInline, xbrlQnameLengthsInline - (5*xbrlQnameCountInline)), messageCode="info", level=logging.DEBUG)
numDupStyles = sum(1 for n in styleAttrCountsInlineWithCsv.values() if n > 1)
bytesSaveableByCssInlineWithCsv = sum(len(s)*(n-1) for s,n in styleAttrCountsInlineWithCsv.items() if n > 1)
cntlr.addToLog("Excepting facts eligible for CSV, number of duplicate styles {:,}, bytes saveable by CSS {:,}, len of all non-ix-hidden @styles {:,}".format(numDupStyles, bytesSaveableByCssInlineWithCsv, totalStyleLenInlineWithCsv), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Excepting facts eligible for CSV, number of XBRL QNames {:,}, bytes saveable by EBA-style element names {:,}".format(xbrlQnameCountInlineWithCsv, xbrlQnameLengthsInlineWithCsv - (5*xbrlQnameCountInlineWithCsv)), messageCode="info", level=logging.DEBUG)
def leastDecimals(binding, localNames):
nonNilFacts = [binding[ln] for ln in localNames if not binding[ln].isNil]
if nonNilFacts:
return min((inferredDecimals(f) for f in nonNilFacts))
return floatINF
def sectTreeRel(self, parentConcept, n, sectCalc2RelSet, inferredParentValues, visited, dimQN=None):
childRels = sectCalc2RelSet.fromModelObject(parentConcept)
if childRels:
visited.add(parentConcept)
inferredChildValues = {}
# setup summation bind keys for child objects
sumParentBindKeys = self.sumConceptBindKeys[parentConcept]
boundSumKeys = set() # these are contributing fact keys, parent may be inferred
boundSums = defaultdict(intervalZero)
boundSummationItems = defaultdict(list)
boundPerKeys = set()
boundPers = defaultdict(intervalZero)
boundDurationItems = defaultdict(list)
boundAggKeys = set()
boundAggs = defaultdict(intervalZero)
boundAggItems = defaultdict(list)
boundAggConcepts = defaultdict(set)
for rel in childRels:
childConcept = rel.toModelObject
if childConcept not in visited:
if rel.arcrole == summationItem:
if not self.sumInit:
self.sumBindFacts()
boundSumKeys |= self.sumConceptBindKeys[childConcept]
elif rel.arcrole == balanceChanges:
if not self.perInit:
self.perBindFacts()
boundPerKeys |= self.perConceptBindKeys[childConcept] # these are only duration items
elif rel.arcrole == domainMember:
boundAggKeys |= self.aggConceptBindKeys[dimQN]
domQN = parentConcept.qname
elif rel.arcrole == aggregationDomain: # this is in visited
dimQN = rel.arcElement.prefixedNameQname(rel.get("dimension"))
if dimQN not in self.aggDimInit:
self.aggBindFacts(dimQN) # bind each referenced dimension's contexts
# depth-first descent calc tree and process item after descent
for rel in childRels:
childConcept = rel.toModelObject
if childConcept not in visited:
# depth-first descent
self.sectTreeRel(childConcept, n+1, sectCalc2RelSet, inferredChildValues, visited, dimQN)
# post-descent summation (allows use of inferred value)
if rel.arcrole == summationItem:
weight = rel.weightDecimal
for sumKey in boundSumKeys:
cntx, unit = sumKey
factKey = (childConcept, cntx, unit)
if factKey in self.sumBoundFacts:
for f in self.sumBoundFacts[factKey]:
addInterval(boundSums, sumKey, intervalValue(f), weight)
boundSummationItems[sumKey].append(f)
elif factKey in inferredChildValues:
addInterval(boundSums, sumKey, inferredChildValues[factKey], weight)
elif factKey in inferredParentValues:
addInterval(boundSums, sumKey, inferredParentValues[factKey], weight)
elif rel.arcrole == balanceChanges:
weight = rel.weightDecimal
for perKey in boundPerKeys:
hCntx, unit, start, end = perKey
factKey = (childConcept, hCntx, unit, start, end)
if factKey in self.perBoundFacts:
for f in self.perBoundFacts[factKey]:
addInterval(boundPers, perKey, intervalValue(f), weight)
boundDurationItems[perKey].append(f)
elif factKey in inferredChildValues:
addInterval(boundPers, perKey, inferredChildValues[factKey], weight)
elif factKey in inferredParentValues:
addInterval(boundPers, perKey, inferredParentValues[factKey], weight)
elif rel.arcrole == domainMember:
memQN = childConcept.qname
for aggKey in boundAggKeys:
hCntx, unit = aggKey
dimMemKey = (hCntx, unit, dimQN, memQN)
if dimMemKey in self.aggBoundFacts:
for f in self.aggBoundFacts[dimMemKey]:
a, b = intervalValue(f)
factDomKey = (f.concept, hCntx, unit, dimQN, domQN)
addInterval(boundAggs, factDomKey, intervalValue(f))
boundAggItems[aggKey].append(f)
boundAggConcepts[aggKey].add(f.concept)
elif rel.arcrole == aggregationDomain: # this is in visited
childRelSet = self.modelXbrl.relationshipSet(domainMember,rel.get("targetRole"))
self.sectTreeRel(childConcept, n+1, childRelSet, inferredParentValues, {None}, dimQN) # infer global to section
# process child items bound to this calc subtree
for sumKey in boundSumKeys:
cntx, unit = sumKey
factKey = (parentConcept, cntx, unit)
ia, ib = boundSums[sumKey]
if factKey in self.sumBoundFacts:
for f in self.sumBoundFacts[factKey]:
d = inferredDecimals(f)
sa, sb = intervalValue(f, d)
if ((ia is NIL) ^ (sa is NIL)) or ((ia is not NIL) and (sb < ia or sa > ib)):
self.modelXbrl.log('INCONSISTENCY', "calc2e:summationInconsistency",
_("Summation inconsistent from %(concept)s in section %(section)s reported sum %(reportedSum)s, computed sum %(computedSum)s context %(contextID)s unit %(unitID)s unreported contributing items %(unreportedContributors)s"),
modelObject=boundSummationItems[sumKey],
concept=parentConcept.qname, section=self.section,
reportedSum=self.formatInterval(sa, sb, d),
computedSum=self.formatInterval(ia, ib, d),
contextID=f.context.id, unitID=f.unit.id,
unreportedContributors=", ".join(str(c.qname) # list the missing/unreported contributors in relationship order
for r in childRels
for c in (r.toModelObject,)
if r.arcrole == summationItem and c is not None and
(c, cntx, unit) not in self.sumBoundFacts)
or "none")
elif inferredParentValues is not None: # value was inferred, return to parent level
inferredParentValues[factKey] = (ia, ib)
for perKey in boundPerKeys:
hCntx, unit, start, end = perKey
ia, ib = boundPers[perKey]
endBalA = endBalB = ZERO
endFactKey = (parentConcept, hCntx, unit, None, end)
if endFactKey in self.perBoundFacts:
for f in self.perBoundFacts[endFactKey]:
if f.isNil:
endBalA = endBalB = NIL
d = 0
break
d = inferredDecimals(f)
a, b = intervalValue(f,d)
endBalA += a
endBalB += b
foundStartingFact = (endBalA is NIL)
while not foundStartingFact:
startFactKey = (parentConcept, hCntx, unit, None, start)
if startFactKey in self.perBoundFacts:
for f in self.perBoundFacts[startFactKey]:
if f.isNil:
endBalA = endBalB = NIL
foundStartingFact = True
break
a, b = intervalValue(f)
endBalA -= a
endBalB -= b
foundStartingFact = True
break
if not foundStartingFact:
# infer backing up one period
_nomPer = nominalPeriod(end - start)
foundEarlierAdjacentPeriodStart = False
for _start in self.durationPeriodStarts.get(_nomPer, ()):
if nominalPeriod(start - _start) == _nomPer: # it's preceding period
end = start
start = _start
perKey = hCntx, unit, start, end
if perKey in boundPerKeys:
chngs = boundPers[perKey]
ia += chngs[0]
ib += chngs[1]
foundEarlierAdjacentPeriodStart = True
break
if not foundEarlierAdjacentPeriodStart:
break
if ((ia is NIL) ^ (endBalA is NIL)) or ((ia is not NIL) and (endBalB < ia or endBalA > ib)):
self.modelXbrl.log('INCONSISTENCY', "calc2e:balanceInconsistency",
_("Balance inconsistent from %(concept)s in section %(section)s reported sum %(reportedSum)s, computed sum %(computedSum)s context %(contextID)s unit %(unitID)s unreported contributing items %(unreportedContributors)s"),
modelObject=boundDurationItems[perKey],
concept=parentConcept.qname, section=self.section,
reportedSum=self.formatInterval(endBalA, endBalB, d),
computedSum=self.formatInterval(ia, ib, d),
contextID=f.context.id, unitID=f.unit.id,
unreportedContributors=", ".join(str(c.qname) # list the missing/unreported contributors in relationship order
for r in childRels
for c in (r.toModelObject,)
if r.arcrole == balanceChanges and c is not None and
(c, hCntx, unit, start, end) not in self.perBoundFacts)
or "none")
for aggKey in boundAggKeys:
hCntx, unit = aggKey
for concept in sorted(boundAggConcepts[aggKey], key=lambda c:c.objectIndex): # repeatable errors
factDomKey = (concept, hCntx, unit, dimQN, domQN)
ia, ib = boundAggs[factDomKey]
if factDomKey in self.aggBoundConceptFacts:
for f in self.aggBoundConceptFacts[factDomKey]:
d = inferredDecimals(f)
sa, sb = intervalValue(f, d)
if ((ia is NIL) ^ (sa is NIL)) or ((ia is not NIL) and (sb < ia or sa > ib)):
self.modelXbrl.log('INCONSISTENCY', "calc2e:aggregationInconsistency",
_("Aggregation inconsistent for %(concept)s, domain %(domain)s in section %(section)s reported sum %(reportedSum)s, computed sum %(computedSum)s context %(contextID)s unit %(unitID)s unreported contributing members %(unreportedContributors)s"),
modelObject=boundAggItems[factDomKey],
concept=concept.qname,
domain=parentConcept.qname, section=self.section,
reportedSum=self.formatInterval(sa, sb, d),
computedSum=self.formatInterval(ia, ib, d),
contextID=f.context.id, unitID=f.unit.id,
unreportedContributors=", ".join(str(c.qname) # list the missing/unreported contributors in relationship order
for r in childRels
for c in (r.toModelObject,)
if r.arcrole == domainMember and c is not None and
(concept, hCntx, unit, dimQN, c.qname) not in self.aggBoundConceptFacts)
or "none")
elif inferredParentValues is not None: # value was inferred, return to parent level
# allow to be retrieved by factDomKey
inferredParentValues[factDomKey] = (ia, ib)
if self.modelXbrl.qnameDimensionDefaults.get(dimQN) == domQN:
cntxKey = (hCntx, dimQN, domQN)
if cntxKey in self.eqCntx:
cntx = self.eqCntx[cntxKey]
else:
cntx = self.aggTotalContext(hCntx, dimQN, domQN)
self.eqCntx[cntxKey] = cntx
if cntx is not None:
# allow to be retrieved by fact line item context key
self.eqCntx[(hCntx, dimQN, domQN)] = cntx
inferredParentValues[(concept, cntx, unit)] = (ia, ib)
visited.remove(parentConcept)
def factAspects(fact):
aspects = OrderedDict()
if hasId and fact.id:
aspects["id"] = fact.id
elif (fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or
(isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))):
aspects["id"] = "f{}".format(fact.objectIndex)
parent = fact.getparent()
concept = fact.concept
_csvType = "Value"
if not fact.isTuple:
if concept is not None:
_baseXsdType = concept.baseXsdType
if _baseXsdType == "XBRLI_DATEUNION":
if getattr(fact.xValue, "dateOnly", False):
_baseXsdType = "date"
else:
_baseXsdType = "dateTime"
aspects["baseType"] = "xsd:{}".format(_baseXsdType)
_csvType = baseTypes.get(_baseXsdType, _baseXsdType) + "Value"
if concept.baseXbrliType in ("string", "normalizedString",
"token") and fact.xmlLang:
aspects[qnOimLangAspect] = fact.xmlLang
if fact.isItem:
if fact.isNil:
_value = None
_strValue = "nil"
else:
_inferredDecimals = inferredDecimals(fact)
_value = oimValue(fact.xValue, _inferredDecimals)
_strValue = str(_value)
if not isCSVorXL:
aspects["value"] = _strValue
if fact.concept is not None and fact.concept.isNumeric:
_numValue = fact.xValue
if isinstance(_numValue, Decimal) and not isinf(
_numValue) and not isnan(_numValue):
if _numValue == _numValue.to_integral():
_numValue = int(_numValue)
else:
_numValue = float(_numValue)
if isCSVorXL:
aspects[_csvType] = _numValue
else:
aspects["numericValue"] = _numValue
if not fact.isNil:
if isinf(_inferredDecimals):
if isJSON: _accuracy = "infinity"
elif isCSVorXL: _accuracy = "INF"
else:
_accuracy = _inferredDecimals
aspects["accuracy"] = _accuracy
elif isinstance(_value, bool):
aspects["booleanValue"] = _value
elif isCSVorXL:
aspects[_csvType] = _strValue
aspects[qnOimConceptAspect] = oimValue(fact.qname)
cntx = fact.context
if cntx is not None:
if cntx.entityIdentifierElement is not None:
aspects[qnOimEntityAspect] = oimValue(
qname(*cntx.entityIdentifier))
if cntx.period is not None:
if isJSON:
aspects[qnOimPeriodAspect] = oimPeriodValue(cntx)
elif isCSVorXL:
_periodValue = oimPeriodValue(cntx)
aspects[qnOimPeriodStartAspect] = _periodValue["start"]
aspects[qnOimPeriodEndAspect] = _periodValue["end"]
for _qn, dim in sorted(cntx.qnameDims.items(),
key=lambda item: item[0]):
aspects[dim.dimensionQname] = (
oimValue(dim.memberQname)
if dim.isExplicit else None if dim.typedMember.get(
"{http://www.w3.org/2001/XMLSchema-instance}nil") in (
"true", "1") else dim.typedMember.stringValue)
unit = fact.unit
if unit is not None:
_mMul, _mDiv = unit.measures
_sMul = '*'.join(
oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m)))
if _mDiv:
_sDiv = '*'.join(
oimValue(m)
for m in sorted(_mDiv, key=lambda m: oimValue(m)))
if len(_mDiv) > 1:
if len(_mMul) > 1:
_sUnit = "({})/({})".format(_sMul, _sDiv)
else:
_sUnit = "{}/({})".format(_sMul, _sDiv)
else:
if len(_mMul) > 1:
_sUnit = "({})/{}".format(_sMul, _sDiv)
else:
_sUnit = "{}/{}".format(_sMul, _sDiv)
else:
_sUnit = _sMul
aspects[qnOimUnitAspect] = _sUnit
if parent.qname != XbrlConst.qnXbrliXbrl:
aspects[
qnOimTupleParentAspect] = parent.id if parent.id else "f{}".format(
parent.objectIndex)
aspects[qnOimTupleOrderAspect] = elementIndex(fact)
if isJSON:
_footnotes = factFootnotes(fact)
if _footnotes:
aspects["footnotes"] = _footnotes
return aspects
def intervalValue(fact, dec=None): # value in decimals
if fact.isNil:
return NILinterval
if dec is None:
dec = inferredDecimals(fact)
return rangeValue(fact.value, dec)
def factAspects(fact):
oimFact = OrderedDict()
aspects = OrderedDict()
if isCSVorXL:
oimFact["id"] = fact.id or "f{}".format(fact.objectIndex)
parent = fact.getparent()
concept = fact.concept
aspects[str(qnOimConceptAspect)] = oimValue(concept.qname)
if concept is not None:
if concept.type.isOimTextFactType and fact.xmlLang:
aspects[str(qnOimLangAspect)] = fact.xmlLang
if fact.isItem:
if fact.isNil:
_value = None
else:
_inferredDecimals = inferredDecimals(fact)
_value = oimValue(fact.xValue, _inferredDecimals)
oimFact["value"] = _value
if fact.concept is not None and fact.concept.isNumeric:
_numValue = fact.xValue
if isinstance(_numValue, Decimal) and not isinf(
_numValue) and not isnan(_numValue):
if _numValue == _numValue.to_integral():
_numValue = int(_numValue)
else:
_numValue = float(_numValue)
if not fact.isNil:
if not isinf(
_inferredDecimals): # accuracy omitted if infinite
oimFact["decimals"] = _inferredDecimals
oimFact["dimensions"] = aspects
cntx = fact.context
if cntx is not None:
if cntx.entityIdentifierElement is not None and cntx.entityIdentifier != ENTITY_NA_QNAME:
aspects[str(qnOimEntityAspect)] = oimValue(
qname(*cntx.entityIdentifier))
if cntx.period is not None and not cntx.isForeverPeriod:
aspects.update(oimPeriodValue(cntx))
for _qn, dim in sorted(cntx.qnameDims.items(),
key=lambda item: item[0]):
if dim.isExplicit:
dimVal = oimValue(dim.memberQname)
else: # typed
if dim.typedMember.get(
"{http://www.w3.org/2001/XMLSchema-instance}nil"
) in ("true", "1"):
dimVal = None
else:
dimVal = dim.typedMember.stringValue
aspects[str(dim.dimensionQname)] = dimVal
unit = fact.unit
if unit is not None:
_mMul, _mDiv = unit.measures
_sMul = '*'.join(
oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m)))
if _mDiv:
_sDiv = '*'.join(
oimValue(m)
for m in sorted(_mDiv, key=lambda m: oimValue(m)))
if len(_mDiv) > 1:
if len(_mMul) > 1:
_sUnit = "({})/({})".format(_sMul, _sDiv)
else:
_sUnit = "{}/({})".format(_sMul, _sDiv)
else:
if len(_mMul) > 1:
_sUnit = "({})/{}".format(_sMul, _sDiv)
else:
_sUnit = "{}/{}".format(_sMul, _sDiv)
else:
_sUnit = _sMul
if _sUnit != "xbrli:pure":
aspects[str(qnOimUnitAspect)] = _sUnit
# Tuples removed from xBRL-JSON
#if parent.qname != XbrlConst.qnXbrliXbrl:
# aspects[str(qnOimTupleParentAspect)] = parent.id if parent.id else "f{}".format(parent.objectIndex)
# aspects[str(qnOimTupleOrderAspect)] = elementIndex(fact)
if isJSON:
factFootnotes(fact, oimFact=oimFact)
return oimFact
def validate(self):
modelXbrl = self.modelXbrl
if not modelXbrl.contexts or not modelXbrl.facts:
return # skip if no contexts or facts
if not self.val.validateInferDecimals: # infering precision is now contrary to XBRL REC section 5.2.5.2
modelXbrl.error("calc2e:inferringPrecision","Calc2 requires inferring decimals.")
return
startedAt = time.time()
# check balance attributes and weights, same as XBRL 2.1
for rel in modelXbrl.relationshipSet(calc2Arcroles).modelRelationships:
weight = rel.weight
fromConcept = rel.fromModelObject
toConcept = rel.toModelObject
if fromConcept is not None and toConcept is not None:
if rel.arcrole == aggregationDomain:
rel.dimension = rel.arcElement.prefixedNameQname(rel.get("dimension"))
if rel.dimension is None or not modelXbrl.qnameConcepts[rel.dimension].isDimensionItem:
modelXbrl.error("calc2e:invalidAggregationDimension",
_("Aggregation-domain relationship has invalid dimension %(dimension)s in link role %(linkrole)s"),
modelObject=rel,
dimension=rel.get("dimension"), linkrole=ELR)
elif fromConcept != toConcept or not fromConcept.isDomainMember:
modelXbrl.error("calc2e:invalidAggregationDomain",
_("Calculation relationship has invalid domain %(domain)s in link role %(linkrole)s"),
modelObject=rel,
domain=fromConcept, linkrole=ELR)
continue
if rel.arcrole == balanceChanges:
if fromConcept.periodType != "instant" or toConcept.periodType != "duration":
modelXbrl.error("calc2e:invalidBalanceChangesPeriodType",
_("Balance-changes relationship must have instant source concept and duration target concept in link role %(linkrole)s"),
modelObject=rel, linkrole=ELR)
if weight not in (1, -1):
modelXbrl.error("calc2e:invalidWeight",
_("Calculation relationship has invalid weight from %(source)s to %(target)s in link role %(linkrole)s"),
modelObject=rel,
source=fromConcept.qname, target=toConcept.qname, linkrole=ELR)
fromBalance = fromConcept.balance
toBalance = toConcept.balance
if fromBalance and toBalance:
if (fromBalance == toBalance and weight < 0) or \
(fromBalance != toBalance and weight > 0):
modelXbrl.error("calc2e:balanceCalcWeightIllegal" +
("Negative" if weight < 0 else "Positive"),
_("Calculation relationship has illegal weight %(weight)s from %(source)s, %(sourceBalance)s, to %(target)s, %(targetBalance)s, in link role %(linkrole)s (per 5.1.1.2 Table 6)"),
modelObject=rel, weight=weight,
source=fromConcept.qname, target=toConcept.qname, linkrole=rel.linkrole,
sourceBalance=fromBalance, targetBalance=toBalance,
messageCodes=("calc2e:balanceCalcWeightIllegalNegative", "calc2:balanceCalcWeightIllegalPositive"))
if not fromConcept.isNumeric or not toConcept.isNumeric:
modelXbrl.error("calc2e:nonNumericCalc",
_("Calculation relationship has illegal concept from %(source)s%(sourceNumericDecorator)s to %(target)s%(targetNumericDecorator)s in link role %(linkrole)s"),
modelObject=rel,
source=fromConcept.qname, target=toConcept.qname, linkrole=rel.linkrole,
sourceNumericDecorator="" if fromConcept.isNumeric else _(" (non-numeric)"),
targetNumericDecorator="" if toConcept.isNumeric else _(" (non-numeric)"))
# identify equal contexts
uniqueCntxHashes = {}
self.modelXbrl.profileActivity()
for cntx in modelXbrl.contexts.values():
h = hash( (cntx.periodHash, cntx.entityIdentifierHash, cntx.dimsHash) ) # OIM-compatible hash
if h in uniqueCntxHashes:
if cntx.isEqualTo(uniqueCntxHashes[h]):
self.eqCntx[cntx] = uniqueCntxHashes[h]
else:
uniqueCntxHashes[h] = cntx
del uniqueCntxHashes
self.modelXbrl.profileActivity("... identify aspect equal contexts", minTimeToShow=1.0)
# identify equal units
uniqueUnitHashes = {}
for unit in self.modelXbrl.units.values():
h = unit.hash
if h in uniqueUnitHashes:
if unit.isEqualTo(uniqueUnitHashes[h]):
self.eqUnit[unit] = uniqueUnitHashes[h]
else:
uniqueUnitHashes[h] = unit
del uniqueUnitHashes
self.modelXbrl.profileActivity("... identify equal units", minTimeToShow=1.0)
sectObjs = sorted(set(rel.fromModelObject # only have numerics with context and unit
for rel in modelXbrl.relationshipSet(sectionFact).modelRelationships
if rel.fromModelObject is not None and rel.fromModelObject.concept is not None),
key=lambda s: (s.concept.label(), s.objectIndex)) # sort into document order for consistent error messages
if not sectObjs:
self.modelXbrl.error("calc2e:noSections",
"Instance contains no sections, nothing to validate.",
modelObject=modelXbrl)
# check by section
factByConceptCntxUnit = OrderedDefaultDict(list) # sort into document order for consistent error messages
self.sectionFacts = []
for sectObj in sectObjs:
#print ("section {}".format(sectObj.concept.label()))
self.section = sectObj.concept.label()
sectLinkRoles = tuple(sectObj.concept.get(calc2linkroles,"").split())
factByConceptCntxUnit.clear()
for f in sorted((rel.toModelObject # sort into document order for consistent error messages
for rel in modelXbrl.relationshipSet(sectionFact,sectLinkRoles).fromModelObject(sectObj)
if rel.toModelObject is not None and # numeric facts with context and unit
rel.fromModelObject is not None and
rel.toModelObject.concept is not None and
rel.toModelObject.context is not None and
rel.toModelObject.unit is not None),
key=lambda f: f.objectIndex):
factByConceptCntxUnit[f.qname, self.eqCntx.get(f.context,f.context), self.eqUnit.get(f.unit,f.unit)].append(f)
for fList in factByConceptCntxUnit.values():
f0 = fList[0]
if len(fList) == 1:
self.sectionFacts.append(f0)
else:
if any(f.isNil for f in fList):
_inConsistent = not all(f.isNil for f in fList)
if _inConsistent: # pick a nil fact for f0 for calc validation
for f in fList:
if f.isNil:
f0 = f
break
elif all(inferredDecimals(f) == inferredDecimals(f0) for f in fList[1:]): # same decimals
v0 = intervalValue(f0)
_inConsistent = not all(intervalValue(f) == v0 for f in fList[1:])
else: # not all have same decimals
d0 = inferredDecimals(f0)
aMax, bMin = intervalValue(f0, d0)
for f in fList[1:]:
df = inferredDecimals(f0)
a, b = intervalValue(f, df)
if a > aMax: aMax = a
if b < bMin: bMin = b
if df > d0: # take most accurate fact in section
f0 = f
d0 = df
_inConsistent = (bMin < aMax)
if _inConsistent:
modelXbrl.error("calc2e:inconsistentDuplicateInSection",
"Section %(section)s contained %(fact)s inconsistent in contexts equivalent to %(contextID)s: values %(values)s",
modelObject=fList, section=sectObj.concept.label(), fact=f0.qname, contextID=f0.contextID, values=", ".join(strTruncate(f.value, 128) for f in fList))
self.sectionFacts.append(f0)
# sectionFacts now in document order and deduplicated
#print("section {} facts {}".format(sectObj.concept.label(), ", ".join(str(f.qname)+"="+f.value for f in self.sectionFacts)))
# depth-first calc tree
sectCalc2RelSet = modelXbrl.relationshipSet(calc2Arcroles, sectLinkRoles)
# indexers for section based on calc2 arcrole
self.sumInit = False
self.sumConceptBindKeys.clear()
self.sumBoundFacts.clear()
self.perInit = False
self.perConceptBindKeys.clear()
self.perBoundFacts.clear()
self.durationPeriodStarts.clear()
self.aggDimInit = set()
self.aggConceptBindKeys.clear()
self.aggBoundFacts.clear()
self.aggBoundConceptFacts.clear()
self.aggDimInit.clear()
inferredValues = {}
for rootConcept in sorted(sectCalc2RelSet.rootConcepts,
key=lambda r: sectCalc2RelSet.fromModelObject(r)[0].order):
self.sectTreeRel(rootConcept, 1, sectCalc2RelSet, inferredValues, {rootConcept, None})
def _decimals(node, sphinxContext, args):
fact = factArg(node, sphinxContext, args, 0)
return inferredDecimals(fact)
def validateXbrlFinally(val, *args, **kwargs):
if not (val.validateESEFplugin):
return
_xhtmlNs = "{{{}}}".format(xhtml)
_xhtmlNsLen = len(_xhtmlNs)
modelXbrl = val.modelXbrl
modelDocument = modelXbrl.modelDocument
_statusMsg = _("validating {0} filing rules").format(
val.disclosureSystem.name)
modelXbrl.profileActivity()
modelXbrl.modelManager.showStatus(_statusMsg)
reportXmlLang = None
firstRootmostXmlLangDepth = 9999999
_ifrsNs = None
for targetNs in modelXbrl.namespaceDocs.keys():
if ifrsNsPattern.match(targetNs):
_ifrsNs = targetNs
if not _ifrsNs:
modelXbrl.error("ESEF.RTS.ifrsRequired",
_("RTS on ESEF requires IFRS taxonomy."),
modelObject=modelXbrl)
return
esefPrimaryStatementPlaceholders = set(
qname(_ifrsNs, n) for n in esefPrimaryStatementPlaceholderNames)
esefMandatoryElements2020 = set(
qname(_ifrsNs, n) for n in esefMandatoryElementNames2020)
if modelDocument.type == ModelDocument.Type.INSTANCE:
modelXbrl.error("ESEF.I.1.instanceShallBeInlineXBRL",
_("RTS on ESEF requires inline XBRL instances."),
modelObject=modelXbrl)
checkFilingDimensions(
val) # sets up val.primaryItems and val.domainMembers
val.hasExtensionSchema = val.hasExtensionPre = val.hasExtensionCal = val.hasExtensionDef = val.hasExtensionLbl = False
checkFilingDTS(val, modelXbrl.modelDocument, [])
modelXbrl.profileActivity("... filer DTS checks", minTimeToShow=1.0)
if not (val.hasExtensionSchema and val.hasExtensionPre
and val.hasExtensionCal and val.hasExtensionDef
and val.hasExtensionLbl):
missingFiles = []
if not val.hasExtensionSchema: missingFiles.append("schema file")
if not val.hasExtensionPre:
missingFiles.append("presentation linkbase")
if not val.hasExtensionCal: missingFiles.append("calculation linkbase")
if not val.hasExtensionDef: missingFiles.append("definition linkbase")
if not val.hasExtensionLbl: missingFiles.append("label linkbase")
modelXbrl.warning(
"ESEF.3.1.1.extensionTaxonomyWrongFilesStructure",
_("Extension taxonomies MUST consist of at least a schema file and presentation, calculation, definition and label linkbases"
": missing %(missingFiles)s"),
modelObject=modelXbrl,
missingFiles=", ".join(missingFiles))
#if modelDocument.type == ModelDocument.Type.INLINEXBRLDOCUMENTSET:
# # reports only under reports, none elsewhere
# modelXbrl.fileSource.dir
if modelDocument.type in (ModelDocument.Type.INLINEXBRL,
ModelDocument.Type.INLINEXBRLDOCUMENTSET,
ModelDocument.Type.INSTANCE):
footnotesRelationshipSet = modelXbrl.relationshipSet("XBRL-footnotes")
orphanedFootnotes = set()
noLangFootnotes = set()
factLangFootnotes = defaultdict(set)
footnoteRoleErrors = set()
transformRegistryErrors = set()
def checkFootnote(elt, text):
if text: # non-empty footnote must be linked to a fact if not empty
if not any(
isinstance(rel.fromModelObject, ModelFact) for rel in
footnotesRelationshipSet.toModelObject(elt)):
orphanedFootnotes.add(elt)
lang = elt.xmlLang
if not lang:
noLangFootnotes.add(elt)
else:
for rel in footnotesRelationshipSet.toModelObject(elt):
if rel.fromModelObject is not None:
factLangFootnotes[rel.fromModelObject].add(lang)
if elt.role != XbrlConst.footnote or not all(
rel.arcrole == XbrlConst.factFootnote
and rel.linkrole == XbrlConst.defaultLinkRole
for rel in footnotesRelationshipSet.toModelObject(elt)):
footnoteRoleErrors.add(elt)
# check file name of each inline document (which might be below a top-level IXDS)
for doc in modelXbrl.urlDocs.values():
if doc.type == ModelDocument.Type.INLINEXBRL:
_baseName, _baseExt = os.path.splitext(doc.basename)
if _baseExt not in (".xhtml", ".html"):
modelXbrl.warning(
"ESEF.RTS.Art.3.fileNameExtension",
_("FileName SHOULD have the extension .xhtml or .html: %(fileName)s"
),
modelObject=doc,
fileName=doc.basename)
docinfo = doc.xmlRootElement.getroottree().docinfo
if " html" in docinfo.doctype:
modelXbrl.warning(
"ESEF.RTS.Art.3.htmlDoctype",
_("Doctype SHOULD NOT be html: %(fileName)s"),
modelObject=doc,
fileName=doc.basename)
if modelDocument.type in (ModelDocument.Type.INLINEXBRL,
ModelDocument.Type.INLINEXBRLDOCUMENTSET):
hiddenEltIds = {}
presentedHiddenEltIds = defaultdict(list)
eligibleForTransformHiddenFacts = []
requiredToDisplayFacts = []
requiredToDisplayFactIds = {}
firstIxdsDoc = True
for ixdsHtmlRootElt in modelXbrl.ixdsHtmlElements: # ix root elements for all ix docs in IXDS
ixNStag = ixdsHtmlRootElt.modelDocument.ixNStag
ixTags = set(ixNStag + ln
for ln in ("nonNumeric", "nonFraction",
"references", "relationship"))
ixTextTags = set(ixNStag + ln
for ln in ("nonFraction", "continuation",
"footnote"))
ixExcludeTag = ixNStag + "exclude"
ixTupleTag = ixNStag + "tuple"
ixFractionTag = ixNStag + "fraction"
for elt, depth in etreeIterWithDepth(ixdsHtmlRootElt):
eltTag = elt.tag
if isinstance(
elt,
(_ElementTree, _Comment, _ProcessingInstruction)):
continue # comment or other non-parsed element
else:
eltTag = elt.tag
if eltTag.startswith(_xhtmlNs):
eltTag = eltTag[_xhtmlNsLen:]
if firstIxdsDoc and (not reportXmlLang or depth <
firstRootmostXmlLangDepth):
xmlLang = elt.get(
"{http://www.w3.org/XML/1998/namespace}lang"
)
if xmlLang:
reportXmlLang = xmlLang
firstRootmostXmlLangDepth = depth
if ((eltTag in ("object", "script"))
or (eltTag == "a"
and "javascript:" in elt.get("href", ""))
or (eltTag == "img"
and "javascript:" in elt.get("src", ""))):
modelXbrl.error(
"ESEF.2.5.1.executableCodePresent",
_("Inline XBRL documents MUST NOT contain executable code: %(element)s"
),
modelObject=elt,
element=eltTag)
elif eltTag == "img":
src = elt.get("src", "").strip()
hasParentIxTextTag = False # check if image is in an ix text-bearing element
_ancestorElt = elt
while (_ancestorElt is not None):
if _ancestorElt.tag == ixExcludeTag: # excluded from any parent text-bearing ix element
break
if _ancestorElt.tag in ixTextTags:
hasParentIxTextTag = True
break
_ancestorElt = _ancestorElt.getparent()
if scheme(src) in ("http", "https", "ftp"):
modelXbrl.error(
"ESEF.3.5.1.inlinXbrlContainsExternalReferences",
_("Inline XBRL instance documents MUST NOT contain any reference pointing to resources outside the reporting package: %(element)s"
),
modelObject=elt,
element=eltTag)
elif not src.startswith("data:image"):
if hasParentIxTextTag:
modelXbrl.error(
"ESEF.2.5.1.imageInIXbrlElementNotEmbedded",
_("Images appearing within an inline XBRL element MUST be embedded regardless of their size."
),
modelObject=elt)
else:
# presume it to be an image file, check image contents
try:
base = elt.modelDocument.baseForElement(
elt)
normalizedUri = elt.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
src, base)
if not elt.modelXbrl.fileSource.isInArchive(
normalizedUri):
normalizedUri = elt.modelXbrl.modelManager.cntlr.webCache.getfilename(
normalizedUri)
imglen = 0
with elt.modelXbrl.fileSource.file(
normalizedUri,
binary=True)[0] as fh:
imglen += len(fh.read())
if imglen < browserMaxBase64ImageLength:
modelXbrl.error(
"ESEF.2.5.1.embeddedImageNotUsingBase64Encoding",
_("Images MUST be included in the XHTML document as a base64 encoded string unless their size exceeds support of browsers (%(maxImageSize)s): %(file)s."
),
modelObject=elt,
maxImageSize=
browserMaxBase64ImageLength,
file=os.path.basename(
normalizedUri))
except IOError as err:
modelXbrl.error(
"ESEF.2.5.1.imageFileCannotBeLoaded",
_("Image file which isn't openable '%(src)s', error: %(error)s"
),
modelObject=elt,
src=src,
error=err)
elif not any(
src.startswith(m)
for m in allowedImgMimeTypes):
modelXbrl.error(
"ESEF.2.5.1.embeddedImageNotUsingBase64Encoding",
_("Images MUST be included in the XHTML document as a base64 encoded string, encoding disallowed: %(src)s."
),
modelObject=elt,
src=attrValue[:128])
elif eltTag == "a":
href = elt.get("href", "").strip()
if scheme(href) in ("http", "https", "ftp"):
modelXbrl.error(
"ESEF.3.5.1.inlinXbrlContainsExternalReferences",
_("Inline XBRL instance documents MUST NOT contain any reference pointing to resources outside the reporting package: %(element)s"
),
modelObject=elt,
element=eltTag)
elif eltTag == "base" or elt.tag == "{http://www.w3.org/XML/1998/namespace}base":
modelXbrl.error(
"ESEF.2.4.2.htmlOrXmlBaseUsed",
_("The HTML <base> elements and xml:base attributes MUST NOT be used in the Inline XBRL document."
),
modelObject=elt,
element=eltTag)
elif eltTag == "link" and elt.get(
"type") == "text/css":
if len(modelXbrl.ixdsHtmlElements) > 1:
f = elt.get("href")
if not f or isHttpUrl(f) or os.path.isabs(f):
modelXbrl.warning(
"ESEF.2.5.4.externalCssReportPackage",
_("The CSS file should be physically stored within the report package: %{file}s."
),
modelObject=elt,
file=f)
else:
modelXbrl.error(
"ESEF.2.5.4.externalCssFileForSingleIXbrlDocument",
_("Where an Inline XBRL document set contains a single document, the CSS MUST be embedded within the document."
),
modelObject=elt,
element=eltTag)
elif eltTag == "style" and elt.get(
"type") == "text/css":
if len(modelXbrl.ixdsHtmlElements) > 1:
modelXbrl.warning(
"ESEF.2.5.4.embeddedCssForMultiHtmlIXbrlDocumentSets",
_("Where an Inline XBRL document set contains multiple documents, the CSS SHOULD be defined in a separate file."
),
modelObject=elt,
element=eltTag)
if eltTag in ixTags and elt.get("target"):
modelXbrl.error(
"ESEF.2.5.3.targetAttributeUsed",
_("Target attribute MUST not be used: element %(localName)s, target attribute %(target)s."
),
modelObject=elt,
localName=elt.elementQname,
target=elt.get("target"))
if eltTag == ixTupleTag:
modelXbrl.error(
"ESEF.2.4.1.tupleElementUsed",
_("The ix:tuple element MUST not be used in the Inline XBRL document: %(qname)s."
),
modelObject=elt,
qname=elt.qname)
if eltTag == ixFractionTag:
modelXbrl.error(
"ESEF.2.4.1.fractionElementUsed",
_("The ix:fraction element MUST not be used in the Inline XBRL document."
),
modelObject=elt)
if elt.get("{http://www.w3.org/XML/1998/namespace}base"
) is not None:
modelXbrl.error(
"ESEF.2.4.1.xmlBaseUsed",
_("xml:base attributes MUST NOT be used in the Inline XBRL document: element %(localName)s, base attribute %(base)s."
),
modelObject=elt,
localName=elt.elementQname,
base=elt.get(
"{http://www.w3.org/XML/1998/namespace}base"))
if isinstance(elt, ModelInlineFootnote):
checkFootnote(elt, elt.value)
elif isinstance(
elt, ModelResource
) and elt.qname == XbrlConst.qnLinkFootnote:
checkFootnote(elt, elt.value)
elif isinstance(elt, ModelInlineFact):
if elt.format is not None and elt.format.namespaceURI not in IXT_NAMESPACES:
transformRegistryErrors.add(elt)
for ixHiddenElt in ixdsHtmlRootElt.iterdescendants(
tag=ixNStag + "hidden"):
for tag in (ixNStag + "nonNumeric",
ixNStag + "nonFraction"):
for ixElt in ixHiddenElt.iterdescendants(tag=tag):
if (
getattr(ixElt, "xValid",
0) >= VALID # may not be validated
): # add future "and" conditions on elements which can be in hidden
if (ixElt.concept.baseXsdType
not in untransformableTypes
and not ixElt.isNil):
eligibleForTransformHiddenFacts.append(
ixElt)
elif ixElt.id is None:
requiredToDisplayFacts.append(ixElt)
if ixElt.id:
hiddenEltIds[ixElt.id] = ixElt
firstIxdsDoc = False
if eligibleForTransformHiddenFacts:
modelXbrl.warning(
"ESEF.2.4.1.transformableElementIncludedInHiddenSection",
_("The ix:hidden section of Inline XBRL document MUST not include elements eligible for transformation. "
"%(countEligible)s fact(s) were eligible for transformation: %(elements)s"
),
modelObject=eligibleForTransformHiddenFacts,
countEligible=len(eligibleForTransformHiddenFacts),
elements=", ".join(
sorted(
set(
str(f.qname)
for f in eligibleForTransformHiddenFacts))))
for ixdsHtmlRootElt in modelXbrl.ixdsHtmlElements:
for ixElt in ixdsHtmlRootElt.getroottree().iterfind(
"//{http://www.w3.org/1999/xhtml}*[@style]"):
hiddenFactRefMatch = styleIxHiddenPattern.match(
ixElt.get("style", ""))
if hiddenFactRefMatch:
hiddenFactRef = hiddenFactRefMatch.group(2)
if hiddenFactRef not in hiddenEltIds:
modelXbrl.error(
"ESEF.2.4.1.esefIxHiddenStyleNotLinkingFactInHiddenSection",
_("\"-esef-ix-hidden\" style identifies @id, %(id)s of a fact that is not in ix:hidden section."
),
modelObject=ixElt,
id=hiddenFactRef)
else:
presentedHiddenEltIds[hiddenFactRef].append(ixElt)
for hiddenEltId, ixElt in hiddenEltIds.items():
if (hiddenEltId not in presentedHiddenEltIds
and getattr(ixElt, "xValid", 0) >= VALID
and # may not be validated
(ixElt.concept.baseXsdType in untransformableTypes
or ixElt.isNil)):
requiredToDisplayFacts.append(ixElt)
if requiredToDisplayFacts:
modelXbrl.warning(
"ESEF.2.4.1.factInHiddenSectionNotInReport",
_("The ix:hidden section contains %(countUnreferenced)s fact(s) whose @id is not applied on any \"-esef-ix- hidden\" style: %(elements)s"
),
modelObject=requiredToDisplayFacts,
countUnreferenced=len(requiredToDisplayFacts),
elements=", ".join(
sorted(
set(str(f.qname)
for f in requiredToDisplayFacts))))
del eligibleForTransformHiddenFacts, hiddenEltIds, presentedHiddenEltIds, requiredToDisplayFacts
elif modelDocument.type == ModelDocument.Type.INSTANCE:
for elt in modelDocument.xmlRootElement.iter():
if elt.qname == XbrlConst.qnLinkFootnote: # for now assume no private elements extend link:footnote
checkFootnote(elt, elt.stringValue)
contextsWithDisallowedOCEs = []
contextsWithDisallowedOCEcontent = []
contextsWithPeriodTime = []
contextsWithPeriodTimeZone = []
contextIdentifiers = defaultdict(list)
nonStandardTypedDimensions = defaultdict(set)
for context in modelXbrl.contexts.values():
for elt in context.iterdescendants(
"{http://www.xbrl.org/2003/instance}startDate",
"{http://www.xbrl.org/2003/instance}endDate",
"{http://www.xbrl.org/2003/instance}instant"):
m = datetimePattern.match(elt.stringValue)
if m:
if m.group(1):
contextsWithPeriodTime.append(context)
if m.group(3):
contextsWithPeriodTimeZone.append(context)
for elt in context.iterdescendants(
"{http://www.xbrl.org/2003/instance}segment"):
contextsWithDisallowedOCEs.append(context)
break
for elt in context.iterdescendants(
"{http://www.xbrl.org/2003/instance}scenario"):
if isinstance(elt, ModelObject):
if any(True for child in elt.iterchildren()
if isinstance(child, ModelObject)
and child.tag not in (
"{http://xbrl.org/2006/xbrldi}explicitMember",
"{http://xbrl.org/2006/xbrldi}typedMember")):
contextsWithDisallowedOCEcontent.append(context)
# check periods here
contextIdentifiers[context.entityIdentifier].append(context)
if contextsWithDisallowedOCEs:
modelXbrl.error(
"ESEF.2.1.3.segmentUsed",
_("xbrli:segment container MUST NOT be used in contexts: %(contextIds)s"
),
modelObject=contextsWithDisallowedOCEs,
contextIds=", ".join(c.id for c in contextsWithDisallowedOCEs))
if contextsWithDisallowedOCEcontent:
modelXbrl.error(
"ESEF.2.1.3.scenarioContainsNonDimensionalContent",
_("xbrli:scenario in contexts MUST NOT contain any other content than defined in XBRL Dimensions specification: %(contextIds)s"
),
modelObject=contextsWithDisallowedOCEcontent,
contextIds=", ".join(
c.id for c in contextsWithDisallowedOCEcontent))
if len(contextIdentifiers) > 1:
modelXbrl.error(
"ESEF.2.1.4.multipleIdentifiers",
_("All entity identifiers in contexts MUST have identical content: %(contextIdentifiers)s"
),
modelObject=modelXbrl,
contextIds=", ".join(i[1] for i in contextIdentifiers))
for (contextScheme,
contextIdentifier), contextElts in contextIdentifiers.items():
if contextScheme != iso17442:
modelXbrl.warning(
"ESEF.2.1.1.nonLEIContextScheme",
_("The scheme attribute of the xbrli:identifier element should have \"%(leiScheme)s\" as its content: %(contextScheme)s"
),
modelObject=contextElts,
contextScheme=contextScheme,
leiScheme=iso17442)
else:
leiValidity = LeiUtil.checkLei(contextIdentifier)
if leiValidity == LeiUtil.LEI_INVALID_LEXICAL:
modelXbrl.warning(
"ESEF.2.1.1.invalidIdentifierFormat",
_("The LEI context identifier has an invalid format: %(identifier)s"
),
modelObject=contextElts,
identifier=contextIdentifier)
elif leiValidity == LeiUtil.LEI_INVALID_CHECKSUM:
modelXbrl.warning(
"ESEF.2.1.1.invalidIdentifier",
_("The LEI context identifier has checksum error: %(identifier)s"
),
modelObject=contextElts,
identifier=contextIdentifier)
if contextsWithPeriodTime:
modelXbrl.warning(
"ESEF.2.1.2.periodWithTimeContent",
_("Context period startDate, endDate and instant elements should be in whole days without time: %(contextIds)s"
),
modelObject=contextsWithPeriodTime,
contextIds=", ".join(c.id for c in contextsWithPeriodTime))
if contextsWithPeriodTimeZone:
modelXbrl.warning(
"ESEF.2.1.2.periodWithTimeZone",
_("Context period startDate, endDate and instant elements should be in whole days without a timezone: %(contextIds)s"
),
modelObject=contextsWithPeriodTimeZone,
contextIds=", ".join(c.id for c in contextsWithPeriodTimeZone))
# identify unique contexts and units
mapContext = {}
mapUnit = {}
uniqueContextHashes = {}
for context in modelXbrl.contexts.values():
h = context.contextDimAwareHash
if h in uniqueContextHashes:
if context.isEqualTo(uniqueContextHashes[h]):
mapContext[context] = uniqueContextHashes[h]
else:
uniqueContextHashes[h] = context
del uniqueContextHashes
uniqueUnitHashes = {}
utrValidator = ValidateUtr(modelXbrl)
utrUnitIds = set(
u.unitId
for unitItemType in utrValidator.utrItemTypeEntries.values()
for u in unitItemType.values())
for unit in modelXbrl.units.values():
h = unit.hash
if h in uniqueUnitHashes:
if unit.isEqualTo(uniqueUnitHashes[h]):
mapUnit[unit] = uniqueUnitHashes[h]
else:
uniqueUnitHashes[h] = unit
# check if any custom measure is in UTR
for measureTerm in unit.measures:
for measure in measureTerm:
ns = measure.namespaceURI
if ns != XbrlConst.iso4217 and not ns.startswith(
"http://www.xbrl.org/"):
if measure.localName in utrUnitIds:
modelXbrl.error(
"ESEF.RTS.III.1.G1-7-1.customUnitInUtr",
_("Custom measure SHOULD NOT duplicate a UnitID of UTR: %(measure)s"
),
modelObject=unit,
measure=measure)
del uniqueUnitHashes
reportedMandatory = set()
precisionFacts = set()
numFactsByConceptContextUnit = defaultdict(list)
textFactsByConceptContext = defaultdict(list)
footnotesRelationshipSet = modelXbrl.relationshipSet(
XbrlConst.factFootnote, XbrlConst.defaultLinkRole)
noLangFacts = []
textFactsMissingReportLang = []
conceptsUsed = set()
for qn, facts in modelXbrl.factsByQname.items():
if qn in mandatory:
reportedMandatory.add(qn)
for f in facts:
if f.precision is not None:
precisionFacts.add(f)
if f.isNumeric:
numFactsByConceptContextUnit[(f.qname,
mapContext.get(
f.context, f.context),
mapUnit.get(
f.unit,
f.unit))].append(f)
if f.concept is not None and not f.isNil and f.xValid >= VALID and f.xValue > 1 and f.concept.type is not None and (
f.concept.type.qname == PERCENT_TYPE
or f.concept.type.isDerivedFrom(PERCENT_TYPE)):
modelXbrl.warning(
"ESEF.2.2.2.percentGreaterThan100",
_("A percent fact should have value <= 100: %(element)s in context %(context)s value %(value)s"
),
modelObject=f,
element=f.qname,
context=f.context.id,
value=f.xValue)
elif f.concept is not None and f.concept.type is not None:
if f.concept.type.isOimTextFactType:
if not f.xmlLang:
noLangFacts.append(f)
elif f.context is not None:
textFactsByConceptContext[(
f.qname, mapContext.get(f.context,
f.context))].append(f)
conceptsUsed.add(f.concept)
if f.context is not None:
for dim in f.context.qnameDims.values():
conceptsUsed.add(dim.dimension)
if dim.isExplicit:
conceptsUsed.add(dim.member)
elif dim.isTyped:
conceptsUsed.add(dim.typedMember)
if noLangFacts:
modelXbrl.error(
"ESEF.2.5.2.undefinedLanguageForTextFact",
_("Each tagged text fact MUST have the 'xml:lang' attribute assigned or inherited."
),
modelObject=noLangFacts)
# missing report lang text facts
if reportXmlLang:
for fList in textFactsByConceptContext.values():
if not any(f.xmlLang == reportXmlLang for f in fList):
modelXbrl.error(
"ESEF.2.5.2.taggedTextFactOnlyInLanguagesOtherThanLanguageOfAReport",
_("Each tagged text fact MUST have the 'xml:lang' provided in at least the language of the report: %(element)s"
),
modelObject=fList,
element=fList[0].qname)
# 2.2.4 test
for fList in numFactsByConceptContextUnit.values():
if len(fList) > 1:
f0 = fList[0]
if any(f.isNil for f in fList):
_inConsistent = not all(f.isNil for f in fList)
elif all(
inferredDecimals(f) == inferredDecimals(f0)
for f in fList[1:]): # same decimals
v0 = rangeValue(f0.value)
_inConsistent = not all(
rangeValue(f.value) == v0 for f in fList[1:])
else: # not all have same decimals
aMax, bMin = rangeValue(f0.value, inferredDecimals(f0))
for f in fList[1:]:
a, b = rangeValue(f.value, inferredDecimals(f))
if a > aMax: aMax = a
if b < bMin: bMin = b
_inConsistent = (bMin < aMax)
if _inConsistent:
modelXbrl.error(
("ESEF.2.2.4.inconsistentDuplicateNumericFactInInlineXbrlDocument"
),
"Inconsistent duplicate numeric facts MUST NOT appear in the content of an inline XBRL document. %(fact)s that was used more than once in contexts equivalent to %(contextID)s: values %(values)s. ",
modelObject=fList,
fact=f0.qname,
contextID=f0.contextID,
values=", ".join(
strTruncate(f.value, 128) for f in fList))
if precisionFacts:
modelXbrl.warning(
"ESEF.2.2.1.precisionAttributeUsed",
_("The accuracy of numeric facts SHOULD be defined with the 'decimals' attribute rather than the 'precision' attribute: %(elements)s."
),
modelObject=precisionFacts,
elements=", ".join(sorted(
str(e.qname) for e in precisionFacts)))
missingElements = (mandatory - reportedMandatory)
if missingElements:
modelXbrl.error(
"ESEF.???.missingRequiredElements",
_("Required elements missing from document: %(elements)s."),
modelObject=modelXbrl,
elements=", ".join(sorted(str(qn) for qn in missingElements)))
if transformRegistryErrors:
modelXbrl.warning(
"ESEF.2.2.3.transformRegistry",
_("ESMA recommends applying the latest available version of the Transformation Rules Registry marked with 'Recommendation' status for these elements: %(elements)s."
),
modelObject=transformRegistryErrors,
elements=", ".join(
sorted(
str(fact.qname) for fact in transformRegistryErrors)))
if orphanedFootnotes:
modelXbrl.error(
"ESEF.2.3.1.unusedFootnote",
_("Non-empty footnotes must be connected to fact(s)."),
modelObject=orphanedFootnotes)
if noLangFootnotes:
modelXbrl.error(
"ESEF.2.3.1.undefinedLanguageForFootnote",
_("Each footnote MUST have the 'xml:lang' attribute whose value corresponds to the language of the text in the content of the respective footnote."
),
modelObject=noLangFootnotes)
nonDefLangFtFacts = set(f for f, langs in factLangFootnotes.items()
if reportXmlLang not in langs)
if nonDefLangFtFacts:
modelXbrl.error(
"ESEF.2.3.1.footnoteOnlyInLanguagesOtherThanLanguageOfAReport",
_("Each fact MUST have at least one footnote with 'xml:lang' attribute whose value corresponds to the language of the text in the content of the respective footnote: %(qnames)s."
),
modelObject=nonDefLangFtFacts,
qnames=", ".join(
sorted(str(f.qname) for f in nonDefLangFtFacts)))
del nonDefLangFtFacts
if footnoteRoleErrors:
modelXbrl.error(
"ESEF.2.3.1.nonStandardRoleForFootnote",
_("The xlink:role attribute of a link:footnote and link:footnoteLink element as well as xlink:arcrole attribute of a link:footnoteArc MUST be defined in the XBRL Specification 2.1."
),
modelObject=footnoteRoleErrors)
nonStdFootnoteElts = list()
for modelLink in modelXbrl.baseSets[("XBRL-footnotes", None, None,
None)]:
for elt in modelLink.iterchildren():
if isinstance(
elt, (_ElementTree, _Comment, _ProcessingInstruction)):
continue # comment or other non-parsed element
if elt.qname not in FOOTNOTE_LINK_CHILDREN:
nonStdFootnoteElts.append(elt)
if nonStdFootnoteElts:
modelXbrl.error(
"ESEF.2.3.2.nonStandardElementInFootnote",
_("A link:footnoteLink element MUST have no children other than link:loc, link:footnote, and link:footnoteArc."
),
modelObject=nonStdFootnoteElts)
for qn in modelXbrl.qnameDimensionDefaults.values():
conceptsUsed.add(modelXbrl.qnameConcepts.get(qn))
# unused elements in linkbases
for arcroles, err in (
((parentChild, ),
"elementsNotUsedForTaggingAppliedInPresentationLinkbase"),
((summationItem, ),
"elementsNotUsedForTaggingAppliedInCalculationLinkbase"),
((dimensionDomain, domainMember),
"elementsNotUsedForTaggingAppliedInDefinitionLinkbase")):
unreportedLbElts = set()
for arcrole in arcroles:
for rel in modelXbrl.relationshipSet(
arcrole).modelRelationships:
fr = rel.fromModelObject
to = rel.toModelObject
if arcrole in (parentChild, summationItem):
if fr is not None and not fr.isAbstract and fr not in conceptsUsed and isExtension(
val, rel):
unreportedLbElts.add(fr)
if to is not None and not to.isAbstract and to not in conceptsUsed and isExtension(
val, rel):
unreportedLbElts.add(to)
elif arcrole == dimensionDomain: # dimension, always abstract
if fr is not None and fr not in conceptsUsed and isExtension(
val, rel):
unreportedLbElts.add(fr)
if to is not None and rel.isUsable and to not in conceptsUsed and isExtension(
val, rel):
unreportedLbElts.add(to)
elif arcrole == domainMember:
if to is not None and not fr.isAbstract and rel.isUsable and to not in conceptsUsed and isExtension(
val, rel):
unreportedLbElts.add(to)
if unreportedLbElts:
modelXbrl.error(
"ESEF.3.4.6." + err,
_("All usable concepts in extension taxonomy relationships MUST be applied by tagged facts: %(elements)s."
),
modelObject=unreportedLbElts,
elements=", ".join(
sorted((str(c.qname) for c in unreportedLbElts))))
# 3.4.4 check for presentation preferred labels
missingConceptLabels = defaultdict(set) # by role
pfsConceptsRootInPreLB = set()
def checkLabels(parent, relSet, labelrole, visited):
if not parent.label(
labelrole, lang=reportXmlLang, fallbackToQname=False):
if parent.name != "NotesAccountingPoliciesAndMandatoryTags": # TEMPORARY TBD remove
missingConceptLabels[labelrole].add(parent)
visited.add(parent)
conceptRels = defaultdict(
list) # counts for concepts without preferred label role
for rel in relSet.fromModelObject(parent):
child = rel.toModelObject
if child is not None:
labelrole = rel.preferredLabel
if not labelrole:
conceptRels[child].append(rel)
if child not in visited:
checkLabels(child, relSet, labelrole, visited)
for concept, rels in conceptRels.items():
if len(rels) > 1:
modelXbrl.warning(
"ESEF.3.4.4.missingPreferredLabelRole",
_("Preferred label role SHOULD be used when concept is duplicated in same presentation tree location: %(qname)s."
),
modelObject=rels + [concept],
qname=concept.qname)
visited.remove(parent)
for ELR in modelXbrl.relationshipSet(parentChild).linkRoleUris:
relSet = modelXbrl.relationshipSet(parentChild, ELR)
for rootConcept in relSet.rootConcepts:
checkLabels(rootConcept, relSet, None, set())
# check for PFS element which isn't an orphan
if rootConcept.qname in esefPrimaryStatementPlaceholders and relSet.fromModelObject(
rootConcept):
pfsConceptsRootInPreLB.add(rootConcept)
for labelrole, concepts in missingConceptLabels.items():
modelXbrl.warning(
"ESEF.3.4.5.missingLabelForRoleInReportLanguage",
_("Label for %(role)s role SHOULD be available in report language for concepts: %(qnames)s."
),
modelObject=concepts,
qnames=", ".join(str(c.qname) for c in concepts),
role=os.path.basename(labelrole) if labelrole else "standard")
if not pfsConceptsRootInPreLB:
# no PFS statements were recognized
modelXbrl.error(
"ESEF.RTS.Annex.II.Par.1.Par.7.missingPrimaryFinancialStatement",
_("A primary financial statement placeholder element MUST be a root of a presentation linkbase tree."
),
modelObject=modelXbrl)
# dereference
del missingConceptLabels, pfsConceptsRootInPreLB
# mandatory factc RTS Annex II
missingMandatoryElements = esefMandatoryElements2020 - modelXbrl.factsByQname.keys(
)
if missingMandatoryElements:
modelXbrl.error(
"ESEF.RTS.Annex.II.Par.2.missingMandatoryMarkups",
_("Mandatory elements to be marked up are missing: %(qnames)s."
),
modelObject=missingMandatoryElements,
qnames=", ".join(
sorted(str(qn) for qn in missingMandatoryElements)))
# duplicated core taxonomy elements
for name, concepts in modelXbrl.nameConcepts.items():
if len(concepts) > 1:
i = None # ifrs Concept
for c in concepts:
if c.qname.namespaceURI == _ifrsNs:
i = c
break
if i is not None:
for c in concepts:
if c != i and c.balance == i.balance and c.periodType == i.periodType:
modelXbrl.error(
"ESEF.RTS.Annex.IV.Par.4.2.extensionElementDuplicatesCoreElement",
_("Extension elements must not duplicate the existing elements from the core taxonomy and be identifiable %(qname)s."
),
modelObject=(c, i),
qname=c.qname)
modelXbrl.profileActivity(_statusMsg, minTimeToShow=0.0)
modelXbrl.modelManager.showStatus(None)
def validate(val, modelXbrl, infosetModelXbrl):
infoset = infosetModelXbrl.modelDocument
if infoset.type == Type.INSTANCE:
# compare facts (assumed out of order)
infosetFacts = defaultdict(list)
for fact in infosetModelXbrl.facts:
infosetFacts[fact.qname].append(fact)
if len(modelXbrl.factsInInstance) != len(
infosetModelXbrl.factsInInstance):
modelXbrl.error(
"arelle:infosetTest",
_("Fact counts mismatch, testcase instance %(foundFactCount)s, infoset instance %(expectedFactCount)s"
),
modelObject=(modelXbrl.modelDocument,
infosetModelXbrl.modelDocument),
foundFactCount=len(modelXbrl.factsInInstance),
expectedFactCount=len(infosetModelXbrl.factsInInstance))
else:
for i, instFact in enumerate(modelXbrl.facts):
infosetFact = None
for fact in infosetFacts[instFact.qname]:
if fact.isTuple and fact.isDuplicateOf(instFact,
deemP0Equal=True):
infosetFact = fact
break
elif fact.isItem and fact.isVEqualTo(instFact,
deemP0Equal=True):
infosetFact = fact
break
if infosetFact is None: # takes precision/decimals into account
if fact is not None:
fact.isVEqualTo(instFact, deemP0Equal=True)
modelXbrl.error(
"arelle:infosetTest",
_("Fact %(factNumber)s mismatch %(concept)s"),
modelObject=instFact,
factNumber=(i + 1),
concept=instFact.qname)
else:
ptvPeriodType = infosetFact.get(
"{http://www.xbrl.org/2003/ptv}periodType")
ptvBalance = infosetFact.get(
"{http://www.xbrl.org/2003/ptv}balance")
ptvDecimals = infosetFact.get(
"{http://www.xbrl.org/2003/ptv}decimals")
ptvPrecision = infosetFact.get(
"{http://www.xbrl.org/2003/ptv}precision")
if ptvPeriodType and ptvPeriodType != instFact.concept.periodType:
modelXbrl.error(
"arelle:infosetTest",
_("Fact %(factNumber)s periodType mismatch %(concept)s expected %(expectedPeriodType)s found %(foundPeriodType)s"
),
modelObject=(instFact, infosetFact),
factNumber=(i + 1),
concept=instFact.qname,
expectedPeriodType=ptvPeriodType,
foundPeriodType=instFact.concept.periodType)
if ptvBalance and ptvBalance != instFact.concept.balance:
modelXbrl.error(
"arelle:infosetTest",
_("Fact %(factNumber)s balance mismatch %(concept)s expected %(expectedBalance)s found %(foundBalance)s"
),
modelObject=(instFact, infosetFact),
factNumber=(i + 1),
concept=instFact.qname,
expectedBalance=ptvBalance,
foundBalance=instFact.concept.balance)
if ptvDecimals and ptvDecimals != str(
inferredDecimals(fact)):
modelXbrl.error(
"arelle:infosetTest",
_("Fact %(factNumber)s inferred decimals mismatch %(concept)s expected %(expectedDecimals)s found %(inferredDecimals)s"
),
modelObject=(instFact, infosetFact),
factNumber=(i + 1),
concept=instFact.qname,
expectedDecimals=ptvDecimals,
inferredDecimals=str(inferredDecimals(fact)))
if ptvPrecision and ptvPrecision != str(
inferredPrecision(fact)):
modelXbrl.error(
"arelle:infosetTest",
_("Fact %(factNumber)s inferred precision mismatch %(concept)s expected %(expectedPrecision)s found %(inferredPrecision)s"
),
modelObject=(instFact, infosetFact),
factNumber=(i + 1),
concept=instFact.qname,
expectedPrecisions=ptvPrecision,
inferredPrecision=str(inferredPrecision(fact)))
elif infoset.type == Type.ARCSINFOSET:
# compare arcs
for arcElt in XmlUtil.children(infoset.xmlRootElement,
"http://www.xbrl.org/2003/ptv", "arc"):
linkType = arcElt.get("linkType")
arcRole = arcElt.get("arcRole")
extRole = arcElt.get("extRole")
fromObj = resolvePath(modelXbrl, arcElt.get("fromPath"))
if fromObj is None:
modelXbrl.error("arelle:infosetTest",
_("Arc fromPath not found: %(fromPath)s"),
modelObject=arcElt,
fromPath=arcElt.get("fromPath"))
continue
if linkType in ("label", "reference"):
labelLang = arcElt.get("labelLang")
resRole = arcElt.get("resRole")
if linkType == "label":
expectedLabel = XmlUtil.text(arcElt)
foundLabel = fromObj.label(preferredLabel=resRole,
fallbackToQname=False,
lang=None,
strip=True,
linkrole=extRole)
if foundLabel != expectedLabel:
modelXbrl.error(
"arelle:infosetTest",
_("Label expected='%(expectedLabel)s', found='%(foundLabel)s'"
),
modelObject=arcElt,
expectedLabel=expectedLabel,
foundLabel=foundLabel)
continue
elif linkType == "reference":
expectedRef = XmlUtil.innerText(arcElt)
referenceFound = False
for refrel in modelXbrl.relationshipSet(
XbrlConst.conceptReference,
extRole).fromModelObject(fromObj):
ref = refrel.toModelObject
if resRole == ref.role:
foundRef = XmlUtil.innerText(ref)
if foundRef != expectedRef:
modelXbrl.error(
"arelle:infosetTest",
_("Reference inner text expected='%(expectedRef)s, found='%(foundRef)s'"
),
modelObject=arcElt,
expectedRef=expectedRef,
foundRef=foundRef)
referenceFound = True
break
if referenceFound:
continue
modelXbrl.error(
"arelle:infosetTest",
_("%(linkType)s not found containing '%(text)s' linkRole %(linkRole)s"
),
modelObject=arcElt,
linkType=linkType.title(),
text=XmlUtil.innerText(arcElt),
linkRole=extRole)
else:
toObj = resolvePath(modelXbrl, arcElt.get("toPath"))
if toObj is None:
modelXbrl.error("arelle:infosetTest",
_("Arc toPath not found: %(toPath)s"),
modelObject=arcElt,
toPath=arcElt.get("toPath"))
continue
weight = arcElt.get("weight")
if weight is not None:
weight = float(weight)
order = arcElt.get("order")
if order is not None:
order = float(order)
preferredLabel = arcElt.get("preferredLabel")
found = False
for rel in modelXbrl.relationshipSet(
arcRole, extRole).fromModelObject(fromObj):
if (rel.toModelObject == toObj
and (weight is None or rel.weight == weight)
and (order is None or rel.order == order)):
found = True
if not found:
modelXbrl.error(
"arelle:infosetTest",
_("Arc not found: from %(fromPath)s, to %(toPath)s, role %(arcRole)s, linkRole $(extRole)s"
),
modelObject=arcElt,
fromPath=arcElt.get("fromPath"),
toPath=arcElt.get("toPath"),
arcRole=arcRole,
linkRole=extRole)
continue
# validate dimensions of each fact
factElts = XmlUtil.children(modelXbrl.modelDocument.xmlRootElement,
None, "*")
for itemElt in XmlUtil.children(infoset.xmlRootElement, None, "item"):
try:
qnElt = XmlUtil.child(itemElt, None, "qnElement")
factQname = qname(qnElt, XmlUtil.text(qnElt))
sPointer = int(XmlUtil.child(itemElt, None, "sPointer").text)
factElt = factElts[sPointer - 1] # 1-based xpath indexing
if factElt.qname != factQname:
modelXbrl.error(
"arelle:infosetTest",
_("Fact %(sPointer)s mismatch Qname, expected %(qnElt)s, observed %(factQname)s"
),
modelObject=itemElt,
sPointer=sPointer,
qnElt=factQname,
factQname=factElt.qname)
elif not factElt.isItem or factElt.context is None:
modelXbrl.error(
"arelle:infosetTest",
_("Fact %(sPointer)s has no context: %(qnElt)s"),
modelObject=(itemElt, factElt),
sPointer=sPointer,
qnElt=factQname)
else:
context = factElt.context
memberElts = XmlUtil.children(itemElt, None, "member")
numNonDefaults = 0
for memberElt in memberElts:
dimElt = XmlUtil.child(memberElt, None, "qnDimension")
qnDim = qname(dimElt, XmlUtil.text(dimElt))
isDefault = XmlUtil.text(
XmlUtil.child(memberElt, None,
"bDefaulted")) == "true"
if not isDefault:
numNonDefaults += 1
if not (
(qnDim in context.qnameDims and not isDefault) or
(qnDim in factElt.modelXbrl.qnameDimensionDefaults
and isDefault)):
modelXbrl.error(
"arelle:infosetTest",
_("Fact %(sPointer)s (qnElt)s dimension mismatch %(qnDim)s"
),
modelObject=(itemElt, factElt, context),
sPointer=sPointer,
qnElt=factQname,
qnDim=qnDim)
if numNonDefaults != len(context.qnameDims):
modelXbrl.error(
"arelle:infosetTest",
_("Fact %(sPointer)s (qnElt)s dimensions count mismatch"
),
modelObject=(itemElt, factElt, context),
sPointer=sPointer,
qnElt=factQname)
except (IndexError, ValueError, AttributeError) as err:
modelXbrl.error(
"arelle:infosetTest",
_("Invalid entity fact dimensions infoset sPointer: %(test)s, error details: %(error)s"
),
modelObject=itemElt,
test=XmlUtil.innerTextList(itemElt),
error=str(err))
def addFact(self, f):
if f.id is None:
f.set("id", "ixv-%d" % (self.idGen))
self.idGen += 1
conceptName = self.nsmap.qname(f.qname)
scheme, ident = f.context.entityIdentifier
aspects = {
"c":
conceptName,
"e":
self.nsmap.qname(
QName(self.nsmap.getPrefix(scheme, "e"), scheme, ident)),
}
factData = {
"a": aspects,
}
if f.isNil:
factData["v"] = None
elif f.concept is not None and f.concept.isEnumeration:
qnEnums = f.xValue
if not isinstance(qnEnums, list):
qnEnums = (qnEnums, )
factData["v"] = " ".join(self.nsmap.qname(qn) for qn in qnEnums)
for qn in qnEnums:
self.addConcept(self.dts.qnameConcepts.get(qn))
else:
factData["v"] = f.value
if f.value == INVALIDixVALUE:
factData["err"] = 'INVALID_IX_VALUE'
if f.format is not None:
factData["f"] = str(f.format)
if f.isNumeric:
if f.unit is not None and len(f.unit.measures[0]):
# XXX does not support complex units
unit = self.nsmap.qname(f.unit.measures[0][0])
aspects["u"] = unit
else:
# The presence of the unit aspect is used by the viewer to
# identify numeric facts. If the fact has no unit (invalid
# XBRL, but we want to support it for draft documents),
# include the unit aspect with a null value.
aspects["u"] = None
d = inferredDecimals(f)
if d != float("INF") and not math.isnan(d):
factData["d"] = d
for d, v in f.context.qnameDims.items():
if v.memberQname is not None:
aspects[self.nsmap.qname(v.dimensionQname)] = self.nsmap.qname(
v.memberQname)
self.addConcept(v.member)
self.addConcept(v.dimension, dimensionType="e")
elif v.typedMember is not None:
aspects[self.nsmap.qname(
v.dimensionQname)] = v.typedMember.text
self.addConcept(v.dimension, dimensionType="t")
if f.context.isForeverPeriod:
aspects["p"] = "f"
elif f.context.isInstantPeriod and f.context.instantDatetime is not None:
aspects["p"] = self.dateFormat(
f.context.instantDatetime.isoformat())
elif f.context.isStartEndPeriod and f.context.startDatetime is not None and f.context.endDatetime is not None:
aspects["p"] = "%s/%s" % (
self.dateFormat(f.context.startDatetime.isoformat()),
self.dateFormat(f.context.endDatetime.isoformat()))
frels = self.footnoteRelationshipSet.fromModelObject(f)
if frels:
for frel in frels:
if frel.toModelObject is not None:
factData.setdefault("fn", []).append(frel.toModelObject.id)
self.taxonomyData["facts"][f.id] = factData
self.addConcept(f.concept)
