def dimValKey(cntx, typedDim=False, behaveAsTypedDims=emptySet, restrictToDims=None):
return '|'.join(sorted("{}({})".format(dim.dimensionQname,
dim.memberQname if dim.isExplicit and dim not in behaveAsTypedDims
else dim.memberQname if typedDim and not dim.isTyped
else xmlstring(dim.typedMember, stripXmlns=True) if typedDim
else '*' )
for dim in cntx.qnameDims.values()
if not restrictToDims or str(dim.dimensionQname) in restrictToDims))
def metDimTypedKey(fact):
cntx = fact.context
key = "MET({})".format(fact.qname)
if cntx is not None and cntx.qnameDims:
key += '|' + '|'.join(sorted("{}({})".format(dim.dimensionQname,
dim.memberQname if dim.isExplicit
else "nil" if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1")
else xmlstring(dim.typedMember, stripXmlns=True))
for dim in cntx.qnameDims.values()))
return key
def metDimTypedKey(fact):
cntx = fact.context
key = "MET({})".format(fact.qname)
if cntx is not None and cntx.qnameDims:
key += '|' + '|'.join(
sorted("{}({})".format(
dim.dimensionQname, dim.memberQname if dim.
isExplicit else "nil" if dim.typedMember.
get("{http://www.w3.org/2001/XMLSchema-instance}nil") in
("true", "1") else xmlstring(dim.typedMember, stripXmlns=True))
for dim in cntx.qnameDims.values()))
return key
def factFootnotes(fact):
footnotes = []
for footnoteRel in footnotesRelationshipSet.fromModelObject(fact):
footnote = OrderedDict((("group", footnoteRel.linkrole),
("footnoteType", footnoteRel.arcrole)))
footnotes.append(footnote)
if isCSVorXL:
footnote["factId"] = fact.id if fact.id else "f{}".format(fact.objectIndex)
toObj = footnoteRel.toModelObject
if isinstance(toObj, ModelFact):
footnote["factRef"] = toObj.id if toObj.id else "f{}".format(toObj.objectIndex)
else:
footnote["footnote"] = xmlstring(toObj, stripXmlns=True, contentsOnly=True, includeText=True)
if toObj.xmlLang:
footnote["language"] = toObj.xmlLang
return footnotes
def factFootnotes(fact):
footnotes = []
for footnoteRel in footnotesRelationshipSet.fromModelObject(fact):
footnote = OrderedDict((("group", footnoteRel.arcrole),))
footnotes.append(footnote)
if isCSV:
footnote["factId"] = fact.id if fact.id else "f{}".format(fact.objectIndex)
toObj = footnoteRel.toModelObject
if isinstance(toObj, ModelFact):
footnote["factRef"] = toObj.id if toObj.id else "f{}".format(toObj.objectIndex)
else:
footnote["footnoteType"] = toObj.role
footnote["footnote"] = xmlstring(toObj, stripXmlns=True, contentsOnly=True, includeText=True)
if toObj.xmlLang:
footnote["language"] = toObj.xmlLang
return footnotes
def factAspects(fact):
aspects = OrderedDict()
if hasId and fact.id:
aspects["id"] = fact.id
elif fact.isTuple or footnotesRelationshipSet.toModelObject(fact):
aspects["id"] = "f{}".format(fact.objectIndex)
parent = fact.getparent()
concept = fact.concept
if not fact.isTuple:
if concept is not None:
_baseXsdType = concept.baseXsdType
if _baseXsdType == "XBRLI_DATEUNION":
if getattr(fact.xValue, "dateOnly", False):
_baseXsdType = "date"
else:
_baseXsdType = "dateTime"
aspects["baseType"] = "xs:{}".format(_baseXsdType)
if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang:
aspects[qnOimLangAspect] = fact.xmlLang
aspects[qnOimTypeAspect] = concept.baseXbrliType
if fact.isItem:
if fact.isNil:
_value = None
_strValue = "nil"
else:
_inferredDecimals = inferredDecimals(fact)
_value = oimValue(fact.xValue, _inferredDecimals)
_strValue = str(_value)
aspects["value"] = _strValue
if fact.concept is not None and fact.concept.isNumeric:
_numValue = fact.xValue
if isinstance(_numValue, Decimal) and not isinf(_numValue) and not isnan(_numValue):
if _numValue == _numValue.to_integral():
_numValue = int(_numValue)
else:
_numValue = float(_numValue)
aspects["numericValue"] = _numValue
if not fact.isNil:
aspects["accuracy"] = "infinity" if isinf(_inferredDecimals) else _inferredDecimals
elif isinstance(_value, bool):
aspects["booleanValue"] = _value
aspects[qnOimConceptAspect] = oimValue(fact.qname)
cntx = fact.context
if cntx is not None:
if cntx.entityIdentifierElement is not None:
aspects[qnOimEntityAspect] = oimValue(qname(*cntx.entityIdentifier))
if cntx.period is not None:
aspects[qnOimPeriodAspect] = oimPeriodValue(cntx)
for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]):
aspects[dim.dimensionQname] = (oimValue(dim.memberQname) if dim.isExplicit
else None if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1")
else dim.typedMember.stringValue)
unit = fact.unit
if unit is not None:
_mMul, _mDiv = unit.measures
if isJSON:
aspects[qnOimUnitAspect] = { # use tuple instead of list for hashability
"numerators": tuple(oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m)))
}
if _mDiv:
aspects[qnOimUnitAspect]["denominators"] = tuple(oimValue(m) for m in sorted(_mDiv, key=lambda m: oimValue(m)))
else: # CSV
if _mMul:
aspects[qnOimUnitMulAspect] = ",".join(oimValue(m)
for m in sorted(_mMul, key=lambda m: q(m)))
if _mDiv:
aspects[qnOimUnitDivAspect] = ",".join(oimValue(m)
for m in sorted(_mDiv, key=lambda m: str(m)))
if parent.qname != XbrlConst.qnXbrliXbrl:
aspects[qnOimTupleParentAspect] = parent.id if parent.id else "f{}".format(parent.objectIndex)
aspects[qnOimTupleOrderAspect] = elementIndex(fact)
footnotes = []
for footnoteRel in footnotesRelationshipSet.fromModelObject(fact):
footnote = {"group": footnoteRel.arcrole}
footnotes.append(footnote)
toObj = footnoteRel.toModelObject
if isinstance(toObj, ModelFact):
footnote["factRef"] = toObj.id if toObj.id else "f{}".format(toObj.objectIndex)
else:
footnote["footnoteType"] = toObj.role
footnote["footnote"] = xmlstring(toObj, stripXmlns=True, contentsOnly=True, includeText=True)
if toObj.xmlLang:
footnote["language"] = toObj.xmlLang
if footnotes:
aspects["footnotes"] = footnotes
return aspects
def saveLoadableOIM(modelXbrl, oimFile, oimStyle, oimQNameSeparator):
isJSON = oimFile.endswith(".json")
isCSV = oimFile.endswith(".csv")
namespacePrefixes = {}
def compileQname(qname):
if qname.namespaceURI not in namespacePrefixes:
namespacePrefixes[qname.namespaceURI] = qname.prefix or ""
aspectsDefined = {
qnOimConceptAspect,
qnOimLocationAspect,
qnOimValueAspect,
qnOimPeriodAspect,
qnOimEntityAspect}
def oimValue(object, decimals=None):
if isinstance(object, QName):
if oimQNameSeparator == "clark":
return object.clarkNotation;
if object.namespaceURI not in namespacePrefixes:
if object.prefix:
namespacePrefixes[object.namespaceURI] = object.prefix
else:
_prefix = "_{}".format(sum(1 for p in namespacePrefixes if p.startswith("_")))
namespacePrefixes[object.namespaceURI] = _prefix
return "{}{}{}".format(namespacePrefixes[object.namespaceURI],
oimQNameSeparator,
object.localName)
if isinstance(object, Decimal):
try:
if decimals is not None and not isnan(decimals) and not isinf(decimals):
if decimals != 0:
object = object / (TEN ** -decimals)
return "{}e{}".format(object, -decimals)
else:
return "{}".format(object) # force to string to prevent json floating error
except:
return str(object)
if isinstance(object, (DateTime, YearMonthDuration, DayTimeDuration, Time,
gYearMonth, gMonthDay, gYear, gMonth, gDay)):
return str(object)
return object
def oimPeriodValue(cntx):
if cntx.isForeverPeriod:
return "forever"
elif cntx.isStartEndPeriod:
return "{}/{}".format(dateunionValue(cntx.startDatetime, dateOnlyHour=0),
dateunionValue(cntx.endDatetime, subtractOneDay=True, dateOnlyHour=24))
else: # instant
return "PT0S/{}".format(dateunionValue(cntx.endDatetime, subtractOneDay=True, dateOnlyHour=24))
hasId = False
hasLocation = False # may be optional based on style?
hasType = True
hasLang = False
hasUnits = False
hasUnitMulMeasures = False
hasUnitDivMeasures = False
hasTuple = False
#compile QNames in instance for OIM
for fact in modelXbrl.factsInInstance:
if fact.id:
hasId = True
concept = fact.concept
if concept is not None:
if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang:
hasLang = True
compileQname(fact.qname)
if hasattr(fact, "xValue") and isinstance(fact.xValue, QName):
compileQname(fact.xValue)
unit = fact.unit
if unit is not None:
hasUnits = True
if unit.measures[0]:
hasUnitMulMeasures = True
if unit.measures[1]:
hasUnitDivMeasures = True
if fact.modelTupleFacts:
hasTuple = True
entitySchemePrefixes = {}
for cntx in modelXbrl.contexts.values():
if cntx.entityIdentifierElement is not None:
scheme = cntx.entityIdentifier[0]
if scheme not in entitySchemePrefixes:
if not entitySchemePrefixes: # first one is just scheme
if scheme == "http://www.sec.gov/CIK":
_schemePrefix = "cik"
elif scheme == "http://standard.iso.org/iso/17442":
_schemePrefix = "lei"
else:
_schemePrefix = "scheme"
else:
_schemePrefix = "scheme-{}".format(len(entitySchemePrefixes) + 1)
entitySchemePrefixes[scheme] = _schemePrefix
namespacePrefixes[scheme] = _schemePrefix
for dim in cntx.qnameDims.values():
compileQname(dim.dimensionQname)
aspectsDefined.add(dim.dimensionQname)
if dim.isExplicit:
compileQname(dim.memberQname)
for unit in modelXbrl.units.values():
if unit is not None:
for measures in unit.measures:
for measure in measures:
compileQname(measure)
if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[XbrlConst.xbrli] != "xbrli":
namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix
if hasId: aspectsDefined.add(qnOimIdAspect)
if hasLang: aspectsDefined.add(qnOimLangAspect)
if hasTuple: aspectsDefined.add(qnOimTupleAspect)
if hasUnits: aspectsDefined.add(qnOimUnitAspect)
if hasUnitMulMeasures: aspectsDefined.add(qnOimUnitMulAspect)
if hasUnitDivMeasures: aspectsDefined.add(qnOimUnitDivAspect)
# compile footnotes and relationships
factRelationships = []
factFootnotes = []
for rel in modelXbrl.relationshipSet(modelXbrl, "XBRL-footnotes").modelRelationships:
oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole}
factRelationships.append(oimRel)
oimRel["fromIds"] = [obj.id if obj.id
else elementChildSequence(obj)
for obj in rel.fromModelObjects]
oimRel["toIds"] = [obj.id if obj.id
else elementChildSequence(obj)
for obj in rel.toModelObjects]
_order = rel.arcElement.get("order")
if _order is not None:
oimRel["order"] = _order
for obj in rel.toModelObjects:
if isinstance(obj, ModelResource): # footnote
oimFootnote = {"role": obj.role,
"id": obj.id if obj.id
else elementChildSequence(obj),
# value needs work for html elements and for inline footnotes
"value": xmlstring(obj, stripXmlns=True)}
if obj.xmlLang:
oimFootnote["lang"] = obj.xmlLang
factFootnotes.append(oimFootnote)
oimFootnote
dtsReferences = [
{"type": "schema" if doc.type == ModelDocument.Type.SCHEMA
else "linkbase" if doc.type == ModelDocument.Type.LINKBASE
else "other",
"href": doc.basename}
for doc,ref in modelXbrl.modelDocument.referencesDocument.items()
if ref.referringModelObject.qname in SCHEMA_LB_REFS]
roleTypes = [
{"type": "role" if ref.referringModelObject.localName == "roleRef" else "arcroleRef",
"href": ref.referringModelObject["href"]}
for doc,ref in modelXbrl.modelDocument.referencesDocument.items()
if ref.referringModelObject.qname in ROLE_REFS]
def factAspects(fact):
aspects = {qnOimConceptAspect: oimValue(fact.qname)}
if hasId and fact.id:
aspects[qnOimIdAspect] = fact.id
if hasLocation:
aspects[qnOimLocationAspect] = elementChildSequence(fact)
concept = fact.concept
if concept is not None:
if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang:
aspects[qnOimLangAspect] = fact.xmlLang
aspects[qnOimTypeAspect] = concept.baseXbrliType
if fact.isItem:
aspects[qnOimValueAspect] = (NILVALUE if fact.isNil else
oimValue(fact.xValue, inferredDecimals(fact)))
cntx = fact.context
if cntx is not None:
if cntx.entityIdentifierElement is not None:
aspects[qnOimEntityAspect] = oimValue(qname(*cntx.entityIdentifier))
if cntx.period is not None:
aspects[qnOimPeriodAspect] = oimPeriodValue(cntx)
for dim in cntx.qnameDims.values():
aspects[dim.dimensionQname] = (oimValue(dim.memberQname) if dim.isExplicit
else dim.typedMember.stringValue)
unit = fact.unit
if unit is not None:
_mMul, _mDiv = unit.measures
if isJSON:
aspects[qnOimUnitAspect] = ( # use tuple instead of list for hashability
tuple(oimValue(m) for m in sorted(_mMul, key=lambda m: str(m))),
tuple(oimValue(m) for m in sorted(_mDiv, key=lambda m: str(m))))
else: # CSV
if _mMul:
aspects[qnOimUnitMulAspect] = ",".join(oimValue(m)
for m in sorted(_mMul, key=lambda m: str(m)))
if _mDiv:
aspects[qnOimUnitDivAspect] = ",".join(oimValue(m)
for m in sorted(_mDiv, key=lambda m: str(m)))
return aspects
if isJSON:
# save JSON
oim = {} # top level of oim json output
oimFacts = []
oimReport = []
oimReport.append({"url": modelXbrl.modelDocument.uri})
if oimQNameSeparator != "clark":
oimReport.append({"prefixMap": dict((p,ns) for ns,p in namespacePrefixes.items())})
oimReport.append({"DTSreferences": dtsReferences})
oimReport.append({"roleTypes": roleTypes})
oimReport.append({"facts": oimFacts})
oimReport.append({"footnotes": factFootnotes})
oimReport.append({"relationships": factRelationships})
if oimStyle == "flat":
def saveFlatJsonFacts(facts, oimFacts):
for fact in facts:
oimFact = factAspects(fact)
if fact.modelTupleFacts:
tupleFacts = []
oimFact[qnOimTupleAspect] = tupleFacts
saveFlatJsonFacts(fact.modelTupleFacts, tupleFacts)
oimFacts.append(dict((oimValue(k),v) for k,v in oimFact.items()))
saveFlatJsonFacts(modelXbrl.facts, oimFacts)
elif oimStyle == "clustered":
# build aspect-value usage per fact for every fact
categoricalAspectValueSets = {} # for each aspect, value facts-set
aspectIndex = {}
indexAspect = {}
def addCategoricalAspect(aspectQn):
i = len(aspectIndex)
aspectIndex[aspectQn] = i
indexAspect[i] = oimValue(aspectQn)
categoricalAspectValueSets[i] = defaultdict(set)
addCategoricalAspect(qnOimConceptAspect)
addCategoricalAspect(qnOimEntityAspect)
addCategoricalAspect(qnOimPeriodAspect)
for aspectQn in aspectsDefined:
if aspectQn.namespaceURI != nsOim or aspectQn in (
qnOimIdAspect, qnOimLangAspect, qnOimUnitAspect):
addCategoricalAspect(aspectQn)
for fact in modelXbrl.facts:
fact._factAspectValues = {}
fact._factAspectSet = set()
for aspectQn, value in factAspects(fact).items():
if aspectQn in aspectIndex:
i = aspectIndex[aspectQn]
v = oimValue(value)
categoricalAspectValueSets[i][v].add(fact)
fact._factAspectValues[i] = v
fact._factAspectSet.add(i)
# order aspectValues by largest population
maxAspectValuePopulation = [(aspectIndex, max(len(factSet) for factSet in oimValueFacts.values()))
for aspectIndex, oimValueFacts in categoricalAspectValueSets.items()]
maxAspectValuePopulation.sort(key=lambda ai_max: -ai_max[1])
factsClustered = set()
_aspectValue = {}
def clusterAspect(_avpi, _data):
if _avpi >= len(maxAspectValuePopulation):
return # end of aspects
_ai = maxAspectValuePopulation[_avpi][0]
for _v, _vFactsSet in categoricalAspectValueSets[_ai].items():
_aspectValue[_ai] = _v
_nestedData = []
_nestedAspect = {indexAspect[_ai]: _v, "data": _nestedData}
for _fact in _vFactsSet - factsClustered:
if (_fact._factAspectSet == _aspectValue.keys() and
all([_fact._factAspectValues[__ai] == _aspectValue[__ai]
for __ai in _aspectValue])):
_factAspects = factAspects(_fact)
_oimFactItem = {oimValue(qnOimValueAspect): _factAspects[qnOimValueAspect]}
if hasLocation:
_oimFactItem[oimValue(qnOimLocationAspect)] = _factAspects[qnOimLocationAspect]
if hasType:
_oimFactItem[oimValue(qnOimTypeAspect)] = _factAspects[qnOimTypeAspect]
_nestedData.append(_oimFactItem)
factsClustered.add(_fact)
clusterAspect(_avpi+1, _nestedData)
if _nestedData:
_data.append(_nestedAspect)
del _aspectValue[_ai]
clusterAspect(0, oimFacts)
with open(oimFile, "w", encoding="utf-8") as fh:
fh.write(json.dumps(oimReport, ensure_ascii=False, indent=1, sort_keys=True))
elif isCSV:
# save CSV
# levels of tuple nesting
def tupleDepth(facts, parentDepth):
_levelDepth = parentDepth
for fact in facts:
_factDepth = tupleDepth(fact.modelTupleFacts, parentDepth + 1)
if _factDepth > _levelDepth:
_levelDepth = _factDepth
return _levelDepth
maxDepth = tupleDepth(modelXbrl.facts, 0)
aspectQnCol = {oimValue(qnOimConceptAspect): maxDepth - 1}
aspectsHeader = [oimValue(qnOimConceptAspect)]
for i in range(maxDepth - 1):
aspectsHeader.append(None)
def addAspectQnCol(aspectQn):
aspectQnCol[aspectQn] = len(aspectsHeader)
aspectsHeader.append(oimValue(aspectQn))
# pre-ordered aspect columns
if hasId:
addAspectQnCol(qnOimIdAspect)
if hasLocation:
addAspectQnCol(qnOimLocationAspect)
if hasType:
addAspectQnCol(qnOimTypeAspect)
addAspectQnCol(qnOimValueAspect)
if qnOimEntityAspect in aspectsDefined:
addAspectQnCol(qnOimEntityAspect)
if qnOimPeriodAspect in aspectsDefined:
addAspectQnCol(qnOimPeriodAspect)
if qnOimUnitMulAspect in aspectsDefined:
addAspectQnCol(qnOimUnitMulAspect)
if qnOimUnitDivAspect in aspectsDefined:
addAspectQnCol(qnOimUnitDivAspect)
for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)):
if aspectQn.namespaceURI != nsOim:
addAspectQnCol(aspectQn)
def aspectCols(fact, depth):
cols = [None for i in range(len(aspectsHeader))]
for aspectQn, aspectValue in factAspects(fact).items():
if aspectQn == qnOimConceptAspect:
cols[depth - 1] = aspectValue
elif aspectQn in aspectQnCol:
cols[aspectQnCol[aspectQn]] = aspectValue
return cols
# save facts
csvFile = open(oimFile, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
csvWriter.writerow(aspectsHeader)
def saveCSVfacts(facts, thisDepth):
for fact in facts:
csvWriter.writerow(aspectCols(fact, thisDepth))
saveCSVfacts(fact.modelTupleFacts, thisDepth + 1)
saveCSVfacts(modelXbrl.facts, 1)
csvFile.close()
# save namespaces
if oimQNameSeparator == "clark":
csvFile = open(oimFile.replace(".csv", "-prefixMap.csv"), csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
csvWriter.writerow(("prefix", "mappedURI"))
for namespaceURI, prefix in sorted(namespacePrefixes.items(), key=lambda item: item[1]):
csvWriter.writerow((prefix, namespaceURI))
csvFile.close()
# save dts references
csvFile = open(oimFile.replace(".csv", "-dts.csv"), csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
csvWriter.writerow(("type", "href"))
for oimRef in dtsReferences:
csvWriter.writerow((oimRef["type"], oimRef["href"]))
csvFile.close()
# save role and arc type references
if roleTypes:
csvFile = open(oimFile.replace(".csv", "-roleTypes.csv"), csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
csvWriter.writerow(("type", "href"))
for oimRef in roleTypes:
csvWriter.writerow((oimRef["type"], oimRef["href"]))
csvFile.close()
# save relationships
csvFile = open(oimFile.replace(".csv", "-relationships.csv"), csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
hasOrder = any(hasattribute(imRel,"order") for oimRel in factRelationships)
csvWriter.writerow(("fromIds", "toIds", "linkrole", "arcrole") +
(("order",) if hasOrder else ()))
for oimRel in factRelationships:
csvWriter.writerow((",".join(oimRel["fromIds"]),
",".join(oimRel["toIds"]),
oimRel["linkrole"],
oimRel["arcrole"]) +
((oimRel.get("order",None),) if hasOrder else ()))
csvFile.close()
# save footnotes
csvFile = open(oimFile.replace(".csv", "-footnotes.csv"), csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
hasLang = any(hasattribute(oimFnt,"lang") for oimFnt in factFootnotes)
csvWriter.writerow(("id", "role") + (("lang",) if hasLang else ()) + ("value",))
for oimFnt in factFootnotes:
csvWriter.writerow((oimFtn["id"], oimFtn["role"]) +
((oimFtn.get("lang",None),) if hasLang else ()) +
(oimFtn["value"],))
csvFile.close()
def saveLoadableOIM(modelXbrl, oimFile, oimStyle, oimQNameSeparator):
isJSON = oimFile.endswith(".json")
isCSV = oimFile.endswith(".csv")
namespacePrefixes = {}
def compileQname(qname):
if qname.namespaceURI not in namespacePrefixes:
namespacePrefixes[qname.namespaceURI] = qname.prefix or ""
aspectsDefined = {
qnOimConceptAspect, qnOimLocationAspect, qnOimValueAspect,
qnOimPeriodAspect, qnOimEntityAspect
}
def oimValue(object, decimals=None):
if isinstance(object, QName):
if oimQNameSeparator == "clark":
return object.clarkNotation
if object.namespaceURI not in namespacePrefixes:
if object.prefix:
namespacePrefixes[object.namespaceURI] = object.prefix
else:
_prefix = "_{}".format(
sum(1 for p in namespacePrefixes if p.startswith("_")))
namespacePrefixes[object.namespaceURI] = _prefix
return "{}{}{}".format(namespacePrefixes[object.namespaceURI],
oimQNameSeparator, object.localName)
if isinstance(object, Decimal):
try:
if decimals is not None and not isnan(decimals) and not isinf(
decimals):
if decimals != 0:
object = object / (TEN**-decimals)
return "{}e{}".format(object, -decimals)
else:
return "{}".format(
object
) # force to string to prevent json floating error
except:
return str(object)
if isinstance(object,
(DateTime, YearMonthDuration, DayTimeDuration, Time,
gYearMonth, gMonthDay, gYear, gMonth, gDay)):
return str(object)
return object
def oimPeriodValue(cntx):
if cntx.isForeverPeriod:
return "forever"
elif cntx.isStartEndPeriod:
return "{}/{}".format(
dateunionValue(cntx.startDatetime, dateOnlyHour=0),
dateunionValue(cntx.endDatetime,
subtractOneDay=True,
dateOnlyHour=24))
else: # instant
return "PT0S/{}".format(
dateunionValue(cntx.endDatetime,
subtractOneDay=True,
dateOnlyHour=24))
hasId = False
hasLocation = False # may be optional based on style?
hasType = True
hasLang = False
hasUnits = False
hasUnitMulMeasures = False
hasUnitDivMeasures = False
hasTuple = False
#compile QNames in instance for OIM
for fact in modelXbrl.factsInInstance:
if fact.id:
hasId = True
concept = fact.concept
if concept is not None:
if concept.baseXbrliType in ("string", "normalizedString",
"token") and fact.xmlLang:
hasLang = True
compileQname(fact.qname)
if hasattr(fact, "xValue") and isinstance(fact.xValue, QName):
compileQname(fact.xValue)
unit = fact.unit
if unit is not None:
hasUnits = True
if unit.measures[0]:
hasUnitMulMeasures = True
if unit.measures[1]:
hasUnitDivMeasures = True
if fact.modelTupleFacts:
hasTuple = True
entitySchemePrefixes = {}
for cntx in modelXbrl.contexts.values():
if cntx.entityIdentifierElement is not None:
scheme = cntx.entityIdentifier[0]
if scheme not in entitySchemePrefixes:
if not entitySchemePrefixes: # first one is just scheme
if scheme == "http://www.sec.gov/CIK":
_schemePrefix = "cik"
elif scheme == "http://standard.iso.org/iso/17442":
_schemePrefix = "lei"
else:
_schemePrefix = "scheme"
else:
_schemePrefix = "scheme-{}".format(
len(entitySchemePrefixes) + 1)
entitySchemePrefixes[scheme] = _schemePrefix
namespacePrefixes[scheme] = _schemePrefix
for dim in cntx.qnameDims.values():
compileQname(dim.dimensionQname)
aspectsDefined.add(dim.dimensionQname)
if dim.isExplicit:
compileQname(dim.memberQname)
for unit in modelXbrl.units.values():
if unit is not None:
for measures in unit.measures:
for measure in measures:
compileQname(measure)
if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[
XbrlConst.xbrli] != "xbrli":
namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix
if hasId: aspectsDefined.add(qnOimIdAspect)
if hasLang: aspectsDefined.add(qnOimLangAspect)
if hasTuple: aspectsDefined.add(qnOimTupleAspect)
if hasUnits: aspectsDefined.add(qnOimUnitAspect)
if hasUnitMulMeasures: aspectsDefined.add(qnOimUnitMulAspect)
if hasUnitDivMeasures: aspectsDefined.add(qnOimUnitDivAspect)
# compile footnotes and relationships
factRelationships = []
factFootnotes = []
for rel in modelXbrl.relationshipSet(modelXbrl,
"XBRL-footnotes").modelRelationships:
oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole}
factRelationships.append(oimRel)
oimRel["fromIds"] = [
obj.id if obj.id else elementChildSequence(obj)
for obj in rel.fromModelObjects
]
oimRel["toIds"] = [
obj.id if obj.id else elementChildSequence(obj)
for obj in rel.toModelObjects
]
_order = rel.arcElement.get("order")
if _order is not None:
oimRel["order"] = _order
for obj in rel.toModelObjects:
if isinstance(obj, ModelResource): # footnote
oimFootnote = {
"role": obj.role,
"id": obj.id if obj.id else elementChildSequence(obj),
# value needs work for html elements and for inline footnotes
"value": xmlstring(obj, stripXmlns=True)
}
if obj.xmlLang:
oimFootnote["lang"] = obj.xmlLang
factFootnotes.append(oimFootnote)
oimFootnote
dtsReferences = [{
"type":
"schema" if doc.type == ModelDocument.Type.SCHEMA else
"linkbase" if doc.type == ModelDocument.Type.LINKBASE else "other",
"href":
doc.basename
} for doc, ref in modelXbrl.modelDocument.referencesDocument.items()
if ref.referringModelObject.qname in SCHEMA_LB_REFS]
roleTypes = [{
"type":
"role"
if ref.referringModelObject.localName == "roleRef" else "arcroleRef",
"href":
ref.referringModelObject["href"]
} for doc, ref in modelXbrl.modelDocument.referencesDocument.items()
if ref.referringModelObject.qname in ROLE_REFS]
def factAspects(fact):
aspects = {qnOimConceptAspect: oimValue(fact.qname)}
if hasId and fact.id:
aspects[qnOimIdAspect] = fact.id
if hasLocation:
aspects[qnOimLocationAspect] = elementChildSequence(fact)
concept = fact.concept
if concept is not None:
if concept.baseXbrliType in ("string", "normalizedString",
"token") and fact.xmlLang:
aspects[qnOimLangAspect] = fact.xmlLang
aspects[qnOimTypeAspect] = concept.baseXbrliType
if fact.isItem:
aspects[qnOimValueAspect] = (NILVALUE if fact.isNil else oimValue(
fact.xValue, inferredDecimals(fact)))
cntx = fact.context
if cntx is not None:
if cntx.entityIdentifierElement is not None:
aspects[qnOimEntityAspect] = oimValue(
qname(*cntx.entityIdentifier))
if cntx.period is not None:
aspects[qnOimPeriodAspect] = oimPeriodValue(cntx)
for dim in cntx.qnameDims.values():
aspects[dim.dimensionQname] = (oimValue(dim.memberQname)
if dim.isExplicit else
dim.typedMember.stringValue)
unit = fact.unit
if unit is not None:
_mMul, _mDiv = unit.measures
if isJSON:
aspects[
qnOimUnitAspect] = ( # use tuple instead of list for hashability
tuple(
oimValue(m)
for m in sorted(_mMul, key=lambda m: str(m))),
tuple(
oimValue(m)
for m in sorted(_mDiv, key=lambda m: str(m))))
else: # CSV
if _mMul:
aspects[qnOimUnitMulAspect] = ",".join(
oimValue(m)
for m in sorted(_mMul, key=lambda m: str(m)))
if _mDiv:
aspects[qnOimUnitDivAspect] = ",".join(
oimValue(m)
for m in sorted(_mDiv, key=lambda m: str(m)))
return aspects
if isJSON:
# save JSON
oim = {} # top level of oim json output
oimFacts = []
oimReport = []
oimReport.append({"url": modelXbrl.modelDocument.uri})
if oimQNameSeparator != "clark":
oimReport.append({
"prefixMap":
dict((p, ns) for ns, p in namespacePrefixes.items())
})
oimReport.append({"DTSreferences": dtsReferences})
oimReport.append({"roleTypes": roleTypes})
oimReport.append({"facts": oimFacts})
oimReport.append({"footnotes": factFootnotes})
oimReport.append({"relationships": factRelationships})
if oimStyle == "flat":
def saveFlatJsonFacts(facts, oimFacts):
for fact in facts:
oimFact = factAspects(fact)
if fact.modelTupleFacts:
tupleFacts = []
oimFact[qnOimTupleAspect] = tupleFacts
saveFlatJsonFacts(fact.modelTupleFacts, tupleFacts)
oimFacts.append(
dict((oimValue(k), v) for k, v in oimFact.items()))
saveFlatJsonFacts(modelXbrl.facts, oimFacts)
elif oimStyle == "clustered":
# build aspect-value usage per fact for every fact
categoricalAspectValueSets = {} # for each aspect, value facts-set
aspectIndex = {}
indexAspect = {}
def addCategoricalAspect(aspectQn):
i = len(aspectIndex)
aspectIndex[aspectQn] = i
indexAspect[i] = oimValue(aspectQn)
categoricalAspectValueSets[i] = defaultdict(set)
addCategoricalAspect(qnOimConceptAspect)
addCategoricalAspect(qnOimEntityAspect)
addCategoricalAspect(qnOimPeriodAspect)
for aspectQn in aspectsDefined:
if aspectQn.namespaceURI != nsOim or aspectQn in (
qnOimIdAspect, qnOimLangAspect, qnOimUnitAspect):
addCategoricalAspect(aspectQn)
for fact in modelXbrl.facts:
fact._factAspectValues = {}
fact._factAspectSet = set()
for aspectQn, value in factAspects(fact).items():
if aspectQn in aspectIndex:
i = aspectIndex[aspectQn]
v = oimValue(value)
categoricalAspectValueSets[i][v].add(fact)
fact._factAspectValues[i] = v
fact._factAspectSet.add(i)
# order aspectValues by largest population
maxAspectValuePopulation = [
(aspectIndex,
max(len(factSet) for factSet in oimValueFacts.values()))
for aspectIndex, oimValueFacts in
categoricalAspectValueSets.items()
]
maxAspectValuePopulation.sort(key=lambda ai_max: -ai_max[1])
factsClustered = set()
_aspectValue = {}
def clusterAspect(_avpi, _data):
if _avpi >= len(maxAspectValuePopulation):
return # end of aspects
_ai = maxAspectValuePopulation[_avpi][0]
for _v, _vFactsSet in categoricalAspectValueSets[_ai].items():
_aspectValue[_ai] = _v
_nestedData = []
_nestedAspect = {indexAspect[_ai]: _v, "data": _nestedData}
for _fact in _vFactsSet - factsClustered:
if (_fact._factAspectSet == _aspectValue.keys()
and all([
_fact._factAspectValues[__ai]
== _aspectValue[__ai]
for __ai in _aspectValue
])):
_factAspects = factAspects(_fact)
_oimFactItem = {
oimValue(qnOimValueAspect):
_factAspects[qnOimValueAspect]
}
if hasLocation:
_oimFactItem[oimValue(
qnOimLocationAspect
)] = _factAspects[qnOimLocationAspect]
if hasType:
_oimFactItem[
oimValue(qnOimTypeAspect
)] = _factAspects[qnOimTypeAspect]
_nestedData.append(_oimFactItem)
factsClustered.add(_fact)
clusterAspect(_avpi + 1, _nestedData)
if _nestedData:
_data.append(_nestedAspect)
del _aspectValue[_ai]
clusterAspect(0, oimFacts)
with open(oimFile, "w", encoding="utf-8") as fh:
fh.write(
json.dumps(oimReport,
ensure_ascii=False,
indent=1,
sort_keys=True))
elif isCSV:
# save CSV
# levels of tuple nesting
def tupleDepth(facts, parentDepth):
_levelDepth = parentDepth
for fact in facts:
_factDepth = tupleDepth(fact.modelTupleFacts, parentDepth + 1)
if _factDepth > _levelDepth:
_levelDepth = _factDepth
return _levelDepth
maxDepth = tupleDepth(modelXbrl.facts, 0)
aspectQnCol = {oimValue(qnOimConceptAspect): maxDepth - 1}
aspectsHeader = [oimValue(qnOimConceptAspect)]
for i in range(maxDepth - 1):
aspectsHeader.append(None)
def addAspectQnCol(aspectQn):
aspectQnCol[aspectQn] = len(aspectsHeader)
aspectsHeader.append(oimValue(aspectQn))
# pre-ordered aspect columns
if hasId:
addAspectQnCol(qnOimIdAspect)
if hasLocation:
addAspectQnCol(qnOimLocationAspect)
if hasType:
addAspectQnCol(qnOimTypeAspect)
addAspectQnCol(qnOimValueAspect)
if qnOimEntityAspect in aspectsDefined:
addAspectQnCol(qnOimEntityAspect)
if qnOimPeriodAspect in aspectsDefined:
addAspectQnCol(qnOimPeriodAspect)
if qnOimUnitMulAspect in aspectsDefined:
addAspectQnCol(qnOimUnitMulAspect)
if qnOimUnitDivAspect in aspectsDefined:
addAspectQnCol(qnOimUnitDivAspect)
for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)):
if aspectQn.namespaceURI != nsOim:
addAspectQnCol(aspectQn)
def aspectCols(fact, depth):
cols = [None for i in range(len(aspectsHeader))]
for aspectQn, aspectValue in factAspects(fact).items():
if aspectQn == qnOimConceptAspect:
cols[depth - 1] = aspectValue
elif aspectQn in aspectQnCol:
cols[aspectQnCol[aspectQn]] = aspectValue
return cols
# save facts
csvFile = open(oimFile,
csvOpenMode,
newline=csvOpenNewline,
encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
csvWriter.writerow(aspectsHeader)
def saveCSVfacts(facts, thisDepth):
for fact in facts:
csvWriter.writerow(aspectCols(fact, thisDepth))
saveCSVfacts(fact.modelTupleFacts, thisDepth + 1)
saveCSVfacts(modelXbrl.facts, 1)
csvFile.close()
# save namespaces
if oimQNameSeparator == "clark":
csvFile = open(oimFile.replace(".csv", "-prefixMap.csv"),
csvOpenMode,
newline=csvOpenNewline,
encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
csvWriter.writerow(("prefix", "mappedURI"))
for namespaceURI, prefix in sorted(namespacePrefixes.items(),
key=lambda item: item[1]):
csvWriter.writerow((prefix, namespaceURI))
csvFile.close()
# save dts references
csvFile = open(oimFile.replace(".csv", "-dts.csv"),
csvOpenMode,
newline=csvOpenNewline,
encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
csvWriter.writerow(("type", "href"))
for oimRef in dtsReferences:
csvWriter.writerow((oimRef["type"], oimRef["href"]))
csvFile.close()
# save role and arc type references
if roleTypes:
csvFile = open(oimFile.replace(".csv", "-roleTypes.csv"),
csvOpenMode,
newline=csvOpenNewline,
encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
csvWriter.writerow(("type", "href"))
for oimRef in roleTypes:
csvWriter.writerow((oimRef["type"], oimRef["href"]))
csvFile.close()
# save relationships
csvFile = open(oimFile.replace(".csv", "-relationships.csv"),
csvOpenMode,
newline=csvOpenNewline,
encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
hasOrder = any(
hasattribute(imRel, "order") for oimRel in factRelationships)
csvWriter.writerow(("fromIds", "toIds", "linkrole", "arcrole") +
(("order", ) if hasOrder else ()))
for oimRel in factRelationships:
csvWriter.writerow(
(",".join(oimRel["fromIds"]), ",".join(oimRel["toIds"]),
oimRel["linkrole"], oimRel["arcrole"]) +
((oimRel.get("order", None), ) if hasOrder else ()))
csvFile.close()
# save footnotes
csvFile = open(oimFile.replace(".csv", "-footnotes.csv"),
csvOpenMode,
newline=csvOpenNewline,
encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
hasLang = any(hasattribute(oimFnt, "lang") for oimFnt in factFootnotes)
csvWriter.writerow(("id", "role") + (("lang", ) if hasLang else ()) +
("value", ))
for oimFnt in factFootnotes:
csvWriter.writerow((oimFtn["id"], oimFtn["role"]) + (
(oimFtn.get("lang", None), ) if hasLang else ()) +
(oimFtn["value"], ))
csvFile.close()
def showInfo(cntlr, options, modelXbrl, _entrypoint, *args, **kwargs):
for url, doc in sorted(modelXbrl.urlDocs.items(), key=lambda i: i[0]):
if not any(url.startswith(w) for w in ("https://xbrl.sec.gov/", "http://xbrl.sec.gov/", "http://xbrl.fasb.org/", "http://www.xbrl.org/",
"http://xbrl.ifrs.org/", "http://www.esma.europa.eu/")):
if os.path.exists(doc.filepath): # skip if in an archive or stream
cntlr.addToLog("File {} size {:,}".format(doc.basename, os.path.getsize(doc.filepath)), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Heap memory before loading {:,}".format(memoryAtStartup), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Heap memory after loading {:,}".format(cntlr.memoryUsed), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Time to load {:.2f} seconds".format(time.time() - timeAtStart), messageCode="info", level=logging.DEBUG)
isInlineXbrl = modelXbrl.modelDocument.type in (ModelDocument.Type.INLINEXBRL, ModelDocument.Type.INLINEXBRLDOCUMENTSET)
if isInlineXbrl:
instanceType = "inline XBRL, number of documents {}".format(len(modelXbrl.ixdsHtmlElements))
else:
instanceType = "xBRL-XML"
cntlr.addToLog("Instance type {}".format(instanceType), messageCode="info", level=logging.DEBUG)
numContexts = len(modelXbrl.contexts)
numLongContexts = 0
bytesSaveableInline = 0
bytesSaveableInlineWithCsv = 0
frequencyOfDims = {}
sumNumDims = 0
distinctDurations = set()
distinctInstants = set()
shortContextIdLen = int(math.log10(numContexts or 1)) + 2 # if no contexts, use 1 for log function to work
xbrlQnameCountInline = 0
xbrlQnameCountInlineWithCsv = 0
xbrlQnameLengthsInline = 0
xbrlQnameLengthsInlineWithCsv = 0
for c in modelXbrl.contexts.values():
sumNumDims += len(c.qnameDims)
for d in c.qnameDims.values():
dimQname = str(d.dimensionQname)
frequencyOfDims[dimQname] = frequencyOfDims.get(dimQname,0) + 1
xbrlQnameCountInline += 1
xbrlQnameCountInlineWithCsv += 1
xbrlQnameLengthsInline += len(d.dimensionQname.localName)
xbrlQnameLengthsInlineWithCsv += len(d.dimensionQname.localName)
if c.isInstantPeriod:
distinctInstants.add(c.instantDatetime)
elif c.isStartEndPeriod:
distinctDurations.add((c.startDatetime, c.endDatetime))
if len(c.id) > shortContextIdLen:
bytesSaveableInline += len(c.id) - shortContextIdLen
bytesSaveableInlineWithCsv += len(c.id) - shortContextIdLen
cntlr.addToLog("Number of contexts {:,}".format(numContexts), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Number of distinct durations {:,}".format(len(distinctDurations)), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Number of distinct instants {:,}".format(len(distinctInstants)), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Avg number dimensions per contexts {:,.2f}".format(sumNumDims/numContexts if numContexts else 0), messageCode="info", level=logging.DEBUG)
mostPopularDims = sorted(frequencyOfDims.items(), key=lambda i:"{:0>9},{}".format(999999999-i[1],i[0]))
for dimName, count in mostPopularDims[0:3]:
cntlr.addToLog("Dimension {} used in {:,} contexts".format(dimName, count), messageCode="info", level=logging.DEBUG)
# analyze for tables which could be composed from CSV data
tblFacts = defaultdict(set)
tblNestedTables = defaultdict(set)
factSize = {}
for f in modelXbrl.factsInInstance:
for tdElt in ancestors(f, xhtml, "td"):
factSize[f] = len(xmlstring(tdElt,stripXmlns=True))
break
childTblElt = None
for tblElt in ancestors(f, xhtml, "table"):
tblFacts[tblElt].add(f)
if childTblElt:
tblNestedTables[tblElt].add(childTblElt)
# find tables containing only numeric facts
def tblNestedFactCount(tbl):
c = len(tblFacts.get(tbl, ()))
for nestedTbl in tblNestedTables.get(tbl,()):
c += tblNestedFactCount(nestedTbl)
return c
factsInInstance = len(modelXbrl.factsInInstance)
factsInTables = len(set.union(*(fset for fset in tblFacts.values())))
cntlr.addToLog("Facts in instance: {:,}, facts in tables: {:,}".format(factsInInstance,factsInTables), messageCode="info", level=logging.DEBUG)
numTblsEligible = 0
numFactsEligible = 0
bytesCsvSavings = 0
factsEligibleForCsv = set()
tablesWithEligibleFacts = set()
if tblFacts and factSize:
# find eligible tables, have facts and not nested tables with other facts
for tbl, facts in tblFacts.items():
if len(facts) == tblNestedFactCount(tbl):
s = sum(factSize.get(f,0) for f in facts) - sum(len(str(f.value)) for f in facts)
if s > 10000:
numTblsEligible += 1
bytesCsvSavings += s
numFactsEligible += len(facts)
factsEligibleForCsv |= facts
tablesWithEligibleFacts.add(tbl)
numFacts = 0
numTableTextBlockFacts = 0
lenTableTextBlockFacts = 0
numTextBlockFacts = 0
lenTextBlockFacts = 0
distinctElementsInFacts = set()
factsPerContext = {}
factForConceptContextUnitHash = defaultdict(list)
for f in modelXbrl.factsInInstance:
context = f.context
concept = f.concept
distinctElementsInFacts.add(f.qname)
numFacts += 1
if f.qname.localName.endswith("TableTextBlock"):
numTableTextBlockFacts += 1
lenTableTextBlockFacts += len(f.xValue)
elif f.qname.localName.endswith("TextBlock"):
numTextBlockFacts += 1
lenTextBlockFacts += len(f.xValue)
if context is not None and concept is not None:
factsPerContext[context.id] = factsPerContext.get(context.id,0) + 1
factForConceptContextUnitHash[f.conceptContextUnitHash].append(f)
bytesSaveableInline += len(context.id) - shortContextIdLen
if f not in factsEligibleForCsv:
bytesSaveableInlineWithCsv += len(context.id) - shortContextIdLen
if numTblsEligible:
cntlr.addToLog("Tables eligible for facts in CSV: {:,}, facts eligible for CSV: {:,}, bytes saveable by facts in CSV {:,}".format(numTblsEligible, numFactsEligible, bytesCsvSavings), messageCode="info", level=logging.DEBUG)
else:
cntlr.addToLog("No tables eligible for facts in CSV", messageCode="info", level=logging.DEBUG)
mostPopularContexts = sorted(factsPerContext.items(), key=lambda i:"{:0>9},{}".format(999999999-i[1],i[0]))
cntlr.addToLog("Number of facts {:,}".format(numFacts), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Number of TableTextBlock facts {:,} avg len {:,.0f}".format(numTableTextBlockFacts, lenTableTextBlockFacts/numTableTextBlockFacts if numTableTextBlockFacts else 0), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Number of TextBlock facts {:,} avg len {:,.0f}".format(numTextBlockFacts, lenTextBlockFacts/numTableTextBlockFacts if numTableTextBlockFacts else 0), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Max number facts per context {:,}".format(mostPopularContexts[0][1] if mostPopularContexts else 0), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Avg number facts per context {:,.2f}".format(sum([v for v in factsPerContext.values()])/numContexts if numContexts else 0), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Distinct elements in facts {:,}".format(len(distinctElementsInFacts)), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Number of bytes saveable context id of {} length is {:,}".format(shortContextIdLen, bytesSaveableInline), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Excepting facts eligible for CSV, number of bytes saveable context id of {} length is {:,}".format(shortContextIdLen, bytesSaveableInlineWithCsv), messageCode="info", level=logging.DEBUG)
aspectEqualFacts = defaultdict(list)
decVals = {}
numConsistentDupFacts = numInConsistentDupFacts = 0
for hashEquivalentFacts in factForConceptContextUnitHash.values():
if len(hashEquivalentFacts) > 1:
for f in hashEquivalentFacts:
aspectEqualFacts[(f.qname,f.contextID,f.unitID,
f.xmlLang.lower() if f.concept.type.isWgnStringFactType else None)].append(f)
for fList in aspectEqualFacts.values():
f0 = fList[0]
if f0.concept.isNumeric:
if any(f.isNil for f in fList):
_inConsistent = not all(f.isNil for f in fList)
else: # not all have same decimals
_d = inferredDecimals(f0)
_v = f0.xValue
_inConsistent = isnan(_v) # NaN is incomparable, always makes dups inconsistent
decVals[_d] = _v
aMax, bMin = rangeValue(_v, _d)
for f in fList[1:]:
_d = inferredDecimals(f)
_v = f.xValue
if isnan(_v):
_inConsistent = True
break
if _d in decVals:
_inConsistent |= _v != decVals[_d]
else:
decVals[_d] = _v
a, b = rangeValue(_v, _d)
if a > aMax: aMax = a
if b < bMin: bMin = b
if not _inConsistent:
_inConsistent = (bMin < aMax)
decVals.clear()
else:
_inConsistent = any(not f.isVEqualTo(f0) for f in fList[1:])
if _inConsistent:
numInConsistentDupFacts += 1
else:
numConsistentDupFacts += 1
aspectEqualFacts.clear()
cntlr.addToLog("Number of duplicate facts consistent {:,} inconsistent {:,}".format(numConsistentDupFacts, numInConsistentDupFacts), messageCode="info", level=logging.DEBUG)
styleAttrCountsInline = {}
styleAttrCountsInlineWithCsv = {}
totalStyleLenInline = 0
totalStyleLenInlineWithCsv = 0
continuationElements = {}
ixNsPrefix = "{http://www.xbrl.org/2013/inlineXBRL}"
for ixdsHtmlRootElt in getattr(modelXbrl, "ixdsHtmlElements", ()): # ix root elements if inline
for ixElt in ixdsHtmlRootElt.iterdescendants():
inEligibleTableForCsv = any(p in tablesWithEligibleFacts for p in ixElt.iterancestors("{http://www.w3.org/1999/xhtml}table"))
style = ixElt.get("style")
ixEltTag = str(ixElt.tag)
if style:
styleAttrCountsInline[style] = styleAttrCountsInline.get(style,0) + 1
if not inEligibleTableForCsv:
styleAttrCountsInlineWithCsv[style] = styleAttrCountsInlineWithCsv.get(style,0) + 1
if styleIxHiddenPattern.match(style) is None:
totalStyleLenInline += len(style)
if not inEligibleTableForCsv:
totalStyleLenInlineWithCsv += len(style)
if ixEltTag == "{http://www.xbrl.org/2013/inlineXBRL}continuation" and ixElt.id:
continuationElements[ixElt.id] = ixElt
if ixEltTag.startswith(ixNsPrefix):
localName = ixEltTag[len(ixNsPrefix):]
if localName == "continuation" and ixElt.id:
continuationElements[ixElt.id] = ixElt
elif localName in ("nonFraction", "nonNumeric", "fraction"):
xbrlQnameCountInline += 1
xbrlQnameLengthsInline += len(ixElt.qname.localName)
if not inEligibleTableForCsv:
xbrlQnameCountInlineWithCsv += 1
xbrlQnameLengthsInlineWithCsv += len(ixElt.qname.localName)
elif isinstance(ixElt, ModelFact):
xbrlQnameCountInline += 2
xbrlQnameLengthsInline += len(ixElt.qname.localName)
if not inEligibleTableForCsv:
xbrlQnameCountInlineWithCsv += 2
xbrlQnameLengthsInlineWithCsv += len(ixElt.qname.localName)
def locateContinuation(element, chain=None):
contAt = element.get("continuedAt")
if contAt:
if contAt in continuationElements:
if chain is None: chain = [element]
contElt = continuationElements[contAt]
if contElt not in chain:
chain.append(contElt)
element._continuationElement = contElt
return locateContinuation(contElt, chain)
elif chain: # end of chain
return len(chain)
numContinuations = 0
maxLenLen = 0
maxLenHops = 0
maxHops = 0
maxHopsLen = 0
for f in modelXbrl.factsInInstance:
if f.get("continuedAt"):
numContinuations += 1
_len = len(f.xValue)
_hops = locateContinuation(f)
if _hops > maxHops:
maxHops = _hops
maxHopsLen = _len
if _len > maxLenLen:
maxLenLen = _len
maxLenHops = _hops
cntlr.addToLog("Number of continuation facts {:,}".format(numContinuations), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Longest continuation fact {:,} number of hops {:,}".format(maxLenLen, maxLenHops), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Most continuation hops {:,} fact len {:,}".format(maxHops, maxHopsLen), messageCode="info", level=logging.DEBUG)
numDupStyles = sum(1 for n in styleAttrCountsInline.values() if n > 1)
bytesSaveableByCssInline = sum(len(s)*(n-1) for s,n in styleAttrCountsInline.items() if n > 1)
cntlr.addToLog("Number of duplicate styles {:,}, bytes saveable by CSS {:,}, len of all non-ix-hidden @styles {:,}".format(numDupStyles, bytesSaveableByCssInline, totalStyleLenInline), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Number of XBRL QNames {:,}, bytes saveable by EBA-style element names {:,}".format(xbrlQnameCountInline, xbrlQnameLengthsInline - (5*xbrlQnameCountInline)), messageCode="info", level=logging.DEBUG)
numDupStyles = sum(1 for n in styleAttrCountsInlineWithCsv.values() if n > 1)
bytesSaveableByCssInlineWithCsv = sum(len(s)*(n-1) for s,n in styleAttrCountsInlineWithCsv.items() if n > 1)
cntlr.addToLog("Excepting facts eligible for CSV, number of duplicate styles {:,}, bytes saveable by CSS {:,}, len of all non-ix-hidden @styles {:,}".format(numDupStyles, bytesSaveableByCssInlineWithCsv, totalStyleLenInlineWithCsv), messageCode="info", level=logging.DEBUG)
cntlr.addToLog("Excepting facts eligible for CSV, number of XBRL QNames {:,}, bytes saveable by EBA-style element names {:,}".format(xbrlQnameCountInlineWithCsv, xbrlQnameLengthsInlineWithCsv - (5*xbrlQnameCountInlineWithCsv)), messageCode="info", level=logging.DEBUG)
def dimValKey(cntx, typedDim=False):
return '|'.join(sorted("{}({})".format(dim.dimensionQname,
dim.memberQname if dim.isExplicit
else xmlstring(dim.typedMember, stripXmlns=True) if typedDim
else '*' )
for dim in cntx.qnameDims.values()))
