class IXBRLViewerBuilder:
def __init__(self, dts):
self.nsmap = NamespaceMap()
self.roleMap = NamespaceMap()
self.dts = dts
self.taxonomyData = {
"concepts": {},
"languages": {},
"facts": {},
}
self.footnoteRelationshipSet = ModelRelationshipSet(
dts, "XBRL-footnotes")
def lineWrap(self, s, n=80):
return "\n".join([s[i:i + n] for i in range(0, len(s), n)])
def dateFormat(self, d):
"""
Strip the time component from an ISO date if it's zero
"""
return re.sub("T00:00:00$", "", d)
def escapeJSONForScriptTag(self, s):
"""
JSON encodes XML special characters XML and HTML apply difference escaping rules to content
within script tags and we need our output to be valid XML, but treated as HTML by browsers.
If we allow XML escaping to occur in a script tag, browsers treating
the document as HTML won't unescape it. If we don't escape XML special
characters, it won't be valid XML.
We avoid this whole mess by escaping XML special characters using JSON
string escapes. This is only safe to do because < > and & can't occur
outside a string in JSON. It can't safely be used on JS.
"""
return s.replace("<",
"\\u003C").replace(">",
"\\u003E").replace("&", "\\u0026")
def makeLanguageName(self, langCode):
code = re.sub("-.*", "", langCode)
try:
language = pycountry.languages.lookup(code)
match = re.match(r'^[^-]+-(.*)$', langCode)
name = language.name
if match is not None:
name = "%s (%s)" % (name, match.group(1).upper())
except LookupError:
name = langCode
return name
def addLanguage(self, langCode):
if langCode not in self.taxonomyData["languages"]:
self.taxonomyData["languages"][langCode] = self.makeLanguageName(
langCode)
def addELR(self, elr):
prefix = self.roleMap.getPrefix(elr)
if self.taxonomyData.setdefault("roleDefs", {}).get(prefix,
None) is None:
rt = self.dts.roleTypes[elr]
label = elr
if len(rt) > 0:
label = rt[0].definition
self.taxonomyData["roleDefs"].setdefault(prefix, {})["en"] = label
def addConcept(self, concept, dimensionType=None):
if concept is None:
return
labelsRelationshipSet = self.dts.relationshipSet(
XbrlConst.conceptLabel)
labels = labelsRelationshipSet.fromModelObject(concept)
conceptName = self.nsmap.qname(concept.qname)
if conceptName not in self.taxonomyData["concepts"]:
conceptData = {"labels": {}}
for lr in labels:
l = lr.toModelObject
conceptData["labels"].setdefault(
self.roleMap.getPrefix(l.role),
{})[l.xmlLang.lower()] = l.text
self.addLanguage(l.xmlLang.lower())
refData = []
for _refRel in concept.modelXbrl.relationshipSet(
XbrlConst.conceptReference).fromModelObject(concept):
ref = []
for _refPart in _refRel.toModelObject.iterchildren():
ref.append(
[_refPart.localName,
_refPart.stringValue.strip()])
refData.append(ref)
if len(refData) > 0:
conceptData['r'] = refData
if dimensionType is not None:
conceptData["d"] = dimensionType
self.taxonomyData["concepts"][conceptName] = conceptData
def treeWalk(self, rels, item, indent=0):
for r in rels.fromModelObject(item):
if r.toModelObject is not None:
self.treeWalk(rels, r.toModelObject, indent + 1)
def getRelationships(self):
rels = {}
for baseSetKey, baseSetModelLinks in self.dts.baseSets.items():
arcrole, ELR, linkqname, arcqname = baseSetKey
if arcrole in (XbrlConst.summationItem,
WIDER_NARROWER_ARCROLE) and ELR is not None:
self.addELR(ELR)
rr = dict()
relSet = self.dts.relationshipSet(arcrole, ELR)
for r in relSet.modelRelationships:
if r.fromModelObject is not None and r.toModelObject is not None:
fromKey = self.nsmap.qname(r.fromModelObject.qname)
rel = {
"t": self.nsmap.qname(r.toModelObject.qname),
}
if r.weight is not None:
rel['w'] = r.weight
rr.setdefault(fromKey, []).append(rel)
self.addConcept(r.toModelObject)
self.addConcept(r.fromModelObject)
rels.setdefault(self.roleMap.getPrefix(arcrole),
{})[self.roleMap.getPrefix(ELR)] = rr
return rels
def createViewer(self, scriptUrl="js/dist/ixbrlviewer.js"):
"""
Create an iXBRL file with XBRL data as a JSON blob, and script tags added
"""
dts = self.dts
iv = iXBRLViewer(dts)
idGen = 0
self.roleMap.getPrefix(XbrlConst.standardLabel, "std")
self.roleMap.getPrefix(XbrlConst.documentationLabel, "doc")
self.roleMap.getPrefix(XbrlConst.summationItem, "calc")
self.roleMap.getPrefix(XbrlConst.parentChild, "pres")
self.roleMap.getPrefix(WIDER_NARROWER_ARCROLE, "w-n")
for f in dts.facts:
if f.id is None:
f.set("id", "ixv-%d" % (idGen))
idGen += 1
conceptName = self.nsmap.qname(f.qname)
scheme, ident = f.context.entityIdentifier
aspects = {
"c":
conceptName,
"e":
self.nsmap.qname(
QName(self.nsmap.getPrefix(scheme, "e"), scheme, ident)),
}
factData = {
"v": f.value if not f.isNil else None,
"a": aspects,
}
if f.format is not None:
factData["f"] = str(f.format)
if f.isNumeric:
if f.unit is not None and len(f.unit.measures[0]):
# XXX does not support complex units
unit = self.nsmap.qname(f.unit.measures[0][0])
aspects["u"] = unit
else:
# The presence of the unit aspect is used by the viewer to
# identify numeric facts. If the fact has no unit (invalid
# XBRL, but we want to support it for draft documents),
# include the unit aspect with a null value.
aspects["u"] = None
d = inferredDecimals(f)
if d != float("INF") and not math.isnan(d):
factData["d"] = d
for d, v in f.context.qnameDims.items():
if v.memberQname is not None:
aspects[self.nsmap.qname(
v.dimensionQname)] = self.nsmap.qname(v.memberQname)
self.addConcept(v.member)
self.addConcept(v.dimension, dimensionType="e")
elif v.typedMember is not None:
aspects[self.nsmap.qname(
v.dimensionQname)] = v.typedMember.text
self.addConcept(v.dimension, dimensionType="t")
if f.context.isForeverPeriod:
aspects["p"] = "f"
elif f.context.isInstantPeriod and f.context.instantDatetime is not None:
aspects["p"] = self.dateFormat(
f.context.instantDatetime.isoformat())
elif f.context.isStartEndPeriod and f.context.startDatetime is not None and f.context.endDatetime is not None:
aspects["p"] = "%s/%s" % (
self.dateFormat(f.context.startDatetime.isoformat()),
self.dateFormat(f.context.endDatetime.isoformat()))
frels = self.footnoteRelationshipSet.fromModelObject(f)
if frels:
for frel in frels:
if frel.toModelObject is not None:
factData.setdefault("fn",
[]).append(frel.toModelObject.id)
self.taxonomyData["facts"][f.id] = factData
if f.concept is not None:
self.addConcept(f.concept)
self.taxonomyData["prefixes"] = self.nsmap.prefixmap
self.taxonomyData["roles"] = self.roleMap.prefixmap
self.taxonomyData["rels"] = self.getRelationships()
dts.info("viewer:info", "Creating iXBRL viewer")
if dts.modelDocument.type == Type.INLINEXBRLDOCUMENTSET:
# Sort by object index to preserve order in which files were specified.
docSet = sorted(dts.modelDocument.referencesDocument.keys(),
key=lambda x: x.objectIndex)
docSetFiles = list(
map(lambda x: os.path.basename(x.filepath), docSet))
self.taxonomyData["docSetFiles"] = docSetFiles
for n in range(0, len(docSet)):
iv.addFile(
iXBRLViewerFile(docSetFiles[n], docSet[n].xmlDocument))
xmlDocument = docSet[0].xmlDocument
else:
xmlDocument = dts.modelDocument.xmlDocument
filename = os.path.basename(dts.modelDocument.filepath)
iv.addFile(iXBRLViewerFile(filename, xmlDocument))
taxonomyDataJSON = self.escapeJSONForScriptTag(
json.dumps(self.taxonomyData, indent=1, allow_nan=False))
for child in xmlDocument.getroot():
if child.tag == '{http://www.w3.org/1999/xhtml}body':
child.append(etree.Comment("BEGIN IXBRL VIEWER EXTENSIONS"))
e = etree.fromstring(
"<script xmlns='http://www.w3.org/1999/xhtml' src='%s' type='text/javascript' />"
% scriptUrl)
# Don't self close
e.text = ''
child.append(e)
# Putting this in the header can interfere with character set
# auto detection
e = etree.fromstring(
"<script xmlns='http://www.w3.org/1999/xhtml' type='application/x.ixbrl-viewer+json'></script>"
)
e.text = taxonomyDataJSON
child.append(e)
child.append(etree.Comment("END IXBRL VIEWER EXTENSIONS"))
break
return iv
def saveLoadableOIM(modelXbrl, oimFile, outputZip=None):
isJSON = oimFile.endswith(".json")
isCSV = oimFile.endswith(".csv")
isXL = oimFile.endswith(".xlsx")
isCSVorXL = isCSV or isXL
if not isJSON and not isCSVorXL:
return
namespacePrefixes = {nsOim: "xbrl"}
prefixNamespaces = {}
linkTypeAliases = {}
groupAliases = {}
linkTypePrefixes = {}
linkTypeUris = {}
linkGroupPrefixes = {}
linkGroupUris = {}
def compileQname(qname):
if qname.namespaceURI not in namespacePrefixes:
namespacePrefixes[qname.namespaceURI] = qname.prefix or ""
aspectsDefined = {qnOimConceptAspect, qnOimEntityAspect, qnOimPeriodAspect}
def oimValue(object, decimals=None):
if isinstance(object, QName):
if object.namespaceURI not in namespacePrefixes:
if object.prefix:
namespacePrefixes[object.namespaceURI] = object.prefix
else:
_prefix = "_{}".format(
sum(1 for p in namespacePrefixes if p.startswith("_")))
namespacePrefixes[object.namespaceURI] = _prefix
return "{}:{}".format(namespacePrefixes[object.namespaceURI],
object.localName)
if isinstance(object, (float, Decimal)):
try:
if isinf(object):
return "-INF" if object < 0 else "INF"
elif isnan(object):
return "NaN"
else:
if isinstance(object,
Decimal) and object == object.to_integral():
object = object.quantize(ONE) # drop any .0
return "{}".format(object)
except:
return str(object)
if isinstance(object, bool):
return "true" if object else "false"
if isinstance(
object,
(DateTime, YearMonthDuration, DayTimeDuration, Time, gYearMonth,
gMonthDay, gYear, gMonth, gDay, IsoDuration, int)):
return str(object)
return object
def oimPeriodValue(cntx):
if cntx.isStartEndPeriod:
s = cntx.startDatetime
e = cntx.endDatetime
else: # instant
s = e = cntx.instantDatetime
return ({
str(qnOimPeriodAspect):
("{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}/".format(
s.year, s.month, s.day, s.hour, s.minute, s.second)
if cntx.isStartEndPeriod else "") +
"{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format(
e.year, e.month, e.day, e.hour, e.minute, e.second)
})
hasId = False
hasTuple = False
hasType = True
hasLang = False
hasUnits = False
hasNumeric = False
footnotesRelationshipSet = ModelRelationshipSet(modelXbrl,
"XBRL-footnotes")
#compile QNames in instance for OIM
for fact in modelXbrl.factsInInstance:
if (fact.id or fact.isTuple
or footnotesRelationshipSet.toModelObject(fact) or
(isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))):
hasId = True
concept = fact.concept
if concept is not None:
if concept.isNumeric:
hasNumeric = True
if concept.baseXbrliType in ("string", "normalizedString",
"token") and fact.xmlLang:
hasLang = True
compileQname(fact.qname)
if hasattr(fact, "xValue") and isinstance(fact.xValue, QName):
compileQname(fact.xValue)
unit = fact.unit
if unit is not None:
hasUnits = True
if fact.modelTupleFacts:
hasTuple = True
if hasTuple:
modelXbrl.error("arelleOIMsaver:tuplesNotAllowed",
"Tuples are not allowed in an OIM document",
modelObject=modelXbrl)
return
entitySchemePrefixes = {}
for cntx in modelXbrl.contexts.values():
if cntx.entityIdentifierElement is not None:
scheme = cntx.entityIdentifier[0]
if scheme not in entitySchemePrefixes:
if not entitySchemePrefixes: # first one is just scheme
if scheme == "http://www.sec.gov/CIK":
_schemePrefix = "cik"
elif scheme == "http://standard.iso.org/iso/17442":
_schemePrefix = "lei"
else:
_schemePrefix = "scheme"
else:
_schemePrefix = "scheme{}".format(
len(entitySchemePrefixes) + 1)
entitySchemePrefixes[scheme] = _schemePrefix
namespacePrefixes[scheme] = _schemePrefix
for dim in cntx.qnameDims.values():
compileQname(dim.dimensionQname)
aspectsDefined.add(dim.dimensionQname)
if dim.isExplicit:
compileQname(dim.memberQname)
for unit in modelXbrl.units.values():
if unit is not None:
for measures in unit.measures:
for measure in measures:
compileQname(measure)
if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[
XbrlConst.xbrli] != "xbrli":
namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix
if hasLang: aspectsDefined.add(qnOimLangAspect)
if hasUnits: aspectsDefined.add(qnOimUnitAspect)
for footnoteRel in footnotesRelationshipSet.modelRelationships:
typePrefix = "ftTyp_" + os.path.basename(footnoteRel.arcrole)
if footnoteRel.linkrole == XbrlConst.defaultLinkRole:
groupPrefix = "ftGrp_default"
else:
groupPrefix = "ftGrp_" + os.path.basename(footnoteRel.linkrole)
if footnoteRel.arcrole not in linkTypeAliases:
linkTypeAliases[footnoteRel.arcrole] = typePrefix
if groupPrefix not in groupAliases:
groupAliases[footnoteRel.linkrole] = groupPrefix
dtsReferences = set()
baseUrl = modelXbrl.modelDocument.uri.partition("#")[0]
for doc, ref in sorted(modelXbrl.modelDocument.referencesDocument.items(),
key=lambda _item: _item[0].uri):
if ref.referringModelObject.qname in SCHEMA_LB_REFS:
dtsReferences.add(relativeUri(baseUrl, doc.uri))
for refType in ("role", "arcrole"):
for refElt in sorted(
modelXbrl.modelDocument.xmlRootElement.iterchildren(
"{{http://www.xbrl.org/2003/linkbase}}{}Ref".format(
refType)),
key=lambda elt: elt.get(refType + "URI")):
dtsReferences.add(
refElt.get("{http://www.w3.org/1999/xlink}href").partition("#")
[0])
dtsReferences = sorted(dtsReferences) # turn into list
footnoteFacts = set()
def factFootnotes(fact, oimFact=None, csvLinks=None):
footnotes = []
if isJSON:
oimLinks = {}
elif isCSVorXL:
oimLinks = csvLinks
for footnoteRel in footnotesRelationshipSet.fromModelObject(fact):
srcId = fact.id if fact.id else "f{}".format(fact.objectIndex)
toObj = footnoteRel.toModelObject
# json
typePrefix = linkTypeAliases[footnoteRel.arcrole]
groupPrefix = groupAliases[footnoteRel.linkrole]
if typePrefix not in oimLinks:
oimLinks[typePrefix] = OrderedDict()
_link = oimLinks[typePrefix]
if groupPrefix not in _link:
if isJSON:
_link[groupPrefix] = []
elif isCSVorXL:
_link[groupPrefix] = OrderedDict()
if isJSON:
tgtIdList = _link[groupPrefix]
elif isCSVorXL:
tgtIdList = _link[groupPrefix].setdefault(srcId, [])
tgtId = toObj.id if toObj.id else "f{}".format(toObj.objectIndex)
tgtIdList.append(tgtId)
footnote = OrderedDict((("group", footnoteRel.linkrole),
("footnoteType", footnoteRel.arcrole)))
if isinstance(toObj, ModelFact):
footnote["factRef"] = tgtId
else: # text footnotes
footnote["id"] = tgtId
footnote["footnote"] = toObj.viewText()
if toObj.xmlLang:
footnote["language"] = toObj.xmlLang
footnoteFacts.add(toObj)
footnotes.append(footnote)
if oimLinks:
_links = OrderedDict(
((typePrefix,
OrderedDict(
((groupPrefix, idList)
for groupPrefix, idList in sorted(groups.items()))))
for typePrefix, groups in sorted(oimLinks.items())))
if isJSON:
oimFact["links"] = _links
return footnotes
def factAspects(fact):
oimFact = OrderedDict()
aspects = OrderedDict()
if isCSVorXL:
oimFact["id"] = fact.id or "f{}".format(fact.objectIndex)
parent = fact.getparent()
concept = fact.concept
aspects[str(qnOimConceptAspect)] = oimValue(concept.qname)
if concept is not None:
if concept.type.isOimTextFactType and fact.xmlLang:
aspects[str(qnOimLangAspect)] = fact.xmlLang
if fact.isItem:
if fact.isNil:
_value = None
else:
_inferredDecimals = inferredDecimals(fact)
_value = oimValue(fact.xValue, _inferredDecimals)
oimFact["value"] = _value
if fact.concept is not None and fact.concept.isNumeric:
_numValue = fact.xValue
if isinstance(_numValue, Decimal) and not isinf(
_numValue) and not isnan(_numValue):
if _numValue == _numValue.to_integral():
_numValue = int(_numValue)
else:
_numValue = float(_numValue)
if not fact.isNil:
if not isinf(
_inferredDecimals): # accuracy omitted if infinite
oimFact["decimals"] = _inferredDecimals
oimFact["dimensions"] = aspects
cntx = fact.context
if cntx is not None:
if cntx.entityIdentifierElement is not None and cntx.entityIdentifier != ENTITY_NA_QNAME:
aspects[str(qnOimEntityAspect)] = oimValue(
qname(*cntx.entityIdentifier))
if cntx.period is not None and not cntx.isForeverPeriod:
aspects.update(oimPeriodValue(cntx))
for _qn, dim in sorted(cntx.qnameDims.items(),
key=lambda item: item[0]):
if dim.isExplicit:
dimVal = oimValue(dim.memberQname)
else: # typed
if dim.typedMember.get(
"{http://www.w3.org/2001/XMLSchema-instance}nil"
) in ("true", "1"):
dimVal = None
else:
dimVal = dim.typedMember.stringValue
aspects[str(dim.dimensionQname)] = dimVal
unit = fact.unit
if unit is not None:
_mMul, _mDiv = unit.measures
_sMul = '*'.join(
oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m)))
if _mDiv:
_sDiv = '*'.join(
oimValue(m)
for m in sorted(_mDiv, key=lambda m: oimValue(m)))
if len(_mDiv) > 1:
if len(_mMul) > 1:
_sUnit = "({})/({})".format(_sMul, _sDiv)
else:
_sUnit = "{}/({})".format(_sMul, _sDiv)
else:
if len(_mMul) > 1:
_sUnit = "({})/{}".format(_sMul, _sDiv)
else:
_sUnit = "{}/{}".format(_sMul, _sDiv)
else:
_sUnit = _sMul
if _sUnit != "xbrli:pure":
aspects[str(qnOimUnitAspect)] = _sUnit
# Tuples removed from xBRL-JSON
#if parent.qname != XbrlConst.qnXbrliXbrl:
# aspects[str(qnOimTupleParentAspect)] = parent.id if parent.id else "f{}".format(parent.objectIndex)
# aspects[str(qnOimTupleOrderAspect)] = elementIndex(fact)
if isJSON:
factFootnotes(fact, oimFact=oimFact)
return oimFact
namespaces = OrderedDict((p, ns)
for ns, p in sorted(namespacePrefixes.items(),
key=lambda item: item[1]))
# common metadata
oimReport = OrderedDict() # top level of oim json output
oimReport["documentInfo"] = oimDocInfo = OrderedDict()
oimDocInfo["documentType"] = nsOim + ("/xbrl-json"
if isJSON else "/xbrl-csv")
if isJSON:
oimDocInfo["features"] = oimFeatures = OrderedDict()
oimDocInfo["namespaces"] = namespaces
if linkTypeAliases:
oimDocInfo["linkTypes"] = OrderedDict(
(a, u) for u, a in sorted(linkTypeAliases.items(),
key=lambda item: item[1]))
if linkTypeAliases:
oimDocInfo["linkGroups"] = OrderedDict(
(a, u)
for u, a in sorted(groupAliases.items(), key=lambda item: item[1]))
oimDocInfo["taxonomy"] = dtsReferences
if isJSON:
oimFeatures["xbrl:canonicalValues"] = True
if isJSON:
# save JSON
oimReport["facts"] = oimFacts = OrderedDict()
def saveJsonFacts(facts, oimFacts, parentFact):
for fact in facts:
oimFact = factAspects(fact)
id = fact.id if fact.id else "f{}".format(fact.objectIndex)
oimFacts[id] = oimFact
if fact.modelTupleFacts:
saveJsonFacts(fact.modelTupleFacts, oimFacts, fact)
saveJsonFacts(modelXbrl.facts, oimFacts, None)
# add footnotes as pseudo facts
for ftObj in footnoteFacts:
ftId = ftObj.id if ftObj.id else "f{}".format(ftObj.objectIndex)
oimFacts[ftId] = oimFact = OrderedDict()
oimFact["value"] = ftObj.viewText()
oimFact["dimensions"] = OrderedDict(
(("concept", "xbrl:note"), ("noteId", ftId)))
if ftObj.xmlLang:
oimFact["dimensions"]["language"] = ftObj.xmlLang.lower()
if outputZip:
fh = io.StringIO()
else:
fh = open(oimFile, "w", encoding="utf-8")
fh.write(json.dumps(oimReport, indent=1))
if outputZip:
fh.seek(0)
outputZip.writestr(os.path.basename(oimFile), fh.read())
fh.close()
elif isCSVorXL:
# save CSV
oimReport["tables"] = oimTables = OrderedDict()
oimReport["tableTemplates"] = csvTableTemplates = OrderedDict()
oimTables["facts"] = csvTable = OrderedDict()
csvTable["template"] = "facts"
csvTable["url"] = "tbd"
csvTableTemplates["facts"] = csvTableTemplate = OrderedDict()
csvTableTemplate["rowIdColumn"] = "id"
csvTableTemplate["dimensions"] = csvTableDimensions = OrderedDict()
csvTableTemplate["columns"] = csvFactColumns = OrderedDict()
if footnotesRelationshipSet.modelRelationships:
csvLinks = OrderedDict()
oimReport["links"] = csvLinks
aspectQnCol = {}
aspectsHeader = []
factsColumns = []
def addAspectQnCol(aspectQn):
colQName = str(aspectQn).replace("xbrl:", "")
colNCName = colQName.replace(":", "_")
if colNCName not in aspectQnCol:
aspectQnCol[colNCName] = len(aspectsHeader)
aspectsHeader.append(colNCName)
if colNCName == "value":
csvFactColumns[colNCName] = col = OrderedDict()
col["dimensions"] = OrderedDict()
elif colNCName == "id":
csvFactColumns[colNCName] = {} # empty object
elif colNCName == "decimals":
csvFactColumns[colNCName] = {} # empty object
csvTableTemplate[colQName] = "$" + colNCName
else:
csvFactColumns[colNCName] = {} # empty object
csvTableDimensions[colQName] = "$" + colNCName
# pre-ordered aspect columns
#if hasId:
addAspectQnCol("id")
addAspectQnCol(qnOimConceptAspect)
if hasNumeric:
addAspectQnCol("decimals")
if qnOimEntityAspect in aspectsDefined:
addAspectQnCol(qnOimEntityAspect)
if qnOimPeriodAspect in aspectsDefined:
addAspectQnCol(qnOimPeriodAspect)
if qnOimUnitAspect in aspectsDefined:
addAspectQnCol(qnOimUnitAspect)
for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)):
if aspectQn.namespaceURI != nsOim:
addAspectQnCol(aspectQn)
addAspectQnCol("value")
def aspectCols(fact):
cols = [None for i in range(len(aspectsHeader))]
def setColValues(aspects):
for aspectQn, aspectValue in aspects.items():
colQName = str(aspectQn).replace("xbrl:", "")
colNCName = colQName.replace(":", "_")
if isinstance(aspectValue, dict):
setColValues(aspectValue)
elif colNCName in aspectQnCol:
if aspectValue is None:
_aspectValue = "#nil"
elif aspectValue == "":
_aspectValue = "#empty"
elif isinstance(aspectValue,
str) and aspectValue.startswith("#"):
_aspectValue = "#" + aspectValue
else:
_aspectValue = aspectValue
cols[aspectQnCol[colNCName]] = _aspectValue
setColValues(factAspects(fact))
return cols
# metadata
_open = _writerow = _close = None
if isCSV:
if oimFile.endswith(
"-facts.csv"
): # strip -facts.csv if a prior -facts.csv file was chosen
_baseURL = oimFile[:-10]
elif oimFile.endswith(".csv"):
_baseURL = oimFile[:-4]
else:
_baseURL = oimFile
_csvinfo = {} # open file, writer
def _open(filesuffix, tabname, csvTable=None):
_filename = _baseURL + filesuffix
if csvTable is not None:
csvTable["url"] = os.path.basename(
_filename) # located in same directory with metadata
_csvinfo["file"] = open(_filename,
csvOpenMode,
newline=csvOpenNewline,
encoding='utf-8-sig')
_csvinfo["writer"] = csv.writer(_csvinfo["file"],
dialect="excel")
def _writerow(row, header=False):
_csvinfo["writer"].writerow(row)
def _close():
_csvinfo["file"].close()
_csvinfo.clear()
elif isXL:
headerWidths = {
"concept": 40,
"decimals": 8,
"language": 9,
"value": 50,
"entity": 20,
"period": 20,
"unit": 20,
"metadata": 100
}
from openpyxl import Workbook
from openpyxl.cell.cell import WriteOnlyCell
from openpyxl.styles import Font, PatternFill, Border, Alignment, Color, fills, Side
from openpyxl.worksheet.dimensions import ColumnDimension
hdrCellFill = PatternFill(
patternType=fills.FILL_SOLID, fgColor=Color(
"00FFBF5F")) # Excel's light orange fill color = 00FF990
workbook = Workbook()
# remove pre-existing worksheets
while len(workbook.worksheets) > 0:
workbook.remove(workbook.worksheets[0])
_xlinfo = {} # open file, writer
def _open(filesuffix, tabname, csvTable=None):
if csvTable is not None:
csvTable["url"] = tabname + "!"
_xlinfo["ws"] = workbook.create_sheet(title=tabname)
def _writerow(rowvalues, header=False):
row = []
_ws = _xlinfo["ws"]
for i, v in enumerate(rowvalues):
cell = WriteOnlyCell(_ws, value=v)
if header:
cell.fill = hdrCellFill
cell.alignment = Alignment(horizontal="center",
vertical="center",
wrap_text=True)
colLetter = chr(ord('A') + i)
_ws.column_dimensions[colLetter] = ColumnDimension(
_ws, customWidth=True)
_ws.column_dimensions[
colLetter].width = headerWidths.get(v, 40)
else:
cell.alignment = Alignment(
horizontal="right" if isinstance(v, _NUM_TYPES)
else "center" if isinstance(v, bool) else "left",
vertical="top",
wrap_text=isinstance(v, str))
row.append(cell)
_ws.append(row)
def _close():
_xlinfo.clear()
# save facts
_open("-facts.csv", "facts", csvTable)
_writerow(aspectsHeader, header=True)
def saveCSVfacts(facts):
for fact in facts:
_writerow(aspectCols(fact))
saveCSVfacts(fact.modelTupleFacts)
saveCSVfacts(modelXbrl.facts)
_close()
# save footnotes
if footnotesRelationshipSet.modelRelationships:
footnotes = sorted(
(footnote for fact in modelXbrl.facts
for footnote in factFootnotes(fact, csvLinks=csvLinks)),
key=lambda footnote: footnote["id"])
if footnotes: # text footnotes
oimTables["footnotes"] = csvFtTable = OrderedDict()
csvFtTable["url"] = "tbd"
csvFtTable[
"tableDimensions"] = csvFtTableDimensions = OrderedDict()
csvFtTable["factColumns"] = csvFtFactColumns = OrderedDict()
csvFtFactColumns["footnote"] = csvFtValCol = OrderedDict()
csvFtValCol["id"] = "$id"
csvFtValCol["noteId"] = "$id"
csvFtValCol["concept"] = "xbrl:note"
csvFtValCol["language"] = "$language"
_open("-footnotes.csv", "footnotes", csvFtTable)
cols = ("id", "footnote", "language")
_writerow(cols, header=True)
for footnote in footnotes:
_writerow(tuple((footnote.get(col, "") for col in cols)))
_close()
# save metadata
if isCSV:
with open(_baseURL + "-metadata.json", "w",
encoding="utf-8") as fh:
fh.write(
json.dumps(oimReport,
ensure_ascii=False,
indent=2,
sort_keys=False))
elif isXL:
_open(None, "metadata")
_writerow(["metadata"], header=True)
_writerow([
json.dumps(oimReport,
ensure_ascii=False,
indent=1,
sort_keys=False)
])
_close()
if isXL:
workbook.save(oimFile)
def saveLoadableOIM(modelXbrl, oimFile):
isJSON = oimFile.endswith(".json")
isCSV = oimFile.endswith(".csv")
isXL = oimFile.endswith(".xlsx")
isCSVorXL = isCSV or isXL
if not isJSON and not isCSVorXL:
return
namespacePrefixes = {nsOim: "oim"}
prefixNamespaces = {"oim": nsOim}
def compileQname(qname):
if qname.namespaceURI not in namespacePrefixes:
namespacePrefixes[qname.namespaceURI] = qname.prefix or ""
aspectsDefined = {qnOimConceptAspect, qnOimEntityAspect}
if isJSON:
aspectsDefined.add(qnOimPeriodAspect)
elif isCSVorXL:
aspectsDefined.add(qnOimPeriodStartAspect)
aspectsDefined.add(qnOimPeriodEndAspect)
def oimValue(object, decimals=None):
if isinstance(object, QName):
if object.namespaceURI not in namespacePrefixes:
if object.prefix:
namespacePrefixes[object.namespaceURI] = object.prefix
else:
_prefix = "_{}".format(
sum(1 for p in namespacePrefixes if p.startswith("_")))
namespacePrefixes[object.namespaceURI] = _prefix
return "{}:{}".format(namespacePrefixes[object.namespaceURI],
object.localName)
if isinstance(object, Decimal):
try:
if isinf(object):
return "-INF" if object < 0 else "INF"
elif isnan(num):
return "NaN"
else:
if object == object.to_integral():
object = object.quantize(ONE) # drop any .0
return "{}".format(object)
except:
return str(object)
if isinstance(object, bool):
return object
if isinstance(object,
(DateTime, YearMonthDuration, DayTimeDuration, Time,
gYearMonth, gMonthDay, gYear, gMonth, gDay)):
return str(object)
return object
def oimPeriodValue(cntx):
if cntx.isForeverPeriod:
pass # not supported
elif cntx.isStartEndPeriod:
s = cntx.startDatetime
e = cntx.endDatetime
else: # instant
s = e = cntx.instantDatetime
return {
"start":
"{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format(
s.year, s.month, s.day, s.hour, s.minute, s.second),
"end":
"{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format(
e.year, e.month, e.day, e.hour, e.minute, e.second)
}
hasId = False
hasTuple = False
hasType = True
hasLang = False
hasUnits = False
hasNumeric = False
footnotesRelationshipSet = ModelRelationshipSet(modelXbrl,
"XBRL-footnotes")
factBaseTypes = set()
#compile QNames in instance for OIM
for fact in modelXbrl.factsInInstance:
if (fact.id or fact.isTuple
or footnotesRelationshipSet.toModelObject(fact) or
(isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))):
hasId = True
concept = fact.concept
if concept is not None:
if concept.isNumeric:
hasNumeric = True
if concept.baseXbrliType in ("string", "normalizedString",
"token") and fact.xmlLang:
hasLang = True
_baseXsdType = concept.baseXsdType
if _baseXsdType == "XBRLI_DATEUNION":
if getattr(fact.xValue, "dateOnly", False):
_baseXsdType = "date"
else:
_baseXsdType = "dateTime"
factBaseTypes.add(baseTypes.get(_baseXsdType, _baseXsdType))
compileQname(fact.qname)
if hasattr(fact, "xValue") and isinstance(fact.xValue, QName):
compileQname(fact.xValue)
unit = fact.unit
if unit is not None:
hasUnits = True
if fact.modelTupleFacts:
hasTuple = True
entitySchemePrefixes = {}
for cntx in modelXbrl.contexts.values():
if cntx.entityIdentifierElement is not None:
scheme = cntx.entityIdentifier[0]
if scheme not in entitySchemePrefixes:
if not entitySchemePrefixes: # first one is just scheme
if scheme == "http://www.sec.gov/CIK":
_schemePrefix = "cik"
elif scheme == "http://standard.iso.org/iso/17442":
_schemePrefix = "lei"
else:
_schemePrefix = "scheme"
else:
_schemePrefix = "scheme{}".format(
len(entitySchemePrefixes) + 1)
entitySchemePrefixes[scheme] = _schemePrefix
namespacePrefixes[scheme] = _schemePrefix
for dim in cntx.qnameDims.values():
compileQname(dim.dimensionQname)
aspectsDefined.add(dim.dimensionQname)
if dim.isExplicit:
compileQname(dim.memberQname)
for unit in modelXbrl.units.values():
if unit is not None:
for measures in unit.measures:
for measure in measures:
compileQname(measure)
if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[
XbrlConst.xbrli] != "xbrli":
namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix
namespacePrefixes[XbrlConst.xsd] = "xsd"
if hasLang: aspectsDefined.add(qnOimLangAspect)
if hasTuple:
aspectsDefined.add(qnOimTupleParentAspect)
aspectsDefined.add(qnOimTupleOrderAspect)
if hasUnits: aspectsDefined.add(qnOimUnitAspect)
# compile footnotes and relationships
'''
factRelationships = []
factFootnotes = []
for rel in modelXbrl.relationshipSet(modelXbrl, "XBRL-footnotes").modelRelationships:
oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole}
factRelationships.append(oimRel)
oimRel["fromIds"] = [obj.id if obj.id
else elementChildSequence(obj)
for obj in rel.fromModelObjects]
oimRel["toIds"] = [obj.id if obj.id
else elementChildSequence(obj)
for obj in rel.toModelObjects]
_order = rel.arcElement.get("order")
if _order is not None:
oimRel["order"] = _order
for obj in rel.toModelObjects:
if isinstance(obj, ModelResource): # footnote
oimFootnote = {"role": obj.role,
"id": obj.id if obj.id
else elementChildSequence(obj),
# value needs work for html elements and for inline footnotes
"value": xmlstring(obj, stripXmlns=True)}
if obj.xmlLang:
oimFootnote["lang"] = obj.xmlLang
factFootnotes.append(oimFootnote)
oimFootnote
'''
dtsReferences = [{
"type":
"schema" if doc.type == ModelDocument.Type.SCHEMA else
"linkbase" if doc.type == ModelDocument.Type.LINKBASE else "other",
"href":
doc.uri
} for doc, ref in modelXbrl.modelDocument.referencesDocument.items()
if ref.referringModelObject.qname in SCHEMA_LB_REFS]
'''
roleTypes = [
{"type": "role" if ref.referringModelObject.localName == "roleRef" else "arcroleRef",
"href": ref.referringModelObject["href"]}
for doc,ref in modelXbrl.modelDocument.referencesDocument.items()
if ref.referringModelObject.qname in ROLE_REFS]
'''
def factFootnotes(fact):
footnotes = []
for footnoteRel in footnotesRelationshipSet.fromModelObject(fact):
footnote = OrderedDict((("group", footnoteRel.linkrole),
("footnoteType", footnoteRel.arcrole)))
footnotes.append(footnote)
if isCSVorXL:
footnote["factId"] = fact.id if fact.id else "f{}".format(
fact.objectIndex)
toObj = footnoteRel.toModelObject
if isinstance(toObj, ModelFact):
footnote["factRef"] = toObj.id if toObj.id else "f{}".format(
toObj.objectIndex)
else:
footnote["footnote"] = xmlstring(toObj,
stripXmlns=True,
contentsOnly=True,
includeText=True)
if toObj.xmlLang:
footnote["language"] = toObj.xmlLang
return footnotes
def factAspects(fact):
aspects = OrderedDict()
if hasId and fact.id:
aspects["id"] = fact.id
elif (fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or
(isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))):
aspects["id"] = "f{}".format(fact.objectIndex)
parent = fact.getparent()
concept = fact.concept
_csvType = "Value"
if not fact.isTuple:
if concept is not None:
_baseXsdType = concept.baseXsdType
if _baseXsdType == "XBRLI_DATEUNION":
if getattr(fact.xValue, "dateOnly", False):
_baseXsdType = "date"
else:
_baseXsdType = "dateTime"
aspects["baseType"] = "xsd:{}".format(_baseXsdType)
_csvType = baseTypes.get(_baseXsdType, _baseXsdType) + "Value"
if concept.baseXbrliType in ("string", "normalizedString",
"token") and fact.xmlLang:
aspects[qnOimLangAspect] = fact.xmlLang
if fact.isItem:
if fact.isNil:
_value = None
_strValue = "nil"
else:
_inferredDecimals = inferredDecimals(fact)
_value = oimValue(fact.xValue, _inferredDecimals)
_strValue = str(_value)
if not isCSVorXL:
aspects["value"] = _strValue
if fact.concept is not None and fact.concept.isNumeric:
_numValue = fact.xValue
if isinstance(_numValue, Decimal) and not isinf(
_numValue) and not isnan(_numValue):
if _numValue == _numValue.to_integral():
_numValue = int(_numValue)
else:
_numValue = float(_numValue)
if isCSVorXL:
aspects[_csvType] = _numValue
else:
aspects["numericValue"] = _numValue
if not fact.isNil:
if isinf(_inferredDecimals):
if isJSON: _accuracy = "infinity"
elif isCSVorXL: _accuracy = "INF"
else:
_accuracy = _inferredDecimals
aspects["accuracy"] = _accuracy
elif isinstance(_value, bool):
aspects["booleanValue"] = _value
elif isCSVorXL:
aspects[_csvType] = _strValue
aspects[qnOimConceptAspect] = oimValue(fact.qname)
cntx = fact.context
if cntx is not None:
if cntx.entityIdentifierElement is not None:
aspects[qnOimEntityAspect] = oimValue(
qname(*cntx.entityIdentifier))
if cntx.period is not None:
if isJSON:
aspects[qnOimPeriodAspect] = oimPeriodValue(cntx)
elif isCSVorXL:
_periodValue = oimPeriodValue(cntx)
aspects[qnOimPeriodStartAspect] = _periodValue["start"]
aspects[qnOimPeriodEndAspect] = _periodValue["end"]
for _qn, dim in sorted(cntx.qnameDims.items(),
key=lambda item: item[0]):
aspects[dim.dimensionQname] = (
oimValue(dim.memberQname)
if dim.isExplicit else None if dim.typedMember.get(
"{http://www.w3.org/2001/XMLSchema-instance}nil") in (
"true", "1") else dim.typedMember.stringValue)
unit = fact.unit
if unit is not None:
_mMul, _mDiv = unit.measures
_sMul = '*'.join(
oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m)))
if _mDiv:
_sDiv = '*'.join(
oimValue(m)
for m in sorted(_mDiv, key=lambda m: oimValue(m)))
if len(_mDiv) > 1:
if len(_mMul) > 1:
_sUnit = "({})/({})".format(_sMul, _sDiv)
else:
_sUnit = "{}/({})".format(_sMul, _sDiv)
else:
if len(_mMul) > 1:
_sUnit = "({})/{}".format(_sMul, _sDiv)
else:
_sUnit = "{}/{}".format(_sMul, _sDiv)
else:
_sUnit = _sMul
aspects[qnOimUnitAspect] = _sUnit
if parent.qname != XbrlConst.qnXbrliXbrl:
aspects[
qnOimTupleParentAspect] = parent.id if parent.id else "f{}".format(
parent.objectIndex)
aspects[qnOimTupleOrderAspect] = elementIndex(fact)
if isJSON:
_footnotes = factFootnotes(fact)
if _footnotes:
aspects["footnotes"] = _footnotes
return aspects
if isJSON:
# save JSON
oimReport = OrderedDict() # top level of oim json output
oimFacts = []
oimReport["prefixes"] = OrderedDict(
(p, ns) for ns, p in sorted(namespacePrefixes.items(),
key=lambda item: item[1]))
oimReport["dtsReferences"] = dtsReferences
oimReport["facts"] = oimFacts
def saveJsonFacts(facts, oimFacts, parentFact):
for fact in facts:
oimFact = factAspects(fact)
oimFacts.append(
OrderedDict((str(k), v) for k, v in oimFact.items()))
if fact.modelTupleFacts:
saveJsonFacts(fact.modelTupleFacts, oimFacts, fact)
saveJsonFacts(modelXbrl.facts, oimFacts, None)
with open(oimFile, "w", encoding="utf-8") as fh:
fh.write(json.dumps(oimReport, indent=1))
elif isCSVorXL:
# save CSV
aspectQnCol = {}
aspectsHeader = []
factsColumns = []
def addAspectQnCol(aspectQn):
aspectQnCol[aspectQn] = len(aspectsHeader)
_colName = oimValue(aspectQn)
aspectsHeader.append(_colName)
_colDataType = {
"id": "Name",
"baseType": "Name",
"oim:concept": "QName",
"oim:periodStart": "dateTime",
"oim:periodEnd": "dateTime",
"oim:tupleOrder": "integer",
"numericValue": "decimal",
"accuracy": "decimal",
"booleanValue": "boolean",
"oim:entity": "QName",
"oim:unit": "string"
}.get(_colName)
if _colDataType is None:
if isCSVorXL and _colName.endswith("Value"):
_colDataType = _colName[:-5]
else:
_colDataType = "string"
factsColumns.append(
OrderedDict((("name", _colName), ("datatype", _colDataType))))
# pre-ordered aspect columns
if hasId:
addAspectQnCol("id")
addAspectQnCol(qnOimConceptAspect)
if hasType:
addAspectQnCol("baseType")
if isCSVorXL:
for _baseType in sorted(factBaseTypes):
addAspectQnCol(_baseType + "Value")
else:
addAspectQnCol("stringValue")
addAspectQnCol("numericValue")
addAspectQnCol("booleanValue")
if hasNumeric:
addAspectQnCol("accuracy")
if hasTuple:
addAspectQnCol(qnOimTupleParentAspect)
addAspectQnCol(qnOimTupleOrderAspect)
if qnOimEntityAspect in aspectsDefined:
addAspectQnCol(qnOimEntityAspect)
if qnOimPeriodStartAspect in aspectsDefined:
addAspectQnCol(qnOimPeriodStartAspect)
addAspectQnCol(qnOimPeriodEndAspect)
if qnOimUnitAspect in aspectsDefined:
addAspectQnCol(qnOimUnitAspect)
for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)):
if aspectQn.namespaceURI != nsOim:
addAspectQnCol(aspectQn)
def aspectCols(fact):
cols = [None for i in range(len(aspectsHeader))]
_factAspects = factAspects(fact)
for aspectQn, aspectValue in _factAspects.items():
if aspectQn in aspectQnCol:
cols[aspectQnCol[aspectQn]] = aspectValue
return cols
# metadata
csvTables = []
csvMetadata = OrderedDict(
(("@context", ["http://www.w3.org/ns/csvw", {
"@base": "./"
}]), ("tables", csvTables)))
_open = _writerow = _close = None
_tableinfo = {}
if isCSV:
if oimFile.endswith(
"-facts.csv"
): # strip -facts.csv if a prior -facts.csv file was chosen
_baseURL = oimFile[:-10]
elif oimFile.endswith(".csv"):
_baseURL = oimFile[:-4]
else:
_baseURL = oimFile
_csvinfo = {} # open file, writer
def _open(filesuffix, tabname):
_filename = _tableinfo["url"] = _baseURL + filesuffix
_csvinfo["file"] = open(_filename,
csvOpenMode,
newline=csvOpenNewline,
encoding='utf-8-sig')
_csvinfo["writer"] = csv.writer(_csvinfo["file"],
dialect="excel")
def _writerow(row, header=False):
_csvinfo["writer"].writerow(row)
def _close():
_csvinfo["file"].close()
_csvinfo.clear()
elif isXL:
headerWidths = {
"href": 100,
"oim:concept": 70,
"accuracy": 8,
"baseType": 10,
"language": 9,
"URI": 80,
"stringValue": 60,
"decimalValue": 12,
"booleanValue": 12,
"dateValue": 12,
"dateTimeValue": 20,
"group": 60,
"footnoteType": 40,
"footnote": 70,
"column": 60
}
from openpyxl import Workbook
from openpyxl.writer.write_only import WriteOnlyCell
from openpyxl.styles import Font, PatternFill, Border, Alignment, Color, fills, Side
from openpyxl.worksheet.dimensions import ColumnDimension
hdrCellFill = PatternFill(
patternType=fills.FILL_SOLID, fgColor=Color(
"00FFBF5F")) # Excel's light orange fill color = 00FF990
workbook = Workbook(encoding="utf-8", write_only=True)
_xlinfo = {} # open file, writer
def _open(filesuffix, tabname):
_tableinfo["url"] = tabname
_xlinfo["ws"] = workbook.create_sheet(title=tabname)
def _writerow(rowvalues, header=False):
row = []
_ws = _xlinfo["ws"]
for i, v in enumerate(rowvalues):
cell = WriteOnlyCell(_ws, value=v)
if header:
cell.fill = hdrCellFill
cell.alignment = Alignment(horizontal="center",
vertical="center",
wrap_text=True)
colLetter = chr(ord('A') + i)
_ws.column_dimensions[colLetter] = ColumnDimension(
_ws, customWidth=True)
_ws.column_dimensions[
colLetter].width = headerWidths.get(v, 20)
else:
cell.alignment = Alignment(
horizontal="right" if isinstance(v, _NUM_TYPES)
else "center" if isinstance(v, bool) else "left",
vertical="top",
wrap_text=isinstance(v, str))
row.append(cell)
_ws.append(row)
def _close():
_xlinfo.clear()
# save facts
_open("-facts.csv", "facts")
_writerow(aspectsHeader, header=True)
def saveCSVfacts(facts):
for fact in facts:
_writerow(aspectCols(fact))
saveCSVfacts(fact.modelTupleFacts)
saveCSVfacts(modelXbrl.facts)
_close()
factsTableSchema = OrderedDict((("columns", factsColumns), ))
csvTables.append(
OrderedDict((("url", _tableinfo["url"]), ("tableSchema",
factsTableSchema))))
# save namespaces
_open("-prefixes.csv", "prefixes")
_writerow(("prefix", "URI"), header=True)
for _URI, prefix in sorted(namespacePrefixes.items(),
key=lambda item: item[1]):
_writerow((prefix, _URI))
_close()
nsTableSchema = OrderedDict((("columns", [
OrderedDict((("name", "prefix"), ("datatype", "Name"))),
OrderedDict((("name", "URI"), ("datatype", "anyURI")))
]), ))
csvTables.append(
OrderedDict(
(("url", _tableinfo["url"]), ("tableSchema", nsTableSchema))))
# save dts references
_open("-dtsReferences.csv", "dtsReferences")
_writerow(("type", "href"), header=True)
for oimRef in dtsReferences:
_writerow((oimRef["type"], oimRef["href"]))
_close()
dtsRefTableSchema = OrderedDict((("columns", [
OrderedDict((("name", "type"), ("datatype", "Name"))),
OrderedDict((("name", "href"), ("datatype", "anyURI")))
]), ))
csvTables.append(
OrderedDict((("url", _tableinfo["url"]), ("tableSchema",
dtsRefTableSchema))))
# save footnotes
if footnotesRelationshipSet.modelRelationships:
_open("-footnotes.csv", "footnotes")
cols = ("group", "footnoteType", "factId", "factRef", "footnote",
"language")
_writerow(cols, header=True)
def saveCSVfootnotes(facts):
for fact in facts:
for _footnote in factFootnotes(fact):
_writerow(
tuple((_footnote.get(col, "") for col in cols)))
saveCSVfootnotes(fact.modelTupleFacts)
saveCSVfootnotes(modelXbrl.facts)
_close()
footnoteTableSchema = OrderedDict((("columns", [
OrderedDict((("name", "group"), ("datatype", "anyURI"))),
OrderedDict((("name", "footnoteType"), ("datatype", "Name"))),
OrderedDict((("name", "factId"), ("datatype", "Name"))),
OrderedDict((("name", "factRef"), ("datatype", "Name"))),
OrderedDict((("name", "footnote"), ("datatype", "string"))),
OrderedDict((("name", "language"), ("datatype", "language")))
]), ))
csvTables.append(
OrderedDict((("url", _tableinfo["url"]),
("tableSchema", footnoteTableSchema))))
# save metadata
if isCSV:
with open(_baseURL + "-metadata.csv", "w", encoding="utf-8") as fh:
fh.write(
json.dumps(csvMetadata,
ensure_ascii=False,
indent=1,
sort_keys=False))
elif isXL:
_open(None, "metadata")
_writerow(("table", "column", "datatype"), header=True)
for table in csvTables:
tablename = table["url"]
for column in table["tableSchema"]["columns"]:
_writerow((tablename, column["name"], column["datatype"]))
_close()
if isXL:
workbook.save(oimFile)
def saveLoadableOIM(modelXbrl, oimFile, outputZip=None):
isJSON = oimFile.endswith(".json")
isCSV = oimFile.endswith(".csv")
isXL = oimFile.endswith(".xlsx")
isCSVorXL = isCSV or isXL
if not isJSON and not isCSVorXL:
return
namespacePrefixes = {nsOim: "xbrl"}
prefixNamespaces = {"xbrl": nsOim}
def compileQname(qname):
if qname.namespaceURI not in namespacePrefixes:
namespacePrefixes[qname.namespaceURI] = qname.prefix or ""
aspectsDefined = {
qnOimConceptAspect,
qnOimEntityAspect,
qnOimPeriodStartAspect,
qnOimPeriodEndAspect}
def oimValue(object, decimals=None):
if isinstance(object, QName):
if object.namespaceURI not in namespacePrefixes:
if object.prefix:
namespacePrefixes[object.namespaceURI] = object.prefix
else:
_prefix = "_{}".format(sum(1 for p in namespacePrefixes if p.startswith("_")))
namespacePrefixes[object.namespaceURI] = _prefix
return "{}:{}".format(namespacePrefixes[object.namespaceURI], object.localName)
if isinstance(object, Decimal):
try:
if isinf(object):
return "-INF" if object < 0 else "INF"
elif isnan(num):
return "NaN"
else:
if object == object.to_integral():
object = object.quantize(ONE) # drop any .0
return "{}".format(object)
except:
return str(object)
if isinstance(object, bool):
return "true" if object else "false"
if isinstance(object, (DateTime, YearMonthDuration, DayTimeDuration, Time,
gYearMonth, gMonthDay, gYear, gMonth, gDay,
IsoDuration)):
return str(object)
return object
def oimPeriodValue(cntx):
if cntx.isForeverPeriod:
return OrderedDict() # defaulted
elif cntx.isStartEndPeriod:
s = cntx.startDatetime
e = cntx.endDatetime
else: # instant
s = e = cntx.instantDatetime
if isJSON:
return({str(qnOimPeriodAspect):
("{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}/".format(
s.year, s.month, s.day, s.hour, s.minute, s.second)
if cntx.isStartEndPeriod else "") +
"{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format(
e.year, e.month, e.day, e.hour, e.minute, e.second)
})
# CSV, XL
return OrderedDict(((str(qnOimPeriodStartAspect), "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format(
s.year, s.month, s.day, s.hour, s.minute, s.second)),
(str(qnOimPeriodEndAspect), "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format(
e.year, e.month, e.day, e.hour, e.minute, e.second))))
hasId = False
hasTuple = False
hasType = True
hasLang = False
hasUnits = False
hasNumeric = False
footnotesRelationshipSet = ModelRelationshipSet(modelXbrl, "XBRL-footnotes")
factBaseTypes = set()
#compile QNames in instance for OIM
for fact in modelXbrl.factsInInstance:
if (fact.id or fact.isTuple or
footnotesRelationshipSet.toModelObject(fact) or
(isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))):
hasId = True
concept = fact.concept
if concept is not None:
if concept.isNumeric:
hasNumeric = True
if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang:
hasLang = True
_baseXsdType = concept.baseXsdType
if _baseXsdType == "XBRLI_DATEUNION":
if getattr(fact.xValue, "dateOnly", False):
_baseXsdType = "date"
else:
_baseXsdType = "dateTime"
factBaseTypes.add(baseTypes.get(_baseXsdType,_baseXsdType))
compileQname(fact.qname)
if hasattr(fact, "xValue") and isinstance(fact.xValue, QName):
compileQname(fact.xValue)
unit = fact.unit
if unit is not None:
hasUnits = True
if fact.modelTupleFacts:
hasTuple = True
if hasTuple:
modelXbrl.error("arelleOIMsaver:tuplesNotAllowed",
"Tuples are not allowed in an OIM document",
modelObject=modelXbrl)
return
entitySchemePrefixes = {}
for cntx in modelXbrl.contexts.values():
if cntx.entityIdentifierElement is not None:
scheme = cntx.entityIdentifier[0]
if scheme not in entitySchemePrefixes:
if not entitySchemePrefixes: # first one is just scheme
if scheme == "http://www.sec.gov/CIK":
_schemePrefix = "cik"
elif scheme == "http://standard.iso.org/iso/17442":
_schemePrefix = "lei"
else:
_schemePrefix = "scheme"
else:
_schemePrefix = "scheme{}".format(len(entitySchemePrefixes) + 1)
entitySchemePrefixes[scheme] = _schemePrefix
namespacePrefixes[scheme] = _schemePrefix
for dim in cntx.qnameDims.values():
compileQname(dim.dimensionQname)
aspectsDefined.add(dim.dimensionQname)
if dim.isExplicit:
compileQname(dim.memberQname)
for unit in modelXbrl.units.values():
if unit is not None:
for measures in unit.measures:
for measure in measures:
compileQname(measure)
if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[XbrlConst.xbrli] != "xbrli":
namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix
namespacePrefixes[XbrlConst.xsd] = "xsd"
if hasLang: aspectsDefined.add(qnOimLangAspect)
if hasTuple:
aspectsDefined.add(qnOimTupleParentAspect)
aspectsDefined.add(qnOimTupleOrderAspect)
if hasUnits: aspectsDefined.add(qnOimUnitAspect)
for footnoteRel in footnotesRelationshipSet.modelRelationships:
typePrefix = "ftTyp_" + os.path.basename(footnoteRel.arcrole)
if footnoteRel.linkrole == XbrlConst.defaultLinkRole:
groupPrefix = "ftGrp_default"
else:
groupPrefix = "ftGrp_" + os.path.basename(footnoteRel.linkrole)
if typePrefix not in namespacePrefixes:
namespacePrefixes[footnoteRel.arcrole] = typePrefix
if groupPrefix not in namespacePrefixes:
namespacePrefixes[footnoteRel.linkrole] = groupPrefix
# compile footnotes and relationships
'''
factRelationships = []
factFootnotes = []
for rel in modelXbrl.relationshipSet(modelXbrl, "XBRL-footnotes").modelRelationships:
oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole}
factRelationships.append(oimRel)
oimRel["fromIds"] = [obj.id if obj.id
else elementChildSequence(obj)
for obj in rel.fromModelObjects]
oimRel["toIds"] = [obj.id if obj.id
else elementChildSequence(obj)
for obj in rel.toModelObjects]
_order = rel.arcElement.get("order")
if _order is not None:
oimRel["order"] = _order
for obj in rel.toModelObjects:
if isinstance(obj, ModelResource): # footnote
oimFootnote = {"role": obj.role,
"id": obj.id if obj.id
else elementChildSequence(obj),
# value needs work for html elements and for inline footnotes
"value": xmlstring(obj, stripXmlns=True)}
if obj.xmlLang:
oimFootnote["lang"] = obj.xmlLang
factFootnotes.append(oimFootnote)
oimFootnote
'''
dtsReferences = set()
baseUrl = modelXbrl.modelDocument.uri.partition("#")[0]
for doc,ref in sorted(modelXbrl.modelDocument.referencesDocument.items(),
key=lambda _item:_item[0].uri):
if ref.referringModelObject.qname in SCHEMA_LB_REFS:
dtsReferences.add(relativeUri(baseUrl,doc.uri))
for refType in ("role", "arcrole"):
for refElt in sorted(modelXbrl.modelDocument.xmlRootElement.iterchildren(
"{{http://www.xbrl.org/2003/linkbase}}{}Ref".format(refType)),
key=lambda elt:elt.get(refType+"URI")
):
dtsReferences.add(refElt.get("{http://www.w3.org/1999/xlink}href").partition("#")[0])
dtsReferences = sorted(dtsReferences) # turn into list
footnoteFacts = set()
def factFootnotes(fact, oimFact=None):
footnotes = []
oimLinks = {}
for footnoteRel in footnotesRelationshipSet.fromModelObject(fact):
toObj = footnoteRel.toModelObject
# json
typePrefix = namespacePrefixes[footnoteRel.arcrole]
groupPrefix = namespacePrefixes[footnoteRel.linkrole]
oimLinks.setdefault(typePrefix,{}).setdefault(groupPrefix,[]).append(
toObj.id if toObj.id else "f{}".format(toObj.objectIndex))
# csv/XL
footnote = OrderedDict((("group", footnoteRel.linkrole),
("footnoteType", footnoteRel.arcrole)))
footnotes.append(footnote)
if isCSVorXL:
footnote["factId"] = fact.id if fact.id else "f{}".format(fact.objectIndex)
if isinstance(toObj, ModelFact):
footnote["factRef"] = toObj.id if toObj.id else "f{}".format(toObj.objectIndex)
else:
footnote["footnote"] = toObj.viewText()
if toObj.xmlLang:
footnote["language"] = toObj.xmlLang
footnoteFacts.add(toObj)
if isJSON and oimLinks:
oimFact["links"] = OrderedDict((
(typePrefix, OrderedDict((
(groupPrefix, idList)
for groupPrefix, idList in sorted(groups.items())
)))
for typePrefix, groups in sorted(oimLinks.items())
))
footnotes.sort(key=lambda f:(f["group"],f.get("factId",f.get("factRef")),f.get("language")))
return footnotes
def factAspects(fact):
oimFact = OrderedDict()
aspects = OrderedDict()
if hasId and fact.id:
if fact.isTuple:
oimFact["tupleId"] = fact.id
else:
oimFact["id"] = fact.id
elif (fact.isTuple or
footnotesRelationshipSet.toModelObject(fact) or
(isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))):
oimFact["id"] = "f{}".format(fact.objectIndex)
parent = fact.getparent()
concept = fact.concept
aspects[str(qnOimConceptAspect)] = oimValue(concept.qname)
_csvType = "Value"
if not fact.isTuple:
if concept is not None:
_baseXsdType = concept.baseXsdType
if _baseXsdType == "XBRLI_DATEUNION":
if getattr(fact.xValue, "dateOnly", False):
_baseXsdType = "date"
else:
_baseXsdType = "dateTime"
_csvType = baseTypes.get(_baseXsdType,_baseXsdType) + "Value"
if concept.baseXbrliType in ("stringItemType", "normalizedStringItemType") and fact.xmlLang:
aspects[str(qnOimLangAspect)] = fact.xmlLang
if fact.isItem:
if fact.isNil:
_value = None
else:
_inferredDecimals = inferredDecimals(fact)
_value = oimValue(fact.xValue, _inferredDecimals)
oimFact["value"] = _value
if fact.concept is not None and fact.concept.isNumeric:
_numValue = fact.xValue
if isinstance(_numValue, Decimal) and not isinf(_numValue) and not isnan(_numValue):
if _numValue == _numValue.to_integral():
_numValue = int(_numValue)
else:
_numValue = float(_numValue)
if not fact.isNil:
if not isinf(_inferredDecimals): # accuracy omitted if infinite
oimFact["decimals"] = _inferredDecimals
oimFact["dimensions"] = aspects
cntx = fact.context
if cntx is not None:
if cntx.entityIdentifierElement is not None:
aspects[str(qnOimEntityAspect)] = oimValue(qname(*cntx.entityIdentifier))
if cntx.period is not None:
aspects.update(oimPeriodValue(cntx))
for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]):
if dim.isExplicit:
dimVal = oimValue(dim.memberQname)
else: # typed
if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1"):
dimVal = None
else:
dimVal = dim.typedMember.stringValue
aspects[str(dim.dimensionQname)] = dimVal
unit = fact.unit
if unit is not None:
_mMul, _mDiv = unit.measures
_sMul = '*'.join(oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m)))
if _mDiv:
_sDiv = '*'.join(oimValue(m) for m in sorted(_mDiv, key=lambda m: oimValue(m)))
if len(_mDiv) > 1:
if len(_mMul) > 1:
_sUnit = "({})/({})".format(_sMul,_sDiv)
else:
_sUnit = "{}/({})".format(_sMul,_sDiv)
else:
if len(_mMul) > 1:
_sUnit = "({})/{}".format(_sMul,_sDiv)
else:
_sUnit = "{}/{}".format(_sMul,_sDiv)
else:
_sUnit = _sMul
aspects[str(qnOimUnitAspect)] = _sUnit
# Tuples removed from xBRL-JSON
#if parent.qname != XbrlConst.qnXbrliXbrl:
# aspects[str(qnOimTupleParentAspect)] = parent.id if parent.id else "f{}".format(parent.objectIndex)
# aspects[str(qnOimTupleOrderAspect)] = elementIndex(fact)
if isJSON:
factFootnotes(fact, oimFact)
return oimFact
prefixes = OrderedDict((p,ns) for ns, p in sorted(namespacePrefixes.items(),
key=lambda item: item[1]))
if isJSON:
# save JSON
oimReport = OrderedDict() # top level of oim json output
oimReport["documentInfo"] = oimDocInfo = OrderedDict()
oimReport["facts"] = oimFacts = OrderedDict()
oimDocInfo["documentType"] = nsOim + "/xbrl-json"
oimDocInfo["features"] = oimFeatures = OrderedDict()
oimDocInfo["prefixes"] = prefixes
oimDocInfo["taxonomy"] = dtsReferences
oimFeatures["xbrl:canonicalValues"] = True
def saveJsonFacts(facts, oimFacts, parentFact):
for fact in facts:
oimFact = factAspects(fact)
id = fact.id if fact.id else "f{}".format(fact.objectIndex)
oimFacts[id] = oimFact
if fact.modelTupleFacts:
saveJsonFacts(fact.modelTupleFacts, oimFacts, fact)
saveJsonFacts(modelXbrl.facts, oimFacts, None)
# add footnotes as pseudo facts
for ftObj in footnoteFacts:
ftId = ftObj.id if ftObj.id else "f{}".format(ftObj.objectIndex)
oimFacts[ftId] = oimFact = OrderedDict()
oimFact["value"] = ftObj.viewText()
oimFact["dimensions"] = OrderedDict((("concept", "xbrl:note"),
("noteId", ftId)))
if ftObj.xmlLang:
oimFact["dimensions"]["language"] = ftObj.xmlLang.lower()
if outputZip:
fh = io.StringIO()
else:
fh = open(oimFile, "w", encoding="utf-8")
fh.write(json.dumps(oimReport, indent=1))
if outputZip:
fh.seek(0)
outputZip.writestr(os.path.basename(oimFile),fh.read())
fh.close()
elif isCSVorXL:
# save CSV
aspectQnCol = {}
aspectsHeader = []
factsColumns = []
def addAspectQnCol(aspectQn):
aspectQnCol[str(aspectQn)] = len(aspectsHeader)
_aspectQn = oimValue(aspectQn)
aspectsHeader.append(_aspectQn)
_colName = _aspectQn.replace("xbrl:", "")
_colDataType = {"id": "Name",
"concept": "QName",
"value": "string",
"decimals": "decimal",
"entity": "QName",
"periodStart": "dateTime",
"periodEnd": "dateTime",
"unit": "string",
"tupleId": "Name",
"tupleParent": "Name",
"tupleOrder": "integer"
}.get(_colName, "string")
col = OrderedDict((("name", _colName),
("datatype", _colDataType)))
if _aspectQn == "value":
col["http://xbrl.org/YYYY/model#simpleFactAspects"] = {}
elif _aspectQn == "tupleId":
col["http://xbrl.org/YYYY/model#tupleFactAspects"] = {}
col["http://xbrl.org/YYYY/model#tupleReferenceId"] = "true"
else:
col["http://xbrl.org/YYYY/model#columnAspect"] = _aspectQn
factsColumns.append(col)
# pre-ordered aspect columns
#if hasId:
# addAspectQnCol("id")
addAspectQnCol(qnOimConceptAspect)
addAspectQnCol("value")
if hasNumeric:
addAspectQnCol("decimals")
if hasTuple:
addAspectQnCol("tupleId")
addAspectQnCol(qnOimTupleParentAspect)
addAspectQnCol(qnOimTupleOrderAspect)
if qnOimEntityAspect in aspectsDefined:
addAspectQnCol(qnOimEntityAspect)
if qnOimPeriodStartAspect in aspectsDefined:
addAspectQnCol(qnOimPeriodStartAspect)
addAspectQnCol(qnOimPeriodEndAspect)
if qnOimUnitAspect in aspectsDefined:
addAspectQnCol(qnOimUnitAspect)
for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)):
if aspectQn.namespaceURI != nsOim:
addAspectQnCol(aspectQn)
def aspectCols(fact):
cols = [None for i in range(len(aspectsHeader))]
def setColValues(aspects):
for aspectQn, aspectValue in aspects.items():
if isinstance(aspectValue, dict):
setColValues(aspectValue)
elif aspectQn in aspectQnCol:
if aspectValue is None:
_aspectValue = "#nil"
elif aspectValue == "":
_aspectValue = "#empty"
elif isinstance(aspectValue, str) and aspectValue.startswith("#"):
_aspectValue = "#" + aspectValue
else:
_aspectValue = aspectValue
cols[aspectQnCol[aspectQn]] = _aspectValue
setColValues(factAspects(fact))
return cols
# metadata
csvTables = []
csvMetadata = OrderedDict((("@context", "http://www.w3.org/ns/csvw"),
("http://xbrl.org/YYYY/model#metadata",
OrderedDict((("documentType", "http://xbrl.org/YYYY/xbrl-csv"),
("dtsReferences", dtsReferences),
("prefixes", prefixes)))),
("tables", csvTables)))
_open = _writerow = _close = None
_tableinfo = {}
if isCSV:
if oimFile.endswith("-facts.csv"): # strip -facts.csv if a prior -facts.csv file was chosen
_baseURL = oimFile[:-10]
elif oimFile.endswith(".csv"):
_baseURL = oimFile[:-4]
else:
_baseURL = oimFile
_csvinfo = {} # open file, writer
def _open(filesuffix, tabname):
_filename = _tableinfo["url"] = _baseURL + filesuffix
_csvinfo["file"] = open(_filename, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
_csvinfo["writer"] = csv.writer(_csvinfo["file"], dialect="excel")
def _writerow(row, header=False):
_csvinfo["writer"].writerow(row)
def _close():
_csvinfo["file"].close()
_csvinfo.clear()
elif isXL:
headerWidths = {"href": 100, "xbrl:concept": 70, "decimals": 8, "language": 9, "URI": 80,
"value": 60,
"group": 60, "footnoteType": 40, "footnote": 70, "column": 20,
'conceptAspect': 40, 'tuple': 20, 'simpleFact': 20}
from openpyxl import Workbook
from openpyxl.writer.write_only import WriteOnlyCell
from openpyxl.styles import Font, PatternFill, Border, Alignment, Color, fills, Side
from openpyxl.worksheet.dimensions import ColumnDimension
hdrCellFill = PatternFill(patternType=fills.FILL_SOLID, fgColor=Color("00FFBF5F")) # Excel's light orange fill color = 00FF990
workbook = Workbook()
# remove pre-existing worksheets
while len(workbook.worksheets)>0:
workbook.remove_sheet(workbook.worksheets[0])
_xlinfo = {} # open file, writer
def _open(filesuffix, tabname):
_tableinfo["url"] = tabname
_xlinfo["ws"] = workbook.create_sheet(title=tabname)
def _writerow(rowvalues, header=False):
row = []
_ws = _xlinfo["ws"]
for i, v in enumerate(rowvalues):
cell = WriteOnlyCell(_ws, value=v)
if header:
cell.fill = hdrCellFill
cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True)
colLetter = chr(ord('A') + i)
_ws.column_dimensions[colLetter] = ColumnDimension(_ws, customWidth=True)
_ws.column_dimensions[colLetter].width = headerWidths.get(v, 20)
else:
cell.alignment = Alignment(horizontal="right" if isinstance(v, _NUM_TYPES)
else "center" if isinstance(v, bool)
else "left",
vertical="top",
wrap_text=isinstance(v, str))
row.append(cell)
_ws.append(row)
def _close():
_xlinfo.clear()
# save facts
_open("-facts.csv", "facts")
_writerow(aspectsHeader, header=True)
def saveCSVfacts(facts):
for fact in facts:
_writerow(aspectCols(fact))
saveCSVfacts(fact.modelTupleFacts)
saveCSVfacts(modelXbrl.facts)
_close()
factsTableSchema = OrderedDict((("columns",factsColumns),))
csvTables.append(OrderedDict((("url",_tableinfo["url"]),
("http://xbrl.org/YYYY/model#tableType", "fact"),
("tableSchema",factsTableSchema))))
# save footnotes
if footnotesRelationshipSet.modelRelationships:
_open("-footnotes.csv", "footnotes")
cols = ("group", "footnoteType", "factId", "factRef", "footnote", "language")
_writerow(cols, header=True)
def saveCSVfootnotes(facts):
for fact in facts:
for _footnote in factFootnotes(fact):
_writerow(tuple((_footnote.get(col,"") for col in cols)))
saveCSVfootnotes(fact.modelTupleFacts)
saveCSVfootnotes(modelXbrl.facts)
_close()
footnoteTableSchema = OrderedDict((("columns",[OrderedDict((("name","group"),("datatype","anyURI"))),
OrderedDict((("name","footnoteType"),("datatype","Name"))),
OrderedDict((("name","factId"),("datatype","Name"))),
OrderedDict((("name","factRef"),("datatype","Name"))),
OrderedDict((("name","footnote"),("datatype","string"))),
OrderedDict((("name","language"),("datatype","language")))]),))
csvTables.append(OrderedDict((("url",_tableinfo["url"]),
("http://xbrl.org/YYYY/model#tableType", "footnote"),
("tableSchema",footnoteTableSchema))))
# save metadata
if isCSV:
with open(_baseURL + "-metadata.json", "w", encoding="utf-8") as fh:
fh.write(json.dumps(csvMetadata, ensure_ascii=False, indent=1, sort_keys=False))
elif isXL:
_open(None, "metadata")
hasColumnAspect = hasSimpleFact = hasTupleFact = False
for table in csvTables:
tablename = table["url"]
for column in table["tableSchema"]["columns"]:
if "http://xbrl.org/YYYY/model#columnAspect" in column:
hasColumnAspect = True
if "http://xbrl.org/YYYY/model#simpleFactAspects" in column:
hasSimpleFact = True
if "http://xbrl.org/YYYY/model#tupleAspects" in column:
hasTupleFact = True
metadataCols = ["table", "column", "datatype"]
if hasColumnAspect:
metadataCols.append("columnAspect")
if hasSimpleFact:
metadataCols.append("simpleFact")
if hasTupleFact:
metadataCols.append("tuple")
_writerow(metadataCols, header=True)
for table in csvTables:
tablename = table["url"]
for column in table["tableSchema"]["columns"]:
row = [tablename, column["name"], column["datatype"]]
if hasColumnAspect:
colAspect = column.get("http://xbrl.org/YYYY/model#columnAspect")
if isinstance(colAspect, str):
row.append(colAspect)
elif isinstance(colAspect, dict):
row.append("\n".join("{} [{}]".format(k, ", ".join(_v for _v in v))
for k, v in dict.items()))
else:
row.append(None)
if hasSimpleFact:
row.append("\u221a" if "http://xbrl.org/YYYY/model#simpleFactAspects" in column else None)
if hasTupleFact:
row.append("\u221a" if "http://xbrl.org/YYYY/model#tupleAspects" in column else None)
_writerow(row)
_close()
if isXL:
workbook.save(oimFile)
def saveLoadableOIM(modelXbrl, oimFile):
isJSON = oimFile.endswith(".json")
isCSV = oimFile.endswith(".csv")
isXL = oimFile.endswith(".xlsx")
isCSVorXL = isCSV or isXL
if not isJSON and not isCSVorXL:
return
namespacePrefixes = {}
prefixNamespaces = {}
def compileQname(qname):
if qname.namespaceURI not in namespacePrefixes:
namespacePrefixes[qname.namespaceURI] = qname.prefix or ""
aspectsDefined = {
qnOimConceptAspect,
qnOimEntityAspect}
if isJSON:
aspectsDefined.add(qnOimPeriodAspect)
elif isCSVorXL:
aspectsDefined.add(qnOimPeriodStartAspect)
aspectsDefined.add(qnOimPeriodEndAspect)
def oimValue(object, decimals=None):
if isinstance(object, QName):
if object.namespaceURI not in namespacePrefixes:
if object.prefix:
namespacePrefixes[object.namespaceURI] = object.prefix
else:
_prefix = "_{}".format(sum(1 for p in namespacePrefixes if p.startswith("_")))
namespacePrefixes[object.namespaceURI] = _prefix
return "{}:{}".format(namespacePrefixes[object.namespaceURI], object.localName)
if isinstance(object, Decimal):
try:
if isinf(object):
return "-INF" if object < 0 else "INF"
elif isnan(num):
return "NaN"
else:
if object == object.to_integral():
object = object.quantize(ONE) # drop any .0
return "{}".format(object)
except:
return str(object)
if isinstance(object, bool):
return object
if isinstance(object, (DateTime, YearMonthDuration, DayTimeDuration, Time,
gYearMonth, gMonthDay, gYear, gMonth, gDay)):
return str(object)
return object
def oimPeriodValue(cntx):
if cntx.isForeverPeriod:
pass # not supported
elif cntx.isStartEndPeriod:
s = cntx.startDatetime
e = cntx.endDatetime
else: # instant
s = e = cntx.instantDatetime
return {"start": "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format(
s.year, s.month, s.day, s.hour, s.minute, s.second),
"end": "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format(
e.year, e.month, e.day, e.hour, e.minute, e.second)}
hasId = False
hasTuple = False
hasType = True
hasLang = False
hasUnits = False
footnotesRelationshipSet = ModelRelationshipSet(modelXbrl, "XBRL-footnotes")
factBaseTypes = set()
#compile QNames in instance for OIM
for fact in modelXbrl.factsInInstance:
if (fact.id or fact.isTuple or
footnotesRelationshipSet.toModelObject(fact) or
(isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))):
hasId = True
concept = fact.concept
if concept is not None:
if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang:
hasLang = True
_baseXsdType = concept.baseXsdType
if _baseXsdType == "XBRLI_DATEUNION":
if getattr(fact.xValue, "dateOnly", False):
_baseXsdType = "date"
else:
_baseXsdType = "dateTime"
factBaseTypes.add(baseTypes.get(_baseXsdType,_baseXsdType))
compileQname(fact.qname)
if hasattr(fact, "xValue") and isinstance(fact.xValue, QName):
compileQname(fact.xValue)
unit = fact.unit
if unit is not None:
hasUnits = True
if fact.modelTupleFacts:
hasTuple = True
entitySchemePrefixes = {}
for cntx in modelXbrl.contexts.values():
if cntx.entityIdentifierElement is not None:
scheme = cntx.entityIdentifier[0]
if scheme not in entitySchemePrefixes:
if not entitySchemePrefixes: # first one is just scheme
if scheme == "http://www.sec.gov/CIK":
_schemePrefix = "cik"
elif scheme == "http://standard.iso.org/iso/17442":
_schemePrefix = "lei"
else:
_schemePrefix = "scheme"
else:
_schemePrefix = "scheme{}".format(len(entitySchemePrefixes) + 1)
entitySchemePrefixes[scheme] = _schemePrefix
namespacePrefixes[scheme] = _schemePrefix
for dim in cntx.qnameDims.values():
compileQname(dim.dimensionQname)
aspectsDefined.add(dim.dimensionQname)
if dim.isExplicit:
compileQname(dim.memberQname)
for unit in modelXbrl.units.values():
if unit is not None:
for measures in unit.measures:
for measure in measures:
compileQname(measure)
if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[XbrlConst.xbrli] != "xbrli":
namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix
namespacePrefixes[XbrlConst.xsd] = "xsd"
if hasLang: aspectsDefined.add(qnOimLangAspect)
if hasTuple:
aspectsDefined.add(qnOimTupleParentAspect)
aspectsDefined.add(qnOimTupleOrderAspect)
if hasUnits: aspectsDefined.add(qnOimUnitAspect)
# compile footnotes and relationships
'''
factRelationships = []
factFootnotes = []
for rel in modelXbrl.relationshipSet(modelXbrl, "XBRL-footnotes").modelRelationships:
oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole}
factRelationships.append(oimRel)
oimRel["fromIds"] = [obj.id if obj.id
else elementChildSequence(obj)
for obj in rel.fromModelObjects]
oimRel["toIds"] = [obj.id if obj.id
else elementChildSequence(obj)
for obj in rel.toModelObjects]
_order = rel.arcElement.get("order")
if _order is not None:
oimRel["order"] = _order
for obj in rel.toModelObjects:
if isinstance(obj, ModelResource): # footnote
oimFootnote = {"role": obj.role,
"id": obj.id if obj.id
else elementChildSequence(obj),
# value needs work for html elements and for inline footnotes
"value": xmlstring(obj, stripXmlns=True)}
if obj.xmlLang:
oimFootnote["lang"] = obj.xmlLang
factFootnotes.append(oimFootnote)
oimFootnote
'''
dtsReferences = [
{"type": "schema" if doc.type == ModelDocument.Type.SCHEMA
else "linkbase" if doc.type == ModelDocument.Type.LINKBASE
else "other",
"href": doc.uri}
for doc,ref in modelXbrl.modelDocument.referencesDocument.items()
if ref.referringModelObject.qname in SCHEMA_LB_REFS]
'''
roleTypes = [
{"type": "role" if ref.referringModelObject.localName == "roleRef" else "arcroleRef",
"href": ref.referringModelObject["href"]}
for doc,ref in modelXbrl.modelDocument.referencesDocument.items()
if ref.referringModelObject.qname in ROLE_REFS]
'''
def factFootnotes(fact):
footnotes = []
for footnoteRel in footnotesRelationshipSet.fromModelObject(fact):
footnote = OrderedDict((("group", footnoteRel.linkrole),
("footnoteType", footnoteRel.arcrole)))
footnotes.append(footnote)
if isCSVorXL:
footnote["factId"] = fact.id if fact.id else "f{}".format(fact.objectIndex)
toObj = footnoteRel.toModelObject
if isinstance(toObj, ModelFact):
footnote["factRef"] = toObj.id if toObj.id else "f{}".format(toObj.objectIndex)
else:
footnote["footnote"] = xmlstring(toObj, stripXmlns=True, contentsOnly=True, includeText=True)
if toObj.xmlLang:
footnote["language"] = toObj.xmlLang
return footnotes
def factAspects(fact):
aspects = OrderedDict()
if hasId and fact.id:
aspects["id"] = fact.id
elif (fact.isTuple or
footnotesRelationshipSet.toModelObject(fact) or
(isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))):
aspects["id"] = "f{}".format(fact.objectIndex)
parent = fact.getparent()
concept = fact.concept
_csvType = "Value"
if not fact.isTuple:
if concept is not None:
_baseXsdType = concept.baseXsdType
if _baseXsdType == "XBRLI_DATEUNION":
if getattr(fact.xValue, "dateOnly", False):
_baseXsdType = "date"
else:
_baseXsdType = "dateTime"
aspects["baseType"] = "xs:{}".format(_baseXsdType)
_csvType = baseTypes.get(_baseXsdType,_baseXsdType) + "Value"
if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang:
aspects[qnOimLangAspect] = fact.xmlLang
if fact.isItem:
if fact.isNil:
_value = None
_strValue = "nil"
else:
_inferredDecimals = inferredDecimals(fact)
_value = oimValue(fact.xValue, _inferredDecimals)
_strValue = str(_value)
if not isCSVorXL:
aspects["value"] = _strValue
if fact.concept is not None and fact.concept.isNumeric:
_numValue = fact.xValue
if isinstance(_numValue, Decimal) and not isinf(_numValue) and not isnan(_numValue):
if _numValue == _numValue.to_integral():
_numValue = int(_numValue)
else:
_numValue = float(_numValue)
if isCSVorXL:
aspects[_csvType] = _numValue
else:
aspects["numericValue"] = _numValue
if not fact.isNil:
if isinf(_inferredDecimals):
if isJSON: _accuracy = "infinity"
elif isCSVorXL: _accuracy = "INF"
else:
_accuracy = _inferredDecimals
aspects["accuracy"] = _accuracy
elif isinstance(_value, bool):
aspects["booleanValue"] = _value
elif isCSVorXL:
aspects[_csvType] = _strValue
aspects[qnOimConceptAspect] = oimValue(fact.qname)
cntx = fact.context
if cntx is not None:
if cntx.entityIdentifierElement is not None:
aspects[qnOimEntityAspect] = oimValue(qname(*cntx.entityIdentifier))
if cntx.period is not None:
if isJSON:
aspects[qnOimPeriodAspect] = oimPeriodValue(cntx)
elif isCSVorXL:
_periodValue = oimPeriodValue(cntx)
aspects[qnOimPeriodStartAspect] = _periodValue["start"]
aspects[qnOimPeriodEndAspect] = _periodValue["end"]
for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]):
aspects[dim.dimensionQname] = (oimValue(dim.memberQname) if dim.isExplicit
else None if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1")
else dim.typedMember.stringValue)
unit = fact.unit
if unit is not None:
_mMul, _mDiv = unit.measures
_sMul = '*'.join(oimValue(m) for m in sorted(_mMul, key=lambda m: str(m)))
if _mDiv:
_sDiv = '*'.join(oimValue(m) for m in sorted(_mDiv, key=lambda m: str(m)))
if len(mDiv) > 1:
if len(mMul) > 1:
_sUnit = "({})/({})".format(_sMul,_sDiv)
else:
_sUnit = "{}/({})".format(_sMul,_sDiv)
else:
if len(mMul) > 1:
_sUnit = "({})/{}".format(_sMul,_sDiv)
else:
_sUnit = "{}/{}".format(_sMul,_sDiv)
else:
_sUnit = _sMul
aspects[qnOimUnitAspect] = _sUnit
if parent.qname != XbrlConst.qnXbrliXbrl:
aspects[qnOimTupleParentAspect] = parent.id if parent.id else "f{}".format(parent.objectIndex)
aspects[qnOimTupleOrderAspect] = elementIndex(fact)
if isJSON:
_footnotes = factFootnotes(fact)
if _footnotes:
aspects["footnotes"] = _footnotes
return aspects
if isJSON:
# save JSON
oimReport = OrderedDict() # top level of oim json output
oimFacts = []
oimReport["prefixes"] = OrderedDict((p,ns) for ns, p in sorted(namespacePrefixes.items(),
key=lambda item: item[1]))
oimReport["dtsReferences"] = dtsReferences
oimReport["facts"] = oimFacts
def saveJsonFacts(facts, oimFacts, parentFact):
for fact in facts:
oimFact = factAspects(fact)
oimFacts.append(OrderedDict((str(k),v) for k,v in oimFact.items()))
if fact.modelTupleFacts:
saveJsonFacts(fact.modelTupleFacts, oimFacts, fact)
saveJsonFacts(modelXbrl.facts, oimFacts, None)
with open(oimFile, "w", encoding="utf-8") as fh:
fh.write(json.dumps(oimReport, ensure_ascii=False, indent=1, sort_keys=False))
elif isCSVorXL:
# save CSV
aspectQnCol = {}
aspectsHeader = []
factsColumns = []
def addAspectQnCol(aspectQn):
aspectQnCol[aspectQn] = len(aspectsHeader)
_colName = oimValue(aspectQn)
aspectsHeader.append(_colName)
_colDataType = {"id": "Name",
"baseType": "Name",
"oim:concept": "QName",
"oim:periodStart": "dateTime",
"oim:periodEnd": "dateTime",
"oim:tupleOrder": "integer",
"numericValue": "decimal",
"accuracy": "decimal",
"booleanValue": "boolean",
"oim:entity": "QName",
"oim:unit": "string"
}.get(_colName)
if _colDataType is None:
if isCSVorXL and _colName.endswith("Value"):
_colDataType = _colName[:-5]
else:
_colDataType = "string"
factsColumns.append(OrderedDict((("name", _colName),
("datatype", _colDataType))))
# pre-ordered aspect columns
if hasId:
addAspectQnCol("id")
addAspectQnCol(qnOimConceptAspect)
if hasType:
addAspectQnCol("baseType")
if isCSVorXL:
for _baseType in sorted(factBaseTypes):
addAspectQnCol(_baseType + "Value")
else:
addAspectQnCol("stringValue")
addAspectQnCol("numericValue")
addAspectQnCol("booleanValue")
addAspectQnCol("accuracy")
if hasTuple:
addAspectQnCol(qnOimTupleParentAspect)
addAspectQnCol(qnOimTupleOrderAspect)
if qnOimEntityAspect in aspectsDefined:
addAspectQnCol(qnOimEntityAspect)
if qnOimPeriodStartAspect in aspectsDefined:
addAspectQnCol(qnOimPeriodStartAspect)
addAspectQnCol(qnOimPeriodEndAspect)
if qnOimUnitAspect in aspectsDefined:
addAspectQnCol(qnOimUnitAspect)
for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)):
if aspectQn.namespaceURI != nsOim:
addAspectQnCol(aspectQn)
def aspectCols(fact):
cols = [None for i in range(len(aspectsHeader))]
_factAspects = factAspects(fact)
for aspectQn, aspectValue in _factAspects.items():
if aspectQn in aspectQnCol:
cols[aspectQnCol[aspectQn]] = aspectValue
return cols
# metadata
csvTables = []
csvMetadata = OrderedDict((("@context",[ "http://www.w3.org/ns/csvw", { "@base": "./" }]),
("tables", csvTables)))
_open = _writerow = _close = None
_tableinfo = {}
if isCSV:
if oimFile.endswith("-facts.csv"): # strip -facts.csv if a prior -facts.csv file was chosen
_baseURL = oimFile[:-10]
elif oimFile.endswith(".csv"):
_baseURL = oimFile[:-4]
else:
_baseURL = oimFile
_csvinfo = {} # open file, writer
def _open(filesuffix, tabname):
_filename = _tableinfo["url"] = _baseURL + filesuffix
_csvinfo["file"] = open(_filename, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
_csvinfo["writer"] = csv.writer(csvFile, dialect="excel")
def _writerow(row, header=False):
_csvinfo["writer"].writerow(row)
def _close():
_csvinfo["writer"].close()
_csvinfo.clear()
elif isXL:
headerWidths = {"href": 100, "oim:concept": 70, "accuracy": 8, "baseType": 10, "language": 9, "URI": 80,
"stringValue": 60, "decimalValue": 12, "booleanValue": 12, "dateValue": 12, "dateTimeValue": 20,
"group": 60, "footnoteType": 40, "footnote": 70, "column": 60}
from openpyxl import Workbook
from openpyxl.writer.write_only import WriteOnlyCell
from openpyxl.styles import Font, PatternFill, Border, Alignment, Color, fills, Side
from openpyxl.worksheet.dimensions import ColumnDimension
hdrCellFill = PatternFill(patternType=fills.FILL_SOLID, fgColor=Color("00FFBF5F")) # Excel's light orange fill color = 00FF990
workbook = Workbook(encoding="utf-8", write_only=True)
_xlinfo = {} # open file, writer
def _open(filesuffix, tabname):
_tableinfo["url"] = tabname
_xlinfo["ws"] = workbook.create_sheet(title=tabname)
def _writerow(rowvalues, header=False):
row = []
_ws = _xlinfo["ws"]
for i, v in enumerate(rowvalues):
cell = WriteOnlyCell(_ws, value=v)
if header:
cell.fill = hdrCellFill
cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True)
colLetter = chr(ord('A') + i)
_ws.column_dimensions[colLetter] = ColumnDimension(_ws, customWidth=True)
_ws.column_dimensions[colLetter].width = headerWidths.get(v, 20)
else:
cell.alignment = Alignment(horizontal="right" if isinstance(v, _NUM_TYPES)
else "center" if isinstance(v, bool)
else "left",
vertical="top",
wrap_text=isinstance(v, str))
row.append(cell)
_ws.append(row)
def _close():
_xlinfo.clear()
# save facts
_open("-facts.csv", "facts")
_writerow(aspectsHeader, header=True)
def saveCSVfacts(facts):
for fact in facts:
_writerow(aspectCols(fact))
saveCSVfacts(fact.modelTupleFacts)
saveCSVfacts(modelXbrl.facts)
_close()
factsTableSchema = OrderedDict((("columns",factsColumns),))
csvTables.append(OrderedDict((("url",_tableinfo["url"]),
("tableSchema",factsTableSchema))))
# save namespaces
_open("-prefixes.csv", "prefixes")
_writerow(("prefix", "URI"), header=True)
for _URI, prefix in sorted(namespacePrefixes.items(), key=lambda item: item[1]):
_writerow((prefix, _URI))
_close()
nsTableSchema = OrderedDict((("columns",[OrderedDict((("name","prefix"),("datatype","Name"))),
OrderedDict((("name","URI"),("datatype","anyURI")))]),))
csvTables.append(OrderedDict((("url",_tableinfo["url"]),
("tableSchema",nsTableSchema))))
# save dts references
_open("-dtsReferences.csv", "dtsReferences")
_writerow(("type", "href"), header=True)
for oimRef in dtsReferences:
_writerow((oimRef["type"], oimRef["href"]))
_close()
dtsRefTableSchema = OrderedDict((("columns",[OrderedDict((("name","type"),("datatype","Name"))),
OrderedDict((("name","href"),("datatype","anyURI")))]),))
csvTables.append(OrderedDict((("url",_tableinfo["url"]),
("tableSchema",dtsRefTableSchema))))
# save footnotes
if footnotesRelationshipSet.modelRelationships:
_open("-footnotes.csv", "footnotes")
cols = ("group", "footnoteType", "factId", "factRef", "footnote", "language")
_writerow(cols, header=True)
def saveCSVfootnotes(facts):
for fact in facts:
for _footnote in factFootnotes(fact):
_writerow(tuple((_footnote.get(col,"") for col in cols)))
saveCSVfootnotes(fact.modelTupleFacts)
saveCSVfootnotes(modelXbrl.facts)
_close()
footnoteTableSchema = OrderedDict((("columns",[OrderedDict((("name","group"),("datatype","anyURI"))),
OrderedDict((("name","footnoteType"),("datatype","Name"))),
OrderedDict((("name","factId"),("datatype","Name"))),
OrderedDict((("name","factRef"),("datatype","Name"))),
OrderedDict((("name","footnote"),("datatype","string"))),
OrderedDict((("name","language"),("datatype","language")))]),))
csvTables.append(OrderedDict((("url",_tableinfo["url"]),
("tableSchema",footnoteTableSchema))))
# save metadata
if isCSV:
with open(_baseURL + "-metadata.csv", "w", encoding="utf-8") as fh:
fh.write(json.dumps(csvMetadata, ensure_ascii=False, indent=1, sort_keys=False))
elif isXL:
_open(None, "metadata")
cols = ("group", "footnoteType", "factId", "factRef", "footnote", "language")
_writerow(("table", "column", "datatype"), header=True)
for table in csvTables:
tablename = table["url"]
for column in table["tableSchema"]["columns"]:
_writerow((tablename, column["name"], column["datatype"]))
_close()
if isXL:
workbook.save(oimFile)
def saveLoadableOIM(modelXbrl, oimFile):
isJSON = oimFile.endswith(".json")
isCSV = oimFile.endswith(".csv")
namespacePrefixes = {}
def compileQname(qname):
if qname.namespaceURI not in namespacePrefixes:
namespacePrefixes[qname.namespaceURI] = qname.prefix or ""
aspectsDefined = {
qnOimConceptAspect,
qnOimEntityAspect,
qnOimTypeAspect}
if isJSON:
aspectsDefined.add(qnOimPeriodAspect)
elif isCSV:
aspectsDefined.add(qnOimPeriodStartAspect)
aspectsDefined.add(qnOimPeriodDurationAspect)
def oimValue(object, decimals=None):
if isinstance(object, QName):
if object.namespaceURI not in namespacePrefixes:
if object.prefix:
namespacePrefixes[object.namespaceURI] = object.prefix
else:
_prefix = "_{}".format(sum(1 for p in namespacePrefixes if p.startswith("_")))
namespacePrefixes[object.namespaceURI] = _prefix
return "{}:{}".format(namespacePrefixes[object.namespaceURI], object.localName)
if isinstance(object, Decimal):
try:
if isinf(object):
return "-INF" if object < 0 else "INF"
elif isnan(num):
return "NaN"
else:
if object == object.to_integral():
object = object.quantize(ONE) # drop any .0
return "{}".format(object)
except:
return str(object)
if isinstance(object, bool):
return object
if isinstance(object, (DateTime, YearMonthDuration, DayTimeDuration, Time,
gYearMonth, gMonthDay, gYear, gMonth, gDay)):
return str(object)
return object
def oimPeriodValue(cntx):
if cntx.isForeverPeriod:
if isCSV:
return "0000-01-01T00:00:00/P9999Y"
return "forever"
elif cntx.isStartEndPeriod:
d = cntx.startDatetime
duration = yearMonthDayTimeDuration(cntx.startDatetime, cntx.endDatetime)
else: # instant
d = cntx.instantDatetime
duration = "PT0S"
return "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}/{6}".format(
d.year, d.month, d.day, d.hour, d.minute, d.second,
duration)
hasId = False
hasTuple = False
hasType = True
hasLang = False
hasUnits = False
hasUnitMulMeasures = False
hasUnitDivMeasures = False
footnotesRelationshipSet = ModelRelationshipSet(modelXbrl, "XBRL-footnotes")
#compile QNames in instance for OIM
for fact in modelXbrl.factsInInstance:
if (fact.id or fact.isTuple or
footnotesRelationshipSet.toModelObject(fact) or
(isCSV and footnotesRelationshipSet.fromModelObject(fact))):
hasId = True
concept = fact.concept
if concept is not None:
if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang:
hasLang = True
compileQname(fact.qname)
if hasattr(fact, "xValue") and isinstance(fact.xValue, QName):
compileQname(fact.xValue)
unit = fact.unit
if unit is not None:
hasUnits = True
if unit.measures[0]:
hasUnitMulMeasures = True
if unit.measures[1]:
hasUnitDivMeasures = True
if fact.modelTupleFacts:
hasTuple = True
entitySchemePrefixes = {}
for cntx in modelXbrl.contexts.values():
if cntx.entityIdentifierElement is not None:
scheme = cntx.entityIdentifier[0]
if scheme not in entitySchemePrefixes:
if not entitySchemePrefixes: # first one is just scheme
if scheme == "http://www.sec.gov/CIK":
_schemePrefix = "cik"
elif scheme == "http://standard.iso.org/iso/17442":
_schemePrefix = "lei"
else:
_schemePrefix = "scheme"
else:
_schemePrefix = "scheme{}".format(len(entitySchemePrefixes) + 1)
entitySchemePrefixes[scheme] = _schemePrefix
namespacePrefixes[scheme] = _schemePrefix
for dim in cntx.qnameDims.values():
compileQname(dim.dimensionQname)
aspectsDefined.add(dim.dimensionQname)
if dim.isExplicit:
compileQname(dim.memberQname)
for unit in modelXbrl.units.values():
if unit is not None:
for measures in unit.measures:
for measure in measures:
compileQname(measure)
if unit.measures[0]:
aspectsDefined.add(qnOimUnitNumeratorsAspect)
if unit.measures[1]:
aspectsDefined.add(qnOimUnitDenominatorsAspect)
if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[XbrlConst.xbrli] != "xbrli":
namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix
namespacePrefixes[XbrlConst.xsd] = "xsd"
if hasLang: aspectsDefined.add(qnOimLangAspect)
if hasTuple:
aspectsDefined.add(qnOimTupleParentAspect)
aspectsDefined.add(qnOimTupleOrderAspect)
if hasUnits: aspectsDefined.add(qnOimUnitAspect)
# compile footnotes and relationships
'''
factRelationships = []
factFootnotes = []
for rel in modelXbrl.relationshipSet(modelXbrl, "XBRL-footnotes").modelRelationships:
oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole}
factRelationships.append(oimRel)
oimRel["fromIds"] = [obj.id if obj.id
else elementChildSequence(obj)
for obj in rel.fromModelObjects]
oimRel["toIds"] = [obj.id if obj.id
else elementChildSequence(obj)
for obj in rel.toModelObjects]
_order = rel.arcElement.get("order")
if _order is not None:
oimRel["order"] = _order
for obj in rel.toModelObjects:
if isinstance(obj, ModelResource): # footnote
oimFootnote = {"role": obj.role,
"id": obj.id if obj.id
else elementChildSequence(obj),
# value needs work for html elements and for inline footnotes
"value": xmlstring(obj, stripXmlns=True)}
if obj.xmlLang:
oimFootnote["lang"] = obj.xmlLang
factFootnotes.append(oimFootnote)
oimFootnote
'''
dtsReferences = [
{"type": "schema" if doc.type == ModelDocument.Type.SCHEMA
else "linkbase" if doc.type == ModelDocument.Type.LINKBASE
else "other",
"href": doc.uri}
for doc,ref in modelXbrl.modelDocument.referencesDocument.items()
if ref.referringModelObject.qname in SCHEMA_LB_REFS]
'''
roleTypes = [
{"type": "role" if ref.referringModelObject.localName == "roleRef" else "arcroleRef",
"href": ref.referringModelObject["href"]}
for doc,ref in modelXbrl.modelDocument.referencesDocument.items()
if ref.referringModelObject.qname in ROLE_REFS]
'''
def factFootnotes(fact):
footnotes = []
for footnoteRel in footnotesRelationshipSet.fromModelObject(fact):
footnote = OrderedDict((("group", footnoteRel.arcrole),))
footnotes.append(footnote)
if isCSV:
footnote["factId"] = fact.id if fact.id else "f{}".format(fact.objectIndex)
toObj = footnoteRel.toModelObject
if isinstance(toObj, ModelFact):
footnote["factRef"] = toObj.id if toObj.id else "f{}".format(toObj.objectIndex)
else:
footnote["footnoteType"] = toObj.role
footnote["footnote"] = xmlstring(toObj, stripXmlns=True, contentsOnly=True, includeText=True)
if toObj.xmlLang:
footnote["language"] = toObj.xmlLang
return footnotes
def factAspects(fact):
aspects = OrderedDict()
if hasId and fact.id:
aspects["id"] = fact.id
elif (fact.isTuple or
footnotesRelationshipSet.toModelObject(fact) or
(isCSV and footnotesRelationshipSet.fromModelObject(fact))):
aspects["id"] = "f{}".format(fact.objectIndex)
parent = fact.getparent()
concept = fact.concept
if not fact.isTuple:
if concept is not None:
_baseXsdType = concept.baseXsdType
if _baseXsdType == "XBRLI_DATEUNION":
if getattr(fact.xValue, "dateOnly", False):
_baseXsdType = "date"
else:
_baseXsdType = "dateTime"
aspects["baseType"] = "xs:{}".format(_baseXsdType)
if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang:
aspects[qnOimLangAspect] = fact.xmlLang
aspects[qnOimTypeAspect] = concept.baseXbrliType
if fact.isItem:
if fact.isNil:
_value = None
_strValue = "nil"
else:
_inferredDecimals = inferredDecimals(fact)
_value = oimValue(fact.xValue, _inferredDecimals)
_strValue = str(_value)
if not isCSV:
aspects["value"] = _strValue
if fact.concept is not None and fact.concept.isNumeric:
_numValue = fact.xValue
if isinstance(_numValue, Decimal) and not isinf(_numValue) and not isnan(_numValue):
if _numValue == _numValue.to_integral():
_numValue = int(_numValue)
else:
_numValue = float(_numValue)
aspects["numericValue"] = _numValue
if not fact.isNil:
if isinf(_inferredDecimals):
if isJSON: _accuracy = "infinity"
elif isCSV: _accuracy = "INF"
else:
_accuracy = _inferredDecimals
aspects["accuracy"] = _inferredDecimals
elif isinstance(_value, bool):
aspects["booleanValue"] = _value
elif isCSV:
aspects["stringValue"] = _strValue
aspects[qnOimConceptAspect] = oimValue(fact.qname)
cntx = fact.context
if cntx is not None:
if cntx.entityIdentifierElement is not None:
aspects[qnOimEntityAspect] = oimValue(qname(*cntx.entityIdentifier))
if cntx.period is not None:
if isJSON:
aspects[qnOimPeriodAspect] = oimPeriodValue(cntx)
elif isCSV:
_periodValue = oimPeriodValue(cntx).split("/") + ["", ""] # default blank if no value
aspects[qnOimPeriodStartAspect] = _periodValue[0]
aspects[qnOimPeriodDurationAspect] = _periodValue[1]
for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]):
aspects[dim.dimensionQname] = (oimValue(dim.memberQname) if dim.isExplicit
else None if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1")
else dim.typedMember.stringValue)
unit = fact.unit
if unit is not None:
_mMul, _mDiv = unit.measures
if isJSON:
aspects[qnOimUnitAspect] = OrderedDict( # use tuple instead of list for hashability
(("numerators", tuple(oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m)))),)
)
if _mDiv:
aspects[qnOimUnitAspect]["denominators"] = tuple(oimValue(m) for m in sorted(_mDiv, key=lambda m: oimValue(m)))
else: # CSV
if _mMul:
aspects[qnOimUnitNumeratorsAspect] = " ".join(oimValue(m)
for m in sorted(_mMul, key=lambda m: str(m)))
if _mDiv:
aspects[qnOimUnitDenominatorsAspect] = " ".join(oimValue(m)
for m in sorted(_mDiv, key=lambda m: str(m)))
if parent.qname != XbrlConst.qnXbrliXbrl:
aspects[qnOimTupleParentAspect] = parent.id if parent.id else "f{}".format(parent.objectIndex)
aspects[qnOimTupleOrderAspect] = elementIndex(fact)
if isJSON:
_footnotes = factFootnotes(fact)
if _footnotes:
aspects["footnotes"] = _footnotes
return aspects
if isJSON:
# save JSON
oimReport = OrderedDict() # top level of oim json output
oimFacts = []
oimReport["prefixes"] = OrderedDict((p,ns) for ns, p in sorted(namespacePrefixes.items(),
key=lambda item: item[1]))
oimReport["dtsReferences"] = dtsReferences
oimReport["facts"] = oimFacts
def saveJsonFacts(facts, oimFacts, parentFact):
for fact in facts:
oimFact = factAspects(fact)
oimFacts.append(OrderedDict((str(k),v) for k,v in oimFact.items()))
if fact.modelTupleFacts:
saveJsonFacts(fact.modelTupleFacts, oimFacts, fact)
saveJsonFacts(modelXbrl.facts, oimFacts, None)
with open(oimFile, "w", encoding="utf-8") as fh:
fh.write(json.dumps(oimReport, ensure_ascii=False, indent=1, sort_keys=False))
elif isCSV:
# save CSV
aspectQnCol = {}
aspectsHeader = []
factsColumns = []
def addAspectQnCol(aspectQn):
aspectQnCol[aspectQn] = len(aspectsHeader)
_colName = oimValue(aspectQn)
aspectsHeader.append(_colName)
_colDataType = {"id": "Name",
"baseType": "Name",
"oim:concept": "Name",
"oim:periodStart": "dateTime", # forever is 0000-01-01T00:00:00
"oim:periodDuration": "duration", # forever is P9999Y
"oim:tupleOrder": "integer",
"numericValue": "decimal",
"accuracy": "decimal",
"booleanValue": "boolean",
"oim:unitNumerators": OrderedDict((("base","Name"), ("separator"," "))),
"oim:unitDenominators": OrderedDict((("base","Name"), ("separator"," "))),
}.get(_colName, "string")
factsColumns.append(OrderedDict((("name", _colName),
("datatype", _colDataType))))
# pre-ordered aspect columns
if hasId:
addAspectQnCol("id")
if hasType:
addAspectQnCol("baseType")
addAspectQnCol("stringValue")
addAspectQnCol("numericValue")
addAspectQnCol("accuracy")
addAspectQnCol("booleanValue")
if hasTuple:
addAspectQnCol(qnOimTupleParentAspect)
addAspectQnCol(qnOimTupleOrderAspect)
addAspectQnCol(qnOimConceptAspect)
if qnOimEntityAspect in aspectsDefined:
addAspectQnCol(qnOimEntityAspect)
if qnOimPeriodStartAspect in aspectsDefined:
addAspectQnCol(qnOimPeriodStartAspect)
addAspectQnCol(qnOimPeriodDurationAspect)
if qnOimUnitNumeratorsAspect in aspectsDefined:
addAspectQnCol(qnOimUnitNumeratorsAspect)
if qnOimUnitDenominatorsAspect in aspectsDefined:
addAspectQnCol(qnOimUnitDenominatorsAspect)
for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)):
if aspectQn.namespaceURI != nsOim:
addAspectQnCol(aspectQn)
def aspectCols(fact):
cols = [None for i in range(len(aspectsHeader))]
_factAspects = factAspects(fact)
for aspectQn, aspectValue in _factAspects.items():
if aspectQn in aspectQnCol:
cols[aspectQnCol[aspectQn]] = aspectValue
return cols
# metadata
csvTables = []
csvMetadata = OrderedDict((("@context",[ "http://www.w3.org/ns/csvw", { "@base": "./" }]),
("tables", csvTables)))
if oimFile.endswith("-facts.csv"): # strip -facts.csv if a prior -facts.csv file was chosen
_baseURL = oimFile[:-10]
elif oimFile.endswith(".csv"):
_baseURL = oimFile[:-4]
else:
_baseURL = oimFile
# save facts
_factsFile = _baseURL + "-facts.csv"
csvFile = open(_factsFile, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
csvWriter.writerow(aspectsHeader)
def saveCSVfacts(facts):
for fact in facts:
csvWriter.writerow(aspectCols(fact))
saveCSVfacts(fact.modelTupleFacts)
saveCSVfacts(modelXbrl.facts)
csvFile.close()
factsTableSchema = OrderedDict((("columns",factsColumns),))
csvTables.append(OrderedDict((("url",os.path.basename(_factsFile)),
("tableSchema",factsTableSchema))))
# save namespaces
_nsFile = _baseURL + "-prefixes.csv"
csvFile = open(_nsFile, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
csvWriter.writerow(("prefix", "URI"))
for _URI, prefix in sorted(namespacePrefixes.items(), key=lambda item: item[1]):
csvWriter.writerow((prefix, _URI))
csvFile.close()
nsTableSchema = OrderedDict((("columns",[OrderedDict((("prefix","string"),
("URI","anyURI")))]),))
csvTables.append(OrderedDict((("url",os.path.basename(_nsFile)),
("tableSchema",nsTableSchema))))
# save dts references
_dtsRefFile = _baseURL + "-dtsReferences.csv"
csvFile = open(_dtsRefFile, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
csvWriter.writerow(("type", "href"))
for oimRef in dtsReferences:
csvWriter.writerow((oimRef["type"], oimRef["href"]))
csvFile.close()
dtsRefTableSchema = OrderedDict((("columns",[OrderedDict((("type","string"),
("href","anyURI")))]),))
csvTables.append(OrderedDict((("url",os.path.basename(_dtsRefFile)),
("tableSchema",dtsRefTableSchema))))
# save footnotes
if footnotesRelationshipSet.modelRelationships:
_footnoteFile = oimFile.replace(".csv", "-footnotes.csv")
csvFile = open(_footnoteFile, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
cols = ("group", "factId", "factRef", "footnoteType", "footnote", "language")
csvWriter.writerow(cols)
def saveCSVfootnotes(facts):
for fact in facts:
for _footnote in factFootnotes(fact):
csvWriter.writerow(tuple((_footnote.get(col,"") for col in cols)))
saveCSVfootnotes(fact.modelTupleFacts)
saveCSVfootnotes(modelXbrl.facts)
csvFile.close()
footnoteTableSchema = OrderedDict((("columns",[OrderedDict((("group","anyURI"),
("factId","Name"),
("factRef","Name"),
("footnoteType","Name"),
("footnote","string"),
("language","language")))]),))
csvTables.append(OrderedDict((("url",os.path.basename(_footnoteFile)),
("tableSchema",footnoteTableSchema))))
# save metadata
with open(_baseURL + "-metadata.csv", "w", encoding="utf-8") as fh:
fh.write(json.dumps(csvMetadata, ensure_ascii=False, indent=1, sort_keys=False))
def saveLoadableOIM(modelXbrl, oimFile, outputZip=None):
isJSON = oimFile.endswith(".json")
isCSV = oimFile.endswith(".csv")
isXL = oimFile.endswith(".xlsx")
isCSVorXL = isCSV or isXL
if not isJSON and not isCSVorXL:
return
namespacePrefixes = {nsOim: "xbrl"}
prefixNamespaces = {"xbrl": nsOim}
def compileQname(qname):
if qname.namespaceURI not in namespacePrefixes:
namespacePrefixes[qname.namespaceURI] = qname.prefix or ""
aspectsDefined = {
qnOimConceptAspect,
qnOimEntityAspect,
qnOimPeriodStartAspect,
qnOimPeriodEndAspect}
def oimValue(object, decimals=None):
if isinstance(object, QName):
if object.namespaceURI not in namespacePrefixes:
if object.prefix:
namespacePrefixes[object.namespaceURI] = object.prefix
else:
_prefix = "_{}".format(sum(1 for p in namespacePrefixes if p.startswith("_")))
namespacePrefixes[object.namespaceURI] = _prefix
return "{}:{}".format(namespacePrefixes[object.namespaceURI], object.localName)
if isinstance(object, Decimal):
try:
if isinf(object):
return "-INF" if object < 0 else "INF"
elif isnan(num):
return "NaN"
else:
if object == object.to_integral():
object = object.quantize(ONE) # drop any .0
return "{}".format(object)
except:
return str(object)
if isinstance(object, bool):
return "true" if object else "false"
if isinstance(object, (DateTime, YearMonthDuration, DayTimeDuration, Time,
gYearMonth, gMonthDay, gYear, gMonth, gDay,
IsoDuration)):
return str(object)
return object
def oimPeriodValue(cntx):
if cntx.isForeverPeriod:
return OrderedDict() # defaulted
elif cntx.isStartEndPeriod:
s = cntx.startDatetime
e = cntx.endDatetime
else: # instant
s = e = cntx.instantDatetime
if isJSON:
return({str(qnOimPeriodAspect):
("{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}/".format(
s.year, s.month, s.day, s.hour, s.minute, s.second)
if cntx.isStartEndPeriod else "") +
"{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format(
e.year, e.month, e.day, e.hour, e.minute, e.second)
})
# CSV, XL
return OrderedDict(((str(qnOimPeriodStartAspect), "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format(
s.year, s.month, s.day, s.hour, s.minute, s.second)),
(str(qnOimPeriodEndAspect), "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format(
e.year, e.month, e.day, e.hour, e.minute, e.second))))
hasId = False
hasTuple = False
hasType = True
hasLang = False
hasUnits = False
hasNumeric = False
footnotesRelationshipSet = ModelRelationshipSet(modelXbrl, "XBRL-footnotes")
factBaseTypes = set()
#compile QNames in instance for OIM
for fact in modelXbrl.factsInInstance:
if (fact.id or fact.isTuple or
footnotesRelationshipSet.toModelObject(fact) or
(isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))):
hasId = True
concept = fact.concept
if concept is not None:
if concept.isNumeric:
hasNumeric = True
if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang:
hasLang = True
_baseXsdType = concept.baseXsdType
if _baseXsdType == "XBRLI_DATEUNION":
if getattr(fact.xValue, "dateOnly", False):
_baseXsdType = "date"
else:
_baseXsdType = "dateTime"
factBaseTypes.add(baseTypes.get(_baseXsdType,_baseXsdType))
compileQname(fact.qname)
if hasattr(fact, "xValue") and isinstance(fact.xValue, QName):
compileQname(fact.xValue)
unit = fact.unit
if unit is not None:
hasUnits = True
if fact.modelTupleFacts:
hasTuple = True
if hasTuple:
modelXbrl.error("arelleOIMsaver:tuplesNotAllowed",
"Tuples are not allowed in an OIM document",
modelObject=modelXbrl)
return
entitySchemePrefixes = {}
for cntx in modelXbrl.contexts.values():
if cntx.entityIdentifierElement is not None:
scheme = cntx.entityIdentifier[0]
if scheme not in entitySchemePrefixes:
if not entitySchemePrefixes: # first one is just scheme
if scheme == "http://www.sec.gov/CIK":
_schemePrefix = "cik"
elif scheme == "http://standard.iso.org/iso/17442":
_schemePrefix = "lei"
else:
_schemePrefix = "scheme"
else:
_schemePrefix = "scheme{}".format(len(entitySchemePrefixes) + 1)
entitySchemePrefixes[scheme] = _schemePrefix
namespacePrefixes[scheme] = _schemePrefix
for dim in cntx.qnameDims.values():
compileQname(dim.dimensionQname)
aspectsDefined.add(dim.dimensionQname)
if dim.isExplicit:
compileQname(dim.memberQname)
for unit in modelXbrl.units.values():
if unit is not None:
for measures in unit.measures:
for measure in measures:
compileQname(measure)
if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[XbrlConst.xbrli] != "xbrli":
namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix
namespacePrefixes[XbrlConst.xsd] = "xsd"
if hasLang: aspectsDefined.add(qnOimLangAspect)
if hasTuple:
aspectsDefined.add(qnOimTupleParentAspect)
aspectsDefined.add(qnOimTupleOrderAspect)
if hasUnits: aspectsDefined.add(qnOimUnitAspect)
for footnoteRel in footnotesRelationshipSet.modelRelationships:
typePrefix = "ftTyp_" + os.path.basename(footnoteRel.arcrole)
if footnoteRel.linkrole == XbrlConst.defaultLinkRole:
groupPrefix = "ftGrp_default"
else:
groupPrefix = "ftGrp_" + os.path.basename(footnoteRel.linkrole)
if typePrefix not in namespacePrefixes:
namespacePrefixes[footnoteRel.arcrole] = typePrefix
if groupPrefix not in namespacePrefixes:
namespacePrefixes[footnoteRel.linkrole] = groupPrefix
# compile footnotes and relationships
'''
factRelationships = []
factFootnotes = []
for rel in modelXbrl.relationshipSet(modelXbrl, "XBRL-footnotes").modelRelationships:
oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole}
factRelationships.append(oimRel)
oimRel["fromIds"] = [obj.id if obj.id
else elementChildSequence(obj)
for obj in rel.fromModelObjects]
oimRel["toIds"] = [obj.id if obj.id
else elementChildSequence(obj)
for obj in rel.toModelObjects]
_order = rel.arcElement.get("order")
if _order is not None:
oimRel["order"] = _order
for obj in rel.toModelObjects:
if isinstance(obj, ModelResource): # footnote
oimFootnote = {"role": obj.role,
"id": obj.id if obj.id
else elementChildSequence(obj),
# value needs work for html elements and for inline footnotes
"value": xmlstring(obj, stripXmlns=True)}
if obj.xmlLang:
oimFootnote["lang"] = obj.xmlLang
factFootnotes.append(oimFootnote)
oimFootnote
'''
dtsReferences = set()
baseUrl = modelXbrl.modelDocument.uri.partition("#")[0]
for doc,ref in sorted(modelXbrl.modelDocument.referencesDocument.items(),
key=lambda _item:_item[0].uri):
if ref.referringModelObject.qname in SCHEMA_LB_REFS:
dtsReferences.add(relativeUri(baseUrl,doc.uri))
for refType in ("role", "arcrole"):
for refElt in sorted(modelXbrl.modelDocument.xmlRootElement.iterchildren(
"{{http://www.xbrl.org/2003/linkbase}}{}Ref".format(refType)),
key=lambda elt:elt.get(refType+"URI")
):
dtsReferences.add(refElt.get("{http://www.w3.org/1999/xlink}href").partition("#")[0])
dtsReferences = sorted(dtsReferences) # turn into list
footnoteFacts = set()
def factFootnotes(fact, oimFact=None):
footnotes = []
oimLinks = {}
for footnoteRel in footnotesRelationshipSet.fromModelObject(fact):
toObj = footnoteRel.toModelObject
# json
typePrefix = namespacePrefixes[footnoteRel.arcrole]
groupPrefix = namespacePrefixes[footnoteRel.linkrole]
oimLinks.setdefault(typePrefix,{}).setdefault(groupPrefix,[]).append(
toObj.id if toObj.id else "f{}".format(toObj.objectIndex))
# csv/XL
footnote = OrderedDict((("group", footnoteRel.linkrole),
("footnoteType", footnoteRel.arcrole)))
footnotes.append(footnote)
if isCSVorXL:
footnote["factId"] = fact.id if fact.id else "f{}".format(fact.objectIndex)
if isinstance(toObj, ModelFact):
footnote["factRef"] = toObj.id if toObj.id else "f{}".format(toObj.objectIndex)
else:
footnote["footnote"] = toObj.viewText()
if toObj.xmlLang:
footnote["language"] = toObj.xmlLang
footnoteFacts.add(toObj)
if isJSON and oimLinks:
oimFact["links"] = OrderedDict((
(typePrefix, OrderedDict((
(groupPrefix, idList)
for groupPrefix, idList in sorted(groups.items())
)))
for typePrefix, groups in sorted(oimLinks.items())
))
footnotes.sort(key=lambda f:(f["group"],f.get("factId",f.get("factRef")),f.get("language")))
return footnotes
def factAspects(fact):
oimFact = OrderedDict()
aspects = OrderedDict()
if hasId and fact.id:
if fact.isTuple:
oimFact["tupleId"] = fact.id
else:
oimFact["id"] = fact.id
elif (fact.isTuple or
footnotesRelationshipSet.toModelObject(fact) or
(isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))):
oimFact["id"] = "f{}".format(fact.objectIndex)
parent = fact.getparent()
concept = fact.concept
aspects[str(qnOimConceptAspect)] = oimValue(concept.qname)
_csvType = "Value"
if not fact.isTuple:
if concept is not None:
_baseXsdType = concept.baseXsdType
if _baseXsdType == "XBRLI_DATEUNION":
if getattr(fact.xValue, "dateOnly", False):
_baseXsdType = "date"
else:
_baseXsdType = "dateTime"
_csvType = baseTypes.get(_baseXsdType,_baseXsdType) + "Value"
if concept.baseXbrliType in ("stringItemType", "normalizedStringItemType") and fact.xmlLang:
aspects[str(qnOimLangAspect)] = fact.xmlLang
if fact.isItem:
if fact.isNil:
_value = None
else:
_inferredDecimals = inferredDecimals(fact)
_value = oimValue(fact.xValue, _inferredDecimals)
oimFact["value"] = _value
if fact.concept is not None and fact.concept.isNumeric:
_numValue = fact.xValue
if isinstance(_numValue, Decimal) and not isinf(_numValue) and not isnan(_numValue):
if _numValue == _numValue.to_integral():
_numValue = int(_numValue)
else:
_numValue = float(_numValue)
if not fact.isNil:
if isinf(_inferredDecimals):
if isJSON: _accuracy = "infinity"
elif isCSVorXL: _accuracy = "INF"
else:
_accuracy = _inferredDecimals
oimFact["accuracy"] = _accuracy
oimFact["aspects"] = aspects
cntx = fact.context
if cntx is not None:
if cntx.entityIdentifierElement is not None:
aspects[str(qnOimEntityAspect)] = oimValue(qname(*cntx.entityIdentifier))
if cntx.period is not None:
aspects.update(oimPeriodValue(cntx))
for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]):
if dim.isExplicit:
dimVal = oimValue(dim.memberQname)
else: # typed
if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1"):
dimVal = None
else:
dimVal = dim.typedMember.stringValue
aspects[str(dim.dimensionQname)] = dimVal
unit = fact.unit
if unit is not None:
_mMul, _mDiv = unit.measures
_sMul = '*'.join(oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m)))
if _mDiv:
_sDiv = '*'.join(oimValue(m) for m in sorted(_mDiv, key=lambda m: oimValue(m)))
if len(_mDiv) > 1:
if len(_mMul) > 1:
_sUnit = "({})/({})".format(_sMul,_sDiv)
else:
_sUnit = "{}/({})".format(_sMul,_sDiv)
else:
if len(_mMul) > 1:
_sUnit = "({})/{}".format(_sMul,_sDiv)
else:
_sUnit = "{}/{}".format(_sMul,_sDiv)
else:
_sUnit = _sMul
aspects[str(qnOimUnitAspect)] = _sUnit
# Tuples removed from xBRL-JSON
#if parent.qname != XbrlConst.qnXbrliXbrl:
# aspects[str(qnOimTupleParentAspect)] = parent.id if parent.id else "f{}".format(parent.objectIndex)
# aspects[str(qnOimTupleOrderAspect)] = elementIndex(fact)
if isJSON:
factFootnotes(fact, oimFact)
return oimFact
prefixes = OrderedDict((p,ns) for ns, p in sorted(namespacePrefixes.items(),
key=lambda item: item[1]))
if isJSON:
# save JSON
oimReport = OrderedDict() # top level of oim json output
oimReport["documentInfo"] = oimDocInfo = OrderedDict()
oimReport["facts"] = oimFacts = OrderedDict()
oimDocInfo["documentType"] = nsOim + "/xbrl-json"
oimDocInfo["features"] = oimFeatures = OrderedDict()
oimDocInfo["prefixes"] = prefixes
oimDocInfo["taxonomy"] = dtsReferences
oimFeatures["xbrl:canonicalValues"] = True
def saveJsonFacts(facts, oimFacts, parentFact):
for fact in facts:
oimFact = factAspects(fact)
id = fact.id if fact.id else "f{}".format(fact.objectIndex)
oimFacts[id] = oimFact
if fact.modelTupleFacts:
saveJsonFacts(fact.modelTupleFacts, oimFacts, fact)
saveJsonFacts(modelXbrl.facts, oimFacts, None)
# add footnotes as pseudo facts
for ftObj in footnoteFacts:
ftId = ftObj.id if ftObj.id else "f{}".format(ftObj.objectIndex)
oimFacts[ftId] = oimFact = OrderedDict()
oimFact["value"] = ftObj.viewText()
oimFact["aspects"] = OrderedDict((("concept", "xbrl:note"),
("noteId", ftId)))
if ftObj.xmlLang:
oimFact["aspects"]["language"] = ftObj.xmlLang.lower()
if outputZip:
fh = io.StringIO()
else:
fh = open(oimFile, "w", encoding="utf-8")
fh.write(json.dumps(oimReport, indent=1))
if outputZip:
fh.seek(0)
outputZip.writestr(os.path.basename(oimFile),fh.read())
fh.close()
elif isCSVorXL:
# save CSV
aspectQnCol = {}
aspectsHeader = []
factsColumns = []
def addAspectQnCol(aspectQn):
aspectQnCol[str(aspectQn)] = len(aspectsHeader)
_aspectQn = oimValue(aspectQn)
aspectsHeader.append(_aspectQn)
_colName = _aspectQn.replace("xbrl:", "")
_colDataType = {"id": "Name",
"concept": "QName",
"value": "string",
"accuracy": "decimal",
"entity": "QName",
"periodStart": "dateTime",
"periodEnd": "dateTime",
"unit": "string",
"tupleId": "Name",
"tupleParent": "Name",
"tupleOrder": "integer"
}.get(_colName, "string")
col = OrderedDict((("name", _colName),
("datatype", _colDataType)))
if _aspectQn == "value":
col["http://xbrl.org/YYYY/model#simpleFactAspects"] = {}
elif _aspectQn == "tupleId":
col["http://xbrl.org/YYYY/model#tupleFactAspects"] = {}
col["http://xbrl.org/YYYY/model#tupleReferenceId"] = "true"
else:
col["http://xbrl.org/YYYY/model#columnAspect"] = _aspectQn
factsColumns.append(col)
# pre-ordered aspect columns
#if hasId:
# addAspectQnCol("id")
addAspectQnCol(qnOimConceptAspect)
addAspectQnCol("value")
if hasNumeric:
addAspectQnCol("accuracy")
if hasTuple:
addAspectQnCol("tupleId")
addAspectQnCol(qnOimTupleParentAspect)
addAspectQnCol(qnOimTupleOrderAspect)
if qnOimEntityAspect in aspectsDefined:
addAspectQnCol(qnOimEntityAspect)
if qnOimPeriodStartAspect in aspectsDefined:
addAspectQnCol(qnOimPeriodStartAspect)
addAspectQnCol(qnOimPeriodEndAspect)
if qnOimUnitAspect in aspectsDefined:
addAspectQnCol(qnOimUnitAspect)
for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)):
if aspectQn.namespaceURI != nsOim:
addAspectQnCol(aspectQn)
def aspectCols(fact):
cols = [None for i in range(len(aspectsHeader))]
def setColValues(aspects):
for aspectQn, aspectValue in aspects.items():
if isinstance(aspectValue, dict):
setColValues(aspectValue)
elif aspectQn in aspectQnCol:
if aspectValue is None:
_aspectValue = "#nil"
elif aspectValue == "":
_aspectValue = "#empty"
elif isinstance(aspectValue, str) and aspectValue.startswith("#"):
_aspectValue = "#" + aspectValue
else:
_aspectValue = aspectValue
cols[aspectQnCol[aspectQn]] = _aspectValue
setColValues(factAspects(fact))
return cols
# metadata
csvTables = []
csvMetadata = OrderedDict((("@context", "http://www.w3.org/ns/csvw"),
("http://xbrl.org/YYYY/model#metadata",
OrderedDict((("documentType", "http://xbrl.org/YYYY/xbrl-csv"),
("dtsReferences", dtsReferences),
("prefixes", prefixes)))),
("tables", csvTables)))
_open = _writerow = _close = None
_tableinfo = {}
if isCSV:
if oimFile.endswith("-facts.csv"): # strip -facts.csv if a prior -facts.csv file was chosen
_baseURL = oimFile[:-10]
elif oimFile.endswith(".csv"):
_baseURL = oimFile[:-4]
else:
_baseURL = oimFile
_csvinfo = {} # open file, writer
def _open(filesuffix, tabname):
_filename = _tableinfo["url"] = _baseURL + filesuffix
_csvinfo["file"] = open(_filename, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
_csvinfo["writer"] = csv.writer(_csvinfo["file"], dialect="excel")
def _writerow(row, header=False):
_csvinfo["writer"].writerow(row)
def _close():
_csvinfo["file"].close()
_csvinfo.clear()
elif isXL:
headerWidths = {"href": 100, "xbrl:concept": 70, "accuracy": 8, "language": 9, "URI": 80,
"value": 60,
"group": 60, "footnoteType": 40, "footnote": 70, "column": 20,
'conceptAspect': 40, 'tuple': 20, 'simpleFact': 20}
from openpyxl import Workbook
from openpyxl.writer.write_only import WriteOnlyCell
from openpyxl.styles import Font, PatternFill, Border, Alignment, Color, fills, Side
from openpyxl.worksheet.dimensions import ColumnDimension
hdrCellFill = PatternFill(patternType=fills.FILL_SOLID, fgColor=Color("00FFBF5F")) # Excel's light orange fill color = 00FF990
workbook = Workbook()
# remove pre-existing worksheets
while len(workbook.worksheets)>0:
workbook.remove_sheet(workbook.worksheets[0])
_xlinfo = {} # open file, writer
def _open(filesuffix, tabname):
_tableinfo["url"] = tabname
_xlinfo["ws"] = workbook.create_sheet(title=tabname)
def _writerow(rowvalues, header=False):
row = []
_ws = _xlinfo["ws"]
for i, v in enumerate(rowvalues):
cell = WriteOnlyCell(_ws, value=v)
if header:
cell.fill = hdrCellFill
cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True)
colLetter = chr(ord('A') + i)
_ws.column_dimensions[colLetter] = ColumnDimension(_ws, customWidth=True)
_ws.column_dimensions[colLetter].width = headerWidths.get(v, 20)
else:
cell.alignment = Alignment(horizontal="right" if isinstance(v, _NUM_TYPES)
else "center" if isinstance(v, bool)
else "left",
vertical="top",
wrap_text=isinstance(v, str))
row.append(cell)
_ws.append(row)
def _close():
_xlinfo.clear()
# save facts
_open("-facts.csv", "facts")
_writerow(aspectsHeader, header=True)
def saveCSVfacts(facts):
for fact in facts:
_writerow(aspectCols(fact))
saveCSVfacts(fact.modelTupleFacts)
saveCSVfacts(modelXbrl.facts)
_close()
factsTableSchema = OrderedDict((("columns",factsColumns),))
csvTables.append(OrderedDict((("url",_tableinfo["url"]),
("http://xbrl.org/YYYY/model#tableType", "fact"),
("tableSchema",factsTableSchema))))
# save footnotes
if footnotesRelationshipSet.modelRelationships:
_open("-footnotes.csv", "footnotes")
cols = ("group", "footnoteType", "factId", "factRef", "footnote", "language")
_writerow(cols, header=True)
def saveCSVfootnotes(facts):
for fact in facts:
for _footnote in factFootnotes(fact):
_writerow(tuple((_footnote.get(col,"") for col in cols)))
saveCSVfootnotes(fact.modelTupleFacts)
saveCSVfootnotes(modelXbrl.facts)
_close()
footnoteTableSchema = OrderedDict((("columns",[OrderedDict((("name","group"),("datatype","anyURI"))),
OrderedDict((("name","footnoteType"),("datatype","Name"))),
OrderedDict((("name","factId"),("datatype","Name"))),
OrderedDict((("name","factRef"),("datatype","Name"))),
OrderedDict((("name","footnote"),("datatype","string"))),
OrderedDict((("name","language"),("datatype","language")))]),))
csvTables.append(OrderedDict((("url",_tableinfo["url"]),
("http://xbrl.org/YYYY/model#tableType", "footnote"),
("tableSchema",footnoteTableSchema))))
# save metadata
if isCSV:
with open(_baseURL + "-metadata.json", "w", encoding="utf-8") as fh:
fh.write(json.dumps(csvMetadata, ensure_ascii=False, indent=1, sort_keys=False))
elif isXL:
_open(None, "metadata")
hasColumnAspect = hasSimpleFact = hasTupleFact = False
for table in csvTables:
tablename = table["url"]
for column in table["tableSchema"]["columns"]:
if "http://xbrl.org/YYYY/model#columnAspect" in column:
hasColumnAspect = True
if "http://xbrl.org/YYYY/model#simpleFactAspects" in column:
hasSimpleFact = True
if "http://xbrl.org/YYYY/model#tupleAspects" in column:
hasTupleFact = True
metadataCols = ["table", "column", "datatype"]
if hasColumnAspect:
metadataCols.append("columnAspect")
if hasSimpleFact:
metadataCols.append("simpleFact")
if hasTupleFact:
metadataCols.append("tuple")
_writerow(metadataCols, header=True)
for table in csvTables:
tablename = table["url"]
for column in table["tableSchema"]["columns"]:
row = [tablename, column["name"], column["datatype"]]
if hasColumnAspect:
colAspect = column.get("http://xbrl.org/YYYY/model#columnAspect")
if isinstance(colAspect, str):
row.append(colAspect)
elif isinstance(colAspect, dict):
row.append("\n".join("{} [{}]".format(k, ", ".join(_v for _v in v))
for k, v in dict.items()))
else:
row.append(None)
if hasSimpleFact:
row.append("\u221a" if "http://xbrl.org/YYYY/model#simpleFactAspects" in column else None)
if hasTupleFact:
row.append("\u221a" if "http://xbrl.org/YYYY/model#tupleAspects" in column else None)
_writerow(row)
_close()
if isXL:
workbook.save(oimFile)