def saveLoadableOIM(modelXbrl, oimFile, outputZip=None):
isJSON = oimFile.endswith(".json")
isCSV = oimFile.endswith(".csv")
isXL = oimFile.endswith(".xlsx")
isCSVorXL = isCSV or isXL
if not isJSON and not isCSVorXL:
return
namespacePrefixes = {nsOim: "xbrl"}
prefixNamespaces = {}
linkTypeAliases = {}
groupAliases = {}
linkTypePrefixes = {}
linkTypeUris = {}
linkGroupPrefixes = {}
linkGroupUris = {}
def compileQname(qname):
if qname.namespaceURI not in namespacePrefixes:
namespacePrefixes[qname.namespaceURI] = qname.prefix or ""
aspectsDefined = {qnOimConceptAspect, qnOimEntityAspect, qnOimPeriodAspect}
def oimValue(object, decimals=None):
if isinstance(object, QName):
if object.namespaceURI not in namespacePrefixes:
if object.prefix:
namespacePrefixes[object.namespaceURI] = object.prefix
else:
_prefix = "_{}".format(
sum(1 for p in namespacePrefixes if p.startswith("_")))
namespacePrefixes[object.namespaceURI] = _prefix
return "{}:{}".format(namespacePrefixes[object.namespaceURI],
object.localName)
if isinstance(object, (float, Decimal)):
try:
if isinf(object):
return "-INF" if object < 0 else "INF"
elif isnan(object):
return "NaN"
else:
if isinstance(object,
Decimal) and object == object.to_integral():
object = object.quantize(ONE) # drop any .0
return "{}".format(object)
except:
return str(object)
if isinstance(object, bool):
return "true" if object else "false"
if isinstance(
object,
(DateTime, YearMonthDuration, DayTimeDuration, Time, gYearMonth,
gMonthDay, gYear, gMonth, gDay, IsoDuration, int)):
return str(object)
return object
def oimPeriodValue(cntx):
if cntx.isStartEndPeriod:
s = cntx.startDatetime
e = cntx.endDatetime
else: # instant
s = e = cntx.instantDatetime
return ({
str(qnOimPeriodAspect):
("{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}/".format(
s.year, s.month, s.day, s.hour, s.minute, s.second)
if cntx.isStartEndPeriod else "") +
"{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format(
e.year, e.month, e.day, e.hour, e.minute, e.second)
})
hasId = False
hasTuple = False
hasType = True
hasLang = False
hasUnits = False
hasNumeric = False
footnotesRelationshipSet = ModelRelationshipSet(modelXbrl,
"XBRL-footnotes")
#compile QNames in instance for OIM
for fact in modelXbrl.factsInInstance:
if (fact.id or fact.isTuple
or footnotesRelationshipSet.toModelObject(fact) or
(isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))):
hasId = True
concept = fact.concept
if concept is not None:
if concept.isNumeric:
hasNumeric = True
if concept.baseXbrliType in ("string", "normalizedString",
"token") and fact.xmlLang:
hasLang = True
compileQname(fact.qname)
if hasattr(fact, "xValue") and isinstance(fact.xValue, QName):
compileQname(fact.xValue)
unit = fact.unit
if unit is not None:
hasUnits = True
if fact.modelTupleFacts:
hasTuple = True
if hasTuple:
modelXbrl.error("arelleOIMsaver:tuplesNotAllowed",
"Tuples are not allowed in an OIM document",
modelObject=modelXbrl)
return
entitySchemePrefixes = {}
for cntx in modelXbrl.contexts.values():
if cntx.entityIdentifierElement is not None:
scheme = cntx.entityIdentifier[0]
if scheme not in entitySchemePrefixes:
if not entitySchemePrefixes: # first one is just scheme
if scheme == "http://www.sec.gov/CIK":
_schemePrefix = "cik"
elif scheme == "http://standard.iso.org/iso/17442":
_schemePrefix = "lei"
else:
_schemePrefix = "scheme"
else:
_schemePrefix = "scheme{}".format(
len(entitySchemePrefixes) + 1)
entitySchemePrefixes[scheme] = _schemePrefix
namespacePrefixes[scheme] = _schemePrefix
for dim in cntx.qnameDims.values():
compileQname(dim.dimensionQname)
aspectsDefined.add(dim.dimensionQname)
if dim.isExplicit:
compileQname(dim.memberQname)
for unit in modelXbrl.units.values():
if unit is not None:
for measures in unit.measures:
for measure in measures:
compileQname(measure)
if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[
XbrlConst.xbrli] != "xbrli":
namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix
if hasLang: aspectsDefined.add(qnOimLangAspect)
if hasUnits: aspectsDefined.add(qnOimUnitAspect)
for footnoteRel in footnotesRelationshipSet.modelRelationships:
typePrefix = "ftTyp_" + os.path.basename(footnoteRel.arcrole)
if footnoteRel.linkrole == XbrlConst.defaultLinkRole:
groupPrefix = "ftGrp_default"
else:
groupPrefix = "ftGrp_" + os.path.basename(footnoteRel.linkrole)
if footnoteRel.arcrole not in linkTypeAliases:
linkTypeAliases[footnoteRel.arcrole] = typePrefix
if groupPrefix not in groupAliases:
groupAliases[footnoteRel.linkrole] = groupPrefix
dtsReferences = set()
baseUrl = modelXbrl.modelDocument.uri.partition("#")[0]
for doc, ref in sorted(modelXbrl.modelDocument.referencesDocument.items(),
key=lambda _item: _item[0].uri):
if ref.referringModelObject.qname in SCHEMA_LB_REFS:
dtsReferences.add(relativeUri(baseUrl, doc.uri))
for refType in ("role", "arcrole"):
for refElt in sorted(
modelXbrl.modelDocument.xmlRootElement.iterchildren(
"{{http://www.xbrl.org/2003/linkbase}}{}Ref".format(
refType)),
key=lambda elt: elt.get(refType + "URI")):
dtsReferences.add(
refElt.get("{http://www.w3.org/1999/xlink}href").partition("#")
[0])
dtsReferences = sorted(dtsReferences) # turn into list
footnoteFacts = set()
def factFootnotes(fact, oimFact=None, csvLinks=None):
footnotes = []
if isJSON:
oimLinks = {}
elif isCSVorXL:
oimLinks = csvLinks
for footnoteRel in footnotesRelationshipSet.fromModelObject(fact):
srcId = fact.id if fact.id else "f{}".format(fact.objectIndex)
toObj = footnoteRel.toModelObject
# json
typePrefix = linkTypeAliases[footnoteRel.arcrole]
groupPrefix = groupAliases[footnoteRel.linkrole]
if typePrefix not in oimLinks:
oimLinks[typePrefix] = OrderedDict()
_link = oimLinks[typePrefix]
if groupPrefix not in _link:
if isJSON:
_link[groupPrefix] = []
elif isCSVorXL:
_link[groupPrefix] = OrderedDict()
if isJSON:
tgtIdList = _link[groupPrefix]
elif isCSVorXL:
tgtIdList = _link[groupPrefix].setdefault(srcId, [])
tgtId = toObj.id if toObj.id else "f{}".format(toObj.objectIndex)
tgtIdList.append(tgtId)
footnote = OrderedDict((("group", footnoteRel.linkrole),
("footnoteType", footnoteRel.arcrole)))
if isinstance(toObj, ModelFact):
footnote["factRef"] = tgtId
else: # text footnotes
footnote["id"] = tgtId
footnote["footnote"] = toObj.viewText()
if toObj.xmlLang:
footnote["language"] = toObj.xmlLang
footnoteFacts.add(toObj)
footnotes.append(footnote)
if oimLinks:
_links = OrderedDict(
((typePrefix,
OrderedDict(
((groupPrefix, idList)
for groupPrefix, idList in sorted(groups.items()))))
for typePrefix, groups in sorted(oimLinks.items())))
if isJSON:
oimFact["links"] = _links
return footnotes
def factAspects(fact):
oimFact = OrderedDict()
aspects = OrderedDict()
if isCSVorXL:
oimFact["id"] = fact.id or "f{}".format(fact.objectIndex)
parent = fact.getparent()
concept = fact.concept
aspects[str(qnOimConceptAspect)] = oimValue(concept.qname)
if concept is not None:
if concept.type.isOimTextFactType and fact.xmlLang:
aspects[str(qnOimLangAspect)] = fact.xmlLang
if fact.isItem:
if fact.isNil:
_value = None
else:
_inferredDecimals = inferredDecimals(fact)
_value = oimValue(fact.xValue, _inferredDecimals)
oimFact["value"] = _value
if fact.concept is not None and fact.concept.isNumeric:
_numValue = fact.xValue
if isinstance(_numValue, Decimal) and not isinf(
_numValue) and not isnan(_numValue):
if _numValue == _numValue.to_integral():
_numValue = int(_numValue)
else:
_numValue = float(_numValue)
if not fact.isNil:
if not isinf(
_inferredDecimals): # accuracy omitted if infinite
oimFact["decimals"] = _inferredDecimals
oimFact["dimensions"] = aspects
cntx = fact.context
if cntx is not None:
if cntx.entityIdentifierElement is not None and cntx.entityIdentifier != ENTITY_NA_QNAME:
aspects[str(qnOimEntityAspect)] = oimValue(
qname(*cntx.entityIdentifier))
if cntx.period is not None and not cntx.isForeverPeriod:
aspects.update(oimPeriodValue(cntx))
for _qn, dim in sorted(cntx.qnameDims.items(),
key=lambda item: item[0]):
if dim.isExplicit:
dimVal = oimValue(dim.memberQname)
else: # typed
if dim.typedMember.get(
"{http://www.w3.org/2001/XMLSchema-instance}nil"
) in ("true", "1"):
dimVal = None
else:
dimVal = dim.typedMember.stringValue
aspects[str(dim.dimensionQname)] = dimVal
unit = fact.unit
if unit is not None:
_mMul, _mDiv = unit.measures
_sMul = '*'.join(
oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m)))
if _mDiv:
_sDiv = '*'.join(
oimValue(m)
for m in sorted(_mDiv, key=lambda m: oimValue(m)))
if len(_mDiv) > 1:
if len(_mMul) > 1:
_sUnit = "({})/({})".format(_sMul, _sDiv)
else:
_sUnit = "{}/({})".format(_sMul, _sDiv)
else:
if len(_mMul) > 1:
_sUnit = "({})/{}".format(_sMul, _sDiv)
else:
_sUnit = "{}/{}".format(_sMul, _sDiv)
else:
_sUnit = _sMul
if _sUnit != "xbrli:pure":
aspects[str(qnOimUnitAspect)] = _sUnit
# Tuples removed from xBRL-JSON
#if parent.qname != XbrlConst.qnXbrliXbrl:
# aspects[str(qnOimTupleParentAspect)] = parent.id if parent.id else "f{}".format(parent.objectIndex)
# aspects[str(qnOimTupleOrderAspect)] = elementIndex(fact)
if isJSON:
factFootnotes(fact, oimFact=oimFact)
return oimFact
namespaces = OrderedDict((p, ns)
for ns, p in sorted(namespacePrefixes.items(),
key=lambda item: item[1]))
# common metadata
oimReport = OrderedDict() # top level of oim json output
oimReport["documentInfo"] = oimDocInfo = OrderedDict()
oimDocInfo["documentType"] = nsOim + ("/xbrl-json"
if isJSON else "/xbrl-csv")
if isJSON:
oimDocInfo["features"] = oimFeatures = OrderedDict()
oimDocInfo["namespaces"] = namespaces
if linkTypeAliases:
oimDocInfo["linkTypes"] = OrderedDict(
(a, u) for u, a in sorted(linkTypeAliases.items(),
key=lambda item: item[1]))
if linkTypeAliases:
oimDocInfo["linkGroups"] = OrderedDict(
(a, u)
for u, a in sorted(groupAliases.items(), key=lambda item: item[1]))
oimDocInfo["taxonomy"] = dtsReferences
if isJSON:
oimFeatures["xbrl:canonicalValues"] = True
if isJSON:
# save JSON
oimReport["facts"] = oimFacts = OrderedDict()
def saveJsonFacts(facts, oimFacts, parentFact):
for fact in facts:
oimFact = factAspects(fact)
id = fact.id if fact.id else "f{}".format(fact.objectIndex)
oimFacts[id] = oimFact
if fact.modelTupleFacts:
saveJsonFacts(fact.modelTupleFacts, oimFacts, fact)
saveJsonFacts(modelXbrl.facts, oimFacts, None)
# add footnotes as pseudo facts
for ftObj in footnoteFacts:
ftId = ftObj.id if ftObj.id else "f{}".format(ftObj.objectIndex)
oimFacts[ftId] = oimFact = OrderedDict()
oimFact["value"] = ftObj.viewText()
oimFact["dimensions"] = OrderedDict(
(("concept", "xbrl:note"), ("noteId", ftId)))
if ftObj.xmlLang:
oimFact["dimensions"]["language"] = ftObj.xmlLang.lower()
if outputZip:
fh = io.StringIO()
else:
fh = open(oimFile, "w", encoding="utf-8")
fh.write(json.dumps(oimReport, indent=1))
if outputZip:
fh.seek(0)
outputZip.writestr(os.path.basename(oimFile), fh.read())
fh.close()
elif isCSVorXL:
# save CSV
oimReport["tables"] = oimTables = OrderedDict()
oimReport["tableTemplates"] = csvTableTemplates = OrderedDict()
oimTables["facts"] = csvTable = OrderedDict()
csvTable["template"] = "facts"
csvTable["url"] = "tbd"
csvTableTemplates["facts"] = csvTableTemplate = OrderedDict()
csvTableTemplate["rowIdColumn"] = "id"
csvTableTemplate["dimensions"] = csvTableDimensions = OrderedDict()
csvTableTemplate["columns"] = csvFactColumns = OrderedDict()
if footnotesRelationshipSet.modelRelationships:
csvLinks = OrderedDict()
oimReport["links"] = csvLinks
aspectQnCol = {}
aspectsHeader = []
factsColumns = []
def addAspectQnCol(aspectQn):
colQName = str(aspectQn).replace("xbrl:", "")
colNCName = colQName.replace(":", "_")
if colNCName not in aspectQnCol:
aspectQnCol[colNCName] = len(aspectsHeader)
aspectsHeader.append(colNCName)
if colNCName == "value":
csvFactColumns[colNCName] = col = OrderedDict()
col["dimensions"] = OrderedDict()
elif colNCName == "id":
csvFactColumns[colNCName] = {} # empty object
elif colNCName == "decimals":
csvFactColumns[colNCName] = {} # empty object
csvTableTemplate[colQName] = "$" + colNCName
else:
csvFactColumns[colNCName] = {} # empty object
csvTableDimensions[colQName] = "$" + colNCName
# pre-ordered aspect columns
#if hasId:
addAspectQnCol("id")
addAspectQnCol(qnOimConceptAspect)
if hasNumeric:
addAspectQnCol("decimals")
if qnOimEntityAspect in aspectsDefined:
addAspectQnCol(qnOimEntityAspect)
if qnOimPeriodAspect in aspectsDefined:
addAspectQnCol(qnOimPeriodAspect)
if qnOimUnitAspect in aspectsDefined:
addAspectQnCol(qnOimUnitAspect)
for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)):
if aspectQn.namespaceURI != nsOim:
addAspectQnCol(aspectQn)
addAspectQnCol("value")
def aspectCols(fact):
cols = [None for i in range(len(aspectsHeader))]
def setColValues(aspects):
for aspectQn, aspectValue in aspects.items():
colQName = str(aspectQn).replace("xbrl:", "")
colNCName = colQName.replace(":", "_")
if isinstance(aspectValue, dict):
setColValues(aspectValue)
elif colNCName in aspectQnCol:
if aspectValue is None:
_aspectValue = "#nil"
elif aspectValue == "":
_aspectValue = "#empty"
elif isinstance(aspectValue,
str) and aspectValue.startswith("#"):
_aspectValue = "#" + aspectValue
else:
_aspectValue = aspectValue
cols[aspectQnCol[colNCName]] = _aspectValue
setColValues(factAspects(fact))
return cols
# metadata
_open = _writerow = _close = None
if isCSV:
if oimFile.endswith(
"-facts.csv"
): # strip -facts.csv if a prior -facts.csv file was chosen
_baseURL = oimFile[:-10]
elif oimFile.endswith(".csv"):
_baseURL = oimFile[:-4]
else:
_baseURL = oimFile
_csvinfo = {} # open file, writer
def _open(filesuffix, tabname, csvTable=None):
_filename = _baseURL + filesuffix
if csvTable is not None:
csvTable["url"] = os.path.basename(
_filename) # located in same directory with metadata
_csvinfo["file"] = open(_filename,
csvOpenMode,
newline=csvOpenNewline,
encoding='utf-8-sig')
_csvinfo["writer"] = csv.writer(_csvinfo["file"],
dialect="excel")
def _writerow(row, header=False):
_csvinfo["writer"].writerow(row)
def _close():
_csvinfo["file"].close()
_csvinfo.clear()
elif isXL:
headerWidths = {
"concept": 40,
"decimals": 8,
"language": 9,
"value": 50,
"entity": 20,
"period": 20,
"unit": 20,
"metadata": 100
}
from openpyxl import Workbook
from openpyxl.cell.cell import WriteOnlyCell
from openpyxl.styles import Font, PatternFill, Border, Alignment, Color, fills, Side
from openpyxl.worksheet.dimensions import ColumnDimension
hdrCellFill = PatternFill(
patternType=fills.FILL_SOLID, fgColor=Color(
"00FFBF5F")) # Excel's light orange fill color = 00FF990
workbook = Workbook()
# remove pre-existing worksheets
while len(workbook.worksheets) > 0:
workbook.remove(workbook.worksheets[0])
_xlinfo = {} # open file, writer
def _open(filesuffix, tabname, csvTable=None):
if csvTable is not None:
csvTable["url"] = tabname + "!"
_xlinfo["ws"] = workbook.create_sheet(title=tabname)
def _writerow(rowvalues, header=False):
row = []
_ws = _xlinfo["ws"]
for i, v in enumerate(rowvalues):
cell = WriteOnlyCell(_ws, value=v)
if header:
cell.fill = hdrCellFill
cell.alignment = Alignment(horizontal="center",
vertical="center",
wrap_text=True)
colLetter = chr(ord('A') + i)
_ws.column_dimensions[colLetter] = ColumnDimension(
_ws, customWidth=True)
_ws.column_dimensions[
colLetter].width = headerWidths.get(v, 40)
else:
cell.alignment = Alignment(
horizontal="right" if isinstance(v, _NUM_TYPES)
else "center" if isinstance(v, bool) else "left",
vertical="top",
wrap_text=isinstance(v, str))
row.append(cell)
_ws.append(row)
def _close():
_xlinfo.clear()
# save facts
_open("-facts.csv", "facts", csvTable)
_writerow(aspectsHeader, header=True)
def saveCSVfacts(facts):
for fact in facts:
_writerow(aspectCols(fact))
saveCSVfacts(fact.modelTupleFacts)
saveCSVfacts(modelXbrl.facts)
_close()
# save footnotes
if footnotesRelationshipSet.modelRelationships:
footnotes = sorted(
(footnote for fact in modelXbrl.facts
for footnote in factFootnotes(fact, csvLinks=csvLinks)),
key=lambda footnote: footnote["id"])
if footnotes: # text footnotes
oimTables["footnotes"] = csvFtTable = OrderedDict()
csvFtTable["url"] = "tbd"
csvFtTable[
"tableDimensions"] = csvFtTableDimensions = OrderedDict()
csvFtTable["factColumns"] = csvFtFactColumns = OrderedDict()
csvFtFactColumns["footnote"] = csvFtValCol = OrderedDict()
csvFtValCol["id"] = "$id"
csvFtValCol["noteId"] = "$id"
csvFtValCol["concept"] = "xbrl:note"
csvFtValCol["language"] = "$language"
_open("-footnotes.csv", "footnotes", csvFtTable)
cols = ("id", "footnote", "language")
_writerow(cols, header=True)
for footnote in footnotes:
_writerow(tuple((footnote.get(col, "") for col in cols)))
_close()
# save metadata
if isCSV:
with open(_baseURL + "-metadata.json", "w",
encoding="utf-8") as fh:
fh.write(
json.dumps(oimReport,
ensure_ascii=False,
indent=2,
sort_keys=False))
elif isXL:
_open(None, "metadata")
_writerow(["metadata"], header=True)
_writerow([
json.dumps(oimReport,
ensure_ascii=False,
indent=1,
sort_keys=False)
])
_close()
if isXL:
workbook.save(oimFile)
def saveLoadableOIM(modelXbrl, oimFile):
isJSON = oimFile.endswith(".json")
isCSV = oimFile.endswith(".csv")
isXL = oimFile.endswith(".xlsx")
isCSVorXL = isCSV or isXL
if not isJSON and not isCSVorXL:
return
namespacePrefixes = {nsOim: "oim"}
prefixNamespaces = {"oim": nsOim}
def compileQname(qname):
if qname.namespaceURI not in namespacePrefixes:
namespacePrefixes[qname.namespaceURI] = qname.prefix or ""
aspectsDefined = {qnOimConceptAspect, qnOimEntityAspect}
if isJSON:
aspectsDefined.add(qnOimPeriodAspect)
elif isCSVorXL:
aspectsDefined.add(qnOimPeriodStartAspect)
aspectsDefined.add(qnOimPeriodEndAspect)
def oimValue(object, decimals=None):
if isinstance(object, QName):
if object.namespaceURI not in namespacePrefixes:
if object.prefix:
namespacePrefixes[object.namespaceURI] = object.prefix
else:
_prefix = "_{}".format(
sum(1 for p in namespacePrefixes if p.startswith("_")))
namespacePrefixes[object.namespaceURI] = _prefix
return "{}:{}".format(namespacePrefixes[object.namespaceURI],
object.localName)
if isinstance(object, Decimal):
try:
if isinf(object):
return "-INF" if object < 0 else "INF"
elif isnan(num):
return "NaN"
else:
if object == object.to_integral():
object = object.quantize(ONE) # drop any .0
return "{}".format(object)
except:
return str(object)
if isinstance(object, bool):
return object
if isinstance(object,
(DateTime, YearMonthDuration, DayTimeDuration, Time,
gYearMonth, gMonthDay, gYear, gMonth, gDay)):
return str(object)
return object
def oimPeriodValue(cntx):
if cntx.isForeverPeriod:
pass # not supported
elif cntx.isStartEndPeriod:
s = cntx.startDatetime
e = cntx.endDatetime
else: # instant
s = e = cntx.instantDatetime
return {
"start":
"{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format(
s.year, s.month, s.day, s.hour, s.minute, s.second),
"end":
"{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format(
e.year, e.month, e.day, e.hour, e.minute, e.second)
}
hasId = False
hasTuple = False
hasType = True
hasLang = False
hasUnits = False
hasNumeric = False
footnotesRelationshipSet = ModelRelationshipSet(modelXbrl,
"XBRL-footnotes")
factBaseTypes = set()
#compile QNames in instance for OIM
for fact in modelXbrl.factsInInstance:
if (fact.id or fact.isTuple
or footnotesRelationshipSet.toModelObject(fact) or
(isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))):
hasId = True
concept = fact.concept
if concept is not None:
if concept.isNumeric:
hasNumeric = True
if concept.baseXbrliType in ("string", "normalizedString",
"token") and fact.xmlLang:
hasLang = True
_baseXsdType = concept.baseXsdType
if _baseXsdType == "XBRLI_DATEUNION":
if getattr(fact.xValue, "dateOnly", False):
_baseXsdType = "date"
else:
_baseXsdType = "dateTime"
factBaseTypes.add(baseTypes.get(_baseXsdType, _baseXsdType))
compileQname(fact.qname)
if hasattr(fact, "xValue") and isinstance(fact.xValue, QName):
compileQname(fact.xValue)
unit = fact.unit
if unit is not None:
hasUnits = True
if fact.modelTupleFacts:
hasTuple = True
entitySchemePrefixes = {}
for cntx in modelXbrl.contexts.values():
if cntx.entityIdentifierElement is not None:
scheme = cntx.entityIdentifier[0]
if scheme not in entitySchemePrefixes:
if not entitySchemePrefixes: # first one is just scheme
if scheme == "http://www.sec.gov/CIK":
_schemePrefix = "cik"
elif scheme == "http://standard.iso.org/iso/17442":
_schemePrefix = "lei"
else:
_schemePrefix = "scheme"
else:
_schemePrefix = "scheme{}".format(
len(entitySchemePrefixes) + 1)
entitySchemePrefixes[scheme] = _schemePrefix
namespacePrefixes[scheme] = _schemePrefix
for dim in cntx.qnameDims.values():
compileQname(dim.dimensionQname)
aspectsDefined.add(dim.dimensionQname)
if dim.isExplicit:
compileQname(dim.memberQname)
for unit in modelXbrl.units.values():
if unit is not None:
for measures in unit.measures:
for measure in measures:
compileQname(measure)
if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[
XbrlConst.xbrli] != "xbrli":
namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix
namespacePrefixes[XbrlConst.xsd] = "xsd"
if hasLang: aspectsDefined.add(qnOimLangAspect)
if hasTuple:
aspectsDefined.add(qnOimTupleParentAspect)
aspectsDefined.add(qnOimTupleOrderAspect)
if hasUnits: aspectsDefined.add(qnOimUnitAspect)
# compile footnotes and relationships
'''
factRelationships = []
factFootnotes = []
for rel in modelXbrl.relationshipSet(modelXbrl, "XBRL-footnotes").modelRelationships:
oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole}
factRelationships.append(oimRel)
oimRel["fromIds"] = [obj.id if obj.id
else elementChildSequence(obj)
for obj in rel.fromModelObjects]
oimRel["toIds"] = [obj.id if obj.id
else elementChildSequence(obj)
for obj in rel.toModelObjects]
_order = rel.arcElement.get("order")
if _order is not None:
oimRel["order"] = _order
for obj in rel.toModelObjects:
if isinstance(obj, ModelResource): # footnote
oimFootnote = {"role": obj.role,
"id": obj.id if obj.id
else elementChildSequence(obj),
# value needs work for html elements and for inline footnotes
"value": xmlstring(obj, stripXmlns=True)}
if obj.xmlLang:
oimFootnote["lang"] = obj.xmlLang
factFootnotes.append(oimFootnote)
oimFootnote
'''
dtsReferences = [{
"type":
"schema" if doc.type == ModelDocument.Type.SCHEMA else
"linkbase" if doc.type == ModelDocument.Type.LINKBASE else "other",
"href":
doc.uri
} for doc, ref in modelXbrl.modelDocument.referencesDocument.items()
if ref.referringModelObject.qname in SCHEMA_LB_REFS]
'''
roleTypes = [
{"type": "role" if ref.referringModelObject.localName == "roleRef" else "arcroleRef",
"href": ref.referringModelObject["href"]}
for doc,ref in modelXbrl.modelDocument.referencesDocument.items()
if ref.referringModelObject.qname in ROLE_REFS]
'''
def factFootnotes(fact):
footnotes = []
for footnoteRel in footnotesRelationshipSet.fromModelObject(fact):
footnote = OrderedDict((("group", footnoteRel.linkrole),
("footnoteType", footnoteRel.arcrole)))
footnotes.append(footnote)
if isCSVorXL:
footnote["factId"] = fact.id if fact.id else "f{}".format(
fact.objectIndex)
toObj = footnoteRel.toModelObject
if isinstance(toObj, ModelFact):
footnote["factRef"] = toObj.id if toObj.id else "f{}".format(
toObj.objectIndex)
else:
footnote["footnote"] = xmlstring(toObj,
stripXmlns=True,
contentsOnly=True,
includeText=True)
if toObj.xmlLang:
footnote["language"] = toObj.xmlLang
return footnotes
def factAspects(fact):
aspects = OrderedDict()
if hasId and fact.id:
aspects["id"] = fact.id
elif (fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or
(isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))):
aspects["id"] = "f{}".format(fact.objectIndex)
parent = fact.getparent()
concept = fact.concept
_csvType = "Value"
if not fact.isTuple:
if concept is not None:
_baseXsdType = concept.baseXsdType
if _baseXsdType == "XBRLI_DATEUNION":
if getattr(fact.xValue, "dateOnly", False):
_baseXsdType = "date"
else:
_baseXsdType = "dateTime"
aspects["baseType"] = "xsd:{}".format(_baseXsdType)
_csvType = baseTypes.get(_baseXsdType, _baseXsdType) + "Value"
if concept.baseXbrliType in ("string", "normalizedString",
"token") and fact.xmlLang:
aspects[qnOimLangAspect] = fact.xmlLang
if fact.isItem:
if fact.isNil:
_value = None
_strValue = "nil"
else:
_inferredDecimals = inferredDecimals(fact)
_value = oimValue(fact.xValue, _inferredDecimals)
_strValue = str(_value)
if not isCSVorXL:
aspects["value"] = _strValue
if fact.concept is not None and fact.concept.isNumeric:
_numValue = fact.xValue
if isinstance(_numValue, Decimal) and not isinf(
_numValue) and not isnan(_numValue):
if _numValue == _numValue.to_integral():
_numValue = int(_numValue)
else:
_numValue = float(_numValue)
if isCSVorXL:
aspects[_csvType] = _numValue
else:
aspects["numericValue"] = _numValue
if not fact.isNil:
if isinf(_inferredDecimals):
if isJSON: _accuracy = "infinity"
elif isCSVorXL: _accuracy = "INF"
else:
_accuracy = _inferredDecimals
aspects["accuracy"] = _accuracy
elif isinstance(_value, bool):
aspects["booleanValue"] = _value
elif isCSVorXL:
aspects[_csvType] = _strValue
aspects[qnOimConceptAspect] = oimValue(fact.qname)
cntx = fact.context
if cntx is not None:
if cntx.entityIdentifierElement is not None:
aspects[qnOimEntityAspect] = oimValue(
qname(*cntx.entityIdentifier))
if cntx.period is not None:
if isJSON:
aspects[qnOimPeriodAspect] = oimPeriodValue(cntx)
elif isCSVorXL:
_periodValue = oimPeriodValue(cntx)
aspects[qnOimPeriodStartAspect] = _periodValue["start"]
aspects[qnOimPeriodEndAspect] = _periodValue["end"]
for _qn, dim in sorted(cntx.qnameDims.items(),
key=lambda item: item[0]):
aspects[dim.dimensionQname] = (
oimValue(dim.memberQname)
if dim.isExplicit else None if dim.typedMember.get(
"{http://www.w3.org/2001/XMLSchema-instance}nil") in (
"true", "1") else dim.typedMember.stringValue)
unit = fact.unit
if unit is not None:
_mMul, _mDiv = unit.measures
_sMul = '*'.join(
oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m)))
if _mDiv:
_sDiv = '*'.join(
oimValue(m)
for m in sorted(_mDiv, key=lambda m: oimValue(m)))
if len(_mDiv) > 1:
if len(_mMul) > 1:
_sUnit = "({})/({})".format(_sMul, _sDiv)
else:
_sUnit = "{}/({})".format(_sMul, _sDiv)
else:
if len(_mMul) > 1:
_sUnit = "({})/{}".format(_sMul, _sDiv)
else:
_sUnit = "{}/{}".format(_sMul, _sDiv)
else:
_sUnit = _sMul
aspects[qnOimUnitAspect] = _sUnit
if parent.qname != XbrlConst.qnXbrliXbrl:
aspects[
qnOimTupleParentAspect] = parent.id if parent.id else "f{}".format(
parent.objectIndex)
aspects[qnOimTupleOrderAspect] = elementIndex(fact)
if isJSON:
_footnotes = factFootnotes(fact)
if _footnotes:
aspects["footnotes"] = _footnotes
return aspects
if isJSON:
# save JSON
oimReport = OrderedDict() # top level of oim json output
oimFacts = []
oimReport["prefixes"] = OrderedDict(
(p, ns) for ns, p in sorted(namespacePrefixes.items(),
key=lambda item: item[1]))
oimReport["dtsReferences"] = dtsReferences
oimReport["facts"] = oimFacts
def saveJsonFacts(facts, oimFacts, parentFact):
for fact in facts:
oimFact = factAspects(fact)
oimFacts.append(
OrderedDict((str(k), v) for k, v in oimFact.items()))
if fact.modelTupleFacts:
saveJsonFacts(fact.modelTupleFacts, oimFacts, fact)
saveJsonFacts(modelXbrl.facts, oimFacts, None)
with open(oimFile, "w", encoding="utf-8") as fh:
fh.write(json.dumps(oimReport, indent=1))
elif isCSVorXL:
# save CSV
aspectQnCol = {}
aspectsHeader = []
factsColumns = []
def addAspectQnCol(aspectQn):
aspectQnCol[aspectQn] = len(aspectsHeader)
_colName = oimValue(aspectQn)
aspectsHeader.append(_colName)
_colDataType = {
"id": "Name",
"baseType": "Name",
"oim:concept": "QName",
"oim:periodStart": "dateTime",
"oim:periodEnd": "dateTime",
"oim:tupleOrder": "integer",
"numericValue": "decimal",
"accuracy": "decimal",
"booleanValue": "boolean",
"oim:entity": "QName",
"oim:unit": "string"
}.get(_colName)
if _colDataType is None:
if isCSVorXL and _colName.endswith("Value"):
_colDataType = _colName[:-5]
else:
_colDataType = "string"
factsColumns.append(
OrderedDict((("name", _colName), ("datatype", _colDataType))))
# pre-ordered aspect columns
if hasId:
addAspectQnCol("id")
addAspectQnCol(qnOimConceptAspect)
if hasType:
addAspectQnCol("baseType")
if isCSVorXL:
for _baseType in sorted(factBaseTypes):
addAspectQnCol(_baseType + "Value")
else:
addAspectQnCol("stringValue")
addAspectQnCol("numericValue")
addAspectQnCol("booleanValue")
if hasNumeric:
addAspectQnCol("accuracy")
if hasTuple:
addAspectQnCol(qnOimTupleParentAspect)
addAspectQnCol(qnOimTupleOrderAspect)
if qnOimEntityAspect in aspectsDefined:
addAspectQnCol(qnOimEntityAspect)
if qnOimPeriodStartAspect in aspectsDefined:
addAspectQnCol(qnOimPeriodStartAspect)
addAspectQnCol(qnOimPeriodEndAspect)
if qnOimUnitAspect in aspectsDefined:
addAspectQnCol(qnOimUnitAspect)
for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)):
if aspectQn.namespaceURI != nsOim:
addAspectQnCol(aspectQn)
def aspectCols(fact):
cols = [None for i in range(len(aspectsHeader))]
_factAspects = factAspects(fact)
for aspectQn, aspectValue in _factAspects.items():
if aspectQn in aspectQnCol:
cols[aspectQnCol[aspectQn]] = aspectValue
return cols
# metadata
csvTables = []
csvMetadata = OrderedDict(
(("@context", ["http://www.w3.org/ns/csvw", {
"@base": "./"
}]), ("tables", csvTables)))
_open = _writerow = _close = None
_tableinfo = {}
if isCSV:
if oimFile.endswith(
"-facts.csv"
): # strip -facts.csv if a prior -facts.csv file was chosen
_baseURL = oimFile[:-10]
elif oimFile.endswith(".csv"):
_baseURL = oimFile[:-4]
else:
_baseURL = oimFile
_csvinfo = {} # open file, writer
def _open(filesuffix, tabname):
_filename = _tableinfo["url"] = _baseURL + filesuffix
_csvinfo["file"] = open(_filename,
csvOpenMode,
newline=csvOpenNewline,
encoding='utf-8-sig')
_csvinfo["writer"] = csv.writer(_csvinfo["file"],
dialect="excel")
def _writerow(row, header=False):
_csvinfo["writer"].writerow(row)
def _close():
_csvinfo["file"].close()
_csvinfo.clear()
elif isXL:
headerWidths = {
"href": 100,
"oim:concept": 70,
"accuracy": 8,
"baseType": 10,
"language": 9,
"URI": 80,
"stringValue": 60,
"decimalValue": 12,
"booleanValue": 12,
"dateValue": 12,
"dateTimeValue": 20,
"group": 60,
"footnoteType": 40,
"footnote": 70,
"column": 60
}
from openpyxl import Workbook
from openpyxl.writer.write_only import WriteOnlyCell
from openpyxl.styles import Font, PatternFill, Border, Alignment, Color, fills, Side
from openpyxl.worksheet.dimensions import ColumnDimension
hdrCellFill = PatternFill(
patternType=fills.FILL_SOLID, fgColor=Color(
"00FFBF5F")) # Excel's light orange fill color = 00FF990
workbook = Workbook(encoding="utf-8", write_only=True)
_xlinfo = {} # open file, writer
def _open(filesuffix, tabname):
_tableinfo["url"] = tabname
_xlinfo["ws"] = workbook.create_sheet(title=tabname)
def _writerow(rowvalues, header=False):
row = []
_ws = _xlinfo["ws"]
for i, v in enumerate(rowvalues):
cell = WriteOnlyCell(_ws, value=v)
if header:
cell.fill = hdrCellFill
cell.alignment = Alignment(horizontal="center",
vertical="center",
wrap_text=True)
colLetter = chr(ord('A') + i)
_ws.column_dimensions[colLetter] = ColumnDimension(
_ws, customWidth=True)
_ws.column_dimensions[
colLetter].width = headerWidths.get(v, 20)
else:
cell.alignment = Alignment(
horizontal="right" if isinstance(v, _NUM_TYPES)
else "center" if isinstance(v, bool) else "left",
vertical="top",
wrap_text=isinstance(v, str))
row.append(cell)
_ws.append(row)
def _close():
_xlinfo.clear()
# save facts
_open("-facts.csv", "facts")
_writerow(aspectsHeader, header=True)
def saveCSVfacts(facts):
for fact in facts:
_writerow(aspectCols(fact))
saveCSVfacts(fact.modelTupleFacts)
saveCSVfacts(modelXbrl.facts)
_close()
factsTableSchema = OrderedDict((("columns", factsColumns), ))
csvTables.append(
OrderedDict((("url", _tableinfo["url"]), ("tableSchema",
factsTableSchema))))
# save namespaces
_open("-prefixes.csv", "prefixes")
_writerow(("prefix", "URI"), header=True)
for _URI, prefix in sorted(namespacePrefixes.items(),
key=lambda item: item[1]):
_writerow((prefix, _URI))
_close()
nsTableSchema = OrderedDict((("columns", [
OrderedDict((("name", "prefix"), ("datatype", "Name"))),
OrderedDict((("name", "URI"), ("datatype", "anyURI")))
]), ))
csvTables.append(
OrderedDict(
(("url", _tableinfo["url"]), ("tableSchema", nsTableSchema))))
# save dts references
_open("-dtsReferences.csv", "dtsReferences")
_writerow(("type", "href"), header=True)
for oimRef in dtsReferences:
_writerow((oimRef["type"], oimRef["href"]))
_close()
dtsRefTableSchema = OrderedDict((("columns", [
OrderedDict((("name", "type"), ("datatype", "Name"))),
OrderedDict((("name", "href"), ("datatype", "anyURI")))
]), ))
csvTables.append(
OrderedDict((("url", _tableinfo["url"]), ("tableSchema",
dtsRefTableSchema))))
# save footnotes
if footnotesRelationshipSet.modelRelationships:
_open("-footnotes.csv", "footnotes")
cols = ("group", "footnoteType", "factId", "factRef", "footnote",
"language")
_writerow(cols, header=True)
def saveCSVfootnotes(facts):
for fact in facts:
for _footnote in factFootnotes(fact):
_writerow(
tuple((_footnote.get(col, "") for col in cols)))
saveCSVfootnotes(fact.modelTupleFacts)
saveCSVfootnotes(modelXbrl.facts)
_close()
footnoteTableSchema = OrderedDict((("columns", [
OrderedDict((("name", "group"), ("datatype", "anyURI"))),
OrderedDict((("name", "footnoteType"), ("datatype", "Name"))),
OrderedDict((("name", "factId"), ("datatype", "Name"))),
OrderedDict((("name", "factRef"), ("datatype", "Name"))),
OrderedDict((("name", "footnote"), ("datatype", "string"))),
OrderedDict((("name", "language"), ("datatype", "language")))
]), ))
csvTables.append(
OrderedDict((("url", _tableinfo["url"]),
("tableSchema", footnoteTableSchema))))
# save metadata
if isCSV:
with open(_baseURL + "-metadata.csv", "w", encoding="utf-8") as fh:
fh.write(
json.dumps(csvMetadata,
ensure_ascii=False,
indent=1,
sort_keys=False))
elif isXL:
_open(None, "metadata")
_writerow(("table", "column", "datatype"), header=True)
for table in csvTables:
tablename = table["url"]
for column in table["tableSchema"]["columns"]:
_writerow((tablename, column["name"], column["datatype"]))
_close()
if isXL:
workbook.save(oimFile)
def saveLoadableOIM(modelXbrl, oimFile, outputZip=None):
isJSON = oimFile.endswith(".json")
isCSV = oimFile.endswith(".csv")
isXL = oimFile.endswith(".xlsx")
isCSVorXL = isCSV or isXL
if not isJSON and not isCSVorXL:
return
namespacePrefixes = {nsOim: "xbrl"}
prefixNamespaces = {"xbrl": nsOim}
def compileQname(qname):
if qname.namespaceURI not in namespacePrefixes:
namespacePrefixes[qname.namespaceURI] = qname.prefix or ""
aspectsDefined = {
qnOimConceptAspect,
qnOimEntityAspect,
qnOimPeriodStartAspect,
qnOimPeriodEndAspect}
def oimValue(object, decimals=None):
if isinstance(object, QName):
if object.namespaceURI not in namespacePrefixes:
if object.prefix:
namespacePrefixes[object.namespaceURI] = object.prefix
else:
_prefix = "_{}".format(sum(1 for p in namespacePrefixes if p.startswith("_")))
namespacePrefixes[object.namespaceURI] = _prefix
return "{}:{}".format(namespacePrefixes[object.namespaceURI], object.localName)
if isinstance(object, Decimal):
try:
if isinf(object):
return "-INF" if object < 0 else "INF"
elif isnan(num):
return "NaN"
else:
if object == object.to_integral():
object = object.quantize(ONE) # drop any .0
return "{}".format(object)
except:
return str(object)
if isinstance(object, bool):
return "true" if object else "false"
if isinstance(object, (DateTime, YearMonthDuration, DayTimeDuration, Time,
gYearMonth, gMonthDay, gYear, gMonth, gDay,
IsoDuration)):
return str(object)
return object
def oimPeriodValue(cntx):
if cntx.isForeverPeriod:
return OrderedDict() # defaulted
elif cntx.isStartEndPeriod:
s = cntx.startDatetime
e = cntx.endDatetime
else: # instant
s = e = cntx.instantDatetime
if isJSON:
return({str(qnOimPeriodAspect):
("{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}/".format(
s.year, s.month, s.day, s.hour, s.minute, s.second)
if cntx.isStartEndPeriod else "") +
"{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format(
e.year, e.month, e.day, e.hour, e.minute, e.second)
})
# CSV, XL
return OrderedDict(((str(qnOimPeriodStartAspect), "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format(
s.year, s.month, s.day, s.hour, s.minute, s.second)),
(str(qnOimPeriodEndAspect), "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format(
e.year, e.month, e.day, e.hour, e.minute, e.second))))
hasId = False
hasTuple = False
hasType = True
hasLang = False
hasUnits = False
hasNumeric = False
footnotesRelationshipSet = ModelRelationshipSet(modelXbrl, "XBRL-footnotes")
factBaseTypes = set()
#compile QNames in instance for OIM
for fact in modelXbrl.factsInInstance:
if (fact.id or fact.isTuple or
footnotesRelationshipSet.toModelObject(fact) or
(isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))):
hasId = True
concept = fact.concept
if concept is not None:
if concept.isNumeric:
hasNumeric = True
if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang:
hasLang = True
_baseXsdType = concept.baseXsdType
if _baseXsdType == "XBRLI_DATEUNION":
if getattr(fact.xValue, "dateOnly", False):
_baseXsdType = "date"
else:
_baseXsdType = "dateTime"
factBaseTypes.add(baseTypes.get(_baseXsdType,_baseXsdType))
compileQname(fact.qname)
if hasattr(fact, "xValue") and isinstance(fact.xValue, QName):
compileQname(fact.xValue)
unit = fact.unit
if unit is not None:
hasUnits = True
if fact.modelTupleFacts:
hasTuple = True
if hasTuple:
modelXbrl.error("arelleOIMsaver:tuplesNotAllowed",
"Tuples are not allowed in an OIM document",
modelObject=modelXbrl)
return
entitySchemePrefixes = {}
for cntx in modelXbrl.contexts.values():
if cntx.entityIdentifierElement is not None:
scheme = cntx.entityIdentifier[0]
if scheme not in entitySchemePrefixes:
if not entitySchemePrefixes: # first one is just scheme
if scheme == "http://www.sec.gov/CIK":
_schemePrefix = "cik"
elif scheme == "http://standard.iso.org/iso/17442":
_schemePrefix = "lei"
else:
_schemePrefix = "scheme"
else:
_schemePrefix = "scheme{}".format(len(entitySchemePrefixes) + 1)
entitySchemePrefixes[scheme] = _schemePrefix
namespacePrefixes[scheme] = _schemePrefix
for dim in cntx.qnameDims.values():
compileQname(dim.dimensionQname)
aspectsDefined.add(dim.dimensionQname)
if dim.isExplicit:
compileQname(dim.memberQname)
for unit in modelXbrl.units.values():
if unit is not None:
for measures in unit.measures:
for measure in measures:
compileQname(measure)
if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[XbrlConst.xbrli] != "xbrli":
namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix
namespacePrefixes[XbrlConst.xsd] = "xsd"
if hasLang: aspectsDefined.add(qnOimLangAspect)
if hasTuple:
aspectsDefined.add(qnOimTupleParentAspect)
aspectsDefined.add(qnOimTupleOrderAspect)
if hasUnits: aspectsDefined.add(qnOimUnitAspect)
for footnoteRel in footnotesRelationshipSet.modelRelationships:
typePrefix = "ftTyp_" + os.path.basename(footnoteRel.arcrole)
if footnoteRel.linkrole == XbrlConst.defaultLinkRole:
groupPrefix = "ftGrp_default"
else:
groupPrefix = "ftGrp_" + os.path.basename(footnoteRel.linkrole)
if typePrefix not in namespacePrefixes:
namespacePrefixes[footnoteRel.arcrole] = typePrefix
if groupPrefix not in namespacePrefixes:
namespacePrefixes[footnoteRel.linkrole] = groupPrefix
# compile footnotes and relationships
'''
factRelationships = []
factFootnotes = []
for rel in modelXbrl.relationshipSet(modelXbrl, "XBRL-footnotes").modelRelationships:
oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole}
factRelationships.append(oimRel)
oimRel["fromIds"] = [obj.id if obj.id
else elementChildSequence(obj)
for obj in rel.fromModelObjects]
oimRel["toIds"] = [obj.id if obj.id
else elementChildSequence(obj)
for obj in rel.toModelObjects]
_order = rel.arcElement.get("order")
if _order is not None:
oimRel["order"] = _order
for obj in rel.toModelObjects:
if isinstance(obj, ModelResource): # footnote
oimFootnote = {"role": obj.role,
"id": obj.id if obj.id
else elementChildSequence(obj),
# value needs work for html elements and for inline footnotes
"value": xmlstring(obj, stripXmlns=True)}
if obj.xmlLang:
oimFootnote["lang"] = obj.xmlLang
factFootnotes.append(oimFootnote)
oimFootnote
'''
dtsReferences = set()
baseUrl = modelXbrl.modelDocument.uri.partition("#")[0]
for doc,ref in sorted(modelXbrl.modelDocument.referencesDocument.items(),
key=lambda _item:_item[0].uri):
if ref.referringModelObject.qname in SCHEMA_LB_REFS:
dtsReferences.add(relativeUri(baseUrl,doc.uri))
for refType in ("role", "arcrole"):
for refElt in sorted(modelXbrl.modelDocument.xmlRootElement.iterchildren(
"{{http://www.xbrl.org/2003/linkbase}}{}Ref".format(refType)),
key=lambda elt:elt.get(refType+"URI")
):
dtsReferences.add(refElt.get("{http://www.w3.org/1999/xlink}href").partition("#")[0])
dtsReferences = sorted(dtsReferences) # turn into list
footnoteFacts = set()
def factFootnotes(fact, oimFact=None):
footnotes = []
oimLinks = {}
for footnoteRel in footnotesRelationshipSet.fromModelObject(fact):
toObj = footnoteRel.toModelObject
# json
typePrefix = namespacePrefixes[footnoteRel.arcrole]
groupPrefix = namespacePrefixes[footnoteRel.linkrole]
oimLinks.setdefault(typePrefix,{}).setdefault(groupPrefix,[]).append(
toObj.id if toObj.id else "f{}".format(toObj.objectIndex))
# csv/XL
footnote = OrderedDict((("group", footnoteRel.linkrole),
("footnoteType", footnoteRel.arcrole)))
footnotes.append(footnote)
if isCSVorXL:
footnote["factId"] = fact.id if fact.id else "f{}".format(fact.objectIndex)
if isinstance(toObj, ModelFact):
footnote["factRef"] = toObj.id if toObj.id else "f{}".format(toObj.objectIndex)
else:
footnote["footnote"] = toObj.viewText()
if toObj.xmlLang:
footnote["language"] = toObj.xmlLang
footnoteFacts.add(toObj)
if isJSON and oimLinks:
oimFact["links"] = OrderedDict((
(typePrefix, OrderedDict((
(groupPrefix, idList)
for groupPrefix, idList in sorted(groups.items())
)))
for typePrefix, groups in sorted(oimLinks.items())
))
footnotes.sort(key=lambda f:(f["group"],f.get("factId",f.get("factRef")),f.get("language")))
return footnotes
def factAspects(fact):
oimFact = OrderedDict()
aspects = OrderedDict()
if hasId and fact.id:
if fact.isTuple:
oimFact["tupleId"] = fact.id
else:
oimFact["id"] = fact.id
elif (fact.isTuple or
footnotesRelationshipSet.toModelObject(fact) or
(isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))):
oimFact["id"] = "f{}".format(fact.objectIndex)
parent = fact.getparent()
concept = fact.concept
aspects[str(qnOimConceptAspect)] = oimValue(concept.qname)
_csvType = "Value"
if not fact.isTuple:
if concept is not None:
_baseXsdType = concept.baseXsdType
if _baseXsdType == "XBRLI_DATEUNION":
if getattr(fact.xValue, "dateOnly", False):
_baseXsdType = "date"
else:
_baseXsdType = "dateTime"
_csvType = baseTypes.get(_baseXsdType,_baseXsdType) + "Value"
if concept.baseXbrliType in ("stringItemType", "normalizedStringItemType") and fact.xmlLang:
aspects[str(qnOimLangAspect)] = fact.xmlLang
if fact.isItem:
if fact.isNil:
_value = None
else:
_inferredDecimals = inferredDecimals(fact)
_value = oimValue(fact.xValue, _inferredDecimals)
oimFact["value"] = _value
if fact.concept is not None and fact.concept.isNumeric:
_numValue = fact.xValue
if isinstance(_numValue, Decimal) and not isinf(_numValue) and not isnan(_numValue):
if _numValue == _numValue.to_integral():
_numValue = int(_numValue)
else:
_numValue = float(_numValue)
if not fact.isNil:
if not isinf(_inferredDecimals): # accuracy omitted if infinite
oimFact["decimals"] = _inferredDecimals
oimFact["dimensions"] = aspects
cntx = fact.context
if cntx is not None:
if cntx.entityIdentifierElement is not None:
aspects[str(qnOimEntityAspect)] = oimValue(qname(*cntx.entityIdentifier))
if cntx.period is not None:
aspects.update(oimPeriodValue(cntx))
for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]):
if dim.isExplicit:
dimVal = oimValue(dim.memberQname)
else: # typed
if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1"):
dimVal = None
else:
dimVal = dim.typedMember.stringValue
aspects[str(dim.dimensionQname)] = dimVal
unit = fact.unit
if unit is not None:
_mMul, _mDiv = unit.measures
_sMul = '*'.join(oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m)))
if _mDiv:
_sDiv = '*'.join(oimValue(m) for m in sorted(_mDiv, key=lambda m: oimValue(m)))
if len(_mDiv) > 1:
if len(_mMul) > 1:
_sUnit = "({})/({})".format(_sMul,_sDiv)
else:
_sUnit = "{}/({})".format(_sMul,_sDiv)
else:
if len(_mMul) > 1:
_sUnit = "({})/{}".format(_sMul,_sDiv)
else:
_sUnit = "{}/{}".format(_sMul,_sDiv)
else:
_sUnit = _sMul
aspects[str(qnOimUnitAspect)] = _sUnit
# Tuples removed from xBRL-JSON
#if parent.qname != XbrlConst.qnXbrliXbrl:
# aspects[str(qnOimTupleParentAspect)] = parent.id if parent.id else "f{}".format(parent.objectIndex)
# aspects[str(qnOimTupleOrderAspect)] = elementIndex(fact)
if isJSON:
factFootnotes(fact, oimFact)
return oimFact
prefixes = OrderedDict((p,ns) for ns, p in sorted(namespacePrefixes.items(),
key=lambda item: item[1]))
if isJSON:
# save JSON
oimReport = OrderedDict() # top level of oim json output
oimReport["documentInfo"] = oimDocInfo = OrderedDict()
oimReport["facts"] = oimFacts = OrderedDict()
oimDocInfo["documentType"] = nsOim + "/xbrl-json"
oimDocInfo["features"] = oimFeatures = OrderedDict()
oimDocInfo["prefixes"] = prefixes
oimDocInfo["taxonomy"] = dtsReferences
oimFeatures["xbrl:canonicalValues"] = True
def saveJsonFacts(facts, oimFacts, parentFact):
for fact in facts:
oimFact = factAspects(fact)
id = fact.id if fact.id else "f{}".format(fact.objectIndex)
oimFacts[id] = oimFact
if fact.modelTupleFacts:
saveJsonFacts(fact.modelTupleFacts, oimFacts, fact)
saveJsonFacts(modelXbrl.facts, oimFacts, None)
# add footnotes as pseudo facts
for ftObj in footnoteFacts:
ftId = ftObj.id if ftObj.id else "f{}".format(ftObj.objectIndex)
oimFacts[ftId] = oimFact = OrderedDict()
oimFact["value"] = ftObj.viewText()
oimFact["dimensions"] = OrderedDict((("concept", "xbrl:note"),
("noteId", ftId)))
if ftObj.xmlLang:
oimFact["dimensions"]["language"] = ftObj.xmlLang.lower()
if outputZip:
fh = io.StringIO()
else:
fh = open(oimFile, "w", encoding="utf-8")
fh.write(json.dumps(oimReport, indent=1))
if outputZip:
fh.seek(0)
outputZip.writestr(os.path.basename(oimFile),fh.read())
fh.close()
elif isCSVorXL:
# save CSV
aspectQnCol = {}
aspectsHeader = []
factsColumns = []
def addAspectQnCol(aspectQn):
aspectQnCol[str(aspectQn)] = len(aspectsHeader)
_aspectQn = oimValue(aspectQn)
aspectsHeader.append(_aspectQn)
_colName = _aspectQn.replace("xbrl:", "")
_colDataType = {"id": "Name",
"concept": "QName",
"value": "string",
"decimals": "decimal",
"entity": "QName",
"periodStart": "dateTime",
"periodEnd": "dateTime",
"unit": "string",
"tupleId": "Name",
"tupleParent": "Name",
"tupleOrder": "integer"
}.get(_colName, "string")
col = OrderedDict((("name", _colName),
("datatype", _colDataType)))
if _aspectQn == "value":
col["http://xbrl.org/YYYY/model#simpleFactAspects"] = {}
elif _aspectQn == "tupleId":
col["http://xbrl.org/YYYY/model#tupleFactAspects"] = {}
col["http://xbrl.org/YYYY/model#tupleReferenceId"] = "true"
else:
col["http://xbrl.org/YYYY/model#columnAspect"] = _aspectQn
factsColumns.append(col)
# pre-ordered aspect columns
#if hasId:
# addAspectQnCol("id")
addAspectQnCol(qnOimConceptAspect)
addAspectQnCol("value")
if hasNumeric:
addAspectQnCol("decimals")
if hasTuple:
addAspectQnCol("tupleId")
addAspectQnCol(qnOimTupleParentAspect)
addAspectQnCol(qnOimTupleOrderAspect)
if qnOimEntityAspect in aspectsDefined:
addAspectQnCol(qnOimEntityAspect)
if qnOimPeriodStartAspect in aspectsDefined:
addAspectQnCol(qnOimPeriodStartAspect)
addAspectQnCol(qnOimPeriodEndAspect)
if qnOimUnitAspect in aspectsDefined:
addAspectQnCol(qnOimUnitAspect)
for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)):
if aspectQn.namespaceURI != nsOim:
addAspectQnCol(aspectQn)
def aspectCols(fact):
cols = [None for i in range(len(aspectsHeader))]
def setColValues(aspects):
for aspectQn, aspectValue in aspects.items():
if isinstance(aspectValue, dict):
setColValues(aspectValue)
elif aspectQn in aspectQnCol:
if aspectValue is None:
_aspectValue = "#nil"
elif aspectValue == "":
_aspectValue = "#empty"
elif isinstance(aspectValue, str) and aspectValue.startswith("#"):
_aspectValue = "#" + aspectValue
else:
_aspectValue = aspectValue
cols[aspectQnCol[aspectQn]] = _aspectValue
setColValues(factAspects(fact))
return cols
# metadata
csvTables = []
csvMetadata = OrderedDict((("@context", "http://www.w3.org/ns/csvw"),
("http://xbrl.org/YYYY/model#metadata",
OrderedDict((("documentType", "http://xbrl.org/YYYY/xbrl-csv"),
("dtsReferences", dtsReferences),
("prefixes", prefixes)))),
("tables", csvTables)))
_open = _writerow = _close = None
_tableinfo = {}
if isCSV:
if oimFile.endswith("-facts.csv"): # strip -facts.csv if a prior -facts.csv file was chosen
_baseURL = oimFile[:-10]
elif oimFile.endswith(".csv"):
_baseURL = oimFile[:-4]
else:
_baseURL = oimFile
_csvinfo = {} # open file, writer
def _open(filesuffix, tabname):
_filename = _tableinfo["url"] = _baseURL + filesuffix
_csvinfo["file"] = open(_filename, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
_csvinfo["writer"] = csv.writer(_csvinfo["file"], dialect="excel")
def _writerow(row, header=False):
_csvinfo["writer"].writerow(row)
def _close():
_csvinfo["file"].close()
_csvinfo.clear()
elif isXL:
headerWidths = {"href": 100, "xbrl:concept": 70, "decimals": 8, "language": 9, "URI": 80,
"value": 60,
"group": 60, "footnoteType": 40, "footnote": 70, "column": 20,
'conceptAspect': 40, 'tuple': 20, 'simpleFact': 20}
from openpyxl import Workbook
from openpyxl.writer.write_only import WriteOnlyCell
from openpyxl.styles import Font, PatternFill, Border, Alignment, Color, fills, Side
from openpyxl.worksheet.dimensions import ColumnDimension
hdrCellFill = PatternFill(patternType=fills.FILL_SOLID, fgColor=Color("00FFBF5F")) # Excel's light orange fill color = 00FF990
workbook = Workbook()
# remove pre-existing worksheets
while len(workbook.worksheets)>0:
workbook.remove_sheet(workbook.worksheets[0])
_xlinfo = {} # open file, writer
def _open(filesuffix, tabname):
_tableinfo["url"] = tabname
_xlinfo["ws"] = workbook.create_sheet(title=tabname)
def _writerow(rowvalues, header=False):
row = []
_ws = _xlinfo["ws"]
for i, v in enumerate(rowvalues):
cell = WriteOnlyCell(_ws, value=v)
if header:
cell.fill = hdrCellFill
cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True)
colLetter = chr(ord('A') + i)
_ws.column_dimensions[colLetter] = ColumnDimension(_ws, customWidth=True)
_ws.column_dimensions[colLetter].width = headerWidths.get(v, 20)
else:
cell.alignment = Alignment(horizontal="right" if isinstance(v, _NUM_TYPES)
else "center" if isinstance(v, bool)
else "left",
vertical="top",
wrap_text=isinstance(v, str))
row.append(cell)
_ws.append(row)
def _close():
_xlinfo.clear()
# save facts
_open("-facts.csv", "facts")
_writerow(aspectsHeader, header=True)
def saveCSVfacts(facts):
for fact in facts:
_writerow(aspectCols(fact))
saveCSVfacts(fact.modelTupleFacts)
saveCSVfacts(modelXbrl.facts)
_close()
factsTableSchema = OrderedDict((("columns",factsColumns),))
csvTables.append(OrderedDict((("url",_tableinfo["url"]),
("http://xbrl.org/YYYY/model#tableType", "fact"),
("tableSchema",factsTableSchema))))
# save footnotes
if footnotesRelationshipSet.modelRelationships:
_open("-footnotes.csv", "footnotes")
cols = ("group", "footnoteType", "factId", "factRef", "footnote", "language")
_writerow(cols, header=True)
def saveCSVfootnotes(facts):
for fact in facts:
for _footnote in factFootnotes(fact):
_writerow(tuple((_footnote.get(col,"") for col in cols)))
saveCSVfootnotes(fact.modelTupleFacts)
saveCSVfootnotes(modelXbrl.facts)
_close()
footnoteTableSchema = OrderedDict((("columns",[OrderedDict((("name","group"),("datatype","anyURI"))),
OrderedDict((("name","footnoteType"),("datatype","Name"))),
OrderedDict((("name","factId"),("datatype","Name"))),
OrderedDict((("name","factRef"),("datatype","Name"))),
OrderedDict((("name","footnote"),("datatype","string"))),
OrderedDict((("name","language"),("datatype","language")))]),))
csvTables.append(OrderedDict((("url",_tableinfo["url"]),
("http://xbrl.org/YYYY/model#tableType", "footnote"),
("tableSchema",footnoteTableSchema))))
# save metadata
if isCSV:
with open(_baseURL + "-metadata.json", "w", encoding="utf-8") as fh:
fh.write(json.dumps(csvMetadata, ensure_ascii=False, indent=1, sort_keys=False))
elif isXL:
_open(None, "metadata")
hasColumnAspect = hasSimpleFact = hasTupleFact = False
for table in csvTables:
tablename = table["url"]
for column in table["tableSchema"]["columns"]:
if "http://xbrl.org/YYYY/model#columnAspect" in column:
hasColumnAspect = True
if "http://xbrl.org/YYYY/model#simpleFactAspects" in column:
hasSimpleFact = True
if "http://xbrl.org/YYYY/model#tupleAspects" in column:
hasTupleFact = True
metadataCols = ["table", "column", "datatype"]
if hasColumnAspect:
metadataCols.append("columnAspect")
if hasSimpleFact:
metadataCols.append("simpleFact")
if hasTupleFact:
metadataCols.append("tuple")
_writerow(metadataCols, header=True)
for table in csvTables:
tablename = table["url"]
for column in table["tableSchema"]["columns"]:
row = [tablename, column["name"], column["datatype"]]
if hasColumnAspect:
colAspect = column.get("http://xbrl.org/YYYY/model#columnAspect")
if isinstance(colAspect, str):
row.append(colAspect)
elif isinstance(colAspect, dict):
row.append("\n".join("{} [{}]".format(k, ", ".join(_v for _v in v))
for k, v in dict.items()))
else:
row.append(None)
if hasSimpleFact:
row.append("\u221a" if "http://xbrl.org/YYYY/model#simpleFactAspects" in column else None)
if hasTupleFact:
row.append("\u221a" if "http://xbrl.org/YYYY/model#tupleAspects" in column else None)
_writerow(row)
_close()
if isXL:
workbook.save(oimFile)
def saveLoadableOIM(modelXbrl, oimFile):
isJSON = oimFile.endswith(".json")
isCSV = oimFile.endswith(".csv")
isXL = oimFile.endswith(".xlsx")
isCSVorXL = isCSV or isXL
if not isJSON and not isCSVorXL:
return
namespacePrefixes = {}
prefixNamespaces = {}
def compileQname(qname):
if qname.namespaceURI not in namespacePrefixes:
namespacePrefixes[qname.namespaceURI] = qname.prefix or ""
aspectsDefined = {
qnOimConceptAspect,
qnOimEntityAspect}
if isJSON:
aspectsDefined.add(qnOimPeriodAspect)
elif isCSVorXL:
aspectsDefined.add(qnOimPeriodStartAspect)
aspectsDefined.add(qnOimPeriodEndAspect)
def oimValue(object, decimals=None):
if isinstance(object, QName):
if object.namespaceURI not in namespacePrefixes:
if object.prefix:
namespacePrefixes[object.namespaceURI] = object.prefix
else:
_prefix = "_{}".format(sum(1 for p in namespacePrefixes if p.startswith("_")))
namespacePrefixes[object.namespaceURI] = _prefix
return "{}:{}".format(namespacePrefixes[object.namespaceURI], object.localName)
if isinstance(object, Decimal):
try:
if isinf(object):
return "-INF" if object < 0 else "INF"
elif isnan(num):
return "NaN"
else:
if object == object.to_integral():
object = object.quantize(ONE) # drop any .0
return "{}".format(object)
except:
return str(object)
if isinstance(object, bool):
return object
if isinstance(object, (DateTime, YearMonthDuration, DayTimeDuration, Time,
gYearMonth, gMonthDay, gYear, gMonth, gDay)):
return str(object)
return object
def oimPeriodValue(cntx):
if cntx.isForeverPeriod:
pass # not supported
elif cntx.isStartEndPeriod:
s = cntx.startDatetime
e = cntx.endDatetime
else: # instant
s = e = cntx.instantDatetime
return {"start": "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format(
s.year, s.month, s.day, s.hour, s.minute, s.second),
"end": "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format(
e.year, e.month, e.day, e.hour, e.minute, e.second)}
hasId = False
hasTuple = False
hasType = True
hasLang = False
hasUnits = False
footnotesRelationshipSet = ModelRelationshipSet(modelXbrl, "XBRL-footnotes")
factBaseTypes = set()
#compile QNames in instance for OIM
for fact in modelXbrl.factsInInstance:
if (fact.id or fact.isTuple or
footnotesRelationshipSet.toModelObject(fact) or
(isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))):
hasId = True
concept = fact.concept
if concept is not None:
if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang:
hasLang = True
_baseXsdType = concept.baseXsdType
if _baseXsdType == "XBRLI_DATEUNION":
if getattr(fact.xValue, "dateOnly", False):
_baseXsdType = "date"
else:
_baseXsdType = "dateTime"
factBaseTypes.add(baseTypes.get(_baseXsdType,_baseXsdType))
compileQname(fact.qname)
if hasattr(fact, "xValue") and isinstance(fact.xValue, QName):
compileQname(fact.xValue)
unit = fact.unit
if unit is not None:
hasUnits = True
if fact.modelTupleFacts:
hasTuple = True
entitySchemePrefixes = {}
for cntx in modelXbrl.contexts.values():
if cntx.entityIdentifierElement is not None:
scheme = cntx.entityIdentifier[0]
if scheme not in entitySchemePrefixes:
if not entitySchemePrefixes: # first one is just scheme
if scheme == "http://www.sec.gov/CIK":
_schemePrefix = "cik"
elif scheme == "http://standard.iso.org/iso/17442":
_schemePrefix = "lei"
else:
_schemePrefix = "scheme"
else:
_schemePrefix = "scheme{}".format(len(entitySchemePrefixes) + 1)
entitySchemePrefixes[scheme] = _schemePrefix
namespacePrefixes[scheme] = _schemePrefix
for dim in cntx.qnameDims.values():
compileQname(dim.dimensionQname)
aspectsDefined.add(dim.dimensionQname)
if dim.isExplicit:
compileQname(dim.memberQname)
for unit in modelXbrl.units.values():
if unit is not None:
for measures in unit.measures:
for measure in measures:
compileQname(measure)
if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[XbrlConst.xbrli] != "xbrli":
namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix
namespacePrefixes[XbrlConst.xsd] = "xsd"
if hasLang: aspectsDefined.add(qnOimLangAspect)
if hasTuple:
aspectsDefined.add(qnOimTupleParentAspect)
aspectsDefined.add(qnOimTupleOrderAspect)
if hasUnits: aspectsDefined.add(qnOimUnitAspect)
# compile footnotes and relationships
'''
factRelationships = []
factFootnotes = []
for rel in modelXbrl.relationshipSet(modelXbrl, "XBRL-footnotes").modelRelationships:
oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole}
factRelationships.append(oimRel)
oimRel["fromIds"] = [obj.id if obj.id
else elementChildSequence(obj)
for obj in rel.fromModelObjects]
oimRel["toIds"] = [obj.id if obj.id
else elementChildSequence(obj)
for obj in rel.toModelObjects]
_order = rel.arcElement.get("order")
if _order is not None:
oimRel["order"] = _order
for obj in rel.toModelObjects:
if isinstance(obj, ModelResource): # footnote
oimFootnote = {"role": obj.role,
"id": obj.id if obj.id
else elementChildSequence(obj),
# value needs work for html elements and for inline footnotes
"value": xmlstring(obj, stripXmlns=True)}
if obj.xmlLang:
oimFootnote["lang"] = obj.xmlLang
factFootnotes.append(oimFootnote)
oimFootnote
'''
dtsReferences = [
{"type": "schema" if doc.type == ModelDocument.Type.SCHEMA
else "linkbase" if doc.type == ModelDocument.Type.LINKBASE
else "other",
"href": doc.uri}
for doc,ref in modelXbrl.modelDocument.referencesDocument.items()
if ref.referringModelObject.qname in SCHEMA_LB_REFS]
'''
roleTypes = [
{"type": "role" if ref.referringModelObject.localName == "roleRef" else "arcroleRef",
"href": ref.referringModelObject["href"]}
for doc,ref in modelXbrl.modelDocument.referencesDocument.items()
if ref.referringModelObject.qname in ROLE_REFS]
'''
def factFootnotes(fact):
footnotes = []
for footnoteRel in footnotesRelationshipSet.fromModelObject(fact):
footnote = OrderedDict((("group", footnoteRel.linkrole),
("footnoteType", footnoteRel.arcrole)))
footnotes.append(footnote)
if isCSVorXL:
footnote["factId"] = fact.id if fact.id else "f{}".format(fact.objectIndex)
toObj = footnoteRel.toModelObject
if isinstance(toObj, ModelFact):
footnote["factRef"] = toObj.id if toObj.id else "f{}".format(toObj.objectIndex)
else:
footnote["footnote"] = xmlstring(toObj, stripXmlns=True, contentsOnly=True, includeText=True)
if toObj.xmlLang:
footnote["language"] = toObj.xmlLang
return footnotes
def factAspects(fact):
aspects = OrderedDict()
if hasId and fact.id:
aspects["id"] = fact.id
elif (fact.isTuple or
footnotesRelationshipSet.toModelObject(fact) or
(isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))):
aspects["id"] = "f{}".format(fact.objectIndex)
parent = fact.getparent()
concept = fact.concept
_csvType = "Value"
if not fact.isTuple:
if concept is not None:
_baseXsdType = concept.baseXsdType
if _baseXsdType == "XBRLI_DATEUNION":
if getattr(fact.xValue, "dateOnly", False):
_baseXsdType = "date"
else:
_baseXsdType = "dateTime"
aspects["baseType"] = "xs:{}".format(_baseXsdType)
_csvType = baseTypes.get(_baseXsdType,_baseXsdType) + "Value"
if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang:
aspects[qnOimLangAspect] = fact.xmlLang
if fact.isItem:
if fact.isNil:
_value = None
_strValue = "nil"
else:
_inferredDecimals = inferredDecimals(fact)
_value = oimValue(fact.xValue, _inferredDecimals)
_strValue = str(_value)
if not isCSVorXL:
aspects["value"] = _strValue
if fact.concept is not None and fact.concept.isNumeric:
_numValue = fact.xValue
if isinstance(_numValue, Decimal) and not isinf(_numValue) and not isnan(_numValue):
if _numValue == _numValue.to_integral():
_numValue = int(_numValue)
else:
_numValue = float(_numValue)
if isCSVorXL:
aspects[_csvType] = _numValue
else:
aspects["numericValue"] = _numValue
if not fact.isNil:
if isinf(_inferredDecimals):
if isJSON: _accuracy = "infinity"
elif isCSVorXL: _accuracy = "INF"
else:
_accuracy = _inferredDecimals
aspects["accuracy"] = _accuracy
elif isinstance(_value, bool):
aspects["booleanValue"] = _value
elif isCSVorXL:
aspects[_csvType] = _strValue
aspects[qnOimConceptAspect] = oimValue(fact.qname)
cntx = fact.context
if cntx is not None:
if cntx.entityIdentifierElement is not None:
aspects[qnOimEntityAspect] = oimValue(qname(*cntx.entityIdentifier))
if cntx.period is not None:
if isJSON:
aspects[qnOimPeriodAspect] = oimPeriodValue(cntx)
elif isCSVorXL:
_periodValue = oimPeriodValue(cntx)
aspects[qnOimPeriodStartAspect] = _periodValue["start"]
aspects[qnOimPeriodEndAspect] = _periodValue["end"]
for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]):
aspects[dim.dimensionQname] = (oimValue(dim.memberQname) if dim.isExplicit
else None if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1")
else dim.typedMember.stringValue)
unit = fact.unit
if unit is not None:
_mMul, _mDiv = unit.measures
_sMul = '*'.join(oimValue(m) for m in sorted(_mMul, key=lambda m: str(m)))
if _mDiv:
_sDiv = '*'.join(oimValue(m) for m in sorted(_mDiv, key=lambda m: str(m)))
if len(mDiv) > 1:
if len(mMul) > 1:
_sUnit = "({})/({})".format(_sMul,_sDiv)
else:
_sUnit = "{}/({})".format(_sMul,_sDiv)
else:
if len(mMul) > 1:
_sUnit = "({})/{}".format(_sMul,_sDiv)
else:
_sUnit = "{}/{}".format(_sMul,_sDiv)
else:
_sUnit = _sMul
aspects[qnOimUnitAspect] = _sUnit
if parent.qname != XbrlConst.qnXbrliXbrl:
aspects[qnOimTupleParentAspect] = parent.id if parent.id else "f{}".format(parent.objectIndex)
aspects[qnOimTupleOrderAspect] = elementIndex(fact)
if isJSON:
_footnotes = factFootnotes(fact)
if _footnotes:
aspects["footnotes"] = _footnotes
return aspects
if isJSON:
# save JSON
oimReport = OrderedDict() # top level of oim json output
oimFacts = []
oimReport["prefixes"] = OrderedDict((p,ns) for ns, p in sorted(namespacePrefixes.items(),
key=lambda item: item[1]))
oimReport["dtsReferences"] = dtsReferences
oimReport["facts"] = oimFacts
def saveJsonFacts(facts, oimFacts, parentFact):
for fact in facts:
oimFact = factAspects(fact)
oimFacts.append(OrderedDict((str(k),v) for k,v in oimFact.items()))
if fact.modelTupleFacts:
saveJsonFacts(fact.modelTupleFacts, oimFacts, fact)
saveJsonFacts(modelXbrl.facts, oimFacts, None)
with open(oimFile, "w", encoding="utf-8") as fh:
fh.write(json.dumps(oimReport, ensure_ascii=False, indent=1, sort_keys=False))
elif isCSVorXL:
# save CSV
aspectQnCol = {}
aspectsHeader = []
factsColumns = []
def addAspectQnCol(aspectQn):
aspectQnCol[aspectQn] = len(aspectsHeader)
_colName = oimValue(aspectQn)
aspectsHeader.append(_colName)
_colDataType = {"id": "Name",
"baseType": "Name",
"oim:concept": "QName",
"oim:periodStart": "dateTime",
"oim:periodEnd": "dateTime",
"oim:tupleOrder": "integer",
"numericValue": "decimal",
"accuracy": "decimal",
"booleanValue": "boolean",
"oim:entity": "QName",
"oim:unit": "string"
}.get(_colName)
if _colDataType is None:
if isCSVorXL and _colName.endswith("Value"):
_colDataType = _colName[:-5]
else:
_colDataType = "string"
factsColumns.append(OrderedDict((("name", _colName),
("datatype", _colDataType))))
# pre-ordered aspect columns
if hasId:
addAspectQnCol("id")
addAspectQnCol(qnOimConceptAspect)
if hasType:
addAspectQnCol("baseType")
if isCSVorXL:
for _baseType in sorted(factBaseTypes):
addAspectQnCol(_baseType + "Value")
else:
addAspectQnCol("stringValue")
addAspectQnCol("numericValue")
addAspectQnCol("booleanValue")
addAspectQnCol("accuracy")
if hasTuple:
addAspectQnCol(qnOimTupleParentAspect)
addAspectQnCol(qnOimTupleOrderAspect)
if qnOimEntityAspect in aspectsDefined:
addAspectQnCol(qnOimEntityAspect)
if qnOimPeriodStartAspect in aspectsDefined:
addAspectQnCol(qnOimPeriodStartAspect)
addAspectQnCol(qnOimPeriodEndAspect)
if qnOimUnitAspect in aspectsDefined:
addAspectQnCol(qnOimUnitAspect)
for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)):
if aspectQn.namespaceURI != nsOim:
addAspectQnCol(aspectQn)
def aspectCols(fact):
cols = [None for i in range(len(aspectsHeader))]
_factAspects = factAspects(fact)
for aspectQn, aspectValue in _factAspects.items():
if aspectQn in aspectQnCol:
cols[aspectQnCol[aspectQn]] = aspectValue
return cols
# metadata
csvTables = []
csvMetadata = OrderedDict((("@context",[ "http://www.w3.org/ns/csvw", { "@base": "./" }]),
("tables", csvTables)))
_open = _writerow = _close = None
_tableinfo = {}
if isCSV:
if oimFile.endswith("-facts.csv"): # strip -facts.csv if a prior -facts.csv file was chosen
_baseURL = oimFile[:-10]
elif oimFile.endswith(".csv"):
_baseURL = oimFile[:-4]
else:
_baseURL = oimFile
_csvinfo = {} # open file, writer
def _open(filesuffix, tabname):
_filename = _tableinfo["url"] = _baseURL + filesuffix
_csvinfo["file"] = open(_filename, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
_csvinfo["writer"] = csv.writer(csvFile, dialect="excel")
def _writerow(row, header=False):
_csvinfo["writer"].writerow(row)
def _close():
_csvinfo["writer"].close()
_csvinfo.clear()
elif isXL:
headerWidths = {"href": 100, "oim:concept": 70, "accuracy": 8, "baseType": 10, "language": 9, "URI": 80,
"stringValue": 60, "decimalValue": 12, "booleanValue": 12, "dateValue": 12, "dateTimeValue": 20,
"group": 60, "footnoteType": 40, "footnote": 70, "column": 60}
from openpyxl import Workbook
from openpyxl.writer.write_only import WriteOnlyCell
from openpyxl.styles import Font, PatternFill, Border, Alignment, Color, fills, Side
from openpyxl.worksheet.dimensions import ColumnDimension
hdrCellFill = PatternFill(patternType=fills.FILL_SOLID, fgColor=Color("00FFBF5F")) # Excel's light orange fill color = 00FF990
workbook = Workbook(encoding="utf-8", write_only=True)
_xlinfo = {} # open file, writer
def _open(filesuffix, tabname):
_tableinfo["url"] = tabname
_xlinfo["ws"] = workbook.create_sheet(title=tabname)
def _writerow(rowvalues, header=False):
row = []
_ws = _xlinfo["ws"]
for i, v in enumerate(rowvalues):
cell = WriteOnlyCell(_ws, value=v)
if header:
cell.fill = hdrCellFill
cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True)
colLetter = chr(ord('A') + i)
_ws.column_dimensions[colLetter] = ColumnDimension(_ws, customWidth=True)
_ws.column_dimensions[colLetter].width = headerWidths.get(v, 20)
else:
cell.alignment = Alignment(horizontal="right" if isinstance(v, _NUM_TYPES)
else "center" if isinstance(v, bool)
else "left",
vertical="top",
wrap_text=isinstance(v, str))
row.append(cell)
_ws.append(row)
def _close():
_xlinfo.clear()
# save facts
_open("-facts.csv", "facts")
_writerow(aspectsHeader, header=True)
def saveCSVfacts(facts):
for fact in facts:
_writerow(aspectCols(fact))
saveCSVfacts(fact.modelTupleFacts)
saveCSVfacts(modelXbrl.facts)
_close()
factsTableSchema = OrderedDict((("columns",factsColumns),))
csvTables.append(OrderedDict((("url",_tableinfo["url"]),
("tableSchema",factsTableSchema))))
# save namespaces
_open("-prefixes.csv", "prefixes")
_writerow(("prefix", "URI"), header=True)
for _URI, prefix in sorted(namespacePrefixes.items(), key=lambda item: item[1]):
_writerow((prefix, _URI))
_close()
nsTableSchema = OrderedDict((("columns",[OrderedDict((("name","prefix"),("datatype","Name"))),
OrderedDict((("name","URI"),("datatype","anyURI")))]),))
csvTables.append(OrderedDict((("url",_tableinfo["url"]),
("tableSchema",nsTableSchema))))
# save dts references
_open("-dtsReferences.csv", "dtsReferences")
_writerow(("type", "href"), header=True)
for oimRef in dtsReferences:
_writerow((oimRef["type"], oimRef["href"]))
_close()
dtsRefTableSchema = OrderedDict((("columns",[OrderedDict((("name","type"),("datatype","Name"))),
OrderedDict((("name","href"),("datatype","anyURI")))]),))
csvTables.append(OrderedDict((("url",_tableinfo["url"]),
("tableSchema",dtsRefTableSchema))))
# save footnotes
if footnotesRelationshipSet.modelRelationships:
_open("-footnotes.csv", "footnotes")
cols = ("group", "footnoteType", "factId", "factRef", "footnote", "language")
_writerow(cols, header=True)
def saveCSVfootnotes(facts):
for fact in facts:
for _footnote in factFootnotes(fact):
_writerow(tuple((_footnote.get(col,"") for col in cols)))
saveCSVfootnotes(fact.modelTupleFacts)
saveCSVfootnotes(modelXbrl.facts)
_close()
footnoteTableSchema = OrderedDict((("columns",[OrderedDict((("name","group"),("datatype","anyURI"))),
OrderedDict((("name","footnoteType"),("datatype","Name"))),
OrderedDict((("name","factId"),("datatype","Name"))),
OrderedDict((("name","factRef"),("datatype","Name"))),
OrderedDict((("name","footnote"),("datatype","string"))),
OrderedDict((("name","language"),("datatype","language")))]),))
csvTables.append(OrderedDict((("url",_tableinfo["url"]),
("tableSchema",footnoteTableSchema))))
# save metadata
if isCSV:
with open(_baseURL + "-metadata.csv", "w", encoding="utf-8") as fh:
fh.write(json.dumps(csvMetadata, ensure_ascii=False, indent=1, sort_keys=False))
elif isXL:
_open(None, "metadata")
cols = ("group", "footnoteType", "factId", "factRef", "footnote", "language")
_writerow(("table", "column", "datatype"), header=True)
for table in csvTables:
tablename = table["url"]
for column in table["tableSchema"]["columns"]:
_writerow((tablename, column["name"], column["datatype"]))
_close()
if isXL:
workbook.save(oimFile)
def saveLoadableOIM(modelXbrl, oimFile):
isJSON = oimFile.endswith(".json")
isCSV = oimFile.endswith(".csv")
namespacePrefixes = {}
def compileQname(qname):
if qname.namespaceURI not in namespacePrefixes:
namespacePrefixes[qname.namespaceURI] = qname.prefix or ""
aspectsDefined = {
qnOimConceptAspect,
qnOimEntityAspect,
qnOimTypeAspect}
if isJSON:
aspectsDefined.add(qnOimPeriodAspect)
elif isCSV:
aspectsDefined.add(qnOimPeriodStartAspect)
aspectsDefined.add(qnOimPeriodDurationAspect)
def oimValue(object, decimals=None):
if isinstance(object, QName):
if object.namespaceURI not in namespacePrefixes:
if object.prefix:
namespacePrefixes[object.namespaceURI] = object.prefix
else:
_prefix = "_{}".format(sum(1 for p in namespacePrefixes if p.startswith("_")))
namespacePrefixes[object.namespaceURI] = _prefix
return "{}:{}".format(namespacePrefixes[object.namespaceURI], object.localName)
if isinstance(object, Decimal):
try:
if isinf(object):
return "-INF" if object < 0 else "INF"
elif isnan(num):
return "NaN"
else:
if object == object.to_integral():
object = object.quantize(ONE) # drop any .0
return "{}".format(object)
except:
return str(object)
if isinstance(object, bool):
return object
if isinstance(object, (DateTime, YearMonthDuration, DayTimeDuration, Time,
gYearMonth, gMonthDay, gYear, gMonth, gDay)):
return str(object)
return object
def oimPeriodValue(cntx):
if cntx.isForeverPeriod:
if isCSV:
return "0000-01-01T00:00:00/P9999Y"
return "forever"
elif cntx.isStartEndPeriod:
d = cntx.startDatetime
duration = yearMonthDayTimeDuration(cntx.startDatetime, cntx.endDatetime)
else: # instant
d = cntx.instantDatetime
duration = "PT0S"
return "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}/{6}".format(
d.year, d.month, d.day, d.hour, d.minute, d.second,
duration)
hasId = False
hasTuple = False
hasType = True
hasLang = False
hasUnits = False
hasUnitMulMeasures = False
hasUnitDivMeasures = False
footnotesRelationshipSet = ModelRelationshipSet(modelXbrl, "XBRL-footnotes")
#compile QNames in instance for OIM
for fact in modelXbrl.factsInInstance:
if (fact.id or fact.isTuple or
footnotesRelationshipSet.toModelObject(fact) or
(isCSV and footnotesRelationshipSet.fromModelObject(fact))):
hasId = True
concept = fact.concept
if concept is not None:
if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang:
hasLang = True
compileQname(fact.qname)
if hasattr(fact, "xValue") and isinstance(fact.xValue, QName):
compileQname(fact.xValue)
unit = fact.unit
if unit is not None:
hasUnits = True
if unit.measures[0]:
hasUnitMulMeasures = True
if unit.measures[1]:
hasUnitDivMeasures = True
if fact.modelTupleFacts:
hasTuple = True
entitySchemePrefixes = {}
for cntx in modelXbrl.contexts.values():
if cntx.entityIdentifierElement is not None:
scheme = cntx.entityIdentifier[0]
if scheme not in entitySchemePrefixes:
if not entitySchemePrefixes: # first one is just scheme
if scheme == "http://www.sec.gov/CIK":
_schemePrefix = "cik"
elif scheme == "http://standard.iso.org/iso/17442":
_schemePrefix = "lei"
else:
_schemePrefix = "scheme"
else:
_schemePrefix = "scheme{}".format(len(entitySchemePrefixes) + 1)
entitySchemePrefixes[scheme] = _schemePrefix
namespacePrefixes[scheme] = _schemePrefix
for dim in cntx.qnameDims.values():
compileQname(dim.dimensionQname)
aspectsDefined.add(dim.dimensionQname)
if dim.isExplicit:
compileQname(dim.memberQname)
for unit in modelXbrl.units.values():
if unit is not None:
for measures in unit.measures:
for measure in measures:
compileQname(measure)
if unit.measures[0]:
aspectsDefined.add(qnOimUnitNumeratorsAspect)
if unit.measures[1]:
aspectsDefined.add(qnOimUnitDenominatorsAspect)
if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[XbrlConst.xbrli] != "xbrli":
namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix
namespacePrefixes[XbrlConst.xsd] = "xsd"
if hasLang: aspectsDefined.add(qnOimLangAspect)
if hasTuple:
aspectsDefined.add(qnOimTupleParentAspect)
aspectsDefined.add(qnOimTupleOrderAspect)
if hasUnits: aspectsDefined.add(qnOimUnitAspect)
# compile footnotes and relationships
'''
factRelationships = []
factFootnotes = []
for rel in modelXbrl.relationshipSet(modelXbrl, "XBRL-footnotes").modelRelationships:
oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole}
factRelationships.append(oimRel)
oimRel["fromIds"] = [obj.id if obj.id
else elementChildSequence(obj)
for obj in rel.fromModelObjects]
oimRel["toIds"] = [obj.id if obj.id
else elementChildSequence(obj)
for obj in rel.toModelObjects]
_order = rel.arcElement.get("order")
if _order is not None:
oimRel["order"] = _order
for obj in rel.toModelObjects:
if isinstance(obj, ModelResource): # footnote
oimFootnote = {"role": obj.role,
"id": obj.id if obj.id
else elementChildSequence(obj),
# value needs work for html elements and for inline footnotes
"value": xmlstring(obj, stripXmlns=True)}
if obj.xmlLang:
oimFootnote["lang"] = obj.xmlLang
factFootnotes.append(oimFootnote)
oimFootnote
'''
dtsReferences = [
{"type": "schema" if doc.type == ModelDocument.Type.SCHEMA
else "linkbase" if doc.type == ModelDocument.Type.LINKBASE
else "other",
"href": doc.uri}
for doc,ref in modelXbrl.modelDocument.referencesDocument.items()
if ref.referringModelObject.qname in SCHEMA_LB_REFS]
'''
roleTypes = [
{"type": "role" if ref.referringModelObject.localName == "roleRef" else "arcroleRef",
"href": ref.referringModelObject["href"]}
for doc,ref in modelXbrl.modelDocument.referencesDocument.items()
if ref.referringModelObject.qname in ROLE_REFS]
'''
def factFootnotes(fact):
footnotes = []
for footnoteRel in footnotesRelationshipSet.fromModelObject(fact):
footnote = OrderedDict((("group", footnoteRel.arcrole),))
footnotes.append(footnote)
if isCSV:
footnote["factId"] = fact.id if fact.id else "f{}".format(fact.objectIndex)
toObj = footnoteRel.toModelObject
if isinstance(toObj, ModelFact):
footnote["factRef"] = toObj.id if toObj.id else "f{}".format(toObj.objectIndex)
else:
footnote["footnoteType"] = toObj.role
footnote["footnote"] = xmlstring(toObj, stripXmlns=True, contentsOnly=True, includeText=True)
if toObj.xmlLang:
footnote["language"] = toObj.xmlLang
return footnotes
def factAspects(fact):
aspects = OrderedDict()
if hasId and fact.id:
aspects["id"] = fact.id
elif (fact.isTuple or
footnotesRelationshipSet.toModelObject(fact) or
(isCSV and footnotesRelationshipSet.fromModelObject(fact))):
aspects["id"] = "f{}".format(fact.objectIndex)
parent = fact.getparent()
concept = fact.concept
if not fact.isTuple:
if concept is not None:
_baseXsdType = concept.baseXsdType
if _baseXsdType == "XBRLI_DATEUNION":
if getattr(fact.xValue, "dateOnly", False):
_baseXsdType = "date"
else:
_baseXsdType = "dateTime"
aspects["baseType"] = "xs:{}".format(_baseXsdType)
if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang:
aspects[qnOimLangAspect] = fact.xmlLang
aspects[qnOimTypeAspect] = concept.baseXbrliType
if fact.isItem:
if fact.isNil:
_value = None
_strValue = "nil"
else:
_inferredDecimals = inferredDecimals(fact)
_value = oimValue(fact.xValue, _inferredDecimals)
_strValue = str(_value)
if not isCSV:
aspects["value"] = _strValue
if fact.concept is not None and fact.concept.isNumeric:
_numValue = fact.xValue
if isinstance(_numValue, Decimal) and not isinf(_numValue) and not isnan(_numValue):
if _numValue == _numValue.to_integral():
_numValue = int(_numValue)
else:
_numValue = float(_numValue)
aspects["numericValue"] = _numValue
if not fact.isNil:
if isinf(_inferredDecimals):
if isJSON: _accuracy = "infinity"
elif isCSV: _accuracy = "INF"
else:
_accuracy = _inferredDecimals
aspects["accuracy"] = _inferredDecimals
elif isinstance(_value, bool):
aspects["booleanValue"] = _value
elif isCSV:
aspects["stringValue"] = _strValue
aspects[qnOimConceptAspect] = oimValue(fact.qname)
cntx = fact.context
if cntx is not None:
if cntx.entityIdentifierElement is not None:
aspects[qnOimEntityAspect] = oimValue(qname(*cntx.entityIdentifier))
if cntx.period is not None:
if isJSON:
aspects[qnOimPeriodAspect] = oimPeriodValue(cntx)
elif isCSV:
_periodValue = oimPeriodValue(cntx).split("/") + ["", ""] # default blank if no value
aspects[qnOimPeriodStartAspect] = _periodValue[0]
aspects[qnOimPeriodDurationAspect] = _periodValue[1]
for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]):
aspects[dim.dimensionQname] = (oimValue(dim.memberQname) if dim.isExplicit
else None if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1")
else dim.typedMember.stringValue)
unit = fact.unit
if unit is not None:
_mMul, _mDiv = unit.measures
if isJSON:
aspects[qnOimUnitAspect] = OrderedDict( # use tuple instead of list for hashability
(("numerators", tuple(oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m)))),)
)
if _mDiv:
aspects[qnOimUnitAspect]["denominators"] = tuple(oimValue(m) for m in sorted(_mDiv, key=lambda m: oimValue(m)))
else: # CSV
if _mMul:
aspects[qnOimUnitNumeratorsAspect] = " ".join(oimValue(m)
for m in sorted(_mMul, key=lambda m: str(m)))
if _mDiv:
aspects[qnOimUnitDenominatorsAspect] = " ".join(oimValue(m)
for m in sorted(_mDiv, key=lambda m: str(m)))
if parent.qname != XbrlConst.qnXbrliXbrl:
aspects[qnOimTupleParentAspect] = parent.id if parent.id else "f{}".format(parent.objectIndex)
aspects[qnOimTupleOrderAspect] = elementIndex(fact)
if isJSON:
_footnotes = factFootnotes(fact)
if _footnotes:
aspects["footnotes"] = _footnotes
return aspects
if isJSON:
# save JSON
oimReport = OrderedDict() # top level of oim json output
oimFacts = []
oimReport["prefixes"] = OrderedDict((p,ns) for ns, p in sorted(namespacePrefixes.items(),
key=lambda item: item[1]))
oimReport["dtsReferences"] = dtsReferences
oimReport["facts"] = oimFacts
def saveJsonFacts(facts, oimFacts, parentFact):
for fact in facts:
oimFact = factAspects(fact)
oimFacts.append(OrderedDict((str(k),v) for k,v in oimFact.items()))
if fact.modelTupleFacts:
saveJsonFacts(fact.modelTupleFacts, oimFacts, fact)
saveJsonFacts(modelXbrl.facts, oimFacts, None)
with open(oimFile, "w", encoding="utf-8") as fh:
fh.write(json.dumps(oimReport, ensure_ascii=False, indent=1, sort_keys=False))
elif isCSV:
# save CSV
aspectQnCol = {}
aspectsHeader = []
factsColumns = []
def addAspectQnCol(aspectQn):
aspectQnCol[aspectQn] = len(aspectsHeader)
_colName = oimValue(aspectQn)
aspectsHeader.append(_colName)
_colDataType = {"id": "Name",
"baseType": "Name",
"oim:concept": "Name",
"oim:periodStart": "dateTime", # forever is 0000-01-01T00:00:00
"oim:periodDuration": "duration", # forever is P9999Y
"oim:tupleOrder": "integer",
"numericValue": "decimal",
"accuracy": "decimal",
"booleanValue": "boolean",
"oim:unitNumerators": OrderedDict((("base","Name"), ("separator"," "))),
"oim:unitDenominators": OrderedDict((("base","Name"), ("separator"," "))),
}.get(_colName, "string")
factsColumns.append(OrderedDict((("name", _colName),
("datatype", _colDataType))))
# pre-ordered aspect columns
if hasId:
addAspectQnCol("id")
if hasType:
addAspectQnCol("baseType")
addAspectQnCol("stringValue")
addAspectQnCol("numericValue")
addAspectQnCol("accuracy")
addAspectQnCol("booleanValue")
if hasTuple:
addAspectQnCol(qnOimTupleParentAspect)
addAspectQnCol(qnOimTupleOrderAspect)
addAspectQnCol(qnOimConceptAspect)
if qnOimEntityAspect in aspectsDefined:
addAspectQnCol(qnOimEntityAspect)
if qnOimPeriodStartAspect in aspectsDefined:
addAspectQnCol(qnOimPeriodStartAspect)
addAspectQnCol(qnOimPeriodDurationAspect)
if qnOimUnitNumeratorsAspect in aspectsDefined:
addAspectQnCol(qnOimUnitNumeratorsAspect)
if qnOimUnitDenominatorsAspect in aspectsDefined:
addAspectQnCol(qnOimUnitDenominatorsAspect)
for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)):
if aspectQn.namespaceURI != nsOim:
addAspectQnCol(aspectQn)
def aspectCols(fact):
cols = [None for i in range(len(aspectsHeader))]
_factAspects = factAspects(fact)
for aspectQn, aspectValue in _factAspects.items():
if aspectQn in aspectQnCol:
cols[aspectQnCol[aspectQn]] = aspectValue
return cols
# metadata
csvTables = []
csvMetadata = OrderedDict((("@context",[ "http://www.w3.org/ns/csvw", { "@base": "./" }]),
("tables", csvTables)))
if oimFile.endswith("-facts.csv"): # strip -facts.csv if a prior -facts.csv file was chosen
_baseURL = oimFile[:-10]
elif oimFile.endswith(".csv"):
_baseURL = oimFile[:-4]
else:
_baseURL = oimFile
# save facts
_factsFile = _baseURL + "-facts.csv"
csvFile = open(_factsFile, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
csvWriter.writerow(aspectsHeader)
def saveCSVfacts(facts):
for fact in facts:
csvWriter.writerow(aspectCols(fact))
saveCSVfacts(fact.modelTupleFacts)
saveCSVfacts(modelXbrl.facts)
csvFile.close()
factsTableSchema = OrderedDict((("columns",factsColumns),))
csvTables.append(OrderedDict((("url",os.path.basename(_factsFile)),
("tableSchema",factsTableSchema))))
# save namespaces
_nsFile = _baseURL + "-prefixes.csv"
csvFile = open(_nsFile, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
csvWriter.writerow(("prefix", "URI"))
for _URI, prefix in sorted(namespacePrefixes.items(), key=lambda item: item[1]):
csvWriter.writerow((prefix, _URI))
csvFile.close()
nsTableSchema = OrderedDict((("columns",[OrderedDict((("prefix","string"),
("URI","anyURI")))]),))
csvTables.append(OrderedDict((("url",os.path.basename(_nsFile)),
("tableSchema",nsTableSchema))))
# save dts references
_dtsRefFile = _baseURL + "-dtsReferences.csv"
csvFile = open(_dtsRefFile, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
csvWriter.writerow(("type", "href"))
for oimRef in dtsReferences:
csvWriter.writerow((oimRef["type"], oimRef["href"]))
csvFile.close()
dtsRefTableSchema = OrderedDict((("columns",[OrderedDict((("type","string"),
("href","anyURI")))]),))
csvTables.append(OrderedDict((("url",os.path.basename(_dtsRefFile)),
("tableSchema",dtsRefTableSchema))))
# save footnotes
if footnotesRelationshipSet.modelRelationships:
_footnoteFile = oimFile.replace(".csv", "-footnotes.csv")
csvFile = open(_footnoteFile, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
cols = ("group", "factId", "factRef", "footnoteType", "footnote", "language")
csvWriter.writerow(cols)
def saveCSVfootnotes(facts):
for fact in facts:
for _footnote in factFootnotes(fact):
csvWriter.writerow(tuple((_footnote.get(col,"") for col in cols)))
saveCSVfootnotes(fact.modelTupleFacts)
saveCSVfootnotes(modelXbrl.facts)
csvFile.close()
footnoteTableSchema = OrderedDict((("columns",[OrderedDict((("group","anyURI"),
("factId","Name"),
("factRef","Name"),
("footnoteType","Name"),
("footnote","string"),
("language","language")))]),))
csvTables.append(OrderedDict((("url",os.path.basename(_footnoteFile)),
("tableSchema",footnoteTableSchema))))
# save metadata
with open(_baseURL + "-metadata.csv", "w", encoding="utf-8") as fh:
fh.write(json.dumps(csvMetadata, ensure_ascii=False, indent=1, sort_keys=False))
def saveLoadableOIM(modelXbrl, oimFile, outputZip=None):
isJSON = oimFile.endswith(".json")
isCSV = oimFile.endswith(".csv")
isXL = oimFile.endswith(".xlsx")
isCSVorXL = isCSV or isXL
if not isJSON and not isCSVorXL:
return
namespacePrefixes = {nsOim: "xbrl"}
prefixNamespaces = {"xbrl": nsOim}
def compileQname(qname):
if qname.namespaceURI not in namespacePrefixes:
namespacePrefixes[qname.namespaceURI] = qname.prefix or ""
aspectsDefined = {
qnOimConceptAspect,
qnOimEntityAspect,
qnOimPeriodStartAspect,
qnOimPeriodEndAspect}
def oimValue(object, decimals=None):
if isinstance(object, QName):
if object.namespaceURI not in namespacePrefixes:
if object.prefix:
namespacePrefixes[object.namespaceURI] = object.prefix
else:
_prefix = "_{}".format(sum(1 for p in namespacePrefixes if p.startswith("_")))
namespacePrefixes[object.namespaceURI] = _prefix
return "{}:{}".format(namespacePrefixes[object.namespaceURI], object.localName)
if isinstance(object, Decimal):
try:
if isinf(object):
return "-INF" if object < 0 else "INF"
elif isnan(num):
return "NaN"
else:
if object == object.to_integral():
object = object.quantize(ONE) # drop any .0
return "{}".format(object)
except:
return str(object)
if isinstance(object, bool):
return "true" if object else "false"
if isinstance(object, (DateTime, YearMonthDuration, DayTimeDuration, Time,
gYearMonth, gMonthDay, gYear, gMonth, gDay,
IsoDuration)):
return str(object)
return object
def oimPeriodValue(cntx):
if cntx.isForeverPeriod:
return OrderedDict() # defaulted
elif cntx.isStartEndPeriod:
s = cntx.startDatetime
e = cntx.endDatetime
else: # instant
s = e = cntx.instantDatetime
if isJSON:
return({str(qnOimPeriodAspect):
("{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}/".format(
s.year, s.month, s.day, s.hour, s.minute, s.second)
if cntx.isStartEndPeriod else "") +
"{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format(
e.year, e.month, e.day, e.hour, e.minute, e.second)
})
# CSV, XL
return OrderedDict(((str(qnOimPeriodStartAspect), "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format(
s.year, s.month, s.day, s.hour, s.minute, s.second)),
(str(qnOimPeriodEndAspect), "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format(
e.year, e.month, e.day, e.hour, e.minute, e.second))))
hasId = False
hasTuple = False
hasType = True
hasLang = False
hasUnits = False
hasNumeric = False
footnotesRelationshipSet = ModelRelationshipSet(modelXbrl, "XBRL-footnotes")
factBaseTypes = set()
#compile QNames in instance for OIM
for fact in modelXbrl.factsInInstance:
if (fact.id or fact.isTuple or
footnotesRelationshipSet.toModelObject(fact) or
(isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))):
hasId = True
concept = fact.concept
if concept is not None:
if concept.isNumeric:
hasNumeric = True
if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang:
hasLang = True
_baseXsdType = concept.baseXsdType
if _baseXsdType == "XBRLI_DATEUNION":
if getattr(fact.xValue, "dateOnly", False):
_baseXsdType = "date"
else:
_baseXsdType = "dateTime"
factBaseTypes.add(baseTypes.get(_baseXsdType,_baseXsdType))
compileQname(fact.qname)
if hasattr(fact, "xValue") and isinstance(fact.xValue, QName):
compileQname(fact.xValue)
unit = fact.unit
if unit is not None:
hasUnits = True
if fact.modelTupleFacts:
hasTuple = True
if hasTuple:
modelXbrl.error("arelleOIMsaver:tuplesNotAllowed",
"Tuples are not allowed in an OIM document",
modelObject=modelXbrl)
return
entitySchemePrefixes = {}
for cntx in modelXbrl.contexts.values():
if cntx.entityIdentifierElement is not None:
scheme = cntx.entityIdentifier[0]
if scheme not in entitySchemePrefixes:
if not entitySchemePrefixes: # first one is just scheme
if scheme == "http://www.sec.gov/CIK":
_schemePrefix = "cik"
elif scheme == "http://standard.iso.org/iso/17442":
_schemePrefix = "lei"
else:
_schemePrefix = "scheme"
else:
_schemePrefix = "scheme{}".format(len(entitySchemePrefixes) + 1)
entitySchemePrefixes[scheme] = _schemePrefix
namespacePrefixes[scheme] = _schemePrefix
for dim in cntx.qnameDims.values():
compileQname(dim.dimensionQname)
aspectsDefined.add(dim.dimensionQname)
if dim.isExplicit:
compileQname(dim.memberQname)
for unit in modelXbrl.units.values():
if unit is not None:
for measures in unit.measures:
for measure in measures:
compileQname(measure)
if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[XbrlConst.xbrli] != "xbrli":
namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix
namespacePrefixes[XbrlConst.xsd] = "xsd"
if hasLang: aspectsDefined.add(qnOimLangAspect)
if hasTuple:
aspectsDefined.add(qnOimTupleParentAspect)
aspectsDefined.add(qnOimTupleOrderAspect)
if hasUnits: aspectsDefined.add(qnOimUnitAspect)
for footnoteRel in footnotesRelationshipSet.modelRelationships:
typePrefix = "ftTyp_" + os.path.basename(footnoteRel.arcrole)
if footnoteRel.linkrole == XbrlConst.defaultLinkRole:
groupPrefix = "ftGrp_default"
else:
groupPrefix = "ftGrp_" + os.path.basename(footnoteRel.linkrole)
if typePrefix not in namespacePrefixes:
namespacePrefixes[footnoteRel.arcrole] = typePrefix
if groupPrefix not in namespacePrefixes:
namespacePrefixes[footnoteRel.linkrole] = groupPrefix
# compile footnotes and relationships
'''
factRelationships = []
factFootnotes = []
for rel in modelXbrl.relationshipSet(modelXbrl, "XBRL-footnotes").modelRelationships:
oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole}
factRelationships.append(oimRel)
oimRel["fromIds"] = [obj.id if obj.id
else elementChildSequence(obj)
for obj in rel.fromModelObjects]
oimRel["toIds"] = [obj.id if obj.id
else elementChildSequence(obj)
for obj in rel.toModelObjects]
_order = rel.arcElement.get("order")
if _order is not None:
oimRel["order"] = _order
for obj in rel.toModelObjects:
if isinstance(obj, ModelResource): # footnote
oimFootnote = {"role": obj.role,
"id": obj.id if obj.id
else elementChildSequence(obj),
# value needs work for html elements and for inline footnotes
"value": xmlstring(obj, stripXmlns=True)}
if obj.xmlLang:
oimFootnote["lang"] = obj.xmlLang
factFootnotes.append(oimFootnote)
oimFootnote
'''
dtsReferences = set()
baseUrl = modelXbrl.modelDocument.uri.partition("#")[0]
for doc,ref in sorted(modelXbrl.modelDocument.referencesDocument.items(),
key=lambda _item:_item[0].uri):
if ref.referringModelObject.qname in SCHEMA_LB_REFS:
dtsReferences.add(relativeUri(baseUrl,doc.uri))
for refType in ("role", "arcrole"):
for refElt in sorted(modelXbrl.modelDocument.xmlRootElement.iterchildren(
"{{http://www.xbrl.org/2003/linkbase}}{}Ref".format(refType)),
key=lambda elt:elt.get(refType+"URI")
):
dtsReferences.add(refElt.get("{http://www.w3.org/1999/xlink}href").partition("#")[0])
dtsReferences = sorted(dtsReferences) # turn into list
footnoteFacts = set()
def factFootnotes(fact, oimFact=None):
footnotes = []
oimLinks = {}
for footnoteRel in footnotesRelationshipSet.fromModelObject(fact):
toObj = footnoteRel.toModelObject
# json
typePrefix = namespacePrefixes[footnoteRel.arcrole]
groupPrefix = namespacePrefixes[footnoteRel.linkrole]
oimLinks.setdefault(typePrefix,{}).setdefault(groupPrefix,[]).append(
toObj.id if toObj.id else "f{}".format(toObj.objectIndex))
# csv/XL
footnote = OrderedDict((("group", footnoteRel.linkrole),
("footnoteType", footnoteRel.arcrole)))
footnotes.append(footnote)
if isCSVorXL:
footnote["factId"] = fact.id if fact.id else "f{}".format(fact.objectIndex)
if isinstance(toObj, ModelFact):
footnote["factRef"] = toObj.id if toObj.id else "f{}".format(toObj.objectIndex)
else:
footnote["footnote"] = toObj.viewText()
if toObj.xmlLang:
footnote["language"] = toObj.xmlLang
footnoteFacts.add(toObj)
if isJSON and oimLinks:
oimFact["links"] = OrderedDict((
(typePrefix, OrderedDict((
(groupPrefix, idList)
for groupPrefix, idList in sorted(groups.items())
)))
for typePrefix, groups in sorted(oimLinks.items())
))
footnotes.sort(key=lambda f:(f["group"],f.get("factId",f.get("factRef")),f.get("language")))
return footnotes
def factAspects(fact):
oimFact = OrderedDict()
aspects = OrderedDict()
if hasId and fact.id:
if fact.isTuple:
oimFact["tupleId"] = fact.id
else:
oimFact["id"] = fact.id
elif (fact.isTuple or
footnotesRelationshipSet.toModelObject(fact) or
(isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))):
oimFact["id"] = "f{}".format(fact.objectIndex)
parent = fact.getparent()
concept = fact.concept
aspects[str(qnOimConceptAspect)] = oimValue(concept.qname)
_csvType = "Value"
if not fact.isTuple:
if concept is not None:
_baseXsdType = concept.baseXsdType
if _baseXsdType == "XBRLI_DATEUNION":
if getattr(fact.xValue, "dateOnly", False):
_baseXsdType = "date"
else:
_baseXsdType = "dateTime"
_csvType = baseTypes.get(_baseXsdType,_baseXsdType) + "Value"
if concept.baseXbrliType in ("stringItemType", "normalizedStringItemType") and fact.xmlLang:
aspects[str(qnOimLangAspect)] = fact.xmlLang
if fact.isItem:
if fact.isNil:
_value = None
else:
_inferredDecimals = inferredDecimals(fact)
_value = oimValue(fact.xValue, _inferredDecimals)
oimFact["value"] = _value
if fact.concept is not None and fact.concept.isNumeric:
_numValue = fact.xValue
if isinstance(_numValue, Decimal) and not isinf(_numValue) and not isnan(_numValue):
if _numValue == _numValue.to_integral():
_numValue = int(_numValue)
else:
_numValue = float(_numValue)
if not fact.isNil:
if isinf(_inferredDecimals):
if isJSON: _accuracy = "infinity"
elif isCSVorXL: _accuracy = "INF"
else:
_accuracy = _inferredDecimals
oimFact["accuracy"] = _accuracy
oimFact["aspects"] = aspects
cntx = fact.context
if cntx is not None:
if cntx.entityIdentifierElement is not None:
aspects[str(qnOimEntityAspect)] = oimValue(qname(*cntx.entityIdentifier))
if cntx.period is not None:
aspects.update(oimPeriodValue(cntx))
for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]):
if dim.isExplicit:
dimVal = oimValue(dim.memberQname)
else: # typed
if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1"):
dimVal = None
else:
dimVal = dim.typedMember.stringValue
aspects[str(dim.dimensionQname)] = dimVal
unit = fact.unit
if unit is not None:
_mMul, _mDiv = unit.measures
_sMul = '*'.join(oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m)))
if _mDiv:
_sDiv = '*'.join(oimValue(m) for m in sorted(_mDiv, key=lambda m: oimValue(m)))
if len(_mDiv) > 1:
if len(_mMul) > 1:
_sUnit = "({})/({})".format(_sMul,_sDiv)
else:
_sUnit = "{}/({})".format(_sMul,_sDiv)
else:
if len(_mMul) > 1:
_sUnit = "({})/{}".format(_sMul,_sDiv)
else:
_sUnit = "{}/{}".format(_sMul,_sDiv)
else:
_sUnit = _sMul
aspects[str(qnOimUnitAspect)] = _sUnit
# Tuples removed from xBRL-JSON
#if parent.qname != XbrlConst.qnXbrliXbrl:
# aspects[str(qnOimTupleParentAspect)] = parent.id if parent.id else "f{}".format(parent.objectIndex)
# aspects[str(qnOimTupleOrderAspect)] = elementIndex(fact)
if isJSON:
factFootnotes(fact, oimFact)
return oimFact
prefixes = OrderedDict((p,ns) for ns, p in sorted(namespacePrefixes.items(),
key=lambda item: item[1]))
if isJSON:
# save JSON
oimReport = OrderedDict() # top level of oim json output
oimReport["documentInfo"] = oimDocInfo = OrderedDict()
oimReport["facts"] = oimFacts = OrderedDict()
oimDocInfo["documentType"] = nsOim + "/xbrl-json"
oimDocInfo["features"] = oimFeatures = OrderedDict()
oimDocInfo["prefixes"] = prefixes
oimDocInfo["taxonomy"] = dtsReferences
oimFeatures["xbrl:canonicalValues"] = True
def saveJsonFacts(facts, oimFacts, parentFact):
for fact in facts:
oimFact = factAspects(fact)
id = fact.id if fact.id else "f{}".format(fact.objectIndex)
oimFacts[id] = oimFact
if fact.modelTupleFacts:
saveJsonFacts(fact.modelTupleFacts, oimFacts, fact)
saveJsonFacts(modelXbrl.facts, oimFacts, None)
# add footnotes as pseudo facts
for ftObj in footnoteFacts:
ftId = ftObj.id if ftObj.id else "f{}".format(ftObj.objectIndex)
oimFacts[ftId] = oimFact = OrderedDict()
oimFact["value"] = ftObj.viewText()
oimFact["aspects"] = OrderedDict((("concept", "xbrl:note"),
("noteId", ftId)))
if ftObj.xmlLang:
oimFact["aspects"]["language"] = ftObj.xmlLang.lower()
if outputZip:
fh = io.StringIO()
else:
fh = open(oimFile, "w", encoding="utf-8")
fh.write(json.dumps(oimReport, indent=1))
if outputZip:
fh.seek(0)
outputZip.writestr(os.path.basename(oimFile),fh.read())
fh.close()
elif isCSVorXL:
# save CSV
aspectQnCol = {}
aspectsHeader = []
factsColumns = []
def addAspectQnCol(aspectQn):
aspectQnCol[str(aspectQn)] = len(aspectsHeader)
_aspectQn = oimValue(aspectQn)
aspectsHeader.append(_aspectQn)
_colName = _aspectQn.replace("xbrl:", "")
_colDataType = {"id": "Name",
"concept": "QName",
"value": "string",
"accuracy": "decimal",
"entity": "QName",
"periodStart": "dateTime",
"periodEnd": "dateTime",
"unit": "string",
"tupleId": "Name",
"tupleParent": "Name",
"tupleOrder": "integer"
}.get(_colName, "string")
col = OrderedDict((("name", _colName),
("datatype", _colDataType)))
if _aspectQn == "value":
col["http://xbrl.org/YYYY/model#simpleFactAspects"] = {}
elif _aspectQn == "tupleId":
col["http://xbrl.org/YYYY/model#tupleFactAspects"] = {}
col["http://xbrl.org/YYYY/model#tupleReferenceId"] = "true"
else:
col["http://xbrl.org/YYYY/model#columnAspect"] = _aspectQn
factsColumns.append(col)
# pre-ordered aspect columns
#if hasId:
# addAspectQnCol("id")
addAspectQnCol(qnOimConceptAspect)
addAspectQnCol("value")
if hasNumeric:
addAspectQnCol("accuracy")
if hasTuple:
addAspectQnCol("tupleId")
addAspectQnCol(qnOimTupleParentAspect)
addAspectQnCol(qnOimTupleOrderAspect)
if qnOimEntityAspect in aspectsDefined:
addAspectQnCol(qnOimEntityAspect)
if qnOimPeriodStartAspect in aspectsDefined:
addAspectQnCol(qnOimPeriodStartAspect)
addAspectQnCol(qnOimPeriodEndAspect)
if qnOimUnitAspect in aspectsDefined:
addAspectQnCol(qnOimUnitAspect)
for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)):
if aspectQn.namespaceURI != nsOim:
addAspectQnCol(aspectQn)
def aspectCols(fact):
cols = [None for i in range(len(aspectsHeader))]
def setColValues(aspects):
for aspectQn, aspectValue in aspects.items():
if isinstance(aspectValue, dict):
setColValues(aspectValue)
elif aspectQn in aspectQnCol:
if aspectValue is None:
_aspectValue = "#nil"
elif aspectValue == "":
_aspectValue = "#empty"
elif isinstance(aspectValue, str) and aspectValue.startswith("#"):
_aspectValue = "#" + aspectValue
else:
_aspectValue = aspectValue
cols[aspectQnCol[aspectQn]] = _aspectValue
setColValues(factAspects(fact))
return cols
# metadata
csvTables = []
csvMetadata = OrderedDict((("@context", "http://www.w3.org/ns/csvw"),
("http://xbrl.org/YYYY/model#metadata",
OrderedDict((("documentType", "http://xbrl.org/YYYY/xbrl-csv"),
("dtsReferences", dtsReferences),
("prefixes", prefixes)))),
("tables", csvTables)))
_open = _writerow = _close = None
_tableinfo = {}
if isCSV:
if oimFile.endswith("-facts.csv"): # strip -facts.csv if a prior -facts.csv file was chosen
_baseURL = oimFile[:-10]
elif oimFile.endswith(".csv"):
_baseURL = oimFile[:-4]
else:
_baseURL = oimFile
_csvinfo = {} # open file, writer
def _open(filesuffix, tabname):
_filename = _tableinfo["url"] = _baseURL + filesuffix
_csvinfo["file"] = open(_filename, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
_csvinfo["writer"] = csv.writer(_csvinfo["file"], dialect="excel")
def _writerow(row, header=False):
_csvinfo["writer"].writerow(row)
def _close():
_csvinfo["file"].close()
_csvinfo.clear()
elif isXL:
headerWidths = {"href": 100, "xbrl:concept": 70, "accuracy": 8, "language": 9, "URI": 80,
"value": 60,
"group": 60, "footnoteType": 40, "footnote": 70, "column": 20,
'conceptAspect': 40, 'tuple': 20, 'simpleFact': 20}
from openpyxl import Workbook
from openpyxl.writer.write_only import WriteOnlyCell
from openpyxl.styles import Font, PatternFill, Border, Alignment, Color, fills, Side
from openpyxl.worksheet.dimensions import ColumnDimension
hdrCellFill = PatternFill(patternType=fills.FILL_SOLID, fgColor=Color("00FFBF5F")) # Excel's light orange fill color = 00FF990
workbook = Workbook()
# remove pre-existing worksheets
while len(workbook.worksheets)>0:
workbook.remove_sheet(workbook.worksheets[0])
_xlinfo = {} # open file, writer
def _open(filesuffix, tabname):
_tableinfo["url"] = tabname
_xlinfo["ws"] = workbook.create_sheet(title=tabname)
def _writerow(rowvalues, header=False):
row = []
_ws = _xlinfo["ws"]
for i, v in enumerate(rowvalues):
cell = WriteOnlyCell(_ws, value=v)
if header:
cell.fill = hdrCellFill
cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True)
colLetter = chr(ord('A') + i)
_ws.column_dimensions[colLetter] = ColumnDimension(_ws, customWidth=True)
_ws.column_dimensions[colLetter].width = headerWidths.get(v, 20)
else:
cell.alignment = Alignment(horizontal="right" if isinstance(v, _NUM_TYPES)
else "center" if isinstance(v, bool)
else "left",
vertical="top",
wrap_text=isinstance(v, str))
row.append(cell)
_ws.append(row)
def _close():
_xlinfo.clear()
# save facts
_open("-facts.csv", "facts")
_writerow(aspectsHeader, header=True)
def saveCSVfacts(facts):
for fact in facts:
_writerow(aspectCols(fact))
saveCSVfacts(fact.modelTupleFacts)
saveCSVfacts(modelXbrl.facts)
_close()
factsTableSchema = OrderedDict((("columns",factsColumns),))
csvTables.append(OrderedDict((("url",_tableinfo["url"]),
("http://xbrl.org/YYYY/model#tableType", "fact"),
("tableSchema",factsTableSchema))))
# save footnotes
if footnotesRelationshipSet.modelRelationships:
_open("-footnotes.csv", "footnotes")
cols = ("group", "footnoteType", "factId", "factRef", "footnote", "language")
_writerow(cols, header=True)
def saveCSVfootnotes(facts):
for fact in facts:
for _footnote in factFootnotes(fact):
_writerow(tuple((_footnote.get(col,"") for col in cols)))
saveCSVfootnotes(fact.modelTupleFacts)
saveCSVfootnotes(modelXbrl.facts)
_close()
footnoteTableSchema = OrderedDict((("columns",[OrderedDict((("name","group"),("datatype","anyURI"))),
OrderedDict((("name","footnoteType"),("datatype","Name"))),
OrderedDict((("name","factId"),("datatype","Name"))),
OrderedDict((("name","factRef"),("datatype","Name"))),
OrderedDict((("name","footnote"),("datatype","string"))),
OrderedDict((("name","language"),("datatype","language")))]),))
csvTables.append(OrderedDict((("url",_tableinfo["url"]),
("http://xbrl.org/YYYY/model#tableType", "footnote"),
("tableSchema",footnoteTableSchema))))
# save metadata
if isCSV:
with open(_baseURL + "-metadata.json", "w", encoding="utf-8") as fh:
fh.write(json.dumps(csvMetadata, ensure_ascii=False, indent=1, sort_keys=False))
elif isXL:
_open(None, "metadata")
hasColumnAspect = hasSimpleFact = hasTupleFact = False
for table in csvTables:
tablename = table["url"]
for column in table["tableSchema"]["columns"]:
if "http://xbrl.org/YYYY/model#columnAspect" in column:
hasColumnAspect = True
if "http://xbrl.org/YYYY/model#simpleFactAspects" in column:
hasSimpleFact = True
if "http://xbrl.org/YYYY/model#tupleAspects" in column:
hasTupleFact = True
metadataCols = ["table", "column", "datatype"]
if hasColumnAspect:
metadataCols.append("columnAspect")
if hasSimpleFact:
metadataCols.append("simpleFact")
if hasTupleFact:
metadataCols.append("tuple")
_writerow(metadataCols, header=True)
for table in csvTables:
tablename = table["url"]
for column in table["tableSchema"]["columns"]:
row = [tablename, column["name"], column["datatype"]]
if hasColumnAspect:
colAspect = column.get("http://xbrl.org/YYYY/model#columnAspect")
if isinstance(colAspect, str):
row.append(colAspect)
elif isinstance(colAspect, dict):
row.append("\n".join("{} [{}]".format(k, ", ".join(_v for _v in v))
for k, v in dict.items()))
else:
row.append(None)
if hasSimpleFact:
row.append("\u221a" if "http://xbrl.org/YYYY/model#simpleFactAspects" in column else None)
if hasTupleFact:
row.append("\u221a" if "http://xbrl.org/YYYY/model#tupleAspects" in column else None)
_writerow(row)
_close()
if isXL:
workbook.save(oimFile)