def saveLoadableOIM(modelXbrl, oimFile, outputZip=None): isJSON = oimFile.endswith(".json") isCSV = oimFile.endswith(".csv") isXL = oimFile.endswith(".xlsx") isCSVorXL = isCSV or isXL if not isJSON and not isCSVorXL: return namespacePrefixes = {nsOim: "xbrl"} prefixNamespaces = {} linkTypeAliases = {} groupAliases = {} linkTypePrefixes = {} linkTypeUris = {} linkGroupPrefixes = {} linkGroupUris = {} def compileQname(qname): if qname.namespaceURI not in namespacePrefixes: namespacePrefixes[qname.namespaceURI] = qname.prefix or "" aspectsDefined = {qnOimConceptAspect, qnOimEntityAspect, qnOimPeriodAspect} def oimValue(object, decimals=None): if isinstance(object, QName): if object.namespaceURI not in namespacePrefixes: if object.prefix: namespacePrefixes[object.namespaceURI] = object.prefix else: _prefix = "_{}".format( sum(1 for p in namespacePrefixes if p.startswith("_"))) namespacePrefixes[object.namespaceURI] = _prefix return "{}:{}".format(namespacePrefixes[object.namespaceURI], object.localName) if isinstance(object, (float, Decimal)): try: if isinf(object): return "-INF" if object < 0 else "INF" elif isnan(object): return "NaN" else: if isinstance(object, Decimal) and object == object.to_integral(): object = object.quantize(ONE) # drop any .0 return "{}".format(object) except: return str(object) if isinstance(object, bool): return "true" if object else "false" if isinstance( object, (DateTime, YearMonthDuration, DayTimeDuration, Time, gYearMonth, gMonthDay, gYear, gMonth, gDay, IsoDuration, int)): return str(object) return object def oimPeriodValue(cntx): if cntx.isStartEndPeriod: s = cntx.startDatetime e = cntx.endDatetime else: # instant s = e = cntx.instantDatetime return ({ str(qnOimPeriodAspect): ("{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}/".format( s.year, s.month, s.day, s.hour, s.minute, s.second) if cntx.isStartEndPeriod else "") + "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format( e.year, e.month, e.day, e.hour, e.minute, e.second) }) hasId = False hasTuple = False hasType = True hasLang = False hasUnits = False hasNumeric = False footnotesRelationshipSet = ModelRelationshipSet(modelXbrl, "XBRL-footnotes") #compile QNames in instance for OIM for fact in modelXbrl.factsInInstance: if (fact.id or fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))): hasId = True concept = fact.concept if concept is not None: if concept.isNumeric: hasNumeric = True if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang: hasLang = True compileQname(fact.qname) if hasattr(fact, "xValue") and isinstance(fact.xValue, QName): compileQname(fact.xValue) unit = fact.unit if unit is not None: hasUnits = True if fact.modelTupleFacts: hasTuple = True if hasTuple: modelXbrl.error("arelleOIMsaver:tuplesNotAllowed", "Tuples are not allowed in an OIM document", modelObject=modelXbrl) return entitySchemePrefixes = {} for cntx in modelXbrl.contexts.values(): if cntx.entityIdentifierElement is not None: scheme = cntx.entityIdentifier[0] if scheme not in entitySchemePrefixes: if not entitySchemePrefixes: # first one is just scheme if scheme == "http://www.sec.gov/CIK": _schemePrefix = "cik" elif scheme == "http://standard.iso.org/iso/17442": _schemePrefix = "lei" else: _schemePrefix = "scheme" else: _schemePrefix = "scheme{}".format( len(entitySchemePrefixes) + 1) entitySchemePrefixes[scheme] = _schemePrefix namespacePrefixes[scheme] = _schemePrefix for dim in cntx.qnameDims.values(): compileQname(dim.dimensionQname) aspectsDefined.add(dim.dimensionQname) if dim.isExplicit: compileQname(dim.memberQname) for unit in modelXbrl.units.values(): if unit is not None: for measures in unit.measures: for measure in measures: compileQname(measure) if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[ XbrlConst.xbrli] != "xbrli": namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix if hasLang: aspectsDefined.add(qnOimLangAspect) if hasUnits: aspectsDefined.add(qnOimUnitAspect) for footnoteRel in footnotesRelationshipSet.modelRelationships: typePrefix = "ftTyp_" + os.path.basename(footnoteRel.arcrole) if footnoteRel.linkrole == XbrlConst.defaultLinkRole: groupPrefix = "ftGrp_default" else: groupPrefix = "ftGrp_" + os.path.basename(footnoteRel.linkrole) if footnoteRel.arcrole not in linkTypeAliases: linkTypeAliases[footnoteRel.arcrole] = typePrefix if groupPrefix not in groupAliases: groupAliases[footnoteRel.linkrole] = groupPrefix dtsReferences = set() baseUrl = modelXbrl.modelDocument.uri.partition("#")[0] for doc, ref in sorted(modelXbrl.modelDocument.referencesDocument.items(), key=lambda _item: _item[0].uri): if ref.referringModelObject.qname in SCHEMA_LB_REFS: dtsReferences.add(relativeUri(baseUrl, doc.uri)) for refType in ("role", "arcrole"): for refElt in sorted( modelXbrl.modelDocument.xmlRootElement.iterchildren( "{{http://www.xbrl.org/2003/linkbase}}{}Ref".format( refType)), key=lambda elt: elt.get(refType + "URI")): dtsReferences.add( refElt.get("{http://www.w3.org/1999/xlink}href").partition("#") [0]) dtsReferences = sorted(dtsReferences) # turn into list footnoteFacts = set() def factFootnotes(fact, oimFact=None, csvLinks=None): footnotes = [] if isJSON: oimLinks = {} elif isCSVorXL: oimLinks = csvLinks for footnoteRel in footnotesRelationshipSet.fromModelObject(fact): srcId = fact.id if fact.id else "f{}".format(fact.objectIndex) toObj = footnoteRel.toModelObject # json typePrefix = linkTypeAliases[footnoteRel.arcrole] groupPrefix = groupAliases[footnoteRel.linkrole] if typePrefix not in oimLinks: oimLinks[typePrefix] = OrderedDict() _link = oimLinks[typePrefix] if groupPrefix not in _link: if isJSON: _link[groupPrefix] = [] elif isCSVorXL: _link[groupPrefix] = OrderedDict() if isJSON: tgtIdList = _link[groupPrefix] elif isCSVorXL: tgtIdList = _link[groupPrefix].setdefault(srcId, []) tgtId = toObj.id if toObj.id else "f{}".format(toObj.objectIndex) tgtIdList.append(tgtId) footnote = OrderedDict((("group", footnoteRel.linkrole), ("footnoteType", footnoteRel.arcrole))) if isinstance(toObj, ModelFact): footnote["factRef"] = tgtId else: # text footnotes footnote["id"] = tgtId footnote["footnote"] = toObj.viewText() if toObj.xmlLang: footnote["language"] = toObj.xmlLang footnoteFacts.add(toObj) footnotes.append(footnote) if oimLinks: _links = OrderedDict( ((typePrefix, OrderedDict( ((groupPrefix, idList) for groupPrefix, idList in sorted(groups.items())))) for typePrefix, groups in sorted(oimLinks.items()))) if isJSON: oimFact["links"] = _links return footnotes def factAspects(fact): oimFact = OrderedDict() aspects = OrderedDict() if isCSVorXL: oimFact["id"] = fact.id or "f{}".format(fact.objectIndex) parent = fact.getparent() concept = fact.concept aspects[str(qnOimConceptAspect)] = oimValue(concept.qname) if concept is not None: if concept.type.isOimTextFactType and fact.xmlLang: aspects[str(qnOimLangAspect)] = fact.xmlLang if fact.isItem: if fact.isNil: _value = None else: _inferredDecimals = inferredDecimals(fact) _value = oimValue(fact.xValue, _inferredDecimals) oimFact["value"] = _value if fact.concept is not None and fact.concept.isNumeric: _numValue = fact.xValue if isinstance(_numValue, Decimal) and not isinf( _numValue) and not isnan(_numValue): if _numValue == _numValue.to_integral(): _numValue = int(_numValue) else: _numValue = float(_numValue) if not fact.isNil: if not isinf( _inferredDecimals): # accuracy omitted if infinite oimFact["decimals"] = _inferredDecimals oimFact["dimensions"] = aspects cntx = fact.context if cntx is not None: if cntx.entityIdentifierElement is not None and cntx.entityIdentifier != ENTITY_NA_QNAME: aspects[str(qnOimEntityAspect)] = oimValue( qname(*cntx.entityIdentifier)) if cntx.period is not None and not cntx.isForeverPeriod: aspects.update(oimPeriodValue(cntx)) for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]): if dim.isExplicit: dimVal = oimValue(dim.memberQname) else: # typed if dim.typedMember.get( "{http://www.w3.org/2001/XMLSchema-instance}nil" ) in ("true", "1"): dimVal = None else: dimVal = dim.typedMember.stringValue aspects[str(dim.dimensionQname)] = dimVal unit = fact.unit if unit is not None: _mMul, _mDiv = unit.measures _sMul = '*'.join( oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m))) if _mDiv: _sDiv = '*'.join( oimValue(m) for m in sorted(_mDiv, key=lambda m: oimValue(m))) if len(_mDiv) > 1: if len(_mMul) > 1: _sUnit = "({})/({})".format(_sMul, _sDiv) else: _sUnit = "{}/({})".format(_sMul, _sDiv) else: if len(_mMul) > 1: _sUnit = "({})/{}".format(_sMul, _sDiv) else: _sUnit = "{}/{}".format(_sMul, _sDiv) else: _sUnit = _sMul if _sUnit != "xbrli:pure": aspects[str(qnOimUnitAspect)] = _sUnit # Tuples removed from xBRL-JSON #if parent.qname != XbrlConst.qnXbrliXbrl: # aspects[str(qnOimTupleParentAspect)] = parent.id if parent.id else "f{}".format(parent.objectIndex) # aspects[str(qnOimTupleOrderAspect)] = elementIndex(fact) if isJSON: factFootnotes(fact, oimFact=oimFact) return oimFact namespaces = OrderedDict((p, ns) for ns, p in sorted(namespacePrefixes.items(), key=lambda item: item[1])) # common metadata oimReport = OrderedDict() # top level of oim json output oimReport["documentInfo"] = oimDocInfo = OrderedDict() oimDocInfo["documentType"] = nsOim + ("/xbrl-json" if isJSON else "/xbrl-csv") if isJSON: oimDocInfo["features"] = oimFeatures = OrderedDict() oimDocInfo["namespaces"] = namespaces if linkTypeAliases: oimDocInfo["linkTypes"] = OrderedDict( (a, u) for u, a in sorted(linkTypeAliases.items(), key=lambda item: item[1])) if linkTypeAliases: oimDocInfo["linkGroups"] = OrderedDict( (a, u) for u, a in sorted(groupAliases.items(), key=lambda item: item[1])) oimDocInfo["taxonomy"] = dtsReferences if isJSON: oimFeatures["xbrl:canonicalValues"] = True if isJSON: # save JSON oimReport["facts"] = oimFacts = OrderedDict() def saveJsonFacts(facts, oimFacts, parentFact): for fact in facts: oimFact = factAspects(fact) id = fact.id if fact.id else "f{}".format(fact.objectIndex) oimFacts[id] = oimFact if fact.modelTupleFacts: saveJsonFacts(fact.modelTupleFacts, oimFacts, fact) saveJsonFacts(modelXbrl.facts, oimFacts, None) # add footnotes as pseudo facts for ftObj in footnoteFacts: ftId = ftObj.id if ftObj.id else "f{}".format(ftObj.objectIndex) oimFacts[ftId] = oimFact = OrderedDict() oimFact["value"] = ftObj.viewText() oimFact["dimensions"] = OrderedDict( (("concept", "xbrl:note"), ("noteId", ftId))) if ftObj.xmlLang: oimFact["dimensions"]["language"] = ftObj.xmlLang.lower() if outputZip: fh = io.StringIO() else: fh = open(oimFile, "w", encoding="utf-8") fh.write(json.dumps(oimReport, indent=1)) if outputZip: fh.seek(0) outputZip.writestr(os.path.basename(oimFile), fh.read()) fh.close() elif isCSVorXL: # save CSV oimReport["tables"] = oimTables = OrderedDict() oimReport["tableTemplates"] = csvTableTemplates = OrderedDict() oimTables["facts"] = csvTable = OrderedDict() csvTable["template"] = "facts" csvTable["url"] = "tbd" csvTableTemplates["facts"] = csvTableTemplate = OrderedDict() csvTableTemplate["rowIdColumn"] = "id" csvTableTemplate["dimensions"] = csvTableDimensions = OrderedDict() csvTableTemplate["columns"] = csvFactColumns = OrderedDict() if footnotesRelationshipSet.modelRelationships: csvLinks = OrderedDict() oimReport["links"] = csvLinks aspectQnCol = {} aspectsHeader = [] factsColumns = [] def addAspectQnCol(aspectQn): colQName = str(aspectQn).replace("xbrl:", "") colNCName = colQName.replace(":", "_") if colNCName not in aspectQnCol: aspectQnCol[colNCName] = len(aspectsHeader) aspectsHeader.append(colNCName) if colNCName == "value": csvFactColumns[colNCName] = col = OrderedDict() col["dimensions"] = OrderedDict() elif colNCName == "id": csvFactColumns[colNCName] = {} # empty object elif colNCName == "decimals": csvFactColumns[colNCName] = {} # empty object csvTableTemplate[colQName] = "$" + colNCName else: csvFactColumns[colNCName] = {} # empty object csvTableDimensions[colQName] = "$" + colNCName # pre-ordered aspect columns #if hasId: addAspectQnCol("id") addAspectQnCol(qnOimConceptAspect) if hasNumeric: addAspectQnCol("decimals") if qnOimEntityAspect in aspectsDefined: addAspectQnCol(qnOimEntityAspect) if qnOimPeriodAspect in aspectsDefined: addAspectQnCol(qnOimPeriodAspect) if qnOimUnitAspect in aspectsDefined: addAspectQnCol(qnOimUnitAspect) for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)): if aspectQn.namespaceURI != nsOim: addAspectQnCol(aspectQn) addAspectQnCol("value") def aspectCols(fact): cols = [None for i in range(len(aspectsHeader))] def setColValues(aspects): for aspectQn, aspectValue in aspects.items(): colQName = str(aspectQn).replace("xbrl:", "") colNCName = colQName.replace(":", "_") if isinstance(aspectValue, dict): setColValues(aspectValue) elif colNCName in aspectQnCol: if aspectValue is None: _aspectValue = "#nil" elif aspectValue == "": _aspectValue = "#empty" elif isinstance(aspectValue, str) and aspectValue.startswith("#"): _aspectValue = "#" + aspectValue else: _aspectValue = aspectValue cols[aspectQnCol[colNCName]] = _aspectValue setColValues(factAspects(fact)) return cols # metadata _open = _writerow = _close = None if isCSV: if oimFile.endswith( "-facts.csv" ): # strip -facts.csv if a prior -facts.csv file was chosen _baseURL = oimFile[:-10] elif oimFile.endswith(".csv"): _baseURL = oimFile[:-4] else: _baseURL = oimFile _csvinfo = {} # open file, writer def _open(filesuffix, tabname, csvTable=None): _filename = _baseURL + filesuffix if csvTable is not None: csvTable["url"] = os.path.basename( _filename) # located in same directory with metadata _csvinfo["file"] = open(_filename, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig') _csvinfo["writer"] = csv.writer(_csvinfo["file"], dialect="excel") def _writerow(row, header=False): _csvinfo["writer"].writerow(row) def _close(): _csvinfo["file"].close() _csvinfo.clear() elif isXL: headerWidths = { "concept": 40, "decimals": 8, "language": 9, "value": 50, "entity": 20, "period": 20, "unit": 20, "metadata": 100 } from openpyxl import Workbook from openpyxl.cell.cell import WriteOnlyCell from openpyxl.styles import Font, PatternFill, Border, Alignment, Color, fills, Side from openpyxl.worksheet.dimensions import ColumnDimension hdrCellFill = PatternFill( patternType=fills.FILL_SOLID, fgColor=Color( "00FFBF5F")) # Excel's light orange fill color = 00FF990 workbook = Workbook() # remove pre-existing worksheets while len(workbook.worksheets) > 0: workbook.remove(workbook.worksheets[0]) _xlinfo = {} # open file, writer def _open(filesuffix, tabname, csvTable=None): if csvTable is not None: csvTable["url"] = tabname + "!" _xlinfo["ws"] = workbook.create_sheet(title=tabname) def _writerow(rowvalues, header=False): row = [] _ws = _xlinfo["ws"] for i, v in enumerate(rowvalues): cell = WriteOnlyCell(_ws, value=v) if header: cell.fill = hdrCellFill cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True) colLetter = chr(ord('A') + i) _ws.column_dimensions[colLetter] = ColumnDimension( _ws, customWidth=True) _ws.column_dimensions[ colLetter].width = headerWidths.get(v, 40) else: cell.alignment = Alignment( horizontal="right" if isinstance(v, _NUM_TYPES) else "center" if isinstance(v, bool) else "left", vertical="top", wrap_text=isinstance(v, str)) row.append(cell) _ws.append(row) def _close(): _xlinfo.clear() # save facts _open("-facts.csv", "facts", csvTable) _writerow(aspectsHeader, header=True) def saveCSVfacts(facts): for fact in facts: _writerow(aspectCols(fact)) saveCSVfacts(fact.modelTupleFacts) saveCSVfacts(modelXbrl.facts) _close() # save footnotes if footnotesRelationshipSet.modelRelationships: footnotes = sorted( (footnote for fact in modelXbrl.facts for footnote in factFootnotes(fact, csvLinks=csvLinks)), key=lambda footnote: footnote["id"]) if footnotes: # text footnotes oimTables["footnotes"] = csvFtTable = OrderedDict() csvFtTable["url"] = "tbd" csvFtTable[ "tableDimensions"] = csvFtTableDimensions = OrderedDict() csvFtTable["factColumns"] = csvFtFactColumns = OrderedDict() csvFtFactColumns["footnote"] = csvFtValCol = OrderedDict() csvFtValCol["id"] = "$id" csvFtValCol["noteId"] = "$id" csvFtValCol["concept"] = "xbrl:note" csvFtValCol["language"] = "$language" _open("-footnotes.csv", "footnotes", csvFtTable) cols = ("id", "footnote", "language") _writerow(cols, header=True) for footnote in footnotes: _writerow(tuple((footnote.get(col, "") for col in cols))) _close() # save metadata if isCSV: with open(_baseURL + "-metadata.json", "w", encoding="utf-8") as fh: fh.write( json.dumps(oimReport, ensure_ascii=False, indent=2, sort_keys=False)) elif isXL: _open(None, "metadata") _writerow(["metadata"], header=True) _writerow([ json.dumps(oimReport, ensure_ascii=False, indent=1, sort_keys=False) ]) _close() if isXL: workbook.save(oimFile)
def saveLoadableOIM(modelXbrl, oimFile): isJSON = oimFile.endswith(".json") isCSV = oimFile.endswith(".csv") isXL = oimFile.endswith(".xlsx") isCSVorXL = isCSV or isXL if not isJSON and not isCSVorXL: return namespacePrefixes = {nsOim: "oim"} prefixNamespaces = {"oim": nsOim} def compileQname(qname): if qname.namespaceURI not in namespacePrefixes: namespacePrefixes[qname.namespaceURI] = qname.prefix or "" aspectsDefined = {qnOimConceptAspect, qnOimEntityAspect} if isJSON: aspectsDefined.add(qnOimPeriodAspect) elif isCSVorXL: aspectsDefined.add(qnOimPeriodStartAspect) aspectsDefined.add(qnOimPeriodEndAspect) def oimValue(object, decimals=None): if isinstance(object, QName): if object.namespaceURI not in namespacePrefixes: if object.prefix: namespacePrefixes[object.namespaceURI] = object.prefix else: _prefix = "_{}".format( sum(1 for p in namespacePrefixes if p.startswith("_"))) namespacePrefixes[object.namespaceURI] = _prefix return "{}:{}".format(namespacePrefixes[object.namespaceURI], object.localName) if isinstance(object, Decimal): try: if isinf(object): return "-INF" if object < 0 else "INF" elif isnan(num): return "NaN" else: if object == object.to_integral(): object = object.quantize(ONE) # drop any .0 return "{}".format(object) except: return str(object) if isinstance(object, bool): return object if isinstance(object, (DateTime, YearMonthDuration, DayTimeDuration, Time, gYearMonth, gMonthDay, gYear, gMonth, gDay)): return str(object) return object def oimPeriodValue(cntx): if cntx.isForeverPeriod: pass # not supported elif cntx.isStartEndPeriod: s = cntx.startDatetime e = cntx.endDatetime else: # instant s = e = cntx.instantDatetime return { "start": "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format( s.year, s.month, s.day, s.hour, s.minute, s.second), "end": "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format( e.year, e.month, e.day, e.hour, e.minute, e.second) } hasId = False hasTuple = False hasType = True hasLang = False hasUnits = False hasNumeric = False footnotesRelationshipSet = ModelRelationshipSet(modelXbrl, "XBRL-footnotes") factBaseTypes = set() #compile QNames in instance for OIM for fact in modelXbrl.factsInInstance: if (fact.id or fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))): hasId = True concept = fact.concept if concept is not None: if concept.isNumeric: hasNumeric = True if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang: hasLang = True _baseXsdType = concept.baseXsdType if _baseXsdType == "XBRLI_DATEUNION": if getattr(fact.xValue, "dateOnly", False): _baseXsdType = "date" else: _baseXsdType = "dateTime" factBaseTypes.add(baseTypes.get(_baseXsdType, _baseXsdType)) compileQname(fact.qname) if hasattr(fact, "xValue") and isinstance(fact.xValue, QName): compileQname(fact.xValue) unit = fact.unit if unit is not None: hasUnits = True if fact.modelTupleFacts: hasTuple = True entitySchemePrefixes = {} for cntx in modelXbrl.contexts.values(): if cntx.entityIdentifierElement is not None: scheme = cntx.entityIdentifier[0] if scheme not in entitySchemePrefixes: if not entitySchemePrefixes: # first one is just scheme if scheme == "http://www.sec.gov/CIK": _schemePrefix = "cik" elif scheme == "http://standard.iso.org/iso/17442": _schemePrefix = "lei" else: _schemePrefix = "scheme" else: _schemePrefix = "scheme{}".format( len(entitySchemePrefixes) + 1) entitySchemePrefixes[scheme] = _schemePrefix namespacePrefixes[scheme] = _schemePrefix for dim in cntx.qnameDims.values(): compileQname(dim.dimensionQname) aspectsDefined.add(dim.dimensionQname) if dim.isExplicit: compileQname(dim.memberQname) for unit in modelXbrl.units.values(): if unit is not None: for measures in unit.measures: for measure in measures: compileQname(measure) if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[ XbrlConst.xbrli] != "xbrli": namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix namespacePrefixes[XbrlConst.xsd] = "xsd" if hasLang: aspectsDefined.add(qnOimLangAspect) if hasTuple: aspectsDefined.add(qnOimTupleParentAspect) aspectsDefined.add(qnOimTupleOrderAspect) if hasUnits: aspectsDefined.add(qnOimUnitAspect) # compile footnotes and relationships ''' factRelationships = [] factFootnotes = [] for rel in modelXbrl.relationshipSet(modelXbrl, "XBRL-footnotes").modelRelationships: oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole} factRelationships.append(oimRel) oimRel["fromIds"] = [obj.id if obj.id else elementChildSequence(obj) for obj in rel.fromModelObjects] oimRel["toIds"] = [obj.id if obj.id else elementChildSequence(obj) for obj in rel.toModelObjects] _order = rel.arcElement.get("order") if _order is not None: oimRel["order"] = _order for obj in rel.toModelObjects: if isinstance(obj, ModelResource): # footnote oimFootnote = {"role": obj.role, "id": obj.id if obj.id else elementChildSequence(obj), # value needs work for html elements and for inline footnotes "value": xmlstring(obj, stripXmlns=True)} if obj.xmlLang: oimFootnote["lang"] = obj.xmlLang factFootnotes.append(oimFootnote) oimFootnote ''' dtsReferences = [{ "type": "schema" if doc.type == ModelDocument.Type.SCHEMA else "linkbase" if doc.type == ModelDocument.Type.LINKBASE else "other", "href": doc.uri } for doc, ref in modelXbrl.modelDocument.referencesDocument.items() if ref.referringModelObject.qname in SCHEMA_LB_REFS] ''' roleTypes = [ {"type": "role" if ref.referringModelObject.localName == "roleRef" else "arcroleRef", "href": ref.referringModelObject["href"]} for doc,ref in modelXbrl.modelDocument.referencesDocument.items() if ref.referringModelObject.qname in ROLE_REFS] ''' def factFootnotes(fact): footnotes = [] for footnoteRel in footnotesRelationshipSet.fromModelObject(fact): footnote = OrderedDict((("group", footnoteRel.linkrole), ("footnoteType", footnoteRel.arcrole))) footnotes.append(footnote) if isCSVorXL: footnote["factId"] = fact.id if fact.id else "f{}".format( fact.objectIndex) toObj = footnoteRel.toModelObject if isinstance(toObj, ModelFact): footnote["factRef"] = toObj.id if toObj.id else "f{}".format( toObj.objectIndex) else: footnote["footnote"] = xmlstring(toObj, stripXmlns=True, contentsOnly=True, includeText=True) if toObj.xmlLang: footnote["language"] = toObj.xmlLang return footnotes def factAspects(fact): aspects = OrderedDict() if hasId and fact.id: aspects["id"] = fact.id elif (fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))): aspects["id"] = "f{}".format(fact.objectIndex) parent = fact.getparent() concept = fact.concept _csvType = "Value" if not fact.isTuple: if concept is not None: _baseXsdType = concept.baseXsdType if _baseXsdType == "XBRLI_DATEUNION": if getattr(fact.xValue, "dateOnly", False): _baseXsdType = "date" else: _baseXsdType = "dateTime" aspects["baseType"] = "xsd:{}".format(_baseXsdType) _csvType = baseTypes.get(_baseXsdType, _baseXsdType) + "Value" if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang: aspects[qnOimLangAspect] = fact.xmlLang if fact.isItem: if fact.isNil: _value = None _strValue = "nil" else: _inferredDecimals = inferredDecimals(fact) _value = oimValue(fact.xValue, _inferredDecimals) _strValue = str(_value) if not isCSVorXL: aspects["value"] = _strValue if fact.concept is not None and fact.concept.isNumeric: _numValue = fact.xValue if isinstance(_numValue, Decimal) and not isinf( _numValue) and not isnan(_numValue): if _numValue == _numValue.to_integral(): _numValue = int(_numValue) else: _numValue = float(_numValue) if isCSVorXL: aspects[_csvType] = _numValue else: aspects["numericValue"] = _numValue if not fact.isNil: if isinf(_inferredDecimals): if isJSON: _accuracy = "infinity" elif isCSVorXL: _accuracy = "INF" else: _accuracy = _inferredDecimals aspects["accuracy"] = _accuracy elif isinstance(_value, bool): aspects["booleanValue"] = _value elif isCSVorXL: aspects[_csvType] = _strValue aspects[qnOimConceptAspect] = oimValue(fact.qname) cntx = fact.context if cntx is not None: if cntx.entityIdentifierElement is not None: aspects[qnOimEntityAspect] = oimValue( qname(*cntx.entityIdentifier)) if cntx.period is not None: if isJSON: aspects[qnOimPeriodAspect] = oimPeriodValue(cntx) elif isCSVorXL: _periodValue = oimPeriodValue(cntx) aspects[qnOimPeriodStartAspect] = _periodValue["start"] aspects[qnOimPeriodEndAspect] = _periodValue["end"] for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]): aspects[dim.dimensionQname] = ( oimValue(dim.memberQname) if dim.isExplicit else None if dim.typedMember.get( "{http://www.w3.org/2001/XMLSchema-instance}nil") in ( "true", "1") else dim.typedMember.stringValue) unit = fact.unit if unit is not None: _mMul, _mDiv = unit.measures _sMul = '*'.join( oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m))) if _mDiv: _sDiv = '*'.join( oimValue(m) for m in sorted(_mDiv, key=lambda m: oimValue(m))) if len(_mDiv) > 1: if len(_mMul) > 1: _sUnit = "({})/({})".format(_sMul, _sDiv) else: _sUnit = "{}/({})".format(_sMul, _sDiv) else: if len(_mMul) > 1: _sUnit = "({})/{}".format(_sMul, _sDiv) else: _sUnit = "{}/{}".format(_sMul, _sDiv) else: _sUnit = _sMul aspects[qnOimUnitAspect] = _sUnit if parent.qname != XbrlConst.qnXbrliXbrl: aspects[ qnOimTupleParentAspect] = parent.id if parent.id else "f{}".format( parent.objectIndex) aspects[qnOimTupleOrderAspect] = elementIndex(fact) if isJSON: _footnotes = factFootnotes(fact) if _footnotes: aspects["footnotes"] = _footnotes return aspects if isJSON: # save JSON oimReport = OrderedDict() # top level of oim json output oimFacts = [] oimReport["prefixes"] = OrderedDict( (p, ns) for ns, p in sorted(namespacePrefixes.items(), key=lambda item: item[1])) oimReport["dtsReferences"] = dtsReferences oimReport["facts"] = oimFacts def saveJsonFacts(facts, oimFacts, parentFact): for fact in facts: oimFact = factAspects(fact) oimFacts.append( OrderedDict((str(k), v) for k, v in oimFact.items())) if fact.modelTupleFacts: saveJsonFacts(fact.modelTupleFacts, oimFacts, fact) saveJsonFacts(modelXbrl.facts, oimFacts, None) with open(oimFile, "w", encoding="utf-8") as fh: fh.write(json.dumps(oimReport, indent=1)) elif isCSVorXL: # save CSV aspectQnCol = {} aspectsHeader = [] factsColumns = [] def addAspectQnCol(aspectQn): aspectQnCol[aspectQn] = len(aspectsHeader) _colName = oimValue(aspectQn) aspectsHeader.append(_colName) _colDataType = { "id": "Name", "baseType": "Name", "oim:concept": "QName", "oim:periodStart": "dateTime", "oim:periodEnd": "dateTime", "oim:tupleOrder": "integer", "numericValue": "decimal", "accuracy": "decimal", "booleanValue": "boolean", "oim:entity": "QName", "oim:unit": "string" }.get(_colName) if _colDataType is None: if isCSVorXL and _colName.endswith("Value"): _colDataType = _colName[:-5] else: _colDataType = "string" factsColumns.append( OrderedDict((("name", _colName), ("datatype", _colDataType)))) # pre-ordered aspect columns if hasId: addAspectQnCol("id") addAspectQnCol(qnOimConceptAspect) if hasType: addAspectQnCol("baseType") if isCSVorXL: for _baseType in sorted(factBaseTypes): addAspectQnCol(_baseType + "Value") else: addAspectQnCol("stringValue") addAspectQnCol("numericValue") addAspectQnCol("booleanValue") if hasNumeric: addAspectQnCol("accuracy") if hasTuple: addAspectQnCol(qnOimTupleParentAspect) addAspectQnCol(qnOimTupleOrderAspect) if qnOimEntityAspect in aspectsDefined: addAspectQnCol(qnOimEntityAspect) if qnOimPeriodStartAspect in aspectsDefined: addAspectQnCol(qnOimPeriodStartAspect) addAspectQnCol(qnOimPeriodEndAspect) if qnOimUnitAspect in aspectsDefined: addAspectQnCol(qnOimUnitAspect) for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)): if aspectQn.namespaceURI != nsOim: addAspectQnCol(aspectQn) def aspectCols(fact): cols = [None for i in range(len(aspectsHeader))] _factAspects = factAspects(fact) for aspectQn, aspectValue in _factAspects.items(): if aspectQn in aspectQnCol: cols[aspectQnCol[aspectQn]] = aspectValue return cols # metadata csvTables = [] csvMetadata = OrderedDict( (("@context", ["http://www.w3.org/ns/csvw", { "@base": "./" }]), ("tables", csvTables))) _open = _writerow = _close = None _tableinfo = {} if isCSV: if oimFile.endswith( "-facts.csv" ): # strip -facts.csv if a prior -facts.csv file was chosen _baseURL = oimFile[:-10] elif oimFile.endswith(".csv"): _baseURL = oimFile[:-4] else: _baseURL = oimFile _csvinfo = {} # open file, writer def _open(filesuffix, tabname): _filename = _tableinfo["url"] = _baseURL + filesuffix _csvinfo["file"] = open(_filename, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig') _csvinfo["writer"] = csv.writer(_csvinfo["file"], dialect="excel") def _writerow(row, header=False): _csvinfo["writer"].writerow(row) def _close(): _csvinfo["file"].close() _csvinfo.clear() elif isXL: headerWidths = { "href": 100, "oim:concept": 70, "accuracy": 8, "baseType": 10, "language": 9, "URI": 80, "stringValue": 60, "decimalValue": 12, "booleanValue": 12, "dateValue": 12, "dateTimeValue": 20, "group": 60, "footnoteType": 40, "footnote": 70, "column": 60 } from openpyxl import Workbook from openpyxl.writer.write_only import WriteOnlyCell from openpyxl.styles import Font, PatternFill, Border, Alignment, Color, fills, Side from openpyxl.worksheet.dimensions import ColumnDimension hdrCellFill = PatternFill( patternType=fills.FILL_SOLID, fgColor=Color( "00FFBF5F")) # Excel's light orange fill color = 00FF990 workbook = Workbook(encoding="utf-8", write_only=True) _xlinfo = {} # open file, writer def _open(filesuffix, tabname): _tableinfo["url"] = tabname _xlinfo["ws"] = workbook.create_sheet(title=tabname) def _writerow(rowvalues, header=False): row = [] _ws = _xlinfo["ws"] for i, v in enumerate(rowvalues): cell = WriteOnlyCell(_ws, value=v) if header: cell.fill = hdrCellFill cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True) colLetter = chr(ord('A') + i) _ws.column_dimensions[colLetter] = ColumnDimension( _ws, customWidth=True) _ws.column_dimensions[ colLetter].width = headerWidths.get(v, 20) else: cell.alignment = Alignment( horizontal="right" if isinstance(v, _NUM_TYPES) else "center" if isinstance(v, bool) else "left", vertical="top", wrap_text=isinstance(v, str)) row.append(cell) _ws.append(row) def _close(): _xlinfo.clear() # save facts _open("-facts.csv", "facts") _writerow(aspectsHeader, header=True) def saveCSVfacts(facts): for fact in facts: _writerow(aspectCols(fact)) saveCSVfacts(fact.modelTupleFacts) saveCSVfacts(modelXbrl.facts) _close() factsTableSchema = OrderedDict((("columns", factsColumns), )) csvTables.append( OrderedDict((("url", _tableinfo["url"]), ("tableSchema", factsTableSchema)))) # save namespaces _open("-prefixes.csv", "prefixes") _writerow(("prefix", "URI"), header=True) for _URI, prefix in sorted(namespacePrefixes.items(), key=lambda item: item[1]): _writerow((prefix, _URI)) _close() nsTableSchema = OrderedDict((("columns", [ OrderedDict((("name", "prefix"), ("datatype", "Name"))), OrderedDict((("name", "URI"), ("datatype", "anyURI"))) ]), )) csvTables.append( OrderedDict( (("url", _tableinfo["url"]), ("tableSchema", nsTableSchema)))) # save dts references _open("-dtsReferences.csv", "dtsReferences") _writerow(("type", "href"), header=True) for oimRef in dtsReferences: _writerow((oimRef["type"], oimRef["href"])) _close() dtsRefTableSchema = OrderedDict((("columns", [ OrderedDict((("name", "type"), ("datatype", "Name"))), OrderedDict((("name", "href"), ("datatype", "anyURI"))) ]), )) csvTables.append( OrderedDict((("url", _tableinfo["url"]), ("tableSchema", dtsRefTableSchema)))) # save footnotes if footnotesRelationshipSet.modelRelationships: _open("-footnotes.csv", "footnotes") cols = ("group", "footnoteType", "factId", "factRef", "footnote", "language") _writerow(cols, header=True) def saveCSVfootnotes(facts): for fact in facts: for _footnote in factFootnotes(fact): _writerow( tuple((_footnote.get(col, "") for col in cols))) saveCSVfootnotes(fact.modelTupleFacts) saveCSVfootnotes(modelXbrl.facts) _close() footnoteTableSchema = OrderedDict((("columns", [ OrderedDict((("name", "group"), ("datatype", "anyURI"))), OrderedDict((("name", "footnoteType"), ("datatype", "Name"))), OrderedDict((("name", "factId"), ("datatype", "Name"))), OrderedDict((("name", "factRef"), ("datatype", "Name"))), OrderedDict((("name", "footnote"), ("datatype", "string"))), OrderedDict((("name", "language"), ("datatype", "language"))) ]), )) csvTables.append( OrderedDict((("url", _tableinfo["url"]), ("tableSchema", footnoteTableSchema)))) # save metadata if isCSV: with open(_baseURL + "-metadata.csv", "w", encoding="utf-8") as fh: fh.write( json.dumps(csvMetadata, ensure_ascii=False, indent=1, sort_keys=False)) elif isXL: _open(None, "metadata") _writerow(("table", "column", "datatype"), header=True) for table in csvTables: tablename = table["url"] for column in table["tableSchema"]["columns"]: _writerow((tablename, column["name"], column["datatype"])) _close() if isXL: workbook.save(oimFile)
def saveLoadableOIM(modelXbrl, oimFile, outputZip=None): isJSON = oimFile.endswith(".json") isCSV = oimFile.endswith(".csv") isXL = oimFile.endswith(".xlsx") isCSVorXL = isCSV or isXL if not isJSON and not isCSVorXL: return namespacePrefixes = {nsOim: "xbrl"} prefixNamespaces = {"xbrl": nsOim} def compileQname(qname): if qname.namespaceURI not in namespacePrefixes: namespacePrefixes[qname.namespaceURI] = qname.prefix or "" aspectsDefined = { qnOimConceptAspect, qnOimEntityAspect, qnOimPeriodStartAspect, qnOimPeriodEndAspect} def oimValue(object, decimals=None): if isinstance(object, QName): if object.namespaceURI not in namespacePrefixes: if object.prefix: namespacePrefixes[object.namespaceURI] = object.prefix else: _prefix = "_{}".format(sum(1 for p in namespacePrefixes if p.startswith("_"))) namespacePrefixes[object.namespaceURI] = _prefix return "{}:{}".format(namespacePrefixes[object.namespaceURI], object.localName) if isinstance(object, Decimal): try: if isinf(object): return "-INF" if object < 0 else "INF" elif isnan(num): return "NaN" else: if object == object.to_integral(): object = object.quantize(ONE) # drop any .0 return "{}".format(object) except: return str(object) if isinstance(object, bool): return "true" if object else "false" if isinstance(object, (DateTime, YearMonthDuration, DayTimeDuration, Time, gYearMonth, gMonthDay, gYear, gMonth, gDay, IsoDuration)): return str(object) return object def oimPeriodValue(cntx): if cntx.isForeverPeriod: return OrderedDict() # defaulted elif cntx.isStartEndPeriod: s = cntx.startDatetime e = cntx.endDatetime else: # instant s = e = cntx.instantDatetime if isJSON: return({str(qnOimPeriodAspect): ("{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}/".format( s.year, s.month, s.day, s.hour, s.minute, s.second) if cntx.isStartEndPeriod else "") + "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format( e.year, e.month, e.day, e.hour, e.minute, e.second) }) # CSV, XL return OrderedDict(((str(qnOimPeriodStartAspect), "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format( s.year, s.month, s.day, s.hour, s.minute, s.second)), (str(qnOimPeriodEndAspect), "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format( e.year, e.month, e.day, e.hour, e.minute, e.second)))) hasId = False hasTuple = False hasType = True hasLang = False hasUnits = False hasNumeric = False footnotesRelationshipSet = ModelRelationshipSet(modelXbrl, "XBRL-footnotes") factBaseTypes = set() #compile QNames in instance for OIM for fact in modelXbrl.factsInInstance: if (fact.id or fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))): hasId = True concept = fact.concept if concept is not None: if concept.isNumeric: hasNumeric = True if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang: hasLang = True _baseXsdType = concept.baseXsdType if _baseXsdType == "XBRLI_DATEUNION": if getattr(fact.xValue, "dateOnly", False): _baseXsdType = "date" else: _baseXsdType = "dateTime" factBaseTypes.add(baseTypes.get(_baseXsdType,_baseXsdType)) compileQname(fact.qname) if hasattr(fact, "xValue") and isinstance(fact.xValue, QName): compileQname(fact.xValue) unit = fact.unit if unit is not None: hasUnits = True if fact.modelTupleFacts: hasTuple = True if hasTuple: modelXbrl.error("arelleOIMsaver:tuplesNotAllowed", "Tuples are not allowed in an OIM document", modelObject=modelXbrl) return entitySchemePrefixes = {} for cntx in modelXbrl.contexts.values(): if cntx.entityIdentifierElement is not None: scheme = cntx.entityIdentifier[0] if scheme not in entitySchemePrefixes: if not entitySchemePrefixes: # first one is just scheme if scheme == "http://www.sec.gov/CIK": _schemePrefix = "cik" elif scheme == "http://standard.iso.org/iso/17442": _schemePrefix = "lei" else: _schemePrefix = "scheme" else: _schemePrefix = "scheme{}".format(len(entitySchemePrefixes) + 1) entitySchemePrefixes[scheme] = _schemePrefix namespacePrefixes[scheme] = _schemePrefix for dim in cntx.qnameDims.values(): compileQname(dim.dimensionQname) aspectsDefined.add(dim.dimensionQname) if dim.isExplicit: compileQname(dim.memberQname) for unit in modelXbrl.units.values(): if unit is not None: for measures in unit.measures: for measure in measures: compileQname(measure) if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[XbrlConst.xbrli] != "xbrli": namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix namespacePrefixes[XbrlConst.xsd] = "xsd" if hasLang: aspectsDefined.add(qnOimLangAspect) if hasTuple: aspectsDefined.add(qnOimTupleParentAspect) aspectsDefined.add(qnOimTupleOrderAspect) if hasUnits: aspectsDefined.add(qnOimUnitAspect) for footnoteRel in footnotesRelationshipSet.modelRelationships: typePrefix = "ftTyp_" + os.path.basename(footnoteRel.arcrole) if footnoteRel.linkrole == XbrlConst.defaultLinkRole: groupPrefix = "ftGrp_default" else: groupPrefix = "ftGrp_" + os.path.basename(footnoteRel.linkrole) if typePrefix not in namespacePrefixes: namespacePrefixes[footnoteRel.arcrole] = typePrefix if groupPrefix not in namespacePrefixes: namespacePrefixes[footnoteRel.linkrole] = groupPrefix # compile footnotes and relationships ''' factRelationships = [] factFootnotes = [] for rel in modelXbrl.relationshipSet(modelXbrl, "XBRL-footnotes").modelRelationships: oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole} factRelationships.append(oimRel) oimRel["fromIds"] = [obj.id if obj.id else elementChildSequence(obj) for obj in rel.fromModelObjects] oimRel["toIds"] = [obj.id if obj.id else elementChildSequence(obj) for obj in rel.toModelObjects] _order = rel.arcElement.get("order") if _order is not None: oimRel["order"] = _order for obj in rel.toModelObjects: if isinstance(obj, ModelResource): # footnote oimFootnote = {"role": obj.role, "id": obj.id if obj.id else elementChildSequence(obj), # value needs work for html elements and for inline footnotes "value": xmlstring(obj, stripXmlns=True)} if obj.xmlLang: oimFootnote["lang"] = obj.xmlLang factFootnotes.append(oimFootnote) oimFootnote ''' dtsReferences = set() baseUrl = modelXbrl.modelDocument.uri.partition("#")[0] for doc,ref in sorted(modelXbrl.modelDocument.referencesDocument.items(), key=lambda _item:_item[0].uri): if ref.referringModelObject.qname in SCHEMA_LB_REFS: dtsReferences.add(relativeUri(baseUrl,doc.uri)) for refType in ("role", "arcrole"): for refElt in sorted(modelXbrl.modelDocument.xmlRootElement.iterchildren( "{{http://www.xbrl.org/2003/linkbase}}{}Ref".format(refType)), key=lambda elt:elt.get(refType+"URI") ): dtsReferences.add(refElt.get("{http://www.w3.org/1999/xlink}href").partition("#")[0]) dtsReferences = sorted(dtsReferences) # turn into list footnoteFacts = set() def factFootnotes(fact, oimFact=None): footnotes = [] oimLinks = {} for footnoteRel in footnotesRelationshipSet.fromModelObject(fact): toObj = footnoteRel.toModelObject # json typePrefix = namespacePrefixes[footnoteRel.arcrole] groupPrefix = namespacePrefixes[footnoteRel.linkrole] oimLinks.setdefault(typePrefix,{}).setdefault(groupPrefix,[]).append( toObj.id if toObj.id else "f{}".format(toObj.objectIndex)) # csv/XL footnote = OrderedDict((("group", footnoteRel.linkrole), ("footnoteType", footnoteRel.arcrole))) footnotes.append(footnote) if isCSVorXL: footnote["factId"] = fact.id if fact.id else "f{}".format(fact.objectIndex) if isinstance(toObj, ModelFact): footnote["factRef"] = toObj.id if toObj.id else "f{}".format(toObj.objectIndex) else: footnote["footnote"] = toObj.viewText() if toObj.xmlLang: footnote["language"] = toObj.xmlLang footnoteFacts.add(toObj) if isJSON and oimLinks: oimFact["links"] = OrderedDict(( (typePrefix, OrderedDict(( (groupPrefix, idList) for groupPrefix, idList in sorted(groups.items()) ))) for typePrefix, groups in sorted(oimLinks.items()) )) footnotes.sort(key=lambda f:(f["group"],f.get("factId",f.get("factRef")),f.get("language"))) return footnotes def factAspects(fact): oimFact = OrderedDict() aspects = OrderedDict() if hasId and fact.id: if fact.isTuple: oimFact["tupleId"] = fact.id else: oimFact["id"] = fact.id elif (fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))): oimFact["id"] = "f{}".format(fact.objectIndex) parent = fact.getparent() concept = fact.concept aspects[str(qnOimConceptAspect)] = oimValue(concept.qname) _csvType = "Value" if not fact.isTuple: if concept is not None: _baseXsdType = concept.baseXsdType if _baseXsdType == "XBRLI_DATEUNION": if getattr(fact.xValue, "dateOnly", False): _baseXsdType = "date" else: _baseXsdType = "dateTime" _csvType = baseTypes.get(_baseXsdType,_baseXsdType) + "Value" if concept.baseXbrliType in ("stringItemType", "normalizedStringItemType") and fact.xmlLang: aspects[str(qnOimLangAspect)] = fact.xmlLang if fact.isItem: if fact.isNil: _value = None else: _inferredDecimals = inferredDecimals(fact) _value = oimValue(fact.xValue, _inferredDecimals) oimFact["value"] = _value if fact.concept is not None and fact.concept.isNumeric: _numValue = fact.xValue if isinstance(_numValue, Decimal) and not isinf(_numValue) and not isnan(_numValue): if _numValue == _numValue.to_integral(): _numValue = int(_numValue) else: _numValue = float(_numValue) if not fact.isNil: if not isinf(_inferredDecimals): # accuracy omitted if infinite oimFact["decimals"] = _inferredDecimals oimFact["dimensions"] = aspects cntx = fact.context if cntx is not None: if cntx.entityIdentifierElement is not None: aspects[str(qnOimEntityAspect)] = oimValue(qname(*cntx.entityIdentifier)) if cntx.period is not None: aspects.update(oimPeriodValue(cntx)) for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]): if dim.isExplicit: dimVal = oimValue(dim.memberQname) else: # typed if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1"): dimVal = None else: dimVal = dim.typedMember.stringValue aspects[str(dim.dimensionQname)] = dimVal unit = fact.unit if unit is not None: _mMul, _mDiv = unit.measures _sMul = '*'.join(oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m))) if _mDiv: _sDiv = '*'.join(oimValue(m) for m in sorted(_mDiv, key=lambda m: oimValue(m))) if len(_mDiv) > 1: if len(_mMul) > 1: _sUnit = "({})/({})".format(_sMul,_sDiv) else: _sUnit = "{}/({})".format(_sMul,_sDiv) else: if len(_mMul) > 1: _sUnit = "({})/{}".format(_sMul,_sDiv) else: _sUnit = "{}/{}".format(_sMul,_sDiv) else: _sUnit = _sMul aspects[str(qnOimUnitAspect)] = _sUnit # Tuples removed from xBRL-JSON #if parent.qname != XbrlConst.qnXbrliXbrl: # aspects[str(qnOimTupleParentAspect)] = parent.id if parent.id else "f{}".format(parent.objectIndex) # aspects[str(qnOimTupleOrderAspect)] = elementIndex(fact) if isJSON: factFootnotes(fact, oimFact) return oimFact prefixes = OrderedDict((p,ns) for ns, p in sorted(namespacePrefixes.items(), key=lambda item: item[1])) if isJSON: # save JSON oimReport = OrderedDict() # top level of oim json output oimReport["documentInfo"] = oimDocInfo = OrderedDict() oimReport["facts"] = oimFacts = OrderedDict() oimDocInfo["documentType"] = nsOim + "/xbrl-json" oimDocInfo["features"] = oimFeatures = OrderedDict() oimDocInfo["prefixes"] = prefixes oimDocInfo["taxonomy"] = dtsReferences oimFeatures["xbrl:canonicalValues"] = True def saveJsonFacts(facts, oimFacts, parentFact): for fact in facts: oimFact = factAspects(fact) id = fact.id if fact.id else "f{}".format(fact.objectIndex) oimFacts[id] = oimFact if fact.modelTupleFacts: saveJsonFacts(fact.modelTupleFacts, oimFacts, fact) saveJsonFacts(modelXbrl.facts, oimFacts, None) # add footnotes as pseudo facts for ftObj in footnoteFacts: ftId = ftObj.id if ftObj.id else "f{}".format(ftObj.objectIndex) oimFacts[ftId] = oimFact = OrderedDict() oimFact["value"] = ftObj.viewText() oimFact["dimensions"] = OrderedDict((("concept", "xbrl:note"), ("noteId", ftId))) if ftObj.xmlLang: oimFact["dimensions"]["language"] = ftObj.xmlLang.lower() if outputZip: fh = io.StringIO() else: fh = open(oimFile, "w", encoding="utf-8") fh.write(json.dumps(oimReport, indent=1)) if outputZip: fh.seek(0) outputZip.writestr(os.path.basename(oimFile),fh.read()) fh.close() elif isCSVorXL: # save CSV aspectQnCol = {} aspectsHeader = [] factsColumns = [] def addAspectQnCol(aspectQn): aspectQnCol[str(aspectQn)] = len(aspectsHeader) _aspectQn = oimValue(aspectQn) aspectsHeader.append(_aspectQn) _colName = _aspectQn.replace("xbrl:", "") _colDataType = {"id": "Name", "concept": "QName", "value": "string", "decimals": "decimal", "entity": "QName", "periodStart": "dateTime", "periodEnd": "dateTime", "unit": "string", "tupleId": "Name", "tupleParent": "Name", "tupleOrder": "integer" }.get(_colName, "string") col = OrderedDict((("name", _colName), ("datatype", _colDataType))) if _aspectQn == "value": col["http://xbrl.org/YYYY/model#simpleFactAspects"] = {} elif _aspectQn == "tupleId": col["http://xbrl.org/YYYY/model#tupleFactAspects"] = {} col["http://xbrl.org/YYYY/model#tupleReferenceId"] = "true" else: col["http://xbrl.org/YYYY/model#columnAspect"] = _aspectQn factsColumns.append(col) # pre-ordered aspect columns #if hasId: # addAspectQnCol("id") addAspectQnCol(qnOimConceptAspect) addAspectQnCol("value") if hasNumeric: addAspectQnCol("decimals") if hasTuple: addAspectQnCol("tupleId") addAspectQnCol(qnOimTupleParentAspect) addAspectQnCol(qnOimTupleOrderAspect) if qnOimEntityAspect in aspectsDefined: addAspectQnCol(qnOimEntityAspect) if qnOimPeriodStartAspect in aspectsDefined: addAspectQnCol(qnOimPeriodStartAspect) addAspectQnCol(qnOimPeriodEndAspect) if qnOimUnitAspect in aspectsDefined: addAspectQnCol(qnOimUnitAspect) for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)): if aspectQn.namespaceURI != nsOim: addAspectQnCol(aspectQn) def aspectCols(fact): cols = [None for i in range(len(aspectsHeader))] def setColValues(aspects): for aspectQn, aspectValue in aspects.items(): if isinstance(aspectValue, dict): setColValues(aspectValue) elif aspectQn in aspectQnCol: if aspectValue is None: _aspectValue = "#nil" elif aspectValue == "": _aspectValue = "#empty" elif isinstance(aspectValue, str) and aspectValue.startswith("#"): _aspectValue = "#" + aspectValue else: _aspectValue = aspectValue cols[aspectQnCol[aspectQn]] = _aspectValue setColValues(factAspects(fact)) return cols # metadata csvTables = [] csvMetadata = OrderedDict((("@context", "http://www.w3.org/ns/csvw"), ("http://xbrl.org/YYYY/model#metadata", OrderedDict((("documentType", "http://xbrl.org/YYYY/xbrl-csv"), ("dtsReferences", dtsReferences), ("prefixes", prefixes)))), ("tables", csvTables))) _open = _writerow = _close = None _tableinfo = {} if isCSV: if oimFile.endswith("-facts.csv"): # strip -facts.csv if a prior -facts.csv file was chosen _baseURL = oimFile[:-10] elif oimFile.endswith(".csv"): _baseURL = oimFile[:-4] else: _baseURL = oimFile _csvinfo = {} # open file, writer def _open(filesuffix, tabname): _filename = _tableinfo["url"] = _baseURL + filesuffix _csvinfo["file"] = open(_filename, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig') _csvinfo["writer"] = csv.writer(_csvinfo["file"], dialect="excel") def _writerow(row, header=False): _csvinfo["writer"].writerow(row) def _close(): _csvinfo["file"].close() _csvinfo.clear() elif isXL: headerWidths = {"href": 100, "xbrl:concept": 70, "decimals": 8, "language": 9, "URI": 80, "value": 60, "group": 60, "footnoteType": 40, "footnote": 70, "column": 20, 'conceptAspect': 40, 'tuple': 20, 'simpleFact': 20} from openpyxl import Workbook from openpyxl.writer.write_only import WriteOnlyCell from openpyxl.styles import Font, PatternFill, Border, Alignment, Color, fills, Side from openpyxl.worksheet.dimensions import ColumnDimension hdrCellFill = PatternFill(patternType=fills.FILL_SOLID, fgColor=Color("00FFBF5F")) # Excel's light orange fill color = 00FF990 workbook = Workbook() # remove pre-existing worksheets while len(workbook.worksheets)>0: workbook.remove_sheet(workbook.worksheets[0]) _xlinfo = {} # open file, writer def _open(filesuffix, tabname): _tableinfo["url"] = tabname _xlinfo["ws"] = workbook.create_sheet(title=tabname) def _writerow(rowvalues, header=False): row = [] _ws = _xlinfo["ws"] for i, v in enumerate(rowvalues): cell = WriteOnlyCell(_ws, value=v) if header: cell.fill = hdrCellFill cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True) colLetter = chr(ord('A') + i) _ws.column_dimensions[colLetter] = ColumnDimension(_ws, customWidth=True) _ws.column_dimensions[colLetter].width = headerWidths.get(v, 20) else: cell.alignment = Alignment(horizontal="right" if isinstance(v, _NUM_TYPES) else "center" if isinstance(v, bool) else "left", vertical="top", wrap_text=isinstance(v, str)) row.append(cell) _ws.append(row) def _close(): _xlinfo.clear() # save facts _open("-facts.csv", "facts") _writerow(aspectsHeader, header=True) def saveCSVfacts(facts): for fact in facts: _writerow(aspectCols(fact)) saveCSVfacts(fact.modelTupleFacts) saveCSVfacts(modelXbrl.facts) _close() factsTableSchema = OrderedDict((("columns",factsColumns),)) csvTables.append(OrderedDict((("url",_tableinfo["url"]), ("http://xbrl.org/YYYY/model#tableType", "fact"), ("tableSchema",factsTableSchema)))) # save footnotes if footnotesRelationshipSet.modelRelationships: _open("-footnotes.csv", "footnotes") cols = ("group", "footnoteType", "factId", "factRef", "footnote", "language") _writerow(cols, header=True) def saveCSVfootnotes(facts): for fact in facts: for _footnote in factFootnotes(fact): _writerow(tuple((_footnote.get(col,"") for col in cols))) saveCSVfootnotes(fact.modelTupleFacts) saveCSVfootnotes(modelXbrl.facts) _close() footnoteTableSchema = OrderedDict((("columns",[OrderedDict((("name","group"),("datatype","anyURI"))), OrderedDict((("name","footnoteType"),("datatype","Name"))), OrderedDict((("name","factId"),("datatype","Name"))), OrderedDict((("name","factRef"),("datatype","Name"))), OrderedDict((("name","footnote"),("datatype","string"))), OrderedDict((("name","language"),("datatype","language")))]),)) csvTables.append(OrderedDict((("url",_tableinfo["url"]), ("http://xbrl.org/YYYY/model#tableType", "footnote"), ("tableSchema",footnoteTableSchema)))) # save metadata if isCSV: with open(_baseURL + "-metadata.json", "w", encoding="utf-8") as fh: fh.write(json.dumps(csvMetadata, ensure_ascii=False, indent=1, sort_keys=False)) elif isXL: _open(None, "metadata") hasColumnAspect = hasSimpleFact = hasTupleFact = False for table in csvTables: tablename = table["url"] for column in table["tableSchema"]["columns"]: if "http://xbrl.org/YYYY/model#columnAspect" in column: hasColumnAspect = True if "http://xbrl.org/YYYY/model#simpleFactAspects" in column: hasSimpleFact = True if "http://xbrl.org/YYYY/model#tupleAspects" in column: hasTupleFact = True metadataCols = ["table", "column", "datatype"] if hasColumnAspect: metadataCols.append("columnAspect") if hasSimpleFact: metadataCols.append("simpleFact") if hasTupleFact: metadataCols.append("tuple") _writerow(metadataCols, header=True) for table in csvTables: tablename = table["url"] for column in table["tableSchema"]["columns"]: row = [tablename, column["name"], column["datatype"]] if hasColumnAspect: colAspect = column.get("http://xbrl.org/YYYY/model#columnAspect") if isinstance(colAspect, str): row.append(colAspect) elif isinstance(colAspect, dict): row.append("\n".join("{} [{}]".format(k, ", ".join(_v for _v in v)) for k, v in dict.items())) else: row.append(None) if hasSimpleFact: row.append("\u221a" if "http://xbrl.org/YYYY/model#simpleFactAspects" in column else None) if hasTupleFact: row.append("\u221a" if "http://xbrl.org/YYYY/model#tupleAspects" in column else None) _writerow(row) _close() if isXL: workbook.save(oimFile)
def saveLoadableOIM(modelXbrl, oimFile): isJSON = oimFile.endswith(".json") isCSV = oimFile.endswith(".csv") isXL = oimFile.endswith(".xlsx") isCSVorXL = isCSV or isXL if not isJSON and not isCSVorXL: return namespacePrefixes = {} prefixNamespaces = {} def compileQname(qname): if qname.namespaceURI not in namespacePrefixes: namespacePrefixes[qname.namespaceURI] = qname.prefix or "" aspectsDefined = { qnOimConceptAspect, qnOimEntityAspect} if isJSON: aspectsDefined.add(qnOimPeriodAspect) elif isCSVorXL: aspectsDefined.add(qnOimPeriodStartAspect) aspectsDefined.add(qnOimPeriodEndAspect) def oimValue(object, decimals=None): if isinstance(object, QName): if object.namespaceURI not in namespacePrefixes: if object.prefix: namespacePrefixes[object.namespaceURI] = object.prefix else: _prefix = "_{}".format(sum(1 for p in namespacePrefixes if p.startswith("_"))) namespacePrefixes[object.namespaceURI] = _prefix return "{}:{}".format(namespacePrefixes[object.namespaceURI], object.localName) if isinstance(object, Decimal): try: if isinf(object): return "-INF" if object < 0 else "INF" elif isnan(num): return "NaN" else: if object == object.to_integral(): object = object.quantize(ONE) # drop any .0 return "{}".format(object) except: return str(object) if isinstance(object, bool): return object if isinstance(object, (DateTime, YearMonthDuration, DayTimeDuration, Time, gYearMonth, gMonthDay, gYear, gMonth, gDay)): return str(object) return object def oimPeriodValue(cntx): if cntx.isForeverPeriod: pass # not supported elif cntx.isStartEndPeriod: s = cntx.startDatetime e = cntx.endDatetime else: # instant s = e = cntx.instantDatetime return {"start": "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format( s.year, s.month, s.day, s.hour, s.minute, s.second), "end": "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format( e.year, e.month, e.day, e.hour, e.minute, e.second)} hasId = False hasTuple = False hasType = True hasLang = False hasUnits = False footnotesRelationshipSet = ModelRelationshipSet(modelXbrl, "XBRL-footnotes") factBaseTypes = set() #compile QNames in instance for OIM for fact in modelXbrl.factsInInstance: if (fact.id or fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))): hasId = True concept = fact.concept if concept is not None: if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang: hasLang = True _baseXsdType = concept.baseXsdType if _baseXsdType == "XBRLI_DATEUNION": if getattr(fact.xValue, "dateOnly", False): _baseXsdType = "date" else: _baseXsdType = "dateTime" factBaseTypes.add(baseTypes.get(_baseXsdType,_baseXsdType)) compileQname(fact.qname) if hasattr(fact, "xValue") and isinstance(fact.xValue, QName): compileQname(fact.xValue) unit = fact.unit if unit is not None: hasUnits = True if fact.modelTupleFacts: hasTuple = True entitySchemePrefixes = {} for cntx in modelXbrl.contexts.values(): if cntx.entityIdentifierElement is not None: scheme = cntx.entityIdentifier[0] if scheme not in entitySchemePrefixes: if not entitySchemePrefixes: # first one is just scheme if scheme == "http://www.sec.gov/CIK": _schemePrefix = "cik" elif scheme == "http://standard.iso.org/iso/17442": _schemePrefix = "lei" else: _schemePrefix = "scheme" else: _schemePrefix = "scheme{}".format(len(entitySchemePrefixes) + 1) entitySchemePrefixes[scheme] = _schemePrefix namespacePrefixes[scheme] = _schemePrefix for dim in cntx.qnameDims.values(): compileQname(dim.dimensionQname) aspectsDefined.add(dim.dimensionQname) if dim.isExplicit: compileQname(dim.memberQname) for unit in modelXbrl.units.values(): if unit is not None: for measures in unit.measures: for measure in measures: compileQname(measure) if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[XbrlConst.xbrli] != "xbrli": namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix namespacePrefixes[XbrlConst.xsd] = "xsd" if hasLang: aspectsDefined.add(qnOimLangAspect) if hasTuple: aspectsDefined.add(qnOimTupleParentAspect) aspectsDefined.add(qnOimTupleOrderAspect) if hasUnits: aspectsDefined.add(qnOimUnitAspect) # compile footnotes and relationships ''' factRelationships = [] factFootnotes = [] for rel in modelXbrl.relationshipSet(modelXbrl, "XBRL-footnotes").modelRelationships: oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole} factRelationships.append(oimRel) oimRel["fromIds"] = [obj.id if obj.id else elementChildSequence(obj) for obj in rel.fromModelObjects] oimRel["toIds"] = [obj.id if obj.id else elementChildSequence(obj) for obj in rel.toModelObjects] _order = rel.arcElement.get("order") if _order is not None: oimRel["order"] = _order for obj in rel.toModelObjects: if isinstance(obj, ModelResource): # footnote oimFootnote = {"role": obj.role, "id": obj.id if obj.id else elementChildSequence(obj), # value needs work for html elements and for inline footnotes "value": xmlstring(obj, stripXmlns=True)} if obj.xmlLang: oimFootnote["lang"] = obj.xmlLang factFootnotes.append(oimFootnote) oimFootnote ''' dtsReferences = [ {"type": "schema" if doc.type == ModelDocument.Type.SCHEMA else "linkbase" if doc.type == ModelDocument.Type.LINKBASE else "other", "href": doc.uri} for doc,ref in modelXbrl.modelDocument.referencesDocument.items() if ref.referringModelObject.qname in SCHEMA_LB_REFS] ''' roleTypes = [ {"type": "role" if ref.referringModelObject.localName == "roleRef" else "arcroleRef", "href": ref.referringModelObject["href"]} for doc,ref in modelXbrl.modelDocument.referencesDocument.items() if ref.referringModelObject.qname in ROLE_REFS] ''' def factFootnotes(fact): footnotes = [] for footnoteRel in footnotesRelationshipSet.fromModelObject(fact): footnote = OrderedDict((("group", footnoteRel.linkrole), ("footnoteType", footnoteRel.arcrole))) footnotes.append(footnote) if isCSVorXL: footnote["factId"] = fact.id if fact.id else "f{}".format(fact.objectIndex) toObj = footnoteRel.toModelObject if isinstance(toObj, ModelFact): footnote["factRef"] = toObj.id if toObj.id else "f{}".format(toObj.objectIndex) else: footnote["footnote"] = xmlstring(toObj, stripXmlns=True, contentsOnly=True, includeText=True) if toObj.xmlLang: footnote["language"] = toObj.xmlLang return footnotes def factAspects(fact): aspects = OrderedDict() if hasId and fact.id: aspects["id"] = fact.id elif (fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))): aspects["id"] = "f{}".format(fact.objectIndex) parent = fact.getparent() concept = fact.concept _csvType = "Value" if not fact.isTuple: if concept is not None: _baseXsdType = concept.baseXsdType if _baseXsdType == "XBRLI_DATEUNION": if getattr(fact.xValue, "dateOnly", False): _baseXsdType = "date" else: _baseXsdType = "dateTime" aspects["baseType"] = "xs:{}".format(_baseXsdType) _csvType = baseTypes.get(_baseXsdType,_baseXsdType) + "Value" if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang: aspects[qnOimLangAspect] = fact.xmlLang if fact.isItem: if fact.isNil: _value = None _strValue = "nil" else: _inferredDecimals = inferredDecimals(fact) _value = oimValue(fact.xValue, _inferredDecimals) _strValue = str(_value) if not isCSVorXL: aspects["value"] = _strValue if fact.concept is not None and fact.concept.isNumeric: _numValue = fact.xValue if isinstance(_numValue, Decimal) and not isinf(_numValue) and not isnan(_numValue): if _numValue == _numValue.to_integral(): _numValue = int(_numValue) else: _numValue = float(_numValue) if isCSVorXL: aspects[_csvType] = _numValue else: aspects["numericValue"] = _numValue if not fact.isNil: if isinf(_inferredDecimals): if isJSON: _accuracy = "infinity" elif isCSVorXL: _accuracy = "INF" else: _accuracy = _inferredDecimals aspects["accuracy"] = _accuracy elif isinstance(_value, bool): aspects["booleanValue"] = _value elif isCSVorXL: aspects[_csvType] = _strValue aspects[qnOimConceptAspect] = oimValue(fact.qname) cntx = fact.context if cntx is not None: if cntx.entityIdentifierElement is not None: aspects[qnOimEntityAspect] = oimValue(qname(*cntx.entityIdentifier)) if cntx.period is not None: if isJSON: aspects[qnOimPeriodAspect] = oimPeriodValue(cntx) elif isCSVorXL: _periodValue = oimPeriodValue(cntx) aspects[qnOimPeriodStartAspect] = _periodValue["start"] aspects[qnOimPeriodEndAspect] = _periodValue["end"] for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]): aspects[dim.dimensionQname] = (oimValue(dim.memberQname) if dim.isExplicit else None if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1") else dim.typedMember.stringValue) unit = fact.unit if unit is not None: _mMul, _mDiv = unit.measures _sMul = '*'.join(oimValue(m) for m in sorted(_mMul, key=lambda m: str(m))) if _mDiv: _sDiv = '*'.join(oimValue(m) for m in sorted(_mDiv, key=lambda m: str(m))) if len(mDiv) > 1: if len(mMul) > 1: _sUnit = "({})/({})".format(_sMul,_sDiv) else: _sUnit = "{}/({})".format(_sMul,_sDiv) else: if len(mMul) > 1: _sUnit = "({})/{}".format(_sMul,_sDiv) else: _sUnit = "{}/{}".format(_sMul,_sDiv) else: _sUnit = _sMul aspects[qnOimUnitAspect] = _sUnit if parent.qname != XbrlConst.qnXbrliXbrl: aspects[qnOimTupleParentAspect] = parent.id if parent.id else "f{}".format(parent.objectIndex) aspects[qnOimTupleOrderAspect] = elementIndex(fact) if isJSON: _footnotes = factFootnotes(fact) if _footnotes: aspects["footnotes"] = _footnotes return aspects if isJSON: # save JSON oimReport = OrderedDict() # top level of oim json output oimFacts = [] oimReport["prefixes"] = OrderedDict((p,ns) for ns, p in sorted(namespacePrefixes.items(), key=lambda item: item[1])) oimReport["dtsReferences"] = dtsReferences oimReport["facts"] = oimFacts def saveJsonFacts(facts, oimFacts, parentFact): for fact in facts: oimFact = factAspects(fact) oimFacts.append(OrderedDict((str(k),v) for k,v in oimFact.items())) if fact.modelTupleFacts: saveJsonFacts(fact.modelTupleFacts, oimFacts, fact) saveJsonFacts(modelXbrl.facts, oimFacts, None) with open(oimFile, "w", encoding="utf-8") as fh: fh.write(json.dumps(oimReport, ensure_ascii=False, indent=1, sort_keys=False)) elif isCSVorXL: # save CSV aspectQnCol = {} aspectsHeader = [] factsColumns = [] def addAspectQnCol(aspectQn): aspectQnCol[aspectQn] = len(aspectsHeader) _colName = oimValue(aspectQn) aspectsHeader.append(_colName) _colDataType = {"id": "Name", "baseType": "Name", "oim:concept": "QName", "oim:periodStart": "dateTime", "oim:periodEnd": "dateTime", "oim:tupleOrder": "integer", "numericValue": "decimal", "accuracy": "decimal", "booleanValue": "boolean", "oim:entity": "QName", "oim:unit": "string" }.get(_colName) if _colDataType is None: if isCSVorXL and _colName.endswith("Value"): _colDataType = _colName[:-5] else: _colDataType = "string" factsColumns.append(OrderedDict((("name", _colName), ("datatype", _colDataType)))) # pre-ordered aspect columns if hasId: addAspectQnCol("id") addAspectQnCol(qnOimConceptAspect) if hasType: addAspectQnCol("baseType") if isCSVorXL: for _baseType in sorted(factBaseTypes): addAspectQnCol(_baseType + "Value") else: addAspectQnCol("stringValue") addAspectQnCol("numericValue") addAspectQnCol("booleanValue") addAspectQnCol("accuracy") if hasTuple: addAspectQnCol(qnOimTupleParentAspect) addAspectQnCol(qnOimTupleOrderAspect) if qnOimEntityAspect in aspectsDefined: addAspectQnCol(qnOimEntityAspect) if qnOimPeriodStartAspect in aspectsDefined: addAspectQnCol(qnOimPeriodStartAspect) addAspectQnCol(qnOimPeriodEndAspect) if qnOimUnitAspect in aspectsDefined: addAspectQnCol(qnOimUnitAspect) for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)): if aspectQn.namespaceURI != nsOim: addAspectQnCol(aspectQn) def aspectCols(fact): cols = [None for i in range(len(aspectsHeader))] _factAspects = factAspects(fact) for aspectQn, aspectValue in _factAspects.items(): if aspectQn in aspectQnCol: cols[aspectQnCol[aspectQn]] = aspectValue return cols # metadata csvTables = [] csvMetadata = OrderedDict((("@context",[ "http://www.w3.org/ns/csvw", { "@base": "./" }]), ("tables", csvTables))) _open = _writerow = _close = None _tableinfo = {} if isCSV: if oimFile.endswith("-facts.csv"): # strip -facts.csv if a prior -facts.csv file was chosen _baseURL = oimFile[:-10] elif oimFile.endswith(".csv"): _baseURL = oimFile[:-4] else: _baseURL = oimFile _csvinfo = {} # open file, writer def _open(filesuffix, tabname): _filename = _tableinfo["url"] = _baseURL + filesuffix _csvinfo["file"] = open(_filename, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig') _csvinfo["writer"] = csv.writer(csvFile, dialect="excel") def _writerow(row, header=False): _csvinfo["writer"].writerow(row) def _close(): _csvinfo["writer"].close() _csvinfo.clear() elif isXL: headerWidths = {"href": 100, "oim:concept": 70, "accuracy": 8, "baseType": 10, "language": 9, "URI": 80, "stringValue": 60, "decimalValue": 12, "booleanValue": 12, "dateValue": 12, "dateTimeValue": 20, "group": 60, "footnoteType": 40, "footnote": 70, "column": 60} from openpyxl import Workbook from openpyxl.writer.write_only import WriteOnlyCell from openpyxl.styles import Font, PatternFill, Border, Alignment, Color, fills, Side from openpyxl.worksheet.dimensions import ColumnDimension hdrCellFill = PatternFill(patternType=fills.FILL_SOLID, fgColor=Color("00FFBF5F")) # Excel's light orange fill color = 00FF990 workbook = Workbook(encoding="utf-8", write_only=True) _xlinfo = {} # open file, writer def _open(filesuffix, tabname): _tableinfo["url"] = tabname _xlinfo["ws"] = workbook.create_sheet(title=tabname) def _writerow(rowvalues, header=False): row = [] _ws = _xlinfo["ws"] for i, v in enumerate(rowvalues): cell = WriteOnlyCell(_ws, value=v) if header: cell.fill = hdrCellFill cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True) colLetter = chr(ord('A') + i) _ws.column_dimensions[colLetter] = ColumnDimension(_ws, customWidth=True) _ws.column_dimensions[colLetter].width = headerWidths.get(v, 20) else: cell.alignment = Alignment(horizontal="right" if isinstance(v, _NUM_TYPES) else "center" if isinstance(v, bool) else "left", vertical="top", wrap_text=isinstance(v, str)) row.append(cell) _ws.append(row) def _close(): _xlinfo.clear() # save facts _open("-facts.csv", "facts") _writerow(aspectsHeader, header=True) def saveCSVfacts(facts): for fact in facts: _writerow(aspectCols(fact)) saveCSVfacts(fact.modelTupleFacts) saveCSVfacts(modelXbrl.facts) _close() factsTableSchema = OrderedDict((("columns",factsColumns),)) csvTables.append(OrderedDict((("url",_tableinfo["url"]), ("tableSchema",factsTableSchema)))) # save namespaces _open("-prefixes.csv", "prefixes") _writerow(("prefix", "URI"), header=True) for _URI, prefix in sorted(namespacePrefixes.items(), key=lambda item: item[1]): _writerow((prefix, _URI)) _close() nsTableSchema = OrderedDict((("columns",[OrderedDict((("name","prefix"),("datatype","Name"))), OrderedDict((("name","URI"),("datatype","anyURI")))]),)) csvTables.append(OrderedDict((("url",_tableinfo["url"]), ("tableSchema",nsTableSchema)))) # save dts references _open("-dtsReferences.csv", "dtsReferences") _writerow(("type", "href"), header=True) for oimRef in dtsReferences: _writerow((oimRef["type"], oimRef["href"])) _close() dtsRefTableSchema = OrderedDict((("columns",[OrderedDict((("name","type"),("datatype","Name"))), OrderedDict((("name","href"),("datatype","anyURI")))]),)) csvTables.append(OrderedDict((("url",_tableinfo["url"]), ("tableSchema",dtsRefTableSchema)))) # save footnotes if footnotesRelationshipSet.modelRelationships: _open("-footnotes.csv", "footnotes") cols = ("group", "footnoteType", "factId", "factRef", "footnote", "language") _writerow(cols, header=True) def saveCSVfootnotes(facts): for fact in facts: for _footnote in factFootnotes(fact): _writerow(tuple((_footnote.get(col,"") for col in cols))) saveCSVfootnotes(fact.modelTupleFacts) saveCSVfootnotes(modelXbrl.facts) _close() footnoteTableSchema = OrderedDict((("columns",[OrderedDict((("name","group"),("datatype","anyURI"))), OrderedDict((("name","footnoteType"),("datatype","Name"))), OrderedDict((("name","factId"),("datatype","Name"))), OrderedDict((("name","factRef"),("datatype","Name"))), OrderedDict((("name","footnote"),("datatype","string"))), OrderedDict((("name","language"),("datatype","language")))]),)) csvTables.append(OrderedDict((("url",_tableinfo["url"]), ("tableSchema",footnoteTableSchema)))) # save metadata if isCSV: with open(_baseURL + "-metadata.csv", "w", encoding="utf-8") as fh: fh.write(json.dumps(csvMetadata, ensure_ascii=False, indent=1, sort_keys=False)) elif isXL: _open(None, "metadata") cols = ("group", "footnoteType", "factId", "factRef", "footnote", "language") _writerow(("table", "column", "datatype"), header=True) for table in csvTables: tablename = table["url"] for column in table["tableSchema"]["columns"]: _writerow((tablename, column["name"], column["datatype"])) _close() if isXL: workbook.save(oimFile)
def saveLoadableOIM(modelXbrl, oimFile): isJSON = oimFile.endswith(".json") isCSV = oimFile.endswith(".csv") namespacePrefixes = {} def compileQname(qname): if qname.namespaceURI not in namespacePrefixes: namespacePrefixes[qname.namespaceURI] = qname.prefix or "" aspectsDefined = { qnOimConceptAspect, qnOimEntityAspect, qnOimTypeAspect} if isJSON: aspectsDefined.add(qnOimPeriodAspect) elif isCSV: aspectsDefined.add(qnOimPeriodStartAspect) aspectsDefined.add(qnOimPeriodDurationAspect) def oimValue(object, decimals=None): if isinstance(object, QName): if object.namespaceURI not in namespacePrefixes: if object.prefix: namespacePrefixes[object.namespaceURI] = object.prefix else: _prefix = "_{}".format(sum(1 for p in namespacePrefixes if p.startswith("_"))) namespacePrefixes[object.namespaceURI] = _prefix return "{}:{}".format(namespacePrefixes[object.namespaceURI], object.localName) if isinstance(object, Decimal): try: if isinf(object): return "-INF" if object < 0 else "INF" elif isnan(num): return "NaN" else: if object == object.to_integral(): object = object.quantize(ONE) # drop any .0 return "{}".format(object) except: return str(object) if isinstance(object, bool): return object if isinstance(object, (DateTime, YearMonthDuration, DayTimeDuration, Time, gYearMonth, gMonthDay, gYear, gMonth, gDay)): return str(object) return object def oimPeriodValue(cntx): if cntx.isForeverPeriod: if isCSV: return "0000-01-01T00:00:00/P9999Y" return "forever" elif cntx.isStartEndPeriod: d = cntx.startDatetime duration = yearMonthDayTimeDuration(cntx.startDatetime, cntx.endDatetime) else: # instant d = cntx.instantDatetime duration = "PT0S" return "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}/{6}".format( d.year, d.month, d.day, d.hour, d.minute, d.second, duration) hasId = False hasTuple = False hasType = True hasLang = False hasUnits = False hasUnitMulMeasures = False hasUnitDivMeasures = False footnotesRelationshipSet = ModelRelationshipSet(modelXbrl, "XBRL-footnotes") #compile QNames in instance for OIM for fact in modelXbrl.factsInInstance: if (fact.id or fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSV and footnotesRelationshipSet.fromModelObject(fact))): hasId = True concept = fact.concept if concept is not None: if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang: hasLang = True compileQname(fact.qname) if hasattr(fact, "xValue") and isinstance(fact.xValue, QName): compileQname(fact.xValue) unit = fact.unit if unit is not None: hasUnits = True if unit.measures[0]: hasUnitMulMeasures = True if unit.measures[1]: hasUnitDivMeasures = True if fact.modelTupleFacts: hasTuple = True entitySchemePrefixes = {} for cntx in modelXbrl.contexts.values(): if cntx.entityIdentifierElement is not None: scheme = cntx.entityIdentifier[0] if scheme not in entitySchemePrefixes: if not entitySchemePrefixes: # first one is just scheme if scheme == "http://www.sec.gov/CIK": _schemePrefix = "cik" elif scheme == "http://standard.iso.org/iso/17442": _schemePrefix = "lei" else: _schemePrefix = "scheme" else: _schemePrefix = "scheme{}".format(len(entitySchemePrefixes) + 1) entitySchemePrefixes[scheme] = _schemePrefix namespacePrefixes[scheme] = _schemePrefix for dim in cntx.qnameDims.values(): compileQname(dim.dimensionQname) aspectsDefined.add(dim.dimensionQname) if dim.isExplicit: compileQname(dim.memberQname) for unit in modelXbrl.units.values(): if unit is not None: for measures in unit.measures: for measure in measures: compileQname(measure) if unit.measures[0]: aspectsDefined.add(qnOimUnitNumeratorsAspect) if unit.measures[1]: aspectsDefined.add(qnOimUnitDenominatorsAspect) if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[XbrlConst.xbrli] != "xbrli": namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix namespacePrefixes[XbrlConst.xsd] = "xsd" if hasLang: aspectsDefined.add(qnOimLangAspect) if hasTuple: aspectsDefined.add(qnOimTupleParentAspect) aspectsDefined.add(qnOimTupleOrderAspect) if hasUnits: aspectsDefined.add(qnOimUnitAspect) # compile footnotes and relationships ''' factRelationships = [] factFootnotes = [] for rel in modelXbrl.relationshipSet(modelXbrl, "XBRL-footnotes").modelRelationships: oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole} factRelationships.append(oimRel) oimRel["fromIds"] = [obj.id if obj.id else elementChildSequence(obj) for obj in rel.fromModelObjects] oimRel["toIds"] = [obj.id if obj.id else elementChildSequence(obj) for obj in rel.toModelObjects] _order = rel.arcElement.get("order") if _order is not None: oimRel["order"] = _order for obj in rel.toModelObjects: if isinstance(obj, ModelResource): # footnote oimFootnote = {"role": obj.role, "id": obj.id if obj.id else elementChildSequence(obj), # value needs work for html elements and for inline footnotes "value": xmlstring(obj, stripXmlns=True)} if obj.xmlLang: oimFootnote["lang"] = obj.xmlLang factFootnotes.append(oimFootnote) oimFootnote ''' dtsReferences = [ {"type": "schema" if doc.type == ModelDocument.Type.SCHEMA else "linkbase" if doc.type == ModelDocument.Type.LINKBASE else "other", "href": doc.uri} for doc,ref in modelXbrl.modelDocument.referencesDocument.items() if ref.referringModelObject.qname in SCHEMA_LB_REFS] ''' roleTypes = [ {"type": "role" if ref.referringModelObject.localName == "roleRef" else "arcroleRef", "href": ref.referringModelObject["href"]} for doc,ref in modelXbrl.modelDocument.referencesDocument.items() if ref.referringModelObject.qname in ROLE_REFS] ''' def factFootnotes(fact): footnotes = [] for footnoteRel in footnotesRelationshipSet.fromModelObject(fact): footnote = OrderedDict((("group", footnoteRel.arcrole),)) footnotes.append(footnote) if isCSV: footnote["factId"] = fact.id if fact.id else "f{}".format(fact.objectIndex) toObj = footnoteRel.toModelObject if isinstance(toObj, ModelFact): footnote["factRef"] = toObj.id if toObj.id else "f{}".format(toObj.objectIndex) else: footnote["footnoteType"] = toObj.role footnote["footnote"] = xmlstring(toObj, stripXmlns=True, contentsOnly=True, includeText=True) if toObj.xmlLang: footnote["language"] = toObj.xmlLang return footnotes def factAspects(fact): aspects = OrderedDict() if hasId and fact.id: aspects["id"] = fact.id elif (fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSV and footnotesRelationshipSet.fromModelObject(fact))): aspects["id"] = "f{}".format(fact.objectIndex) parent = fact.getparent() concept = fact.concept if not fact.isTuple: if concept is not None: _baseXsdType = concept.baseXsdType if _baseXsdType == "XBRLI_DATEUNION": if getattr(fact.xValue, "dateOnly", False): _baseXsdType = "date" else: _baseXsdType = "dateTime" aspects["baseType"] = "xs:{}".format(_baseXsdType) if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang: aspects[qnOimLangAspect] = fact.xmlLang aspects[qnOimTypeAspect] = concept.baseXbrliType if fact.isItem: if fact.isNil: _value = None _strValue = "nil" else: _inferredDecimals = inferredDecimals(fact) _value = oimValue(fact.xValue, _inferredDecimals) _strValue = str(_value) if not isCSV: aspects["value"] = _strValue if fact.concept is not None and fact.concept.isNumeric: _numValue = fact.xValue if isinstance(_numValue, Decimal) and not isinf(_numValue) and not isnan(_numValue): if _numValue == _numValue.to_integral(): _numValue = int(_numValue) else: _numValue = float(_numValue) aspects["numericValue"] = _numValue if not fact.isNil: if isinf(_inferredDecimals): if isJSON: _accuracy = "infinity" elif isCSV: _accuracy = "INF" else: _accuracy = _inferredDecimals aspects["accuracy"] = _inferredDecimals elif isinstance(_value, bool): aspects["booleanValue"] = _value elif isCSV: aspects["stringValue"] = _strValue aspects[qnOimConceptAspect] = oimValue(fact.qname) cntx = fact.context if cntx is not None: if cntx.entityIdentifierElement is not None: aspects[qnOimEntityAspect] = oimValue(qname(*cntx.entityIdentifier)) if cntx.period is not None: if isJSON: aspects[qnOimPeriodAspect] = oimPeriodValue(cntx) elif isCSV: _periodValue = oimPeriodValue(cntx).split("/") + ["", ""] # default blank if no value aspects[qnOimPeriodStartAspect] = _periodValue[0] aspects[qnOimPeriodDurationAspect] = _periodValue[1] for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]): aspects[dim.dimensionQname] = (oimValue(dim.memberQname) if dim.isExplicit else None if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1") else dim.typedMember.stringValue) unit = fact.unit if unit is not None: _mMul, _mDiv = unit.measures if isJSON: aspects[qnOimUnitAspect] = OrderedDict( # use tuple instead of list for hashability (("numerators", tuple(oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m)))),) ) if _mDiv: aspects[qnOimUnitAspect]["denominators"] = tuple(oimValue(m) for m in sorted(_mDiv, key=lambda m: oimValue(m))) else: # CSV if _mMul: aspects[qnOimUnitNumeratorsAspect] = " ".join(oimValue(m) for m in sorted(_mMul, key=lambda m: str(m))) if _mDiv: aspects[qnOimUnitDenominatorsAspect] = " ".join(oimValue(m) for m in sorted(_mDiv, key=lambda m: str(m))) if parent.qname != XbrlConst.qnXbrliXbrl: aspects[qnOimTupleParentAspect] = parent.id if parent.id else "f{}".format(parent.objectIndex) aspects[qnOimTupleOrderAspect] = elementIndex(fact) if isJSON: _footnotes = factFootnotes(fact) if _footnotes: aspects["footnotes"] = _footnotes return aspects if isJSON: # save JSON oimReport = OrderedDict() # top level of oim json output oimFacts = [] oimReport["prefixes"] = OrderedDict((p,ns) for ns, p in sorted(namespacePrefixes.items(), key=lambda item: item[1])) oimReport["dtsReferences"] = dtsReferences oimReport["facts"] = oimFacts def saveJsonFacts(facts, oimFacts, parentFact): for fact in facts: oimFact = factAspects(fact) oimFacts.append(OrderedDict((str(k),v) for k,v in oimFact.items())) if fact.modelTupleFacts: saveJsonFacts(fact.modelTupleFacts, oimFacts, fact) saveJsonFacts(modelXbrl.facts, oimFacts, None) with open(oimFile, "w", encoding="utf-8") as fh: fh.write(json.dumps(oimReport, ensure_ascii=False, indent=1, sort_keys=False)) elif isCSV: # save CSV aspectQnCol = {} aspectsHeader = [] factsColumns = [] def addAspectQnCol(aspectQn): aspectQnCol[aspectQn] = len(aspectsHeader) _colName = oimValue(aspectQn) aspectsHeader.append(_colName) _colDataType = {"id": "Name", "baseType": "Name", "oim:concept": "Name", "oim:periodStart": "dateTime", # forever is 0000-01-01T00:00:00 "oim:periodDuration": "duration", # forever is P9999Y "oim:tupleOrder": "integer", "numericValue": "decimal", "accuracy": "decimal", "booleanValue": "boolean", "oim:unitNumerators": OrderedDict((("base","Name"), ("separator"," "))), "oim:unitDenominators": OrderedDict((("base","Name"), ("separator"," "))), }.get(_colName, "string") factsColumns.append(OrderedDict((("name", _colName), ("datatype", _colDataType)))) # pre-ordered aspect columns if hasId: addAspectQnCol("id") if hasType: addAspectQnCol("baseType") addAspectQnCol("stringValue") addAspectQnCol("numericValue") addAspectQnCol("accuracy") addAspectQnCol("booleanValue") if hasTuple: addAspectQnCol(qnOimTupleParentAspect) addAspectQnCol(qnOimTupleOrderAspect) addAspectQnCol(qnOimConceptAspect) if qnOimEntityAspect in aspectsDefined: addAspectQnCol(qnOimEntityAspect) if qnOimPeriodStartAspect in aspectsDefined: addAspectQnCol(qnOimPeriodStartAspect) addAspectQnCol(qnOimPeriodDurationAspect) if qnOimUnitNumeratorsAspect in aspectsDefined: addAspectQnCol(qnOimUnitNumeratorsAspect) if qnOimUnitDenominatorsAspect in aspectsDefined: addAspectQnCol(qnOimUnitDenominatorsAspect) for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)): if aspectQn.namespaceURI != nsOim: addAspectQnCol(aspectQn) def aspectCols(fact): cols = [None for i in range(len(aspectsHeader))] _factAspects = factAspects(fact) for aspectQn, aspectValue in _factAspects.items(): if aspectQn in aspectQnCol: cols[aspectQnCol[aspectQn]] = aspectValue return cols # metadata csvTables = [] csvMetadata = OrderedDict((("@context",[ "http://www.w3.org/ns/csvw", { "@base": "./" }]), ("tables", csvTables))) if oimFile.endswith("-facts.csv"): # strip -facts.csv if a prior -facts.csv file was chosen _baseURL = oimFile[:-10] elif oimFile.endswith(".csv"): _baseURL = oimFile[:-4] else: _baseURL = oimFile # save facts _factsFile = _baseURL + "-facts.csv" csvFile = open(_factsFile, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig') csvWriter = csv.writer(csvFile, dialect="excel") csvWriter.writerow(aspectsHeader) def saveCSVfacts(facts): for fact in facts: csvWriter.writerow(aspectCols(fact)) saveCSVfacts(fact.modelTupleFacts) saveCSVfacts(modelXbrl.facts) csvFile.close() factsTableSchema = OrderedDict((("columns",factsColumns),)) csvTables.append(OrderedDict((("url",os.path.basename(_factsFile)), ("tableSchema",factsTableSchema)))) # save namespaces _nsFile = _baseURL + "-prefixes.csv" csvFile = open(_nsFile, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig') csvWriter = csv.writer(csvFile, dialect="excel") csvWriter.writerow(("prefix", "URI")) for _URI, prefix in sorted(namespacePrefixes.items(), key=lambda item: item[1]): csvWriter.writerow((prefix, _URI)) csvFile.close() nsTableSchema = OrderedDict((("columns",[OrderedDict((("prefix","string"), ("URI","anyURI")))]),)) csvTables.append(OrderedDict((("url",os.path.basename(_nsFile)), ("tableSchema",nsTableSchema)))) # save dts references _dtsRefFile = _baseURL + "-dtsReferences.csv" csvFile = open(_dtsRefFile, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig') csvWriter = csv.writer(csvFile, dialect="excel") csvWriter.writerow(("type", "href")) for oimRef in dtsReferences: csvWriter.writerow((oimRef["type"], oimRef["href"])) csvFile.close() dtsRefTableSchema = OrderedDict((("columns",[OrderedDict((("type","string"), ("href","anyURI")))]),)) csvTables.append(OrderedDict((("url",os.path.basename(_dtsRefFile)), ("tableSchema",dtsRefTableSchema)))) # save footnotes if footnotesRelationshipSet.modelRelationships: _footnoteFile = oimFile.replace(".csv", "-footnotes.csv") csvFile = open(_footnoteFile, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig') csvWriter = csv.writer(csvFile, dialect="excel") cols = ("group", "factId", "factRef", "footnoteType", "footnote", "language") csvWriter.writerow(cols) def saveCSVfootnotes(facts): for fact in facts: for _footnote in factFootnotes(fact): csvWriter.writerow(tuple((_footnote.get(col,"") for col in cols))) saveCSVfootnotes(fact.modelTupleFacts) saveCSVfootnotes(modelXbrl.facts) csvFile.close() footnoteTableSchema = OrderedDict((("columns",[OrderedDict((("group","anyURI"), ("factId","Name"), ("factRef","Name"), ("footnoteType","Name"), ("footnote","string"), ("language","language")))]),)) csvTables.append(OrderedDict((("url",os.path.basename(_footnoteFile)), ("tableSchema",footnoteTableSchema)))) # save metadata with open(_baseURL + "-metadata.csv", "w", encoding="utf-8") as fh: fh.write(json.dumps(csvMetadata, ensure_ascii=False, indent=1, sort_keys=False))
def saveLoadableOIM(modelXbrl, oimFile, outputZip=None): isJSON = oimFile.endswith(".json") isCSV = oimFile.endswith(".csv") isXL = oimFile.endswith(".xlsx") isCSVorXL = isCSV or isXL if not isJSON and not isCSVorXL: return namespacePrefixes = {nsOim: "xbrl"} prefixNamespaces = {"xbrl": nsOim} def compileQname(qname): if qname.namespaceURI not in namespacePrefixes: namespacePrefixes[qname.namespaceURI] = qname.prefix or "" aspectsDefined = { qnOimConceptAspect, qnOimEntityAspect, qnOimPeriodStartAspect, qnOimPeriodEndAspect} def oimValue(object, decimals=None): if isinstance(object, QName): if object.namespaceURI not in namespacePrefixes: if object.prefix: namespacePrefixes[object.namespaceURI] = object.prefix else: _prefix = "_{}".format(sum(1 for p in namespacePrefixes if p.startswith("_"))) namespacePrefixes[object.namespaceURI] = _prefix return "{}:{}".format(namespacePrefixes[object.namespaceURI], object.localName) if isinstance(object, Decimal): try: if isinf(object): return "-INF" if object < 0 else "INF" elif isnan(num): return "NaN" else: if object == object.to_integral(): object = object.quantize(ONE) # drop any .0 return "{}".format(object) except: return str(object) if isinstance(object, bool): return "true" if object else "false" if isinstance(object, (DateTime, YearMonthDuration, DayTimeDuration, Time, gYearMonth, gMonthDay, gYear, gMonth, gDay, IsoDuration)): return str(object) return object def oimPeriodValue(cntx): if cntx.isForeverPeriod: return OrderedDict() # defaulted elif cntx.isStartEndPeriod: s = cntx.startDatetime e = cntx.endDatetime else: # instant s = e = cntx.instantDatetime if isJSON: return({str(qnOimPeriodAspect): ("{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}/".format( s.year, s.month, s.day, s.hour, s.minute, s.second) if cntx.isStartEndPeriod else "") + "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format( e.year, e.month, e.day, e.hour, e.minute, e.second) }) # CSV, XL return OrderedDict(((str(qnOimPeriodStartAspect), "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format( s.year, s.month, s.day, s.hour, s.minute, s.second)), (str(qnOimPeriodEndAspect), "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format( e.year, e.month, e.day, e.hour, e.minute, e.second)))) hasId = False hasTuple = False hasType = True hasLang = False hasUnits = False hasNumeric = False footnotesRelationshipSet = ModelRelationshipSet(modelXbrl, "XBRL-footnotes") factBaseTypes = set() #compile QNames in instance for OIM for fact in modelXbrl.factsInInstance: if (fact.id or fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))): hasId = True concept = fact.concept if concept is not None: if concept.isNumeric: hasNumeric = True if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang: hasLang = True _baseXsdType = concept.baseXsdType if _baseXsdType == "XBRLI_DATEUNION": if getattr(fact.xValue, "dateOnly", False): _baseXsdType = "date" else: _baseXsdType = "dateTime" factBaseTypes.add(baseTypes.get(_baseXsdType,_baseXsdType)) compileQname(fact.qname) if hasattr(fact, "xValue") and isinstance(fact.xValue, QName): compileQname(fact.xValue) unit = fact.unit if unit is not None: hasUnits = True if fact.modelTupleFacts: hasTuple = True if hasTuple: modelXbrl.error("arelleOIMsaver:tuplesNotAllowed", "Tuples are not allowed in an OIM document", modelObject=modelXbrl) return entitySchemePrefixes = {} for cntx in modelXbrl.contexts.values(): if cntx.entityIdentifierElement is not None: scheme = cntx.entityIdentifier[0] if scheme not in entitySchemePrefixes: if not entitySchemePrefixes: # first one is just scheme if scheme == "http://www.sec.gov/CIK": _schemePrefix = "cik" elif scheme == "http://standard.iso.org/iso/17442": _schemePrefix = "lei" else: _schemePrefix = "scheme" else: _schemePrefix = "scheme{}".format(len(entitySchemePrefixes) + 1) entitySchemePrefixes[scheme] = _schemePrefix namespacePrefixes[scheme] = _schemePrefix for dim in cntx.qnameDims.values(): compileQname(dim.dimensionQname) aspectsDefined.add(dim.dimensionQname) if dim.isExplicit: compileQname(dim.memberQname) for unit in modelXbrl.units.values(): if unit is not None: for measures in unit.measures: for measure in measures: compileQname(measure) if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[XbrlConst.xbrli] != "xbrli": namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix namespacePrefixes[XbrlConst.xsd] = "xsd" if hasLang: aspectsDefined.add(qnOimLangAspect) if hasTuple: aspectsDefined.add(qnOimTupleParentAspect) aspectsDefined.add(qnOimTupleOrderAspect) if hasUnits: aspectsDefined.add(qnOimUnitAspect) for footnoteRel in footnotesRelationshipSet.modelRelationships: typePrefix = "ftTyp_" + os.path.basename(footnoteRel.arcrole) if footnoteRel.linkrole == XbrlConst.defaultLinkRole: groupPrefix = "ftGrp_default" else: groupPrefix = "ftGrp_" + os.path.basename(footnoteRel.linkrole) if typePrefix not in namespacePrefixes: namespacePrefixes[footnoteRel.arcrole] = typePrefix if groupPrefix not in namespacePrefixes: namespacePrefixes[footnoteRel.linkrole] = groupPrefix # compile footnotes and relationships ''' factRelationships = [] factFootnotes = [] for rel in modelXbrl.relationshipSet(modelXbrl, "XBRL-footnotes").modelRelationships: oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole} factRelationships.append(oimRel) oimRel["fromIds"] = [obj.id if obj.id else elementChildSequence(obj) for obj in rel.fromModelObjects] oimRel["toIds"] = [obj.id if obj.id else elementChildSequence(obj) for obj in rel.toModelObjects] _order = rel.arcElement.get("order") if _order is not None: oimRel["order"] = _order for obj in rel.toModelObjects: if isinstance(obj, ModelResource): # footnote oimFootnote = {"role": obj.role, "id": obj.id if obj.id else elementChildSequence(obj), # value needs work for html elements and for inline footnotes "value": xmlstring(obj, stripXmlns=True)} if obj.xmlLang: oimFootnote["lang"] = obj.xmlLang factFootnotes.append(oimFootnote) oimFootnote ''' dtsReferences = set() baseUrl = modelXbrl.modelDocument.uri.partition("#")[0] for doc,ref in sorted(modelXbrl.modelDocument.referencesDocument.items(), key=lambda _item:_item[0].uri): if ref.referringModelObject.qname in SCHEMA_LB_REFS: dtsReferences.add(relativeUri(baseUrl,doc.uri)) for refType in ("role", "arcrole"): for refElt in sorted(modelXbrl.modelDocument.xmlRootElement.iterchildren( "{{http://www.xbrl.org/2003/linkbase}}{}Ref".format(refType)), key=lambda elt:elt.get(refType+"URI") ): dtsReferences.add(refElt.get("{http://www.w3.org/1999/xlink}href").partition("#")[0]) dtsReferences = sorted(dtsReferences) # turn into list footnoteFacts = set() def factFootnotes(fact, oimFact=None): footnotes = [] oimLinks = {} for footnoteRel in footnotesRelationshipSet.fromModelObject(fact): toObj = footnoteRel.toModelObject # json typePrefix = namespacePrefixes[footnoteRel.arcrole] groupPrefix = namespacePrefixes[footnoteRel.linkrole] oimLinks.setdefault(typePrefix,{}).setdefault(groupPrefix,[]).append( toObj.id if toObj.id else "f{}".format(toObj.objectIndex)) # csv/XL footnote = OrderedDict((("group", footnoteRel.linkrole), ("footnoteType", footnoteRel.arcrole))) footnotes.append(footnote) if isCSVorXL: footnote["factId"] = fact.id if fact.id else "f{}".format(fact.objectIndex) if isinstance(toObj, ModelFact): footnote["factRef"] = toObj.id if toObj.id else "f{}".format(toObj.objectIndex) else: footnote["footnote"] = toObj.viewText() if toObj.xmlLang: footnote["language"] = toObj.xmlLang footnoteFacts.add(toObj) if isJSON and oimLinks: oimFact["links"] = OrderedDict(( (typePrefix, OrderedDict(( (groupPrefix, idList) for groupPrefix, idList in sorted(groups.items()) ))) for typePrefix, groups in sorted(oimLinks.items()) )) footnotes.sort(key=lambda f:(f["group"],f.get("factId",f.get("factRef")),f.get("language"))) return footnotes def factAspects(fact): oimFact = OrderedDict() aspects = OrderedDict() if hasId and fact.id: if fact.isTuple: oimFact["tupleId"] = fact.id else: oimFact["id"] = fact.id elif (fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))): oimFact["id"] = "f{}".format(fact.objectIndex) parent = fact.getparent() concept = fact.concept aspects[str(qnOimConceptAspect)] = oimValue(concept.qname) _csvType = "Value" if not fact.isTuple: if concept is not None: _baseXsdType = concept.baseXsdType if _baseXsdType == "XBRLI_DATEUNION": if getattr(fact.xValue, "dateOnly", False): _baseXsdType = "date" else: _baseXsdType = "dateTime" _csvType = baseTypes.get(_baseXsdType,_baseXsdType) + "Value" if concept.baseXbrliType in ("stringItemType", "normalizedStringItemType") and fact.xmlLang: aspects[str(qnOimLangAspect)] = fact.xmlLang if fact.isItem: if fact.isNil: _value = None else: _inferredDecimals = inferredDecimals(fact) _value = oimValue(fact.xValue, _inferredDecimals) oimFact["value"] = _value if fact.concept is not None and fact.concept.isNumeric: _numValue = fact.xValue if isinstance(_numValue, Decimal) and not isinf(_numValue) and not isnan(_numValue): if _numValue == _numValue.to_integral(): _numValue = int(_numValue) else: _numValue = float(_numValue) if not fact.isNil: if isinf(_inferredDecimals): if isJSON: _accuracy = "infinity" elif isCSVorXL: _accuracy = "INF" else: _accuracy = _inferredDecimals oimFact["accuracy"] = _accuracy oimFact["aspects"] = aspects cntx = fact.context if cntx is not None: if cntx.entityIdentifierElement is not None: aspects[str(qnOimEntityAspect)] = oimValue(qname(*cntx.entityIdentifier)) if cntx.period is not None: aspects.update(oimPeriodValue(cntx)) for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]): if dim.isExplicit: dimVal = oimValue(dim.memberQname) else: # typed if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1"): dimVal = None else: dimVal = dim.typedMember.stringValue aspects[str(dim.dimensionQname)] = dimVal unit = fact.unit if unit is not None: _mMul, _mDiv = unit.measures _sMul = '*'.join(oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m))) if _mDiv: _sDiv = '*'.join(oimValue(m) for m in sorted(_mDiv, key=lambda m: oimValue(m))) if len(_mDiv) > 1: if len(_mMul) > 1: _sUnit = "({})/({})".format(_sMul,_sDiv) else: _sUnit = "{}/({})".format(_sMul,_sDiv) else: if len(_mMul) > 1: _sUnit = "({})/{}".format(_sMul,_sDiv) else: _sUnit = "{}/{}".format(_sMul,_sDiv) else: _sUnit = _sMul aspects[str(qnOimUnitAspect)] = _sUnit # Tuples removed from xBRL-JSON #if parent.qname != XbrlConst.qnXbrliXbrl: # aspects[str(qnOimTupleParentAspect)] = parent.id if parent.id else "f{}".format(parent.objectIndex) # aspects[str(qnOimTupleOrderAspect)] = elementIndex(fact) if isJSON: factFootnotes(fact, oimFact) return oimFact prefixes = OrderedDict((p,ns) for ns, p in sorted(namespacePrefixes.items(), key=lambda item: item[1])) if isJSON: # save JSON oimReport = OrderedDict() # top level of oim json output oimReport["documentInfo"] = oimDocInfo = OrderedDict() oimReport["facts"] = oimFacts = OrderedDict() oimDocInfo["documentType"] = nsOim + "/xbrl-json" oimDocInfo["features"] = oimFeatures = OrderedDict() oimDocInfo["prefixes"] = prefixes oimDocInfo["taxonomy"] = dtsReferences oimFeatures["xbrl:canonicalValues"] = True def saveJsonFacts(facts, oimFacts, parentFact): for fact in facts: oimFact = factAspects(fact) id = fact.id if fact.id else "f{}".format(fact.objectIndex) oimFacts[id] = oimFact if fact.modelTupleFacts: saveJsonFacts(fact.modelTupleFacts, oimFacts, fact) saveJsonFacts(modelXbrl.facts, oimFacts, None) # add footnotes as pseudo facts for ftObj in footnoteFacts: ftId = ftObj.id if ftObj.id else "f{}".format(ftObj.objectIndex) oimFacts[ftId] = oimFact = OrderedDict() oimFact["value"] = ftObj.viewText() oimFact["aspects"] = OrderedDict((("concept", "xbrl:note"), ("noteId", ftId))) if ftObj.xmlLang: oimFact["aspects"]["language"] = ftObj.xmlLang.lower() if outputZip: fh = io.StringIO() else: fh = open(oimFile, "w", encoding="utf-8") fh.write(json.dumps(oimReport, indent=1)) if outputZip: fh.seek(0) outputZip.writestr(os.path.basename(oimFile),fh.read()) fh.close() elif isCSVorXL: # save CSV aspectQnCol = {} aspectsHeader = [] factsColumns = [] def addAspectQnCol(aspectQn): aspectQnCol[str(aspectQn)] = len(aspectsHeader) _aspectQn = oimValue(aspectQn) aspectsHeader.append(_aspectQn) _colName = _aspectQn.replace("xbrl:", "") _colDataType = {"id": "Name", "concept": "QName", "value": "string", "accuracy": "decimal", "entity": "QName", "periodStart": "dateTime", "periodEnd": "dateTime", "unit": "string", "tupleId": "Name", "tupleParent": "Name", "tupleOrder": "integer" }.get(_colName, "string") col = OrderedDict((("name", _colName), ("datatype", _colDataType))) if _aspectQn == "value": col["http://xbrl.org/YYYY/model#simpleFactAspects"] = {} elif _aspectQn == "tupleId": col["http://xbrl.org/YYYY/model#tupleFactAspects"] = {} col["http://xbrl.org/YYYY/model#tupleReferenceId"] = "true" else: col["http://xbrl.org/YYYY/model#columnAspect"] = _aspectQn factsColumns.append(col) # pre-ordered aspect columns #if hasId: # addAspectQnCol("id") addAspectQnCol(qnOimConceptAspect) addAspectQnCol("value") if hasNumeric: addAspectQnCol("accuracy") if hasTuple: addAspectQnCol("tupleId") addAspectQnCol(qnOimTupleParentAspect) addAspectQnCol(qnOimTupleOrderAspect) if qnOimEntityAspect in aspectsDefined: addAspectQnCol(qnOimEntityAspect) if qnOimPeriodStartAspect in aspectsDefined: addAspectQnCol(qnOimPeriodStartAspect) addAspectQnCol(qnOimPeriodEndAspect) if qnOimUnitAspect in aspectsDefined: addAspectQnCol(qnOimUnitAspect) for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)): if aspectQn.namespaceURI != nsOim: addAspectQnCol(aspectQn) def aspectCols(fact): cols = [None for i in range(len(aspectsHeader))] def setColValues(aspects): for aspectQn, aspectValue in aspects.items(): if isinstance(aspectValue, dict): setColValues(aspectValue) elif aspectQn in aspectQnCol: if aspectValue is None: _aspectValue = "#nil" elif aspectValue == "": _aspectValue = "#empty" elif isinstance(aspectValue, str) and aspectValue.startswith("#"): _aspectValue = "#" + aspectValue else: _aspectValue = aspectValue cols[aspectQnCol[aspectQn]] = _aspectValue setColValues(factAspects(fact)) return cols # metadata csvTables = [] csvMetadata = OrderedDict((("@context", "http://www.w3.org/ns/csvw"), ("http://xbrl.org/YYYY/model#metadata", OrderedDict((("documentType", "http://xbrl.org/YYYY/xbrl-csv"), ("dtsReferences", dtsReferences), ("prefixes", prefixes)))), ("tables", csvTables))) _open = _writerow = _close = None _tableinfo = {} if isCSV: if oimFile.endswith("-facts.csv"): # strip -facts.csv if a prior -facts.csv file was chosen _baseURL = oimFile[:-10] elif oimFile.endswith(".csv"): _baseURL = oimFile[:-4] else: _baseURL = oimFile _csvinfo = {} # open file, writer def _open(filesuffix, tabname): _filename = _tableinfo["url"] = _baseURL + filesuffix _csvinfo["file"] = open(_filename, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig') _csvinfo["writer"] = csv.writer(_csvinfo["file"], dialect="excel") def _writerow(row, header=False): _csvinfo["writer"].writerow(row) def _close(): _csvinfo["file"].close() _csvinfo.clear() elif isXL: headerWidths = {"href": 100, "xbrl:concept": 70, "accuracy": 8, "language": 9, "URI": 80, "value": 60, "group": 60, "footnoteType": 40, "footnote": 70, "column": 20, 'conceptAspect': 40, 'tuple': 20, 'simpleFact': 20} from openpyxl import Workbook from openpyxl.writer.write_only import WriteOnlyCell from openpyxl.styles import Font, PatternFill, Border, Alignment, Color, fills, Side from openpyxl.worksheet.dimensions import ColumnDimension hdrCellFill = PatternFill(patternType=fills.FILL_SOLID, fgColor=Color("00FFBF5F")) # Excel's light orange fill color = 00FF990 workbook = Workbook() # remove pre-existing worksheets while len(workbook.worksheets)>0: workbook.remove_sheet(workbook.worksheets[0]) _xlinfo = {} # open file, writer def _open(filesuffix, tabname): _tableinfo["url"] = tabname _xlinfo["ws"] = workbook.create_sheet(title=tabname) def _writerow(rowvalues, header=False): row = [] _ws = _xlinfo["ws"] for i, v in enumerate(rowvalues): cell = WriteOnlyCell(_ws, value=v) if header: cell.fill = hdrCellFill cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True) colLetter = chr(ord('A') + i) _ws.column_dimensions[colLetter] = ColumnDimension(_ws, customWidth=True) _ws.column_dimensions[colLetter].width = headerWidths.get(v, 20) else: cell.alignment = Alignment(horizontal="right" if isinstance(v, _NUM_TYPES) else "center" if isinstance(v, bool) else "left", vertical="top", wrap_text=isinstance(v, str)) row.append(cell) _ws.append(row) def _close(): _xlinfo.clear() # save facts _open("-facts.csv", "facts") _writerow(aspectsHeader, header=True) def saveCSVfacts(facts): for fact in facts: _writerow(aspectCols(fact)) saveCSVfacts(fact.modelTupleFacts) saveCSVfacts(modelXbrl.facts) _close() factsTableSchema = OrderedDict((("columns",factsColumns),)) csvTables.append(OrderedDict((("url",_tableinfo["url"]), ("http://xbrl.org/YYYY/model#tableType", "fact"), ("tableSchema",factsTableSchema)))) # save footnotes if footnotesRelationshipSet.modelRelationships: _open("-footnotes.csv", "footnotes") cols = ("group", "footnoteType", "factId", "factRef", "footnote", "language") _writerow(cols, header=True) def saveCSVfootnotes(facts): for fact in facts: for _footnote in factFootnotes(fact): _writerow(tuple((_footnote.get(col,"") for col in cols))) saveCSVfootnotes(fact.modelTupleFacts) saveCSVfootnotes(modelXbrl.facts) _close() footnoteTableSchema = OrderedDict((("columns",[OrderedDict((("name","group"),("datatype","anyURI"))), OrderedDict((("name","footnoteType"),("datatype","Name"))), OrderedDict((("name","factId"),("datatype","Name"))), OrderedDict((("name","factRef"),("datatype","Name"))), OrderedDict((("name","footnote"),("datatype","string"))), OrderedDict((("name","language"),("datatype","language")))]),)) csvTables.append(OrderedDict((("url",_tableinfo["url"]), ("http://xbrl.org/YYYY/model#tableType", "footnote"), ("tableSchema",footnoteTableSchema)))) # save metadata if isCSV: with open(_baseURL + "-metadata.json", "w", encoding="utf-8") as fh: fh.write(json.dumps(csvMetadata, ensure_ascii=False, indent=1, sort_keys=False)) elif isXL: _open(None, "metadata") hasColumnAspect = hasSimpleFact = hasTupleFact = False for table in csvTables: tablename = table["url"] for column in table["tableSchema"]["columns"]: if "http://xbrl.org/YYYY/model#columnAspect" in column: hasColumnAspect = True if "http://xbrl.org/YYYY/model#simpleFactAspects" in column: hasSimpleFact = True if "http://xbrl.org/YYYY/model#tupleAspects" in column: hasTupleFact = True metadataCols = ["table", "column", "datatype"] if hasColumnAspect: metadataCols.append("columnAspect") if hasSimpleFact: metadataCols.append("simpleFact") if hasTupleFact: metadataCols.append("tuple") _writerow(metadataCols, header=True) for table in csvTables: tablename = table["url"] for column in table["tableSchema"]["columns"]: row = [tablename, column["name"], column["datatype"]] if hasColumnAspect: colAspect = column.get("http://xbrl.org/YYYY/model#columnAspect") if isinstance(colAspect, str): row.append(colAspect) elif isinstance(colAspect, dict): row.append("\n".join("{} [{}]".format(k, ", ".join(_v for _v in v)) for k, v in dict.items())) else: row.append(None) if hasSimpleFact: row.append("\u221a" if "http://xbrl.org/YYYY/model#simpleFactAspects" in column else None) if hasTupleFact: row.append("\u221a" if "http://xbrl.org/YYYY/model#tupleAspects" in column else None) _writerow(row) _close() if isXL: workbook.save(oimFile)