class IXBRLViewerBuilder: def __init__(self, dts): self.nsmap = NamespaceMap() self.roleMap = NamespaceMap() self.dts = dts self.taxonomyData = { "concepts": {}, "languages": {}, "facts": {}, } self.footnoteRelationshipSet = ModelRelationshipSet( dts, "XBRL-footnotes") def lineWrap(self, s, n=80): return "\n".join([s[i:i + n] for i in range(0, len(s), n)]) def dateFormat(self, d): """ Strip the time component from an ISO date if it's zero """ return re.sub("T00:00:00$", "", d) def escapeJSONForScriptTag(self, s): """ JSON encodes XML special characters XML and HTML apply difference escaping rules to content within script tags and we need our output to be valid XML, but treated as HTML by browsers. If we allow XML escaping to occur in a script tag, browsers treating the document as HTML won't unescape it. If we don't escape XML special characters, it won't be valid XML. We avoid this whole mess by escaping XML special characters using JSON string escapes. This is only safe to do because < > and & can't occur outside a string in JSON. It can't safely be used on JS. """ return s.replace("<", "\\u003C").replace(">", "\\u003E").replace("&", "\\u0026") def makeLanguageName(self, langCode): code = re.sub("-.*", "", langCode) try: language = pycountry.languages.lookup(code) match = re.match(r'^[^-]+-(.*)$', langCode) name = language.name if match is not None: name = "%s (%s)" % (name, match.group(1).upper()) except LookupError: name = langCode return name def addLanguage(self, langCode): if langCode not in self.taxonomyData["languages"]: self.taxonomyData["languages"][langCode] = self.makeLanguageName( langCode) def addELR(self, elr): prefix = self.roleMap.getPrefix(elr) if self.taxonomyData.setdefault("roleDefs", {}).get(prefix, None) is None: rt = self.dts.roleTypes[elr] label = elr if len(rt) > 0: label = rt[0].definition self.taxonomyData["roleDefs"].setdefault(prefix, {})["en"] = label def addConcept(self, concept, dimensionType=None): if concept is None: return labelsRelationshipSet = self.dts.relationshipSet( XbrlConst.conceptLabel) labels = labelsRelationshipSet.fromModelObject(concept) conceptName = self.nsmap.qname(concept.qname) if conceptName not in self.taxonomyData["concepts"]: conceptData = {"labels": {}} for lr in labels: l = lr.toModelObject conceptData["labels"].setdefault( self.roleMap.getPrefix(l.role), {})[l.xmlLang.lower()] = l.text self.addLanguage(l.xmlLang.lower()) refData = [] for _refRel in concept.modelXbrl.relationshipSet( XbrlConst.conceptReference).fromModelObject(concept): ref = [] for _refPart in _refRel.toModelObject.iterchildren(): ref.append( [_refPart.localName, _refPart.stringValue.strip()]) refData.append(ref) if len(refData) > 0: conceptData['r'] = refData if dimensionType is not None: conceptData["d"] = dimensionType self.taxonomyData["concepts"][conceptName] = conceptData def treeWalk(self, rels, item, indent=0): for r in rels.fromModelObject(item): if r.toModelObject is not None: self.treeWalk(rels, r.toModelObject, indent + 1) def getRelationships(self): rels = {} for baseSetKey, baseSetModelLinks in self.dts.baseSets.items(): arcrole, ELR, linkqname, arcqname = baseSetKey if arcrole in (XbrlConst.summationItem, WIDER_NARROWER_ARCROLE) and ELR is not None: self.addELR(ELR) rr = dict() relSet = self.dts.relationshipSet(arcrole, ELR) for r in relSet.modelRelationships: if r.fromModelObject is not None and r.toModelObject is not None: fromKey = self.nsmap.qname(r.fromModelObject.qname) rel = { "t": self.nsmap.qname(r.toModelObject.qname), } if r.weight is not None: rel['w'] = r.weight rr.setdefault(fromKey, []).append(rel) self.addConcept(r.toModelObject) self.addConcept(r.fromModelObject) rels.setdefault(self.roleMap.getPrefix(arcrole), {})[self.roleMap.getPrefix(ELR)] = rr return rels def createViewer(self, scriptUrl="js/dist/ixbrlviewer.js"): """ Create an iXBRL file with XBRL data as a JSON blob, and script tags added """ dts = self.dts iv = iXBRLViewer(dts) idGen = 0 self.roleMap.getPrefix(XbrlConst.standardLabel, "std") self.roleMap.getPrefix(XbrlConst.documentationLabel, "doc") self.roleMap.getPrefix(XbrlConst.summationItem, "calc") self.roleMap.getPrefix(XbrlConst.parentChild, "pres") self.roleMap.getPrefix(WIDER_NARROWER_ARCROLE, "w-n") for f in dts.facts: if f.id is None: f.set("id", "ixv-%d" % (idGen)) idGen += 1 conceptName = self.nsmap.qname(f.qname) scheme, ident = f.context.entityIdentifier aspects = { "c": conceptName, "e": self.nsmap.qname( QName(self.nsmap.getPrefix(scheme, "e"), scheme, ident)), } factData = { "v": f.value if not f.isNil else None, "a": aspects, } if f.format is not None: factData["f"] = str(f.format) if f.isNumeric: if f.unit is not None and len(f.unit.measures[0]): # XXX does not support complex units unit = self.nsmap.qname(f.unit.measures[0][0]) aspects["u"] = unit else: # The presence of the unit aspect is used by the viewer to # identify numeric facts. If the fact has no unit (invalid # XBRL, but we want to support it for draft documents), # include the unit aspect with a null value. aspects["u"] = None d = inferredDecimals(f) if d != float("INF") and not math.isnan(d): factData["d"] = d for d, v in f.context.qnameDims.items(): if v.memberQname is not None: aspects[self.nsmap.qname( v.dimensionQname)] = self.nsmap.qname(v.memberQname) self.addConcept(v.member) self.addConcept(v.dimension, dimensionType="e") elif v.typedMember is not None: aspects[self.nsmap.qname( v.dimensionQname)] = v.typedMember.text self.addConcept(v.dimension, dimensionType="t") if f.context.isForeverPeriod: aspects["p"] = "f" elif f.context.isInstantPeriod and f.context.instantDatetime is not None: aspects["p"] = self.dateFormat( f.context.instantDatetime.isoformat()) elif f.context.isStartEndPeriod and f.context.startDatetime is not None and f.context.endDatetime is not None: aspects["p"] = "%s/%s" % ( self.dateFormat(f.context.startDatetime.isoformat()), self.dateFormat(f.context.endDatetime.isoformat())) frels = self.footnoteRelationshipSet.fromModelObject(f) if frels: for frel in frels: if frel.toModelObject is not None: factData.setdefault("fn", []).append(frel.toModelObject.id) self.taxonomyData["facts"][f.id] = factData if f.concept is not None: self.addConcept(f.concept) self.taxonomyData["prefixes"] = self.nsmap.prefixmap self.taxonomyData["roles"] = self.roleMap.prefixmap self.taxonomyData["rels"] = self.getRelationships() dts.info("viewer:info", "Creating iXBRL viewer") if dts.modelDocument.type == Type.INLINEXBRLDOCUMENTSET: # Sort by object index to preserve order in which files were specified. docSet = sorted(dts.modelDocument.referencesDocument.keys(), key=lambda x: x.objectIndex) docSetFiles = list( map(lambda x: os.path.basename(x.filepath), docSet)) self.taxonomyData["docSetFiles"] = docSetFiles for n in range(0, len(docSet)): iv.addFile( iXBRLViewerFile(docSetFiles[n], docSet[n].xmlDocument)) xmlDocument = docSet[0].xmlDocument else: xmlDocument = dts.modelDocument.xmlDocument filename = os.path.basename(dts.modelDocument.filepath) iv.addFile(iXBRLViewerFile(filename, xmlDocument)) taxonomyDataJSON = self.escapeJSONForScriptTag( json.dumps(self.taxonomyData, indent=1, allow_nan=False)) for child in xmlDocument.getroot(): if child.tag == '{http://www.w3.org/1999/xhtml}body': child.append(etree.Comment("BEGIN IXBRL VIEWER EXTENSIONS")) e = etree.fromstring( "<script xmlns='http://www.w3.org/1999/xhtml' src='%s' type='text/javascript' />" % scriptUrl) # Don't self close e.text = '' child.append(e) # Putting this in the header can interfere with character set # auto detection e = etree.fromstring( "<script xmlns='http://www.w3.org/1999/xhtml' type='application/x.ixbrl-viewer+json'></script>" ) e.text = taxonomyDataJSON child.append(e) child.append(etree.Comment("END IXBRL VIEWER EXTENSIONS")) break return iv
def saveLoadableOIM(modelXbrl, oimFile, outputZip=None): isJSON = oimFile.endswith(".json") isCSV = oimFile.endswith(".csv") isXL = oimFile.endswith(".xlsx") isCSVorXL = isCSV or isXL if not isJSON and not isCSVorXL: return namespacePrefixes = {nsOim: "xbrl"} prefixNamespaces = {} linkTypeAliases = {} groupAliases = {} linkTypePrefixes = {} linkTypeUris = {} linkGroupPrefixes = {} linkGroupUris = {} def compileQname(qname): if qname.namespaceURI not in namespacePrefixes: namespacePrefixes[qname.namespaceURI] = qname.prefix or "" aspectsDefined = {qnOimConceptAspect, qnOimEntityAspect, qnOimPeriodAspect} def oimValue(object, decimals=None): if isinstance(object, QName): if object.namespaceURI not in namespacePrefixes: if object.prefix: namespacePrefixes[object.namespaceURI] = object.prefix else: _prefix = "_{}".format( sum(1 for p in namespacePrefixes if p.startswith("_"))) namespacePrefixes[object.namespaceURI] = _prefix return "{}:{}".format(namespacePrefixes[object.namespaceURI], object.localName) if isinstance(object, (float, Decimal)): try: if isinf(object): return "-INF" if object < 0 else "INF" elif isnan(object): return "NaN" else: if isinstance(object, Decimal) and object == object.to_integral(): object = object.quantize(ONE) # drop any .0 return "{}".format(object) except: return str(object) if isinstance(object, bool): return "true" if object else "false" if isinstance( object, (DateTime, YearMonthDuration, DayTimeDuration, Time, gYearMonth, gMonthDay, gYear, gMonth, gDay, IsoDuration, int)): return str(object) return object def oimPeriodValue(cntx): if cntx.isStartEndPeriod: s = cntx.startDatetime e = cntx.endDatetime else: # instant s = e = cntx.instantDatetime return ({ str(qnOimPeriodAspect): ("{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}/".format( s.year, s.month, s.day, s.hour, s.minute, s.second) if cntx.isStartEndPeriod else "") + "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format( e.year, e.month, e.day, e.hour, e.minute, e.second) }) hasId = False hasTuple = False hasType = True hasLang = False hasUnits = False hasNumeric = False footnotesRelationshipSet = ModelRelationshipSet(modelXbrl, "XBRL-footnotes") #compile QNames in instance for OIM for fact in modelXbrl.factsInInstance: if (fact.id or fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))): hasId = True concept = fact.concept if concept is not None: if concept.isNumeric: hasNumeric = True if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang: hasLang = True compileQname(fact.qname) if hasattr(fact, "xValue") and isinstance(fact.xValue, QName): compileQname(fact.xValue) unit = fact.unit if unit is not None: hasUnits = True if fact.modelTupleFacts: hasTuple = True if hasTuple: modelXbrl.error("arelleOIMsaver:tuplesNotAllowed", "Tuples are not allowed in an OIM document", modelObject=modelXbrl) return entitySchemePrefixes = {} for cntx in modelXbrl.contexts.values(): if cntx.entityIdentifierElement is not None: scheme = cntx.entityIdentifier[0] if scheme not in entitySchemePrefixes: if not entitySchemePrefixes: # first one is just scheme if scheme == "http://www.sec.gov/CIK": _schemePrefix = "cik" elif scheme == "http://standard.iso.org/iso/17442": _schemePrefix = "lei" else: _schemePrefix = "scheme" else: _schemePrefix = "scheme{}".format( len(entitySchemePrefixes) + 1) entitySchemePrefixes[scheme] = _schemePrefix namespacePrefixes[scheme] = _schemePrefix for dim in cntx.qnameDims.values(): compileQname(dim.dimensionQname) aspectsDefined.add(dim.dimensionQname) if dim.isExplicit: compileQname(dim.memberQname) for unit in modelXbrl.units.values(): if unit is not None: for measures in unit.measures: for measure in measures: compileQname(measure) if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[ XbrlConst.xbrli] != "xbrli": namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix if hasLang: aspectsDefined.add(qnOimLangAspect) if hasUnits: aspectsDefined.add(qnOimUnitAspect) for footnoteRel in footnotesRelationshipSet.modelRelationships: typePrefix = "ftTyp_" + os.path.basename(footnoteRel.arcrole) if footnoteRel.linkrole == XbrlConst.defaultLinkRole: groupPrefix = "ftGrp_default" else: groupPrefix = "ftGrp_" + os.path.basename(footnoteRel.linkrole) if footnoteRel.arcrole not in linkTypeAliases: linkTypeAliases[footnoteRel.arcrole] = typePrefix if groupPrefix not in groupAliases: groupAliases[footnoteRel.linkrole] = groupPrefix dtsReferences = set() baseUrl = modelXbrl.modelDocument.uri.partition("#")[0] for doc, ref in sorted(modelXbrl.modelDocument.referencesDocument.items(), key=lambda _item: _item[0].uri): if ref.referringModelObject.qname in SCHEMA_LB_REFS: dtsReferences.add(relativeUri(baseUrl, doc.uri)) for refType in ("role", "arcrole"): for refElt in sorted( modelXbrl.modelDocument.xmlRootElement.iterchildren( "{{http://www.xbrl.org/2003/linkbase}}{}Ref".format( refType)), key=lambda elt: elt.get(refType + "URI")): dtsReferences.add( refElt.get("{http://www.w3.org/1999/xlink}href").partition("#") [0]) dtsReferences = sorted(dtsReferences) # turn into list footnoteFacts = set() def factFootnotes(fact, oimFact=None, csvLinks=None): footnotes = [] if isJSON: oimLinks = {} elif isCSVorXL: oimLinks = csvLinks for footnoteRel in footnotesRelationshipSet.fromModelObject(fact): srcId = fact.id if fact.id else "f{}".format(fact.objectIndex) toObj = footnoteRel.toModelObject # json typePrefix = linkTypeAliases[footnoteRel.arcrole] groupPrefix = groupAliases[footnoteRel.linkrole] if typePrefix not in oimLinks: oimLinks[typePrefix] = OrderedDict() _link = oimLinks[typePrefix] if groupPrefix not in _link: if isJSON: _link[groupPrefix] = [] elif isCSVorXL: _link[groupPrefix] = OrderedDict() if isJSON: tgtIdList = _link[groupPrefix] elif isCSVorXL: tgtIdList = _link[groupPrefix].setdefault(srcId, []) tgtId = toObj.id if toObj.id else "f{}".format(toObj.objectIndex) tgtIdList.append(tgtId) footnote = OrderedDict((("group", footnoteRel.linkrole), ("footnoteType", footnoteRel.arcrole))) if isinstance(toObj, ModelFact): footnote["factRef"] = tgtId else: # text footnotes footnote["id"] = tgtId footnote["footnote"] = toObj.viewText() if toObj.xmlLang: footnote["language"] = toObj.xmlLang footnoteFacts.add(toObj) footnotes.append(footnote) if oimLinks: _links = OrderedDict( ((typePrefix, OrderedDict( ((groupPrefix, idList) for groupPrefix, idList in sorted(groups.items())))) for typePrefix, groups in sorted(oimLinks.items()))) if isJSON: oimFact["links"] = _links return footnotes def factAspects(fact): oimFact = OrderedDict() aspects = OrderedDict() if isCSVorXL: oimFact["id"] = fact.id or "f{}".format(fact.objectIndex) parent = fact.getparent() concept = fact.concept aspects[str(qnOimConceptAspect)] = oimValue(concept.qname) if concept is not None: if concept.type.isOimTextFactType and fact.xmlLang: aspects[str(qnOimLangAspect)] = fact.xmlLang if fact.isItem: if fact.isNil: _value = None else: _inferredDecimals = inferredDecimals(fact) _value = oimValue(fact.xValue, _inferredDecimals) oimFact["value"] = _value if fact.concept is not None and fact.concept.isNumeric: _numValue = fact.xValue if isinstance(_numValue, Decimal) and not isinf( _numValue) and not isnan(_numValue): if _numValue == _numValue.to_integral(): _numValue = int(_numValue) else: _numValue = float(_numValue) if not fact.isNil: if not isinf( _inferredDecimals): # accuracy omitted if infinite oimFact["decimals"] = _inferredDecimals oimFact["dimensions"] = aspects cntx = fact.context if cntx is not None: if cntx.entityIdentifierElement is not None and cntx.entityIdentifier != ENTITY_NA_QNAME: aspects[str(qnOimEntityAspect)] = oimValue( qname(*cntx.entityIdentifier)) if cntx.period is not None and not cntx.isForeverPeriod: aspects.update(oimPeriodValue(cntx)) for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]): if dim.isExplicit: dimVal = oimValue(dim.memberQname) else: # typed if dim.typedMember.get( "{http://www.w3.org/2001/XMLSchema-instance}nil" ) in ("true", "1"): dimVal = None else: dimVal = dim.typedMember.stringValue aspects[str(dim.dimensionQname)] = dimVal unit = fact.unit if unit is not None: _mMul, _mDiv = unit.measures _sMul = '*'.join( oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m))) if _mDiv: _sDiv = '*'.join( oimValue(m) for m in sorted(_mDiv, key=lambda m: oimValue(m))) if len(_mDiv) > 1: if len(_mMul) > 1: _sUnit = "({})/({})".format(_sMul, _sDiv) else: _sUnit = "{}/({})".format(_sMul, _sDiv) else: if len(_mMul) > 1: _sUnit = "({})/{}".format(_sMul, _sDiv) else: _sUnit = "{}/{}".format(_sMul, _sDiv) else: _sUnit = _sMul if _sUnit != "xbrli:pure": aspects[str(qnOimUnitAspect)] = _sUnit # Tuples removed from xBRL-JSON #if parent.qname != XbrlConst.qnXbrliXbrl: # aspects[str(qnOimTupleParentAspect)] = parent.id if parent.id else "f{}".format(parent.objectIndex) # aspects[str(qnOimTupleOrderAspect)] = elementIndex(fact) if isJSON: factFootnotes(fact, oimFact=oimFact) return oimFact namespaces = OrderedDict((p, ns) for ns, p in sorted(namespacePrefixes.items(), key=lambda item: item[1])) # common metadata oimReport = OrderedDict() # top level of oim json output oimReport["documentInfo"] = oimDocInfo = OrderedDict() oimDocInfo["documentType"] = nsOim + ("/xbrl-json" if isJSON else "/xbrl-csv") if isJSON: oimDocInfo["features"] = oimFeatures = OrderedDict() oimDocInfo["namespaces"] = namespaces if linkTypeAliases: oimDocInfo["linkTypes"] = OrderedDict( (a, u) for u, a in sorted(linkTypeAliases.items(), key=lambda item: item[1])) if linkTypeAliases: oimDocInfo["linkGroups"] = OrderedDict( (a, u) for u, a in sorted(groupAliases.items(), key=lambda item: item[1])) oimDocInfo["taxonomy"] = dtsReferences if isJSON: oimFeatures["xbrl:canonicalValues"] = True if isJSON: # save JSON oimReport["facts"] = oimFacts = OrderedDict() def saveJsonFacts(facts, oimFacts, parentFact): for fact in facts: oimFact = factAspects(fact) id = fact.id if fact.id else "f{}".format(fact.objectIndex) oimFacts[id] = oimFact if fact.modelTupleFacts: saveJsonFacts(fact.modelTupleFacts, oimFacts, fact) saveJsonFacts(modelXbrl.facts, oimFacts, None) # add footnotes as pseudo facts for ftObj in footnoteFacts: ftId = ftObj.id if ftObj.id else "f{}".format(ftObj.objectIndex) oimFacts[ftId] = oimFact = OrderedDict() oimFact["value"] = ftObj.viewText() oimFact["dimensions"] = OrderedDict( (("concept", "xbrl:note"), ("noteId", ftId))) if ftObj.xmlLang: oimFact["dimensions"]["language"] = ftObj.xmlLang.lower() if outputZip: fh = io.StringIO() else: fh = open(oimFile, "w", encoding="utf-8") fh.write(json.dumps(oimReport, indent=1)) if outputZip: fh.seek(0) outputZip.writestr(os.path.basename(oimFile), fh.read()) fh.close() elif isCSVorXL: # save CSV oimReport["tables"] = oimTables = OrderedDict() oimReport["tableTemplates"] = csvTableTemplates = OrderedDict() oimTables["facts"] = csvTable = OrderedDict() csvTable["template"] = "facts" csvTable["url"] = "tbd" csvTableTemplates["facts"] = csvTableTemplate = OrderedDict() csvTableTemplate["rowIdColumn"] = "id" csvTableTemplate["dimensions"] = csvTableDimensions = OrderedDict() csvTableTemplate["columns"] = csvFactColumns = OrderedDict() if footnotesRelationshipSet.modelRelationships: csvLinks = OrderedDict() oimReport["links"] = csvLinks aspectQnCol = {} aspectsHeader = [] factsColumns = [] def addAspectQnCol(aspectQn): colQName = str(aspectQn).replace("xbrl:", "") colNCName = colQName.replace(":", "_") if colNCName not in aspectQnCol: aspectQnCol[colNCName] = len(aspectsHeader) aspectsHeader.append(colNCName) if colNCName == "value": csvFactColumns[colNCName] = col = OrderedDict() col["dimensions"] = OrderedDict() elif colNCName == "id": csvFactColumns[colNCName] = {} # empty object elif colNCName == "decimals": csvFactColumns[colNCName] = {} # empty object csvTableTemplate[colQName] = "$" + colNCName else: csvFactColumns[colNCName] = {} # empty object csvTableDimensions[colQName] = "$" + colNCName # pre-ordered aspect columns #if hasId: addAspectQnCol("id") addAspectQnCol(qnOimConceptAspect) if hasNumeric: addAspectQnCol("decimals") if qnOimEntityAspect in aspectsDefined: addAspectQnCol(qnOimEntityAspect) if qnOimPeriodAspect in aspectsDefined: addAspectQnCol(qnOimPeriodAspect) if qnOimUnitAspect in aspectsDefined: addAspectQnCol(qnOimUnitAspect) for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)): if aspectQn.namespaceURI != nsOim: addAspectQnCol(aspectQn) addAspectQnCol("value") def aspectCols(fact): cols = [None for i in range(len(aspectsHeader))] def setColValues(aspects): for aspectQn, aspectValue in aspects.items(): colQName = str(aspectQn).replace("xbrl:", "") colNCName = colQName.replace(":", "_") if isinstance(aspectValue, dict): setColValues(aspectValue) elif colNCName in aspectQnCol: if aspectValue is None: _aspectValue = "#nil" elif aspectValue == "": _aspectValue = "#empty" elif isinstance(aspectValue, str) and aspectValue.startswith("#"): _aspectValue = "#" + aspectValue else: _aspectValue = aspectValue cols[aspectQnCol[colNCName]] = _aspectValue setColValues(factAspects(fact)) return cols # metadata _open = _writerow = _close = None if isCSV: if oimFile.endswith( "-facts.csv" ): # strip -facts.csv if a prior -facts.csv file was chosen _baseURL = oimFile[:-10] elif oimFile.endswith(".csv"): _baseURL = oimFile[:-4] else: _baseURL = oimFile _csvinfo = {} # open file, writer def _open(filesuffix, tabname, csvTable=None): _filename = _baseURL + filesuffix if csvTable is not None: csvTable["url"] = os.path.basename( _filename) # located in same directory with metadata _csvinfo["file"] = open(_filename, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig') _csvinfo["writer"] = csv.writer(_csvinfo["file"], dialect="excel") def _writerow(row, header=False): _csvinfo["writer"].writerow(row) def _close(): _csvinfo["file"].close() _csvinfo.clear() elif isXL: headerWidths = { "concept": 40, "decimals": 8, "language": 9, "value": 50, "entity": 20, "period": 20, "unit": 20, "metadata": 100 } from openpyxl import Workbook from openpyxl.cell.cell import WriteOnlyCell from openpyxl.styles import Font, PatternFill, Border, Alignment, Color, fills, Side from openpyxl.worksheet.dimensions import ColumnDimension hdrCellFill = PatternFill( patternType=fills.FILL_SOLID, fgColor=Color( "00FFBF5F")) # Excel's light orange fill color = 00FF990 workbook = Workbook() # remove pre-existing worksheets while len(workbook.worksheets) > 0: workbook.remove(workbook.worksheets[0]) _xlinfo = {} # open file, writer def _open(filesuffix, tabname, csvTable=None): if csvTable is not None: csvTable["url"] = tabname + "!" _xlinfo["ws"] = workbook.create_sheet(title=tabname) def _writerow(rowvalues, header=False): row = [] _ws = _xlinfo["ws"] for i, v in enumerate(rowvalues): cell = WriteOnlyCell(_ws, value=v) if header: cell.fill = hdrCellFill cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True) colLetter = chr(ord('A') + i) _ws.column_dimensions[colLetter] = ColumnDimension( _ws, customWidth=True) _ws.column_dimensions[ colLetter].width = headerWidths.get(v, 40) else: cell.alignment = Alignment( horizontal="right" if isinstance(v, _NUM_TYPES) else "center" if isinstance(v, bool) else "left", vertical="top", wrap_text=isinstance(v, str)) row.append(cell) _ws.append(row) def _close(): _xlinfo.clear() # save facts _open("-facts.csv", "facts", csvTable) _writerow(aspectsHeader, header=True) def saveCSVfacts(facts): for fact in facts: _writerow(aspectCols(fact)) saveCSVfacts(fact.modelTupleFacts) saveCSVfacts(modelXbrl.facts) _close() # save footnotes if footnotesRelationshipSet.modelRelationships: footnotes = sorted( (footnote for fact in modelXbrl.facts for footnote in factFootnotes(fact, csvLinks=csvLinks)), key=lambda footnote: footnote["id"]) if footnotes: # text footnotes oimTables["footnotes"] = csvFtTable = OrderedDict() csvFtTable["url"] = "tbd" csvFtTable[ "tableDimensions"] = csvFtTableDimensions = OrderedDict() csvFtTable["factColumns"] = csvFtFactColumns = OrderedDict() csvFtFactColumns["footnote"] = csvFtValCol = OrderedDict() csvFtValCol["id"] = "$id" csvFtValCol["noteId"] = "$id" csvFtValCol["concept"] = "xbrl:note" csvFtValCol["language"] = "$language" _open("-footnotes.csv", "footnotes", csvFtTable) cols = ("id", "footnote", "language") _writerow(cols, header=True) for footnote in footnotes: _writerow(tuple((footnote.get(col, "") for col in cols))) _close() # save metadata if isCSV: with open(_baseURL + "-metadata.json", "w", encoding="utf-8") as fh: fh.write( json.dumps(oimReport, ensure_ascii=False, indent=2, sort_keys=False)) elif isXL: _open(None, "metadata") _writerow(["metadata"], header=True) _writerow([ json.dumps(oimReport, ensure_ascii=False, indent=1, sort_keys=False) ]) _close() if isXL: workbook.save(oimFile)
def saveLoadableOIM(modelXbrl, oimFile): isJSON = oimFile.endswith(".json") isCSV = oimFile.endswith(".csv") isXL = oimFile.endswith(".xlsx") isCSVorXL = isCSV or isXL if not isJSON and not isCSVorXL: return namespacePrefixes = {nsOim: "oim"} prefixNamespaces = {"oim": nsOim} def compileQname(qname): if qname.namespaceURI not in namespacePrefixes: namespacePrefixes[qname.namespaceURI] = qname.prefix or "" aspectsDefined = {qnOimConceptAspect, qnOimEntityAspect} if isJSON: aspectsDefined.add(qnOimPeriodAspect) elif isCSVorXL: aspectsDefined.add(qnOimPeriodStartAspect) aspectsDefined.add(qnOimPeriodEndAspect) def oimValue(object, decimals=None): if isinstance(object, QName): if object.namespaceURI not in namespacePrefixes: if object.prefix: namespacePrefixes[object.namespaceURI] = object.prefix else: _prefix = "_{}".format( sum(1 for p in namespacePrefixes if p.startswith("_"))) namespacePrefixes[object.namespaceURI] = _prefix return "{}:{}".format(namespacePrefixes[object.namespaceURI], object.localName) if isinstance(object, Decimal): try: if isinf(object): return "-INF" if object < 0 else "INF" elif isnan(num): return "NaN" else: if object == object.to_integral(): object = object.quantize(ONE) # drop any .0 return "{}".format(object) except: return str(object) if isinstance(object, bool): return object if isinstance(object, (DateTime, YearMonthDuration, DayTimeDuration, Time, gYearMonth, gMonthDay, gYear, gMonth, gDay)): return str(object) return object def oimPeriodValue(cntx): if cntx.isForeverPeriod: pass # not supported elif cntx.isStartEndPeriod: s = cntx.startDatetime e = cntx.endDatetime else: # instant s = e = cntx.instantDatetime return { "start": "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format( s.year, s.month, s.day, s.hour, s.minute, s.second), "end": "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format( e.year, e.month, e.day, e.hour, e.minute, e.second) } hasId = False hasTuple = False hasType = True hasLang = False hasUnits = False hasNumeric = False footnotesRelationshipSet = ModelRelationshipSet(modelXbrl, "XBRL-footnotes") factBaseTypes = set() #compile QNames in instance for OIM for fact in modelXbrl.factsInInstance: if (fact.id or fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))): hasId = True concept = fact.concept if concept is not None: if concept.isNumeric: hasNumeric = True if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang: hasLang = True _baseXsdType = concept.baseXsdType if _baseXsdType == "XBRLI_DATEUNION": if getattr(fact.xValue, "dateOnly", False): _baseXsdType = "date" else: _baseXsdType = "dateTime" factBaseTypes.add(baseTypes.get(_baseXsdType, _baseXsdType)) compileQname(fact.qname) if hasattr(fact, "xValue") and isinstance(fact.xValue, QName): compileQname(fact.xValue) unit = fact.unit if unit is not None: hasUnits = True if fact.modelTupleFacts: hasTuple = True entitySchemePrefixes = {} for cntx in modelXbrl.contexts.values(): if cntx.entityIdentifierElement is not None: scheme = cntx.entityIdentifier[0] if scheme not in entitySchemePrefixes: if not entitySchemePrefixes: # first one is just scheme if scheme == "http://www.sec.gov/CIK": _schemePrefix = "cik" elif scheme == "http://standard.iso.org/iso/17442": _schemePrefix = "lei" else: _schemePrefix = "scheme" else: _schemePrefix = "scheme{}".format( len(entitySchemePrefixes) + 1) entitySchemePrefixes[scheme] = _schemePrefix namespacePrefixes[scheme] = _schemePrefix for dim in cntx.qnameDims.values(): compileQname(dim.dimensionQname) aspectsDefined.add(dim.dimensionQname) if dim.isExplicit: compileQname(dim.memberQname) for unit in modelXbrl.units.values(): if unit is not None: for measures in unit.measures: for measure in measures: compileQname(measure) if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[ XbrlConst.xbrli] != "xbrli": namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix namespacePrefixes[XbrlConst.xsd] = "xsd" if hasLang: aspectsDefined.add(qnOimLangAspect) if hasTuple: aspectsDefined.add(qnOimTupleParentAspect) aspectsDefined.add(qnOimTupleOrderAspect) if hasUnits: aspectsDefined.add(qnOimUnitAspect) # compile footnotes and relationships ''' factRelationships = [] factFootnotes = [] for rel in modelXbrl.relationshipSet(modelXbrl, "XBRL-footnotes").modelRelationships: oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole} factRelationships.append(oimRel) oimRel["fromIds"] = [obj.id if obj.id else elementChildSequence(obj) for obj in rel.fromModelObjects] oimRel["toIds"] = [obj.id if obj.id else elementChildSequence(obj) for obj in rel.toModelObjects] _order = rel.arcElement.get("order") if _order is not None: oimRel["order"] = _order for obj in rel.toModelObjects: if isinstance(obj, ModelResource): # footnote oimFootnote = {"role": obj.role, "id": obj.id if obj.id else elementChildSequence(obj), # value needs work for html elements and for inline footnotes "value": xmlstring(obj, stripXmlns=True)} if obj.xmlLang: oimFootnote["lang"] = obj.xmlLang factFootnotes.append(oimFootnote) oimFootnote ''' dtsReferences = [{ "type": "schema" if doc.type == ModelDocument.Type.SCHEMA else "linkbase" if doc.type == ModelDocument.Type.LINKBASE else "other", "href": doc.uri } for doc, ref in modelXbrl.modelDocument.referencesDocument.items() if ref.referringModelObject.qname in SCHEMA_LB_REFS] ''' roleTypes = [ {"type": "role" if ref.referringModelObject.localName == "roleRef" else "arcroleRef", "href": ref.referringModelObject["href"]} for doc,ref in modelXbrl.modelDocument.referencesDocument.items() if ref.referringModelObject.qname in ROLE_REFS] ''' def factFootnotes(fact): footnotes = [] for footnoteRel in footnotesRelationshipSet.fromModelObject(fact): footnote = OrderedDict((("group", footnoteRel.linkrole), ("footnoteType", footnoteRel.arcrole))) footnotes.append(footnote) if isCSVorXL: footnote["factId"] = fact.id if fact.id else "f{}".format( fact.objectIndex) toObj = footnoteRel.toModelObject if isinstance(toObj, ModelFact): footnote["factRef"] = toObj.id if toObj.id else "f{}".format( toObj.objectIndex) else: footnote["footnote"] = xmlstring(toObj, stripXmlns=True, contentsOnly=True, includeText=True) if toObj.xmlLang: footnote["language"] = toObj.xmlLang return footnotes def factAspects(fact): aspects = OrderedDict() if hasId and fact.id: aspects["id"] = fact.id elif (fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))): aspects["id"] = "f{}".format(fact.objectIndex) parent = fact.getparent() concept = fact.concept _csvType = "Value" if not fact.isTuple: if concept is not None: _baseXsdType = concept.baseXsdType if _baseXsdType == "XBRLI_DATEUNION": if getattr(fact.xValue, "dateOnly", False): _baseXsdType = "date" else: _baseXsdType = "dateTime" aspects["baseType"] = "xsd:{}".format(_baseXsdType) _csvType = baseTypes.get(_baseXsdType, _baseXsdType) + "Value" if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang: aspects[qnOimLangAspect] = fact.xmlLang if fact.isItem: if fact.isNil: _value = None _strValue = "nil" else: _inferredDecimals = inferredDecimals(fact) _value = oimValue(fact.xValue, _inferredDecimals) _strValue = str(_value) if not isCSVorXL: aspects["value"] = _strValue if fact.concept is not None and fact.concept.isNumeric: _numValue = fact.xValue if isinstance(_numValue, Decimal) and not isinf( _numValue) and not isnan(_numValue): if _numValue == _numValue.to_integral(): _numValue = int(_numValue) else: _numValue = float(_numValue) if isCSVorXL: aspects[_csvType] = _numValue else: aspects["numericValue"] = _numValue if not fact.isNil: if isinf(_inferredDecimals): if isJSON: _accuracy = "infinity" elif isCSVorXL: _accuracy = "INF" else: _accuracy = _inferredDecimals aspects["accuracy"] = _accuracy elif isinstance(_value, bool): aspects["booleanValue"] = _value elif isCSVorXL: aspects[_csvType] = _strValue aspects[qnOimConceptAspect] = oimValue(fact.qname) cntx = fact.context if cntx is not None: if cntx.entityIdentifierElement is not None: aspects[qnOimEntityAspect] = oimValue( qname(*cntx.entityIdentifier)) if cntx.period is not None: if isJSON: aspects[qnOimPeriodAspect] = oimPeriodValue(cntx) elif isCSVorXL: _periodValue = oimPeriodValue(cntx) aspects[qnOimPeriodStartAspect] = _periodValue["start"] aspects[qnOimPeriodEndAspect] = _periodValue["end"] for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]): aspects[dim.dimensionQname] = ( oimValue(dim.memberQname) if dim.isExplicit else None if dim.typedMember.get( "{http://www.w3.org/2001/XMLSchema-instance}nil") in ( "true", "1") else dim.typedMember.stringValue) unit = fact.unit if unit is not None: _mMul, _mDiv = unit.measures _sMul = '*'.join( oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m))) if _mDiv: _sDiv = '*'.join( oimValue(m) for m in sorted(_mDiv, key=lambda m: oimValue(m))) if len(_mDiv) > 1: if len(_mMul) > 1: _sUnit = "({})/({})".format(_sMul, _sDiv) else: _sUnit = "{}/({})".format(_sMul, _sDiv) else: if len(_mMul) > 1: _sUnit = "({})/{}".format(_sMul, _sDiv) else: _sUnit = "{}/{}".format(_sMul, _sDiv) else: _sUnit = _sMul aspects[qnOimUnitAspect] = _sUnit if parent.qname != XbrlConst.qnXbrliXbrl: aspects[ qnOimTupleParentAspect] = parent.id if parent.id else "f{}".format( parent.objectIndex) aspects[qnOimTupleOrderAspect] = elementIndex(fact) if isJSON: _footnotes = factFootnotes(fact) if _footnotes: aspects["footnotes"] = _footnotes return aspects if isJSON: # save JSON oimReport = OrderedDict() # top level of oim json output oimFacts = [] oimReport["prefixes"] = OrderedDict( (p, ns) for ns, p in sorted(namespacePrefixes.items(), key=lambda item: item[1])) oimReport["dtsReferences"] = dtsReferences oimReport["facts"] = oimFacts def saveJsonFacts(facts, oimFacts, parentFact): for fact in facts: oimFact = factAspects(fact) oimFacts.append( OrderedDict((str(k), v) for k, v in oimFact.items())) if fact.modelTupleFacts: saveJsonFacts(fact.modelTupleFacts, oimFacts, fact) saveJsonFacts(modelXbrl.facts, oimFacts, None) with open(oimFile, "w", encoding="utf-8") as fh: fh.write(json.dumps(oimReport, indent=1)) elif isCSVorXL: # save CSV aspectQnCol = {} aspectsHeader = [] factsColumns = [] def addAspectQnCol(aspectQn): aspectQnCol[aspectQn] = len(aspectsHeader) _colName = oimValue(aspectQn) aspectsHeader.append(_colName) _colDataType = { "id": "Name", "baseType": "Name", "oim:concept": "QName", "oim:periodStart": "dateTime", "oim:periodEnd": "dateTime", "oim:tupleOrder": "integer", "numericValue": "decimal", "accuracy": "decimal", "booleanValue": "boolean", "oim:entity": "QName", "oim:unit": "string" }.get(_colName) if _colDataType is None: if isCSVorXL and _colName.endswith("Value"): _colDataType = _colName[:-5] else: _colDataType = "string" factsColumns.append( OrderedDict((("name", _colName), ("datatype", _colDataType)))) # pre-ordered aspect columns if hasId: addAspectQnCol("id") addAspectQnCol(qnOimConceptAspect) if hasType: addAspectQnCol("baseType") if isCSVorXL: for _baseType in sorted(factBaseTypes): addAspectQnCol(_baseType + "Value") else: addAspectQnCol("stringValue") addAspectQnCol("numericValue") addAspectQnCol("booleanValue") if hasNumeric: addAspectQnCol("accuracy") if hasTuple: addAspectQnCol(qnOimTupleParentAspect) addAspectQnCol(qnOimTupleOrderAspect) if qnOimEntityAspect in aspectsDefined: addAspectQnCol(qnOimEntityAspect) if qnOimPeriodStartAspect in aspectsDefined: addAspectQnCol(qnOimPeriodStartAspect) addAspectQnCol(qnOimPeriodEndAspect) if qnOimUnitAspect in aspectsDefined: addAspectQnCol(qnOimUnitAspect) for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)): if aspectQn.namespaceURI != nsOim: addAspectQnCol(aspectQn) def aspectCols(fact): cols = [None for i in range(len(aspectsHeader))] _factAspects = factAspects(fact) for aspectQn, aspectValue in _factAspects.items(): if aspectQn in aspectQnCol: cols[aspectQnCol[aspectQn]] = aspectValue return cols # metadata csvTables = [] csvMetadata = OrderedDict( (("@context", ["http://www.w3.org/ns/csvw", { "@base": "./" }]), ("tables", csvTables))) _open = _writerow = _close = None _tableinfo = {} if isCSV: if oimFile.endswith( "-facts.csv" ): # strip -facts.csv if a prior -facts.csv file was chosen _baseURL = oimFile[:-10] elif oimFile.endswith(".csv"): _baseURL = oimFile[:-4] else: _baseURL = oimFile _csvinfo = {} # open file, writer def _open(filesuffix, tabname): _filename = _tableinfo["url"] = _baseURL + filesuffix _csvinfo["file"] = open(_filename, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig') _csvinfo["writer"] = csv.writer(_csvinfo["file"], dialect="excel") def _writerow(row, header=False): _csvinfo["writer"].writerow(row) def _close(): _csvinfo["file"].close() _csvinfo.clear() elif isXL: headerWidths = { "href": 100, "oim:concept": 70, "accuracy": 8, "baseType": 10, "language": 9, "URI": 80, "stringValue": 60, "decimalValue": 12, "booleanValue": 12, "dateValue": 12, "dateTimeValue": 20, "group": 60, "footnoteType": 40, "footnote": 70, "column": 60 } from openpyxl import Workbook from openpyxl.writer.write_only import WriteOnlyCell from openpyxl.styles import Font, PatternFill, Border, Alignment, Color, fills, Side from openpyxl.worksheet.dimensions import ColumnDimension hdrCellFill = PatternFill( patternType=fills.FILL_SOLID, fgColor=Color( "00FFBF5F")) # Excel's light orange fill color = 00FF990 workbook = Workbook(encoding="utf-8", write_only=True) _xlinfo = {} # open file, writer def _open(filesuffix, tabname): _tableinfo["url"] = tabname _xlinfo["ws"] = workbook.create_sheet(title=tabname) def _writerow(rowvalues, header=False): row = [] _ws = _xlinfo["ws"] for i, v in enumerate(rowvalues): cell = WriteOnlyCell(_ws, value=v) if header: cell.fill = hdrCellFill cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True) colLetter = chr(ord('A') + i) _ws.column_dimensions[colLetter] = ColumnDimension( _ws, customWidth=True) _ws.column_dimensions[ colLetter].width = headerWidths.get(v, 20) else: cell.alignment = Alignment( horizontal="right" if isinstance(v, _NUM_TYPES) else "center" if isinstance(v, bool) else "left", vertical="top", wrap_text=isinstance(v, str)) row.append(cell) _ws.append(row) def _close(): _xlinfo.clear() # save facts _open("-facts.csv", "facts") _writerow(aspectsHeader, header=True) def saveCSVfacts(facts): for fact in facts: _writerow(aspectCols(fact)) saveCSVfacts(fact.modelTupleFacts) saveCSVfacts(modelXbrl.facts) _close() factsTableSchema = OrderedDict((("columns", factsColumns), )) csvTables.append( OrderedDict((("url", _tableinfo["url"]), ("tableSchema", factsTableSchema)))) # save namespaces _open("-prefixes.csv", "prefixes") _writerow(("prefix", "URI"), header=True) for _URI, prefix in sorted(namespacePrefixes.items(), key=lambda item: item[1]): _writerow((prefix, _URI)) _close() nsTableSchema = OrderedDict((("columns", [ OrderedDict((("name", "prefix"), ("datatype", "Name"))), OrderedDict((("name", "URI"), ("datatype", "anyURI"))) ]), )) csvTables.append( OrderedDict( (("url", _tableinfo["url"]), ("tableSchema", nsTableSchema)))) # save dts references _open("-dtsReferences.csv", "dtsReferences") _writerow(("type", "href"), header=True) for oimRef in dtsReferences: _writerow((oimRef["type"], oimRef["href"])) _close() dtsRefTableSchema = OrderedDict((("columns", [ OrderedDict((("name", "type"), ("datatype", "Name"))), OrderedDict((("name", "href"), ("datatype", "anyURI"))) ]), )) csvTables.append( OrderedDict((("url", _tableinfo["url"]), ("tableSchema", dtsRefTableSchema)))) # save footnotes if footnotesRelationshipSet.modelRelationships: _open("-footnotes.csv", "footnotes") cols = ("group", "footnoteType", "factId", "factRef", "footnote", "language") _writerow(cols, header=True) def saveCSVfootnotes(facts): for fact in facts: for _footnote in factFootnotes(fact): _writerow( tuple((_footnote.get(col, "") for col in cols))) saveCSVfootnotes(fact.modelTupleFacts) saveCSVfootnotes(modelXbrl.facts) _close() footnoteTableSchema = OrderedDict((("columns", [ OrderedDict((("name", "group"), ("datatype", "anyURI"))), OrderedDict((("name", "footnoteType"), ("datatype", "Name"))), OrderedDict((("name", "factId"), ("datatype", "Name"))), OrderedDict((("name", "factRef"), ("datatype", "Name"))), OrderedDict((("name", "footnote"), ("datatype", "string"))), OrderedDict((("name", "language"), ("datatype", "language"))) ]), )) csvTables.append( OrderedDict((("url", _tableinfo["url"]), ("tableSchema", footnoteTableSchema)))) # save metadata if isCSV: with open(_baseURL + "-metadata.csv", "w", encoding="utf-8") as fh: fh.write( json.dumps(csvMetadata, ensure_ascii=False, indent=1, sort_keys=False)) elif isXL: _open(None, "metadata") _writerow(("table", "column", "datatype"), header=True) for table in csvTables: tablename = table["url"] for column in table["tableSchema"]["columns"]: _writerow((tablename, column["name"], column["datatype"])) _close() if isXL: workbook.save(oimFile)
def saveLoadableOIM(modelXbrl, oimFile, outputZip=None): isJSON = oimFile.endswith(".json") isCSV = oimFile.endswith(".csv") isXL = oimFile.endswith(".xlsx") isCSVorXL = isCSV or isXL if not isJSON and not isCSVorXL: return namespacePrefixes = {nsOim: "xbrl"} prefixNamespaces = {"xbrl": nsOim} def compileQname(qname): if qname.namespaceURI not in namespacePrefixes: namespacePrefixes[qname.namespaceURI] = qname.prefix or "" aspectsDefined = { qnOimConceptAspect, qnOimEntityAspect, qnOimPeriodStartAspect, qnOimPeriodEndAspect} def oimValue(object, decimals=None): if isinstance(object, QName): if object.namespaceURI not in namespacePrefixes: if object.prefix: namespacePrefixes[object.namespaceURI] = object.prefix else: _prefix = "_{}".format(sum(1 for p in namespacePrefixes if p.startswith("_"))) namespacePrefixes[object.namespaceURI] = _prefix return "{}:{}".format(namespacePrefixes[object.namespaceURI], object.localName) if isinstance(object, Decimal): try: if isinf(object): return "-INF" if object < 0 else "INF" elif isnan(num): return "NaN" else: if object == object.to_integral(): object = object.quantize(ONE) # drop any .0 return "{}".format(object) except: return str(object) if isinstance(object, bool): return "true" if object else "false" if isinstance(object, (DateTime, YearMonthDuration, DayTimeDuration, Time, gYearMonth, gMonthDay, gYear, gMonth, gDay, IsoDuration)): return str(object) return object def oimPeriodValue(cntx): if cntx.isForeverPeriod: return OrderedDict() # defaulted elif cntx.isStartEndPeriod: s = cntx.startDatetime e = cntx.endDatetime else: # instant s = e = cntx.instantDatetime if isJSON: return({str(qnOimPeriodAspect): ("{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}/".format( s.year, s.month, s.day, s.hour, s.minute, s.second) if cntx.isStartEndPeriod else "") + "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format( e.year, e.month, e.day, e.hour, e.minute, e.second) }) # CSV, XL return OrderedDict(((str(qnOimPeriodStartAspect), "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format( s.year, s.month, s.day, s.hour, s.minute, s.second)), (str(qnOimPeriodEndAspect), "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format( e.year, e.month, e.day, e.hour, e.minute, e.second)))) hasId = False hasTuple = False hasType = True hasLang = False hasUnits = False hasNumeric = False footnotesRelationshipSet = ModelRelationshipSet(modelXbrl, "XBRL-footnotes") factBaseTypes = set() #compile QNames in instance for OIM for fact in modelXbrl.factsInInstance: if (fact.id or fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))): hasId = True concept = fact.concept if concept is not None: if concept.isNumeric: hasNumeric = True if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang: hasLang = True _baseXsdType = concept.baseXsdType if _baseXsdType == "XBRLI_DATEUNION": if getattr(fact.xValue, "dateOnly", False): _baseXsdType = "date" else: _baseXsdType = "dateTime" factBaseTypes.add(baseTypes.get(_baseXsdType,_baseXsdType)) compileQname(fact.qname) if hasattr(fact, "xValue") and isinstance(fact.xValue, QName): compileQname(fact.xValue) unit = fact.unit if unit is not None: hasUnits = True if fact.modelTupleFacts: hasTuple = True if hasTuple: modelXbrl.error("arelleOIMsaver:tuplesNotAllowed", "Tuples are not allowed in an OIM document", modelObject=modelXbrl) return entitySchemePrefixes = {} for cntx in modelXbrl.contexts.values(): if cntx.entityIdentifierElement is not None: scheme = cntx.entityIdentifier[0] if scheme not in entitySchemePrefixes: if not entitySchemePrefixes: # first one is just scheme if scheme == "http://www.sec.gov/CIK": _schemePrefix = "cik" elif scheme == "http://standard.iso.org/iso/17442": _schemePrefix = "lei" else: _schemePrefix = "scheme" else: _schemePrefix = "scheme{}".format(len(entitySchemePrefixes) + 1) entitySchemePrefixes[scheme] = _schemePrefix namespacePrefixes[scheme] = _schemePrefix for dim in cntx.qnameDims.values(): compileQname(dim.dimensionQname) aspectsDefined.add(dim.dimensionQname) if dim.isExplicit: compileQname(dim.memberQname) for unit in modelXbrl.units.values(): if unit is not None: for measures in unit.measures: for measure in measures: compileQname(measure) if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[XbrlConst.xbrli] != "xbrli": namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix namespacePrefixes[XbrlConst.xsd] = "xsd" if hasLang: aspectsDefined.add(qnOimLangAspect) if hasTuple: aspectsDefined.add(qnOimTupleParentAspect) aspectsDefined.add(qnOimTupleOrderAspect) if hasUnits: aspectsDefined.add(qnOimUnitAspect) for footnoteRel in footnotesRelationshipSet.modelRelationships: typePrefix = "ftTyp_" + os.path.basename(footnoteRel.arcrole) if footnoteRel.linkrole == XbrlConst.defaultLinkRole: groupPrefix = "ftGrp_default" else: groupPrefix = "ftGrp_" + os.path.basename(footnoteRel.linkrole) if typePrefix not in namespacePrefixes: namespacePrefixes[footnoteRel.arcrole] = typePrefix if groupPrefix not in namespacePrefixes: namespacePrefixes[footnoteRel.linkrole] = groupPrefix # compile footnotes and relationships ''' factRelationships = [] factFootnotes = [] for rel in modelXbrl.relationshipSet(modelXbrl, "XBRL-footnotes").modelRelationships: oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole} factRelationships.append(oimRel) oimRel["fromIds"] = [obj.id if obj.id else elementChildSequence(obj) for obj in rel.fromModelObjects] oimRel["toIds"] = [obj.id if obj.id else elementChildSequence(obj) for obj in rel.toModelObjects] _order = rel.arcElement.get("order") if _order is not None: oimRel["order"] = _order for obj in rel.toModelObjects: if isinstance(obj, ModelResource): # footnote oimFootnote = {"role": obj.role, "id": obj.id if obj.id else elementChildSequence(obj), # value needs work for html elements and for inline footnotes "value": xmlstring(obj, stripXmlns=True)} if obj.xmlLang: oimFootnote["lang"] = obj.xmlLang factFootnotes.append(oimFootnote) oimFootnote ''' dtsReferences = set() baseUrl = modelXbrl.modelDocument.uri.partition("#")[0] for doc,ref in sorted(modelXbrl.modelDocument.referencesDocument.items(), key=lambda _item:_item[0].uri): if ref.referringModelObject.qname in SCHEMA_LB_REFS: dtsReferences.add(relativeUri(baseUrl,doc.uri)) for refType in ("role", "arcrole"): for refElt in sorted(modelXbrl.modelDocument.xmlRootElement.iterchildren( "{{http://www.xbrl.org/2003/linkbase}}{}Ref".format(refType)), key=lambda elt:elt.get(refType+"URI") ): dtsReferences.add(refElt.get("{http://www.w3.org/1999/xlink}href").partition("#")[0]) dtsReferences = sorted(dtsReferences) # turn into list footnoteFacts = set() def factFootnotes(fact, oimFact=None): footnotes = [] oimLinks = {} for footnoteRel in footnotesRelationshipSet.fromModelObject(fact): toObj = footnoteRel.toModelObject # json typePrefix = namespacePrefixes[footnoteRel.arcrole] groupPrefix = namespacePrefixes[footnoteRel.linkrole] oimLinks.setdefault(typePrefix,{}).setdefault(groupPrefix,[]).append( toObj.id if toObj.id else "f{}".format(toObj.objectIndex)) # csv/XL footnote = OrderedDict((("group", footnoteRel.linkrole), ("footnoteType", footnoteRel.arcrole))) footnotes.append(footnote) if isCSVorXL: footnote["factId"] = fact.id if fact.id else "f{}".format(fact.objectIndex) if isinstance(toObj, ModelFact): footnote["factRef"] = toObj.id if toObj.id else "f{}".format(toObj.objectIndex) else: footnote["footnote"] = toObj.viewText() if toObj.xmlLang: footnote["language"] = toObj.xmlLang footnoteFacts.add(toObj) if isJSON and oimLinks: oimFact["links"] = OrderedDict(( (typePrefix, OrderedDict(( (groupPrefix, idList) for groupPrefix, idList in sorted(groups.items()) ))) for typePrefix, groups in sorted(oimLinks.items()) )) footnotes.sort(key=lambda f:(f["group"],f.get("factId",f.get("factRef")),f.get("language"))) return footnotes def factAspects(fact): oimFact = OrderedDict() aspects = OrderedDict() if hasId and fact.id: if fact.isTuple: oimFact["tupleId"] = fact.id else: oimFact["id"] = fact.id elif (fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))): oimFact["id"] = "f{}".format(fact.objectIndex) parent = fact.getparent() concept = fact.concept aspects[str(qnOimConceptAspect)] = oimValue(concept.qname) _csvType = "Value" if not fact.isTuple: if concept is not None: _baseXsdType = concept.baseXsdType if _baseXsdType == "XBRLI_DATEUNION": if getattr(fact.xValue, "dateOnly", False): _baseXsdType = "date" else: _baseXsdType = "dateTime" _csvType = baseTypes.get(_baseXsdType,_baseXsdType) + "Value" if concept.baseXbrliType in ("stringItemType", "normalizedStringItemType") and fact.xmlLang: aspects[str(qnOimLangAspect)] = fact.xmlLang if fact.isItem: if fact.isNil: _value = None else: _inferredDecimals = inferredDecimals(fact) _value = oimValue(fact.xValue, _inferredDecimals) oimFact["value"] = _value if fact.concept is not None and fact.concept.isNumeric: _numValue = fact.xValue if isinstance(_numValue, Decimal) and not isinf(_numValue) and not isnan(_numValue): if _numValue == _numValue.to_integral(): _numValue = int(_numValue) else: _numValue = float(_numValue) if not fact.isNil: if not isinf(_inferredDecimals): # accuracy omitted if infinite oimFact["decimals"] = _inferredDecimals oimFact["dimensions"] = aspects cntx = fact.context if cntx is not None: if cntx.entityIdentifierElement is not None: aspects[str(qnOimEntityAspect)] = oimValue(qname(*cntx.entityIdentifier)) if cntx.period is not None: aspects.update(oimPeriodValue(cntx)) for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]): if dim.isExplicit: dimVal = oimValue(dim.memberQname) else: # typed if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1"): dimVal = None else: dimVal = dim.typedMember.stringValue aspects[str(dim.dimensionQname)] = dimVal unit = fact.unit if unit is not None: _mMul, _mDiv = unit.measures _sMul = '*'.join(oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m))) if _mDiv: _sDiv = '*'.join(oimValue(m) for m in sorted(_mDiv, key=lambda m: oimValue(m))) if len(_mDiv) > 1: if len(_mMul) > 1: _sUnit = "({})/({})".format(_sMul,_sDiv) else: _sUnit = "{}/({})".format(_sMul,_sDiv) else: if len(_mMul) > 1: _sUnit = "({})/{}".format(_sMul,_sDiv) else: _sUnit = "{}/{}".format(_sMul,_sDiv) else: _sUnit = _sMul aspects[str(qnOimUnitAspect)] = _sUnit # Tuples removed from xBRL-JSON #if parent.qname != XbrlConst.qnXbrliXbrl: # aspects[str(qnOimTupleParentAspect)] = parent.id if parent.id else "f{}".format(parent.objectIndex) # aspects[str(qnOimTupleOrderAspect)] = elementIndex(fact) if isJSON: factFootnotes(fact, oimFact) return oimFact prefixes = OrderedDict((p,ns) for ns, p in sorted(namespacePrefixes.items(), key=lambda item: item[1])) if isJSON: # save JSON oimReport = OrderedDict() # top level of oim json output oimReport["documentInfo"] = oimDocInfo = OrderedDict() oimReport["facts"] = oimFacts = OrderedDict() oimDocInfo["documentType"] = nsOim + "/xbrl-json" oimDocInfo["features"] = oimFeatures = OrderedDict() oimDocInfo["prefixes"] = prefixes oimDocInfo["taxonomy"] = dtsReferences oimFeatures["xbrl:canonicalValues"] = True def saveJsonFacts(facts, oimFacts, parentFact): for fact in facts: oimFact = factAspects(fact) id = fact.id if fact.id else "f{}".format(fact.objectIndex) oimFacts[id] = oimFact if fact.modelTupleFacts: saveJsonFacts(fact.modelTupleFacts, oimFacts, fact) saveJsonFacts(modelXbrl.facts, oimFacts, None) # add footnotes as pseudo facts for ftObj in footnoteFacts: ftId = ftObj.id if ftObj.id else "f{}".format(ftObj.objectIndex) oimFacts[ftId] = oimFact = OrderedDict() oimFact["value"] = ftObj.viewText() oimFact["dimensions"] = OrderedDict((("concept", "xbrl:note"), ("noteId", ftId))) if ftObj.xmlLang: oimFact["dimensions"]["language"] = ftObj.xmlLang.lower() if outputZip: fh = io.StringIO() else: fh = open(oimFile, "w", encoding="utf-8") fh.write(json.dumps(oimReport, indent=1)) if outputZip: fh.seek(0) outputZip.writestr(os.path.basename(oimFile),fh.read()) fh.close() elif isCSVorXL: # save CSV aspectQnCol = {} aspectsHeader = [] factsColumns = [] def addAspectQnCol(aspectQn): aspectQnCol[str(aspectQn)] = len(aspectsHeader) _aspectQn = oimValue(aspectQn) aspectsHeader.append(_aspectQn) _colName = _aspectQn.replace("xbrl:", "") _colDataType = {"id": "Name", "concept": "QName", "value": "string", "decimals": "decimal", "entity": "QName", "periodStart": "dateTime", "periodEnd": "dateTime", "unit": "string", "tupleId": "Name", "tupleParent": "Name", "tupleOrder": "integer" }.get(_colName, "string") col = OrderedDict((("name", _colName), ("datatype", _colDataType))) if _aspectQn == "value": col["http://xbrl.org/YYYY/model#simpleFactAspects"] = {} elif _aspectQn == "tupleId": col["http://xbrl.org/YYYY/model#tupleFactAspects"] = {} col["http://xbrl.org/YYYY/model#tupleReferenceId"] = "true" else: col["http://xbrl.org/YYYY/model#columnAspect"] = _aspectQn factsColumns.append(col) # pre-ordered aspect columns #if hasId: # addAspectQnCol("id") addAspectQnCol(qnOimConceptAspect) addAspectQnCol("value") if hasNumeric: addAspectQnCol("decimals") if hasTuple: addAspectQnCol("tupleId") addAspectQnCol(qnOimTupleParentAspect) addAspectQnCol(qnOimTupleOrderAspect) if qnOimEntityAspect in aspectsDefined: addAspectQnCol(qnOimEntityAspect) if qnOimPeriodStartAspect in aspectsDefined: addAspectQnCol(qnOimPeriodStartAspect) addAspectQnCol(qnOimPeriodEndAspect) if qnOimUnitAspect in aspectsDefined: addAspectQnCol(qnOimUnitAspect) for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)): if aspectQn.namespaceURI != nsOim: addAspectQnCol(aspectQn) def aspectCols(fact): cols = [None for i in range(len(aspectsHeader))] def setColValues(aspects): for aspectQn, aspectValue in aspects.items(): if isinstance(aspectValue, dict): setColValues(aspectValue) elif aspectQn in aspectQnCol: if aspectValue is None: _aspectValue = "#nil" elif aspectValue == "": _aspectValue = "#empty" elif isinstance(aspectValue, str) and aspectValue.startswith("#"): _aspectValue = "#" + aspectValue else: _aspectValue = aspectValue cols[aspectQnCol[aspectQn]] = _aspectValue setColValues(factAspects(fact)) return cols # metadata csvTables = [] csvMetadata = OrderedDict((("@context", "http://www.w3.org/ns/csvw"), ("http://xbrl.org/YYYY/model#metadata", OrderedDict((("documentType", "http://xbrl.org/YYYY/xbrl-csv"), ("dtsReferences", dtsReferences), ("prefixes", prefixes)))), ("tables", csvTables))) _open = _writerow = _close = None _tableinfo = {} if isCSV: if oimFile.endswith("-facts.csv"): # strip -facts.csv if a prior -facts.csv file was chosen _baseURL = oimFile[:-10] elif oimFile.endswith(".csv"): _baseURL = oimFile[:-4] else: _baseURL = oimFile _csvinfo = {} # open file, writer def _open(filesuffix, tabname): _filename = _tableinfo["url"] = _baseURL + filesuffix _csvinfo["file"] = open(_filename, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig') _csvinfo["writer"] = csv.writer(_csvinfo["file"], dialect="excel") def _writerow(row, header=False): _csvinfo["writer"].writerow(row) def _close(): _csvinfo["file"].close() _csvinfo.clear() elif isXL: headerWidths = {"href": 100, "xbrl:concept": 70, "decimals": 8, "language": 9, "URI": 80, "value": 60, "group": 60, "footnoteType": 40, "footnote": 70, "column": 20, 'conceptAspect': 40, 'tuple': 20, 'simpleFact': 20} from openpyxl import Workbook from openpyxl.writer.write_only import WriteOnlyCell from openpyxl.styles import Font, PatternFill, Border, Alignment, Color, fills, Side from openpyxl.worksheet.dimensions import ColumnDimension hdrCellFill = PatternFill(patternType=fills.FILL_SOLID, fgColor=Color("00FFBF5F")) # Excel's light orange fill color = 00FF990 workbook = Workbook() # remove pre-existing worksheets while len(workbook.worksheets)>0: workbook.remove_sheet(workbook.worksheets[0]) _xlinfo = {} # open file, writer def _open(filesuffix, tabname): _tableinfo["url"] = tabname _xlinfo["ws"] = workbook.create_sheet(title=tabname) def _writerow(rowvalues, header=False): row = [] _ws = _xlinfo["ws"] for i, v in enumerate(rowvalues): cell = WriteOnlyCell(_ws, value=v) if header: cell.fill = hdrCellFill cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True) colLetter = chr(ord('A') + i) _ws.column_dimensions[colLetter] = ColumnDimension(_ws, customWidth=True) _ws.column_dimensions[colLetter].width = headerWidths.get(v, 20) else: cell.alignment = Alignment(horizontal="right" if isinstance(v, _NUM_TYPES) else "center" if isinstance(v, bool) else "left", vertical="top", wrap_text=isinstance(v, str)) row.append(cell) _ws.append(row) def _close(): _xlinfo.clear() # save facts _open("-facts.csv", "facts") _writerow(aspectsHeader, header=True) def saveCSVfacts(facts): for fact in facts: _writerow(aspectCols(fact)) saveCSVfacts(fact.modelTupleFacts) saveCSVfacts(modelXbrl.facts) _close() factsTableSchema = OrderedDict((("columns",factsColumns),)) csvTables.append(OrderedDict((("url",_tableinfo["url"]), ("http://xbrl.org/YYYY/model#tableType", "fact"), ("tableSchema",factsTableSchema)))) # save footnotes if footnotesRelationshipSet.modelRelationships: _open("-footnotes.csv", "footnotes") cols = ("group", "footnoteType", "factId", "factRef", "footnote", "language") _writerow(cols, header=True) def saveCSVfootnotes(facts): for fact in facts: for _footnote in factFootnotes(fact): _writerow(tuple((_footnote.get(col,"") for col in cols))) saveCSVfootnotes(fact.modelTupleFacts) saveCSVfootnotes(modelXbrl.facts) _close() footnoteTableSchema = OrderedDict((("columns",[OrderedDict((("name","group"),("datatype","anyURI"))), OrderedDict((("name","footnoteType"),("datatype","Name"))), OrderedDict((("name","factId"),("datatype","Name"))), OrderedDict((("name","factRef"),("datatype","Name"))), OrderedDict((("name","footnote"),("datatype","string"))), OrderedDict((("name","language"),("datatype","language")))]),)) csvTables.append(OrderedDict((("url",_tableinfo["url"]), ("http://xbrl.org/YYYY/model#tableType", "footnote"), ("tableSchema",footnoteTableSchema)))) # save metadata if isCSV: with open(_baseURL + "-metadata.json", "w", encoding="utf-8") as fh: fh.write(json.dumps(csvMetadata, ensure_ascii=False, indent=1, sort_keys=False)) elif isXL: _open(None, "metadata") hasColumnAspect = hasSimpleFact = hasTupleFact = False for table in csvTables: tablename = table["url"] for column in table["tableSchema"]["columns"]: if "http://xbrl.org/YYYY/model#columnAspect" in column: hasColumnAspect = True if "http://xbrl.org/YYYY/model#simpleFactAspects" in column: hasSimpleFact = True if "http://xbrl.org/YYYY/model#tupleAspects" in column: hasTupleFact = True metadataCols = ["table", "column", "datatype"] if hasColumnAspect: metadataCols.append("columnAspect") if hasSimpleFact: metadataCols.append("simpleFact") if hasTupleFact: metadataCols.append("tuple") _writerow(metadataCols, header=True) for table in csvTables: tablename = table["url"] for column in table["tableSchema"]["columns"]: row = [tablename, column["name"], column["datatype"]] if hasColumnAspect: colAspect = column.get("http://xbrl.org/YYYY/model#columnAspect") if isinstance(colAspect, str): row.append(colAspect) elif isinstance(colAspect, dict): row.append("\n".join("{} [{}]".format(k, ", ".join(_v for _v in v)) for k, v in dict.items())) else: row.append(None) if hasSimpleFact: row.append("\u221a" if "http://xbrl.org/YYYY/model#simpleFactAspects" in column else None) if hasTupleFact: row.append("\u221a" if "http://xbrl.org/YYYY/model#tupleAspects" in column else None) _writerow(row) _close() if isXL: workbook.save(oimFile)
def saveLoadableOIM(modelXbrl, oimFile): isJSON = oimFile.endswith(".json") isCSV = oimFile.endswith(".csv") isXL = oimFile.endswith(".xlsx") isCSVorXL = isCSV or isXL if not isJSON and not isCSVorXL: return namespacePrefixes = {} prefixNamespaces = {} def compileQname(qname): if qname.namespaceURI not in namespacePrefixes: namespacePrefixes[qname.namespaceURI] = qname.prefix or "" aspectsDefined = { qnOimConceptAspect, qnOimEntityAspect} if isJSON: aspectsDefined.add(qnOimPeriodAspect) elif isCSVorXL: aspectsDefined.add(qnOimPeriodStartAspect) aspectsDefined.add(qnOimPeriodEndAspect) def oimValue(object, decimals=None): if isinstance(object, QName): if object.namespaceURI not in namespacePrefixes: if object.prefix: namespacePrefixes[object.namespaceURI] = object.prefix else: _prefix = "_{}".format(sum(1 for p in namespacePrefixes if p.startswith("_"))) namespacePrefixes[object.namespaceURI] = _prefix return "{}:{}".format(namespacePrefixes[object.namespaceURI], object.localName) if isinstance(object, Decimal): try: if isinf(object): return "-INF" if object < 0 else "INF" elif isnan(num): return "NaN" else: if object == object.to_integral(): object = object.quantize(ONE) # drop any .0 return "{}".format(object) except: return str(object) if isinstance(object, bool): return object if isinstance(object, (DateTime, YearMonthDuration, DayTimeDuration, Time, gYearMonth, gMonthDay, gYear, gMonth, gDay)): return str(object) return object def oimPeriodValue(cntx): if cntx.isForeverPeriod: pass # not supported elif cntx.isStartEndPeriod: s = cntx.startDatetime e = cntx.endDatetime else: # instant s = e = cntx.instantDatetime return {"start": "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format( s.year, s.month, s.day, s.hour, s.minute, s.second), "end": "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format( e.year, e.month, e.day, e.hour, e.minute, e.second)} hasId = False hasTuple = False hasType = True hasLang = False hasUnits = False footnotesRelationshipSet = ModelRelationshipSet(modelXbrl, "XBRL-footnotes") factBaseTypes = set() #compile QNames in instance for OIM for fact in modelXbrl.factsInInstance: if (fact.id or fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))): hasId = True concept = fact.concept if concept is not None: if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang: hasLang = True _baseXsdType = concept.baseXsdType if _baseXsdType == "XBRLI_DATEUNION": if getattr(fact.xValue, "dateOnly", False): _baseXsdType = "date" else: _baseXsdType = "dateTime" factBaseTypes.add(baseTypes.get(_baseXsdType,_baseXsdType)) compileQname(fact.qname) if hasattr(fact, "xValue") and isinstance(fact.xValue, QName): compileQname(fact.xValue) unit = fact.unit if unit is not None: hasUnits = True if fact.modelTupleFacts: hasTuple = True entitySchemePrefixes = {} for cntx in modelXbrl.contexts.values(): if cntx.entityIdentifierElement is not None: scheme = cntx.entityIdentifier[0] if scheme not in entitySchemePrefixes: if not entitySchemePrefixes: # first one is just scheme if scheme == "http://www.sec.gov/CIK": _schemePrefix = "cik" elif scheme == "http://standard.iso.org/iso/17442": _schemePrefix = "lei" else: _schemePrefix = "scheme" else: _schemePrefix = "scheme{}".format(len(entitySchemePrefixes) + 1) entitySchemePrefixes[scheme] = _schemePrefix namespacePrefixes[scheme] = _schemePrefix for dim in cntx.qnameDims.values(): compileQname(dim.dimensionQname) aspectsDefined.add(dim.dimensionQname) if dim.isExplicit: compileQname(dim.memberQname) for unit in modelXbrl.units.values(): if unit is not None: for measures in unit.measures: for measure in measures: compileQname(measure) if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[XbrlConst.xbrli] != "xbrli": namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix namespacePrefixes[XbrlConst.xsd] = "xsd" if hasLang: aspectsDefined.add(qnOimLangAspect) if hasTuple: aspectsDefined.add(qnOimTupleParentAspect) aspectsDefined.add(qnOimTupleOrderAspect) if hasUnits: aspectsDefined.add(qnOimUnitAspect) # compile footnotes and relationships ''' factRelationships = [] factFootnotes = [] for rel in modelXbrl.relationshipSet(modelXbrl, "XBRL-footnotes").modelRelationships: oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole} factRelationships.append(oimRel) oimRel["fromIds"] = [obj.id if obj.id else elementChildSequence(obj) for obj in rel.fromModelObjects] oimRel["toIds"] = [obj.id if obj.id else elementChildSequence(obj) for obj in rel.toModelObjects] _order = rel.arcElement.get("order") if _order is not None: oimRel["order"] = _order for obj in rel.toModelObjects: if isinstance(obj, ModelResource): # footnote oimFootnote = {"role": obj.role, "id": obj.id if obj.id else elementChildSequence(obj), # value needs work for html elements and for inline footnotes "value": xmlstring(obj, stripXmlns=True)} if obj.xmlLang: oimFootnote["lang"] = obj.xmlLang factFootnotes.append(oimFootnote) oimFootnote ''' dtsReferences = [ {"type": "schema" if doc.type == ModelDocument.Type.SCHEMA else "linkbase" if doc.type == ModelDocument.Type.LINKBASE else "other", "href": doc.uri} for doc,ref in modelXbrl.modelDocument.referencesDocument.items() if ref.referringModelObject.qname in SCHEMA_LB_REFS] ''' roleTypes = [ {"type": "role" if ref.referringModelObject.localName == "roleRef" else "arcroleRef", "href": ref.referringModelObject["href"]} for doc,ref in modelXbrl.modelDocument.referencesDocument.items() if ref.referringModelObject.qname in ROLE_REFS] ''' def factFootnotes(fact): footnotes = [] for footnoteRel in footnotesRelationshipSet.fromModelObject(fact): footnote = OrderedDict((("group", footnoteRel.linkrole), ("footnoteType", footnoteRel.arcrole))) footnotes.append(footnote) if isCSVorXL: footnote["factId"] = fact.id if fact.id else "f{}".format(fact.objectIndex) toObj = footnoteRel.toModelObject if isinstance(toObj, ModelFact): footnote["factRef"] = toObj.id if toObj.id else "f{}".format(toObj.objectIndex) else: footnote["footnote"] = xmlstring(toObj, stripXmlns=True, contentsOnly=True, includeText=True) if toObj.xmlLang: footnote["language"] = toObj.xmlLang return footnotes def factAspects(fact): aspects = OrderedDict() if hasId and fact.id: aspects["id"] = fact.id elif (fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))): aspects["id"] = "f{}".format(fact.objectIndex) parent = fact.getparent() concept = fact.concept _csvType = "Value" if not fact.isTuple: if concept is not None: _baseXsdType = concept.baseXsdType if _baseXsdType == "XBRLI_DATEUNION": if getattr(fact.xValue, "dateOnly", False): _baseXsdType = "date" else: _baseXsdType = "dateTime" aspects["baseType"] = "xs:{}".format(_baseXsdType) _csvType = baseTypes.get(_baseXsdType,_baseXsdType) + "Value" if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang: aspects[qnOimLangAspect] = fact.xmlLang if fact.isItem: if fact.isNil: _value = None _strValue = "nil" else: _inferredDecimals = inferredDecimals(fact) _value = oimValue(fact.xValue, _inferredDecimals) _strValue = str(_value) if not isCSVorXL: aspects["value"] = _strValue if fact.concept is not None and fact.concept.isNumeric: _numValue = fact.xValue if isinstance(_numValue, Decimal) and not isinf(_numValue) and not isnan(_numValue): if _numValue == _numValue.to_integral(): _numValue = int(_numValue) else: _numValue = float(_numValue) if isCSVorXL: aspects[_csvType] = _numValue else: aspects["numericValue"] = _numValue if not fact.isNil: if isinf(_inferredDecimals): if isJSON: _accuracy = "infinity" elif isCSVorXL: _accuracy = "INF" else: _accuracy = _inferredDecimals aspects["accuracy"] = _accuracy elif isinstance(_value, bool): aspects["booleanValue"] = _value elif isCSVorXL: aspects[_csvType] = _strValue aspects[qnOimConceptAspect] = oimValue(fact.qname) cntx = fact.context if cntx is not None: if cntx.entityIdentifierElement is not None: aspects[qnOimEntityAspect] = oimValue(qname(*cntx.entityIdentifier)) if cntx.period is not None: if isJSON: aspects[qnOimPeriodAspect] = oimPeriodValue(cntx) elif isCSVorXL: _periodValue = oimPeriodValue(cntx) aspects[qnOimPeriodStartAspect] = _periodValue["start"] aspects[qnOimPeriodEndAspect] = _periodValue["end"] for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]): aspects[dim.dimensionQname] = (oimValue(dim.memberQname) if dim.isExplicit else None if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1") else dim.typedMember.stringValue) unit = fact.unit if unit is not None: _mMul, _mDiv = unit.measures _sMul = '*'.join(oimValue(m) for m in sorted(_mMul, key=lambda m: str(m))) if _mDiv: _sDiv = '*'.join(oimValue(m) for m in sorted(_mDiv, key=lambda m: str(m))) if len(mDiv) > 1: if len(mMul) > 1: _sUnit = "({})/({})".format(_sMul,_sDiv) else: _sUnit = "{}/({})".format(_sMul,_sDiv) else: if len(mMul) > 1: _sUnit = "({})/{}".format(_sMul,_sDiv) else: _sUnit = "{}/{}".format(_sMul,_sDiv) else: _sUnit = _sMul aspects[qnOimUnitAspect] = _sUnit if parent.qname != XbrlConst.qnXbrliXbrl: aspects[qnOimTupleParentAspect] = parent.id if parent.id else "f{}".format(parent.objectIndex) aspects[qnOimTupleOrderAspect] = elementIndex(fact) if isJSON: _footnotes = factFootnotes(fact) if _footnotes: aspects["footnotes"] = _footnotes return aspects if isJSON: # save JSON oimReport = OrderedDict() # top level of oim json output oimFacts = [] oimReport["prefixes"] = OrderedDict((p,ns) for ns, p in sorted(namespacePrefixes.items(), key=lambda item: item[1])) oimReport["dtsReferences"] = dtsReferences oimReport["facts"] = oimFacts def saveJsonFacts(facts, oimFacts, parentFact): for fact in facts: oimFact = factAspects(fact) oimFacts.append(OrderedDict((str(k),v) for k,v in oimFact.items())) if fact.modelTupleFacts: saveJsonFacts(fact.modelTupleFacts, oimFacts, fact) saveJsonFacts(modelXbrl.facts, oimFacts, None) with open(oimFile, "w", encoding="utf-8") as fh: fh.write(json.dumps(oimReport, ensure_ascii=False, indent=1, sort_keys=False)) elif isCSVorXL: # save CSV aspectQnCol = {} aspectsHeader = [] factsColumns = [] def addAspectQnCol(aspectQn): aspectQnCol[aspectQn] = len(aspectsHeader) _colName = oimValue(aspectQn) aspectsHeader.append(_colName) _colDataType = {"id": "Name", "baseType": "Name", "oim:concept": "QName", "oim:periodStart": "dateTime", "oim:periodEnd": "dateTime", "oim:tupleOrder": "integer", "numericValue": "decimal", "accuracy": "decimal", "booleanValue": "boolean", "oim:entity": "QName", "oim:unit": "string" }.get(_colName) if _colDataType is None: if isCSVorXL and _colName.endswith("Value"): _colDataType = _colName[:-5] else: _colDataType = "string" factsColumns.append(OrderedDict((("name", _colName), ("datatype", _colDataType)))) # pre-ordered aspect columns if hasId: addAspectQnCol("id") addAspectQnCol(qnOimConceptAspect) if hasType: addAspectQnCol("baseType") if isCSVorXL: for _baseType in sorted(factBaseTypes): addAspectQnCol(_baseType + "Value") else: addAspectQnCol("stringValue") addAspectQnCol("numericValue") addAspectQnCol("booleanValue") addAspectQnCol("accuracy") if hasTuple: addAspectQnCol(qnOimTupleParentAspect) addAspectQnCol(qnOimTupleOrderAspect) if qnOimEntityAspect in aspectsDefined: addAspectQnCol(qnOimEntityAspect) if qnOimPeriodStartAspect in aspectsDefined: addAspectQnCol(qnOimPeriodStartAspect) addAspectQnCol(qnOimPeriodEndAspect) if qnOimUnitAspect in aspectsDefined: addAspectQnCol(qnOimUnitAspect) for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)): if aspectQn.namespaceURI != nsOim: addAspectQnCol(aspectQn) def aspectCols(fact): cols = [None for i in range(len(aspectsHeader))] _factAspects = factAspects(fact) for aspectQn, aspectValue in _factAspects.items(): if aspectQn in aspectQnCol: cols[aspectQnCol[aspectQn]] = aspectValue return cols # metadata csvTables = [] csvMetadata = OrderedDict((("@context",[ "http://www.w3.org/ns/csvw", { "@base": "./" }]), ("tables", csvTables))) _open = _writerow = _close = None _tableinfo = {} if isCSV: if oimFile.endswith("-facts.csv"): # strip -facts.csv if a prior -facts.csv file was chosen _baseURL = oimFile[:-10] elif oimFile.endswith(".csv"): _baseURL = oimFile[:-4] else: _baseURL = oimFile _csvinfo = {} # open file, writer def _open(filesuffix, tabname): _filename = _tableinfo["url"] = _baseURL + filesuffix _csvinfo["file"] = open(_filename, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig') _csvinfo["writer"] = csv.writer(csvFile, dialect="excel") def _writerow(row, header=False): _csvinfo["writer"].writerow(row) def _close(): _csvinfo["writer"].close() _csvinfo.clear() elif isXL: headerWidths = {"href": 100, "oim:concept": 70, "accuracy": 8, "baseType": 10, "language": 9, "URI": 80, "stringValue": 60, "decimalValue": 12, "booleanValue": 12, "dateValue": 12, "dateTimeValue": 20, "group": 60, "footnoteType": 40, "footnote": 70, "column": 60} from openpyxl import Workbook from openpyxl.writer.write_only import WriteOnlyCell from openpyxl.styles import Font, PatternFill, Border, Alignment, Color, fills, Side from openpyxl.worksheet.dimensions import ColumnDimension hdrCellFill = PatternFill(patternType=fills.FILL_SOLID, fgColor=Color("00FFBF5F")) # Excel's light orange fill color = 00FF990 workbook = Workbook(encoding="utf-8", write_only=True) _xlinfo = {} # open file, writer def _open(filesuffix, tabname): _tableinfo["url"] = tabname _xlinfo["ws"] = workbook.create_sheet(title=tabname) def _writerow(rowvalues, header=False): row = [] _ws = _xlinfo["ws"] for i, v in enumerate(rowvalues): cell = WriteOnlyCell(_ws, value=v) if header: cell.fill = hdrCellFill cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True) colLetter = chr(ord('A') + i) _ws.column_dimensions[colLetter] = ColumnDimension(_ws, customWidth=True) _ws.column_dimensions[colLetter].width = headerWidths.get(v, 20) else: cell.alignment = Alignment(horizontal="right" if isinstance(v, _NUM_TYPES) else "center" if isinstance(v, bool) else "left", vertical="top", wrap_text=isinstance(v, str)) row.append(cell) _ws.append(row) def _close(): _xlinfo.clear() # save facts _open("-facts.csv", "facts") _writerow(aspectsHeader, header=True) def saveCSVfacts(facts): for fact in facts: _writerow(aspectCols(fact)) saveCSVfacts(fact.modelTupleFacts) saveCSVfacts(modelXbrl.facts) _close() factsTableSchema = OrderedDict((("columns",factsColumns),)) csvTables.append(OrderedDict((("url",_tableinfo["url"]), ("tableSchema",factsTableSchema)))) # save namespaces _open("-prefixes.csv", "prefixes") _writerow(("prefix", "URI"), header=True) for _URI, prefix in sorted(namespacePrefixes.items(), key=lambda item: item[1]): _writerow((prefix, _URI)) _close() nsTableSchema = OrderedDict((("columns",[OrderedDict((("name","prefix"),("datatype","Name"))), OrderedDict((("name","URI"),("datatype","anyURI")))]),)) csvTables.append(OrderedDict((("url",_tableinfo["url"]), ("tableSchema",nsTableSchema)))) # save dts references _open("-dtsReferences.csv", "dtsReferences") _writerow(("type", "href"), header=True) for oimRef in dtsReferences: _writerow((oimRef["type"], oimRef["href"])) _close() dtsRefTableSchema = OrderedDict((("columns",[OrderedDict((("name","type"),("datatype","Name"))), OrderedDict((("name","href"),("datatype","anyURI")))]),)) csvTables.append(OrderedDict((("url",_tableinfo["url"]), ("tableSchema",dtsRefTableSchema)))) # save footnotes if footnotesRelationshipSet.modelRelationships: _open("-footnotes.csv", "footnotes") cols = ("group", "footnoteType", "factId", "factRef", "footnote", "language") _writerow(cols, header=True) def saveCSVfootnotes(facts): for fact in facts: for _footnote in factFootnotes(fact): _writerow(tuple((_footnote.get(col,"") for col in cols))) saveCSVfootnotes(fact.modelTupleFacts) saveCSVfootnotes(modelXbrl.facts) _close() footnoteTableSchema = OrderedDict((("columns",[OrderedDict((("name","group"),("datatype","anyURI"))), OrderedDict((("name","footnoteType"),("datatype","Name"))), OrderedDict((("name","factId"),("datatype","Name"))), OrderedDict((("name","factRef"),("datatype","Name"))), OrderedDict((("name","footnote"),("datatype","string"))), OrderedDict((("name","language"),("datatype","language")))]),)) csvTables.append(OrderedDict((("url",_tableinfo["url"]), ("tableSchema",footnoteTableSchema)))) # save metadata if isCSV: with open(_baseURL + "-metadata.csv", "w", encoding="utf-8") as fh: fh.write(json.dumps(csvMetadata, ensure_ascii=False, indent=1, sort_keys=False)) elif isXL: _open(None, "metadata") cols = ("group", "footnoteType", "factId", "factRef", "footnote", "language") _writerow(("table", "column", "datatype"), header=True) for table in csvTables: tablename = table["url"] for column in table["tableSchema"]["columns"]: _writerow((tablename, column["name"], column["datatype"])) _close() if isXL: workbook.save(oimFile)
def saveLoadableOIM(modelXbrl, oimFile): isJSON = oimFile.endswith(".json") isCSV = oimFile.endswith(".csv") namespacePrefixes = {} def compileQname(qname): if qname.namespaceURI not in namespacePrefixes: namespacePrefixes[qname.namespaceURI] = qname.prefix or "" aspectsDefined = { qnOimConceptAspect, qnOimEntityAspect, qnOimTypeAspect} if isJSON: aspectsDefined.add(qnOimPeriodAspect) elif isCSV: aspectsDefined.add(qnOimPeriodStartAspect) aspectsDefined.add(qnOimPeriodDurationAspect) def oimValue(object, decimals=None): if isinstance(object, QName): if object.namespaceURI not in namespacePrefixes: if object.prefix: namespacePrefixes[object.namespaceURI] = object.prefix else: _prefix = "_{}".format(sum(1 for p in namespacePrefixes if p.startswith("_"))) namespacePrefixes[object.namespaceURI] = _prefix return "{}:{}".format(namespacePrefixes[object.namespaceURI], object.localName) if isinstance(object, Decimal): try: if isinf(object): return "-INF" if object < 0 else "INF" elif isnan(num): return "NaN" else: if object == object.to_integral(): object = object.quantize(ONE) # drop any .0 return "{}".format(object) except: return str(object) if isinstance(object, bool): return object if isinstance(object, (DateTime, YearMonthDuration, DayTimeDuration, Time, gYearMonth, gMonthDay, gYear, gMonth, gDay)): return str(object) return object def oimPeriodValue(cntx): if cntx.isForeverPeriod: if isCSV: return "0000-01-01T00:00:00/P9999Y" return "forever" elif cntx.isStartEndPeriod: d = cntx.startDatetime duration = yearMonthDayTimeDuration(cntx.startDatetime, cntx.endDatetime) else: # instant d = cntx.instantDatetime duration = "PT0S" return "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}/{6}".format( d.year, d.month, d.day, d.hour, d.minute, d.second, duration) hasId = False hasTuple = False hasType = True hasLang = False hasUnits = False hasUnitMulMeasures = False hasUnitDivMeasures = False footnotesRelationshipSet = ModelRelationshipSet(modelXbrl, "XBRL-footnotes") #compile QNames in instance for OIM for fact in modelXbrl.factsInInstance: if (fact.id or fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSV and footnotesRelationshipSet.fromModelObject(fact))): hasId = True concept = fact.concept if concept is not None: if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang: hasLang = True compileQname(fact.qname) if hasattr(fact, "xValue") and isinstance(fact.xValue, QName): compileQname(fact.xValue) unit = fact.unit if unit is not None: hasUnits = True if unit.measures[0]: hasUnitMulMeasures = True if unit.measures[1]: hasUnitDivMeasures = True if fact.modelTupleFacts: hasTuple = True entitySchemePrefixes = {} for cntx in modelXbrl.contexts.values(): if cntx.entityIdentifierElement is not None: scheme = cntx.entityIdentifier[0] if scheme not in entitySchemePrefixes: if not entitySchemePrefixes: # first one is just scheme if scheme == "http://www.sec.gov/CIK": _schemePrefix = "cik" elif scheme == "http://standard.iso.org/iso/17442": _schemePrefix = "lei" else: _schemePrefix = "scheme" else: _schemePrefix = "scheme{}".format(len(entitySchemePrefixes) + 1) entitySchemePrefixes[scheme] = _schemePrefix namespacePrefixes[scheme] = _schemePrefix for dim in cntx.qnameDims.values(): compileQname(dim.dimensionQname) aspectsDefined.add(dim.dimensionQname) if dim.isExplicit: compileQname(dim.memberQname) for unit in modelXbrl.units.values(): if unit is not None: for measures in unit.measures: for measure in measures: compileQname(measure) if unit.measures[0]: aspectsDefined.add(qnOimUnitNumeratorsAspect) if unit.measures[1]: aspectsDefined.add(qnOimUnitDenominatorsAspect) if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[XbrlConst.xbrli] != "xbrli": namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix namespacePrefixes[XbrlConst.xsd] = "xsd" if hasLang: aspectsDefined.add(qnOimLangAspect) if hasTuple: aspectsDefined.add(qnOimTupleParentAspect) aspectsDefined.add(qnOimTupleOrderAspect) if hasUnits: aspectsDefined.add(qnOimUnitAspect) # compile footnotes and relationships ''' factRelationships = [] factFootnotes = [] for rel in modelXbrl.relationshipSet(modelXbrl, "XBRL-footnotes").modelRelationships: oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole} factRelationships.append(oimRel) oimRel["fromIds"] = [obj.id if obj.id else elementChildSequence(obj) for obj in rel.fromModelObjects] oimRel["toIds"] = [obj.id if obj.id else elementChildSequence(obj) for obj in rel.toModelObjects] _order = rel.arcElement.get("order") if _order is not None: oimRel["order"] = _order for obj in rel.toModelObjects: if isinstance(obj, ModelResource): # footnote oimFootnote = {"role": obj.role, "id": obj.id if obj.id else elementChildSequence(obj), # value needs work for html elements and for inline footnotes "value": xmlstring(obj, stripXmlns=True)} if obj.xmlLang: oimFootnote["lang"] = obj.xmlLang factFootnotes.append(oimFootnote) oimFootnote ''' dtsReferences = [ {"type": "schema" if doc.type == ModelDocument.Type.SCHEMA else "linkbase" if doc.type == ModelDocument.Type.LINKBASE else "other", "href": doc.uri} for doc,ref in modelXbrl.modelDocument.referencesDocument.items() if ref.referringModelObject.qname in SCHEMA_LB_REFS] ''' roleTypes = [ {"type": "role" if ref.referringModelObject.localName == "roleRef" else "arcroleRef", "href": ref.referringModelObject["href"]} for doc,ref in modelXbrl.modelDocument.referencesDocument.items() if ref.referringModelObject.qname in ROLE_REFS] ''' def factFootnotes(fact): footnotes = [] for footnoteRel in footnotesRelationshipSet.fromModelObject(fact): footnote = OrderedDict((("group", footnoteRel.arcrole),)) footnotes.append(footnote) if isCSV: footnote["factId"] = fact.id if fact.id else "f{}".format(fact.objectIndex) toObj = footnoteRel.toModelObject if isinstance(toObj, ModelFact): footnote["factRef"] = toObj.id if toObj.id else "f{}".format(toObj.objectIndex) else: footnote["footnoteType"] = toObj.role footnote["footnote"] = xmlstring(toObj, stripXmlns=True, contentsOnly=True, includeText=True) if toObj.xmlLang: footnote["language"] = toObj.xmlLang return footnotes def factAspects(fact): aspects = OrderedDict() if hasId and fact.id: aspects["id"] = fact.id elif (fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSV and footnotesRelationshipSet.fromModelObject(fact))): aspects["id"] = "f{}".format(fact.objectIndex) parent = fact.getparent() concept = fact.concept if not fact.isTuple: if concept is not None: _baseXsdType = concept.baseXsdType if _baseXsdType == "XBRLI_DATEUNION": if getattr(fact.xValue, "dateOnly", False): _baseXsdType = "date" else: _baseXsdType = "dateTime" aspects["baseType"] = "xs:{}".format(_baseXsdType) if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang: aspects[qnOimLangAspect] = fact.xmlLang aspects[qnOimTypeAspect] = concept.baseXbrliType if fact.isItem: if fact.isNil: _value = None _strValue = "nil" else: _inferredDecimals = inferredDecimals(fact) _value = oimValue(fact.xValue, _inferredDecimals) _strValue = str(_value) if not isCSV: aspects["value"] = _strValue if fact.concept is not None and fact.concept.isNumeric: _numValue = fact.xValue if isinstance(_numValue, Decimal) and not isinf(_numValue) and not isnan(_numValue): if _numValue == _numValue.to_integral(): _numValue = int(_numValue) else: _numValue = float(_numValue) aspects["numericValue"] = _numValue if not fact.isNil: if isinf(_inferredDecimals): if isJSON: _accuracy = "infinity" elif isCSV: _accuracy = "INF" else: _accuracy = _inferredDecimals aspects["accuracy"] = _inferredDecimals elif isinstance(_value, bool): aspects["booleanValue"] = _value elif isCSV: aspects["stringValue"] = _strValue aspects[qnOimConceptAspect] = oimValue(fact.qname) cntx = fact.context if cntx is not None: if cntx.entityIdentifierElement is not None: aspects[qnOimEntityAspect] = oimValue(qname(*cntx.entityIdentifier)) if cntx.period is not None: if isJSON: aspects[qnOimPeriodAspect] = oimPeriodValue(cntx) elif isCSV: _periodValue = oimPeriodValue(cntx).split("/") + ["", ""] # default blank if no value aspects[qnOimPeriodStartAspect] = _periodValue[0] aspects[qnOimPeriodDurationAspect] = _periodValue[1] for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]): aspects[dim.dimensionQname] = (oimValue(dim.memberQname) if dim.isExplicit else None if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1") else dim.typedMember.stringValue) unit = fact.unit if unit is not None: _mMul, _mDiv = unit.measures if isJSON: aspects[qnOimUnitAspect] = OrderedDict( # use tuple instead of list for hashability (("numerators", tuple(oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m)))),) ) if _mDiv: aspects[qnOimUnitAspect]["denominators"] = tuple(oimValue(m) for m in sorted(_mDiv, key=lambda m: oimValue(m))) else: # CSV if _mMul: aspects[qnOimUnitNumeratorsAspect] = " ".join(oimValue(m) for m in sorted(_mMul, key=lambda m: str(m))) if _mDiv: aspects[qnOimUnitDenominatorsAspect] = " ".join(oimValue(m) for m in sorted(_mDiv, key=lambda m: str(m))) if parent.qname != XbrlConst.qnXbrliXbrl: aspects[qnOimTupleParentAspect] = parent.id if parent.id else "f{}".format(parent.objectIndex) aspects[qnOimTupleOrderAspect] = elementIndex(fact) if isJSON: _footnotes = factFootnotes(fact) if _footnotes: aspects["footnotes"] = _footnotes return aspects if isJSON: # save JSON oimReport = OrderedDict() # top level of oim json output oimFacts = [] oimReport["prefixes"] = OrderedDict((p,ns) for ns, p in sorted(namespacePrefixes.items(), key=lambda item: item[1])) oimReport["dtsReferences"] = dtsReferences oimReport["facts"] = oimFacts def saveJsonFacts(facts, oimFacts, parentFact): for fact in facts: oimFact = factAspects(fact) oimFacts.append(OrderedDict((str(k),v) for k,v in oimFact.items())) if fact.modelTupleFacts: saveJsonFacts(fact.modelTupleFacts, oimFacts, fact) saveJsonFacts(modelXbrl.facts, oimFacts, None) with open(oimFile, "w", encoding="utf-8") as fh: fh.write(json.dumps(oimReport, ensure_ascii=False, indent=1, sort_keys=False)) elif isCSV: # save CSV aspectQnCol = {} aspectsHeader = [] factsColumns = [] def addAspectQnCol(aspectQn): aspectQnCol[aspectQn] = len(aspectsHeader) _colName = oimValue(aspectQn) aspectsHeader.append(_colName) _colDataType = {"id": "Name", "baseType": "Name", "oim:concept": "Name", "oim:periodStart": "dateTime", # forever is 0000-01-01T00:00:00 "oim:periodDuration": "duration", # forever is P9999Y "oim:tupleOrder": "integer", "numericValue": "decimal", "accuracy": "decimal", "booleanValue": "boolean", "oim:unitNumerators": OrderedDict((("base","Name"), ("separator"," "))), "oim:unitDenominators": OrderedDict((("base","Name"), ("separator"," "))), }.get(_colName, "string") factsColumns.append(OrderedDict((("name", _colName), ("datatype", _colDataType)))) # pre-ordered aspect columns if hasId: addAspectQnCol("id") if hasType: addAspectQnCol("baseType") addAspectQnCol("stringValue") addAspectQnCol("numericValue") addAspectQnCol("accuracy") addAspectQnCol("booleanValue") if hasTuple: addAspectQnCol(qnOimTupleParentAspect) addAspectQnCol(qnOimTupleOrderAspect) addAspectQnCol(qnOimConceptAspect) if qnOimEntityAspect in aspectsDefined: addAspectQnCol(qnOimEntityAspect) if qnOimPeriodStartAspect in aspectsDefined: addAspectQnCol(qnOimPeriodStartAspect) addAspectQnCol(qnOimPeriodDurationAspect) if qnOimUnitNumeratorsAspect in aspectsDefined: addAspectQnCol(qnOimUnitNumeratorsAspect) if qnOimUnitDenominatorsAspect in aspectsDefined: addAspectQnCol(qnOimUnitDenominatorsAspect) for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)): if aspectQn.namespaceURI != nsOim: addAspectQnCol(aspectQn) def aspectCols(fact): cols = [None for i in range(len(aspectsHeader))] _factAspects = factAspects(fact) for aspectQn, aspectValue in _factAspects.items(): if aspectQn in aspectQnCol: cols[aspectQnCol[aspectQn]] = aspectValue return cols # metadata csvTables = [] csvMetadata = OrderedDict((("@context",[ "http://www.w3.org/ns/csvw", { "@base": "./" }]), ("tables", csvTables))) if oimFile.endswith("-facts.csv"): # strip -facts.csv if a prior -facts.csv file was chosen _baseURL = oimFile[:-10] elif oimFile.endswith(".csv"): _baseURL = oimFile[:-4] else: _baseURL = oimFile # save facts _factsFile = _baseURL + "-facts.csv" csvFile = open(_factsFile, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig') csvWriter = csv.writer(csvFile, dialect="excel") csvWriter.writerow(aspectsHeader) def saveCSVfacts(facts): for fact in facts: csvWriter.writerow(aspectCols(fact)) saveCSVfacts(fact.modelTupleFacts) saveCSVfacts(modelXbrl.facts) csvFile.close() factsTableSchema = OrderedDict((("columns",factsColumns),)) csvTables.append(OrderedDict((("url",os.path.basename(_factsFile)), ("tableSchema",factsTableSchema)))) # save namespaces _nsFile = _baseURL + "-prefixes.csv" csvFile = open(_nsFile, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig') csvWriter = csv.writer(csvFile, dialect="excel") csvWriter.writerow(("prefix", "URI")) for _URI, prefix in sorted(namespacePrefixes.items(), key=lambda item: item[1]): csvWriter.writerow((prefix, _URI)) csvFile.close() nsTableSchema = OrderedDict((("columns",[OrderedDict((("prefix","string"), ("URI","anyURI")))]),)) csvTables.append(OrderedDict((("url",os.path.basename(_nsFile)), ("tableSchema",nsTableSchema)))) # save dts references _dtsRefFile = _baseURL + "-dtsReferences.csv" csvFile = open(_dtsRefFile, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig') csvWriter = csv.writer(csvFile, dialect="excel") csvWriter.writerow(("type", "href")) for oimRef in dtsReferences: csvWriter.writerow((oimRef["type"], oimRef["href"])) csvFile.close() dtsRefTableSchema = OrderedDict((("columns",[OrderedDict((("type","string"), ("href","anyURI")))]),)) csvTables.append(OrderedDict((("url",os.path.basename(_dtsRefFile)), ("tableSchema",dtsRefTableSchema)))) # save footnotes if footnotesRelationshipSet.modelRelationships: _footnoteFile = oimFile.replace(".csv", "-footnotes.csv") csvFile = open(_footnoteFile, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig') csvWriter = csv.writer(csvFile, dialect="excel") cols = ("group", "factId", "factRef", "footnoteType", "footnote", "language") csvWriter.writerow(cols) def saveCSVfootnotes(facts): for fact in facts: for _footnote in factFootnotes(fact): csvWriter.writerow(tuple((_footnote.get(col,"") for col in cols))) saveCSVfootnotes(fact.modelTupleFacts) saveCSVfootnotes(modelXbrl.facts) csvFile.close() footnoteTableSchema = OrderedDict((("columns",[OrderedDict((("group","anyURI"), ("factId","Name"), ("factRef","Name"), ("footnoteType","Name"), ("footnote","string"), ("language","language")))]),)) csvTables.append(OrderedDict((("url",os.path.basename(_footnoteFile)), ("tableSchema",footnoteTableSchema)))) # save metadata with open(_baseURL + "-metadata.csv", "w", encoding="utf-8") as fh: fh.write(json.dumps(csvMetadata, ensure_ascii=False, indent=1, sort_keys=False))
def saveLoadableOIM(modelXbrl, oimFile, outputZip=None): isJSON = oimFile.endswith(".json") isCSV = oimFile.endswith(".csv") isXL = oimFile.endswith(".xlsx") isCSVorXL = isCSV or isXL if not isJSON and not isCSVorXL: return namespacePrefixes = {nsOim: "xbrl"} prefixNamespaces = {"xbrl": nsOim} def compileQname(qname): if qname.namespaceURI not in namespacePrefixes: namespacePrefixes[qname.namespaceURI] = qname.prefix or "" aspectsDefined = { qnOimConceptAspect, qnOimEntityAspect, qnOimPeriodStartAspect, qnOimPeriodEndAspect} def oimValue(object, decimals=None): if isinstance(object, QName): if object.namespaceURI not in namespacePrefixes: if object.prefix: namespacePrefixes[object.namespaceURI] = object.prefix else: _prefix = "_{}".format(sum(1 for p in namespacePrefixes if p.startswith("_"))) namespacePrefixes[object.namespaceURI] = _prefix return "{}:{}".format(namespacePrefixes[object.namespaceURI], object.localName) if isinstance(object, Decimal): try: if isinf(object): return "-INF" if object < 0 else "INF" elif isnan(num): return "NaN" else: if object == object.to_integral(): object = object.quantize(ONE) # drop any .0 return "{}".format(object) except: return str(object) if isinstance(object, bool): return "true" if object else "false" if isinstance(object, (DateTime, YearMonthDuration, DayTimeDuration, Time, gYearMonth, gMonthDay, gYear, gMonth, gDay, IsoDuration)): return str(object) return object def oimPeriodValue(cntx): if cntx.isForeverPeriod: return OrderedDict() # defaulted elif cntx.isStartEndPeriod: s = cntx.startDatetime e = cntx.endDatetime else: # instant s = e = cntx.instantDatetime if isJSON: return({str(qnOimPeriodAspect): ("{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}/".format( s.year, s.month, s.day, s.hour, s.minute, s.second) if cntx.isStartEndPeriod else "") + "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format( e.year, e.month, e.day, e.hour, e.minute, e.second) }) # CSV, XL return OrderedDict(((str(qnOimPeriodStartAspect), "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format( s.year, s.month, s.day, s.hour, s.minute, s.second)), (str(qnOimPeriodEndAspect), "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format( e.year, e.month, e.day, e.hour, e.minute, e.second)))) hasId = False hasTuple = False hasType = True hasLang = False hasUnits = False hasNumeric = False footnotesRelationshipSet = ModelRelationshipSet(modelXbrl, "XBRL-footnotes") factBaseTypes = set() #compile QNames in instance for OIM for fact in modelXbrl.factsInInstance: if (fact.id or fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))): hasId = True concept = fact.concept if concept is not None: if concept.isNumeric: hasNumeric = True if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang: hasLang = True _baseXsdType = concept.baseXsdType if _baseXsdType == "XBRLI_DATEUNION": if getattr(fact.xValue, "dateOnly", False): _baseXsdType = "date" else: _baseXsdType = "dateTime" factBaseTypes.add(baseTypes.get(_baseXsdType,_baseXsdType)) compileQname(fact.qname) if hasattr(fact, "xValue") and isinstance(fact.xValue, QName): compileQname(fact.xValue) unit = fact.unit if unit is not None: hasUnits = True if fact.modelTupleFacts: hasTuple = True if hasTuple: modelXbrl.error("arelleOIMsaver:tuplesNotAllowed", "Tuples are not allowed in an OIM document", modelObject=modelXbrl) return entitySchemePrefixes = {} for cntx in modelXbrl.contexts.values(): if cntx.entityIdentifierElement is not None: scheme = cntx.entityIdentifier[0] if scheme not in entitySchemePrefixes: if not entitySchemePrefixes: # first one is just scheme if scheme == "http://www.sec.gov/CIK": _schemePrefix = "cik" elif scheme == "http://standard.iso.org/iso/17442": _schemePrefix = "lei" else: _schemePrefix = "scheme" else: _schemePrefix = "scheme{}".format(len(entitySchemePrefixes) + 1) entitySchemePrefixes[scheme] = _schemePrefix namespacePrefixes[scheme] = _schemePrefix for dim in cntx.qnameDims.values(): compileQname(dim.dimensionQname) aspectsDefined.add(dim.dimensionQname) if dim.isExplicit: compileQname(dim.memberQname) for unit in modelXbrl.units.values(): if unit is not None: for measures in unit.measures: for measure in measures: compileQname(measure) if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[XbrlConst.xbrli] != "xbrli": namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix namespacePrefixes[XbrlConst.xsd] = "xsd" if hasLang: aspectsDefined.add(qnOimLangAspect) if hasTuple: aspectsDefined.add(qnOimTupleParentAspect) aspectsDefined.add(qnOimTupleOrderAspect) if hasUnits: aspectsDefined.add(qnOimUnitAspect) for footnoteRel in footnotesRelationshipSet.modelRelationships: typePrefix = "ftTyp_" + os.path.basename(footnoteRel.arcrole) if footnoteRel.linkrole == XbrlConst.defaultLinkRole: groupPrefix = "ftGrp_default" else: groupPrefix = "ftGrp_" + os.path.basename(footnoteRel.linkrole) if typePrefix not in namespacePrefixes: namespacePrefixes[footnoteRel.arcrole] = typePrefix if groupPrefix not in namespacePrefixes: namespacePrefixes[footnoteRel.linkrole] = groupPrefix # compile footnotes and relationships ''' factRelationships = [] factFootnotes = [] for rel in modelXbrl.relationshipSet(modelXbrl, "XBRL-footnotes").modelRelationships: oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole} factRelationships.append(oimRel) oimRel["fromIds"] = [obj.id if obj.id else elementChildSequence(obj) for obj in rel.fromModelObjects] oimRel["toIds"] = [obj.id if obj.id else elementChildSequence(obj) for obj in rel.toModelObjects] _order = rel.arcElement.get("order") if _order is not None: oimRel["order"] = _order for obj in rel.toModelObjects: if isinstance(obj, ModelResource): # footnote oimFootnote = {"role": obj.role, "id": obj.id if obj.id else elementChildSequence(obj), # value needs work for html elements and for inline footnotes "value": xmlstring(obj, stripXmlns=True)} if obj.xmlLang: oimFootnote["lang"] = obj.xmlLang factFootnotes.append(oimFootnote) oimFootnote ''' dtsReferences = set() baseUrl = modelXbrl.modelDocument.uri.partition("#")[0] for doc,ref in sorted(modelXbrl.modelDocument.referencesDocument.items(), key=lambda _item:_item[0].uri): if ref.referringModelObject.qname in SCHEMA_LB_REFS: dtsReferences.add(relativeUri(baseUrl,doc.uri)) for refType in ("role", "arcrole"): for refElt in sorted(modelXbrl.modelDocument.xmlRootElement.iterchildren( "{{http://www.xbrl.org/2003/linkbase}}{}Ref".format(refType)), key=lambda elt:elt.get(refType+"URI") ): dtsReferences.add(refElt.get("{http://www.w3.org/1999/xlink}href").partition("#")[0]) dtsReferences = sorted(dtsReferences) # turn into list footnoteFacts = set() def factFootnotes(fact, oimFact=None): footnotes = [] oimLinks = {} for footnoteRel in footnotesRelationshipSet.fromModelObject(fact): toObj = footnoteRel.toModelObject # json typePrefix = namespacePrefixes[footnoteRel.arcrole] groupPrefix = namespacePrefixes[footnoteRel.linkrole] oimLinks.setdefault(typePrefix,{}).setdefault(groupPrefix,[]).append( toObj.id if toObj.id else "f{}".format(toObj.objectIndex)) # csv/XL footnote = OrderedDict((("group", footnoteRel.linkrole), ("footnoteType", footnoteRel.arcrole))) footnotes.append(footnote) if isCSVorXL: footnote["factId"] = fact.id if fact.id else "f{}".format(fact.objectIndex) if isinstance(toObj, ModelFact): footnote["factRef"] = toObj.id if toObj.id else "f{}".format(toObj.objectIndex) else: footnote["footnote"] = toObj.viewText() if toObj.xmlLang: footnote["language"] = toObj.xmlLang footnoteFacts.add(toObj) if isJSON and oimLinks: oimFact["links"] = OrderedDict(( (typePrefix, OrderedDict(( (groupPrefix, idList) for groupPrefix, idList in sorted(groups.items()) ))) for typePrefix, groups in sorted(oimLinks.items()) )) footnotes.sort(key=lambda f:(f["group"],f.get("factId",f.get("factRef")),f.get("language"))) return footnotes def factAspects(fact): oimFact = OrderedDict() aspects = OrderedDict() if hasId and fact.id: if fact.isTuple: oimFact["tupleId"] = fact.id else: oimFact["id"] = fact.id elif (fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))): oimFact["id"] = "f{}".format(fact.objectIndex) parent = fact.getparent() concept = fact.concept aspects[str(qnOimConceptAspect)] = oimValue(concept.qname) _csvType = "Value" if not fact.isTuple: if concept is not None: _baseXsdType = concept.baseXsdType if _baseXsdType == "XBRLI_DATEUNION": if getattr(fact.xValue, "dateOnly", False): _baseXsdType = "date" else: _baseXsdType = "dateTime" _csvType = baseTypes.get(_baseXsdType,_baseXsdType) + "Value" if concept.baseXbrliType in ("stringItemType", "normalizedStringItemType") and fact.xmlLang: aspects[str(qnOimLangAspect)] = fact.xmlLang if fact.isItem: if fact.isNil: _value = None else: _inferredDecimals = inferredDecimals(fact) _value = oimValue(fact.xValue, _inferredDecimals) oimFact["value"] = _value if fact.concept is not None and fact.concept.isNumeric: _numValue = fact.xValue if isinstance(_numValue, Decimal) and not isinf(_numValue) and not isnan(_numValue): if _numValue == _numValue.to_integral(): _numValue = int(_numValue) else: _numValue = float(_numValue) if not fact.isNil: if isinf(_inferredDecimals): if isJSON: _accuracy = "infinity" elif isCSVorXL: _accuracy = "INF" else: _accuracy = _inferredDecimals oimFact["accuracy"] = _accuracy oimFact["aspects"] = aspects cntx = fact.context if cntx is not None: if cntx.entityIdentifierElement is not None: aspects[str(qnOimEntityAspect)] = oimValue(qname(*cntx.entityIdentifier)) if cntx.period is not None: aspects.update(oimPeriodValue(cntx)) for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]): if dim.isExplicit: dimVal = oimValue(dim.memberQname) else: # typed if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1"): dimVal = None else: dimVal = dim.typedMember.stringValue aspects[str(dim.dimensionQname)] = dimVal unit = fact.unit if unit is not None: _mMul, _mDiv = unit.measures _sMul = '*'.join(oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m))) if _mDiv: _sDiv = '*'.join(oimValue(m) for m in sorted(_mDiv, key=lambda m: oimValue(m))) if len(_mDiv) > 1: if len(_mMul) > 1: _sUnit = "({})/({})".format(_sMul,_sDiv) else: _sUnit = "{}/({})".format(_sMul,_sDiv) else: if len(_mMul) > 1: _sUnit = "({})/{}".format(_sMul,_sDiv) else: _sUnit = "{}/{}".format(_sMul,_sDiv) else: _sUnit = _sMul aspects[str(qnOimUnitAspect)] = _sUnit # Tuples removed from xBRL-JSON #if parent.qname != XbrlConst.qnXbrliXbrl: # aspects[str(qnOimTupleParentAspect)] = parent.id if parent.id else "f{}".format(parent.objectIndex) # aspects[str(qnOimTupleOrderAspect)] = elementIndex(fact) if isJSON: factFootnotes(fact, oimFact) return oimFact prefixes = OrderedDict((p,ns) for ns, p in sorted(namespacePrefixes.items(), key=lambda item: item[1])) if isJSON: # save JSON oimReport = OrderedDict() # top level of oim json output oimReport["documentInfo"] = oimDocInfo = OrderedDict() oimReport["facts"] = oimFacts = OrderedDict() oimDocInfo["documentType"] = nsOim + "/xbrl-json" oimDocInfo["features"] = oimFeatures = OrderedDict() oimDocInfo["prefixes"] = prefixes oimDocInfo["taxonomy"] = dtsReferences oimFeatures["xbrl:canonicalValues"] = True def saveJsonFacts(facts, oimFacts, parentFact): for fact in facts: oimFact = factAspects(fact) id = fact.id if fact.id else "f{}".format(fact.objectIndex) oimFacts[id] = oimFact if fact.modelTupleFacts: saveJsonFacts(fact.modelTupleFacts, oimFacts, fact) saveJsonFacts(modelXbrl.facts, oimFacts, None) # add footnotes as pseudo facts for ftObj in footnoteFacts: ftId = ftObj.id if ftObj.id else "f{}".format(ftObj.objectIndex) oimFacts[ftId] = oimFact = OrderedDict() oimFact["value"] = ftObj.viewText() oimFact["aspects"] = OrderedDict((("concept", "xbrl:note"), ("noteId", ftId))) if ftObj.xmlLang: oimFact["aspects"]["language"] = ftObj.xmlLang.lower() if outputZip: fh = io.StringIO() else: fh = open(oimFile, "w", encoding="utf-8") fh.write(json.dumps(oimReport, indent=1)) if outputZip: fh.seek(0) outputZip.writestr(os.path.basename(oimFile),fh.read()) fh.close() elif isCSVorXL: # save CSV aspectQnCol = {} aspectsHeader = [] factsColumns = [] def addAspectQnCol(aspectQn): aspectQnCol[str(aspectQn)] = len(aspectsHeader) _aspectQn = oimValue(aspectQn) aspectsHeader.append(_aspectQn) _colName = _aspectQn.replace("xbrl:", "") _colDataType = {"id": "Name", "concept": "QName", "value": "string", "accuracy": "decimal", "entity": "QName", "periodStart": "dateTime", "periodEnd": "dateTime", "unit": "string", "tupleId": "Name", "tupleParent": "Name", "tupleOrder": "integer" }.get(_colName, "string") col = OrderedDict((("name", _colName), ("datatype", _colDataType))) if _aspectQn == "value": col["http://xbrl.org/YYYY/model#simpleFactAspects"] = {} elif _aspectQn == "tupleId": col["http://xbrl.org/YYYY/model#tupleFactAspects"] = {} col["http://xbrl.org/YYYY/model#tupleReferenceId"] = "true" else: col["http://xbrl.org/YYYY/model#columnAspect"] = _aspectQn factsColumns.append(col) # pre-ordered aspect columns #if hasId: # addAspectQnCol("id") addAspectQnCol(qnOimConceptAspect) addAspectQnCol("value") if hasNumeric: addAspectQnCol("accuracy") if hasTuple: addAspectQnCol("tupleId") addAspectQnCol(qnOimTupleParentAspect) addAspectQnCol(qnOimTupleOrderAspect) if qnOimEntityAspect in aspectsDefined: addAspectQnCol(qnOimEntityAspect) if qnOimPeriodStartAspect in aspectsDefined: addAspectQnCol(qnOimPeriodStartAspect) addAspectQnCol(qnOimPeriodEndAspect) if qnOimUnitAspect in aspectsDefined: addAspectQnCol(qnOimUnitAspect) for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)): if aspectQn.namespaceURI != nsOim: addAspectQnCol(aspectQn) def aspectCols(fact): cols = [None for i in range(len(aspectsHeader))] def setColValues(aspects): for aspectQn, aspectValue in aspects.items(): if isinstance(aspectValue, dict): setColValues(aspectValue) elif aspectQn in aspectQnCol: if aspectValue is None: _aspectValue = "#nil" elif aspectValue == "": _aspectValue = "#empty" elif isinstance(aspectValue, str) and aspectValue.startswith("#"): _aspectValue = "#" + aspectValue else: _aspectValue = aspectValue cols[aspectQnCol[aspectQn]] = _aspectValue setColValues(factAspects(fact)) return cols # metadata csvTables = [] csvMetadata = OrderedDict((("@context", "http://www.w3.org/ns/csvw"), ("http://xbrl.org/YYYY/model#metadata", OrderedDict((("documentType", "http://xbrl.org/YYYY/xbrl-csv"), ("dtsReferences", dtsReferences), ("prefixes", prefixes)))), ("tables", csvTables))) _open = _writerow = _close = None _tableinfo = {} if isCSV: if oimFile.endswith("-facts.csv"): # strip -facts.csv if a prior -facts.csv file was chosen _baseURL = oimFile[:-10] elif oimFile.endswith(".csv"): _baseURL = oimFile[:-4] else: _baseURL = oimFile _csvinfo = {} # open file, writer def _open(filesuffix, tabname): _filename = _tableinfo["url"] = _baseURL + filesuffix _csvinfo["file"] = open(_filename, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig') _csvinfo["writer"] = csv.writer(_csvinfo["file"], dialect="excel") def _writerow(row, header=False): _csvinfo["writer"].writerow(row) def _close(): _csvinfo["file"].close() _csvinfo.clear() elif isXL: headerWidths = {"href": 100, "xbrl:concept": 70, "accuracy": 8, "language": 9, "URI": 80, "value": 60, "group": 60, "footnoteType": 40, "footnote": 70, "column": 20, 'conceptAspect': 40, 'tuple': 20, 'simpleFact': 20} from openpyxl import Workbook from openpyxl.writer.write_only import WriteOnlyCell from openpyxl.styles import Font, PatternFill, Border, Alignment, Color, fills, Side from openpyxl.worksheet.dimensions import ColumnDimension hdrCellFill = PatternFill(patternType=fills.FILL_SOLID, fgColor=Color("00FFBF5F")) # Excel's light orange fill color = 00FF990 workbook = Workbook() # remove pre-existing worksheets while len(workbook.worksheets)>0: workbook.remove_sheet(workbook.worksheets[0]) _xlinfo = {} # open file, writer def _open(filesuffix, tabname): _tableinfo["url"] = tabname _xlinfo["ws"] = workbook.create_sheet(title=tabname) def _writerow(rowvalues, header=False): row = [] _ws = _xlinfo["ws"] for i, v in enumerate(rowvalues): cell = WriteOnlyCell(_ws, value=v) if header: cell.fill = hdrCellFill cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True) colLetter = chr(ord('A') + i) _ws.column_dimensions[colLetter] = ColumnDimension(_ws, customWidth=True) _ws.column_dimensions[colLetter].width = headerWidths.get(v, 20) else: cell.alignment = Alignment(horizontal="right" if isinstance(v, _NUM_TYPES) else "center" if isinstance(v, bool) else "left", vertical="top", wrap_text=isinstance(v, str)) row.append(cell) _ws.append(row) def _close(): _xlinfo.clear() # save facts _open("-facts.csv", "facts") _writerow(aspectsHeader, header=True) def saveCSVfacts(facts): for fact in facts: _writerow(aspectCols(fact)) saveCSVfacts(fact.modelTupleFacts) saveCSVfacts(modelXbrl.facts) _close() factsTableSchema = OrderedDict((("columns",factsColumns),)) csvTables.append(OrderedDict((("url",_tableinfo["url"]), ("http://xbrl.org/YYYY/model#tableType", "fact"), ("tableSchema",factsTableSchema)))) # save footnotes if footnotesRelationshipSet.modelRelationships: _open("-footnotes.csv", "footnotes") cols = ("group", "footnoteType", "factId", "factRef", "footnote", "language") _writerow(cols, header=True) def saveCSVfootnotes(facts): for fact in facts: for _footnote in factFootnotes(fact): _writerow(tuple((_footnote.get(col,"") for col in cols))) saveCSVfootnotes(fact.modelTupleFacts) saveCSVfootnotes(modelXbrl.facts) _close() footnoteTableSchema = OrderedDict((("columns",[OrderedDict((("name","group"),("datatype","anyURI"))), OrderedDict((("name","footnoteType"),("datatype","Name"))), OrderedDict((("name","factId"),("datatype","Name"))), OrderedDict((("name","factRef"),("datatype","Name"))), OrderedDict((("name","footnote"),("datatype","string"))), OrderedDict((("name","language"),("datatype","language")))]),)) csvTables.append(OrderedDict((("url",_tableinfo["url"]), ("http://xbrl.org/YYYY/model#tableType", "footnote"), ("tableSchema",footnoteTableSchema)))) # save metadata if isCSV: with open(_baseURL + "-metadata.json", "w", encoding="utf-8") as fh: fh.write(json.dumps(csvMetadata, ensure_ascii=False, indent=1, sort_keys=False)) elif isXL: _open(None, "metadata") hasColumnAspect = hasSimpleFact = hasTupleFact = False for table in csvTables: tablename = table["url"] for column in table["tableSchema"]["columns"]: if "http://xbrl.org/YYYY/model#columnAspect" in column: hasColumnAspect = True if "http://xbrl.org/YYYY/model#simpleFactAspects" in column: hasSimpleFact = True if "http://xbrl.org/YYYY/model#tupleAspects" in column: hasTupleFact = True metadataCols = ["table", "column", "datatype"] if hasColumnAspect: metadataCols.append("columnAspect") if hasSimpleFact: metadataCols.append("simpleFact") if hasTupleFact: metadataCols.append("tuple") _writerow(metadataCols, header=True) for table in csvTables: tablename = table["url"] for column in table["tableSchema"]["columns"]: row = [tablename, column["name"], column["datatype"]] if hasColumnAspect: colAspect = column.get("http://xbrl.org/YYYY/model#columnAspect") if isinstance(colAspect, str): row.append(colAspect) elif isinstance(colAspect, dict): row.append("\n".join("{} [{}]".format(k, ", ".join(_v for _v in v)) for k, v in dict.items())) else: row.append(None) if hasSimpleFact: row.append("\u221a" if "http://xbrl.org/YYYY/model#simpleFactAspects" in column else None) if hasTupleFact: row.append("\u221a" if "http://xbrl.org/YYYY/model#tupleAspects" in column else None) _writerow(row) _close() if isXL: workbook.save(oimFile)