def __init__(self, dts): self.nsmap = NamespaceMap() self.roleMap = NamespaceMap() self.dts = dts self.taxonomyData = { "concepts": {}, "languages": {}, "facts": {}, } self.footnoteRelationshipSet = ModelRelationshipSet(dts, "XBRL-footnotes")
def base_dimension_sets(dts): """Get the Xule base dimension sets. This is like the baseSets dictionary of a model. The base dimension set is a dictionary keyed by the drs role and hypercube. The drs role is the role of the initial 'all' relationship or the target role of the initial 'all' relationship if ther eis a target role. The value of the diction is a set of the 'all' relationships. """ _imports() if not hasattr(dts, 'xuleBaseDimensionSets'): dts.xuleBaseDimensionSets = collections.defaultdict(set) for base_set in dts.baseSets: if (base_set[XuleProperties.NETWORK_ARCROLE] == 'http://xbrl.org/int/dim/arcrole/all' and base_set[XuleProperties.NETWORK_ROLE] is not None and base_set[XuleProperties.NETWORK_LINK] is not None and base_set[XuleProperties.NETWORK_ARC] is not None): # This is an 'all' dimension base set find the hypercubes relationship_set =dts.relationshipSets.get(base_set, ModelRelationshipSet(dts, base_set[XuleProperties.NETWORK_ARCROLE], base_set[XuleProperties.NETWORK_ROLE], base_set[XuleProperties.NETWORK_LINK], base_set[XuleProperties.NETWORK_ARC])) for rel in relationship_set.modelRelationships: drs_role = rel.targetRole or base_set[XuleProperties.NETWORK_ROLE] hypercube = rel.toModelObject dts.xuleBaseDimensionSets[(drs_role, hypercube)].add(rel) return dts.xuleBaseDimensionSets
def relationship_set(dts, relationship_set_info): _imports() return (dts.relationshipSets[relationship_set_info] if relationship_set_info in dts.relationshipSets else ModelRelationshipSet(dts, relationship_set_info[XuleProperties.NETWORK_ARCROLE], relationship_set_info[XuleProperties.NETWORK_ROLE], relationship_set_info[XuleProperties.NETWORK_LINK], relationship_set_info[XuleProperties.NETWORK_ARC]))
def validateSetup(val, parameters=None): val.validateEBA = val.validateDisclosureSystem and getattr( val.disclosureSystem, "EBA", False) val.validateEIOPA = val.validateDisclosureSystem and getattr( val.disclosureSystem, "EIOPA", False) if not (val.validateEBA or val.validateEIOPA): return val.validateUTR = False # do not use default UTR validation, it's at error level and not streamable val.utrValidator = ValidateUtr( val.modelXbrl, "WARNING", # EBA specifies SHOULD on UTR validation "EBA.2.23") # override utre error-severity message code val.isEIOPAfullVersion = False modelDocument = val.modelXbrl.modelDocument if modelDocument.type == ModelDocument.Type.INSTANCE: for doc, docRef in modelDocument.referencesDocument.items(): if docRef.referenceType == "href": if docRef.referringModelObject.localName == "schemaRef": _match = schemaRefDatePattern.match(doc.uri) if _match: val.isEIOPAfullVersion = _match.group(1) > "2015-02-28" break val.prefixNamespace = {} val.namespacePrefix = {} val.idObjects = {} val.typedDomainQnames = set() val.typedDomainElements = set() for modelConcept in val.modelXbrl.qnameConcepts.values(): if modelConcept.isTypedDimension: typedDomainElement = modelConcept.typedDomainElement if isinstance(typedDomainElement, ModelConcept): val.typedDomainQnames.add(typedDomainElement.qname) val.typedDomainElements.add(typedDomainElement) val.filingIndicators = {} val.numFilingIndicatorTuples = 0 val.cntxEntities = set() val.cntxDates = defaultdict(set) val.unusedCntxIDs = set() val.unusedUnitIDs = set() val.currenciesUsed = {} val.namespacePrefixesUsed = defaultdict(set) val.prefixesUnused = set( val.modelXbrl.modelDocument.xmlRootElement.nsmap.keys()).copy() val.firstFactObjectIndex = sys.maxsize val.firstFact = None val.footnotesRelationshipSet = ModelRelationshipSet( val.modelXbrl, "XBRL-footnotes")
def groupRelationshipSet(modelXbrl, arcrole, linkrole, linkqname, arcqname): if isinstance(arcrole, (list,tuple)): # (group-name, [arcroles]) arcroles = arcrole[1] relationshipSet = ModelRelationshipSet(modelXbrl, arcroles[0], linkrole, linkqname, arcqname) for arcrole in arcroles[1:]: if arcrole != XbrlConst.arcroleGroupDetect: rels = modelXbrl.relationshipSet(arcrole, linkrole, linkqname, arcqname) if rels: relationshipSet.modelRelationships.extend(rels.modelRelationships) relationshipSet.modelRelationships.sort(key=lambda rel: rel.order) else: relationshipSet = modelXbrl.relationshipSet(arcrole, linkrole, linkqname, arcqname) return relationshipSet
def network(node, sphinxContext, args, linkqname=None, linkrole=None, arcqname=None, arcrole=None): dts = dtsArg(node, sphinxContext, args) return ModelRelationshipSet(dts, arcrole, linkrole=linkrole, linkqname=linkqname, arcqname=arcqname)
def viewFacts(modelXbrl, tabWin, lang=None): modelXbrl.modelManager.showStatus(_("viewing facts")) view = ViewFactList(modelXbrl, tabWin, lang) view.treeView["columns"] = ("sequence", "contextID", "unitID", "decimals", "precision", "language", "footnoted", "value") view.treeView.column("#0", width=200, anchor="w") view.treeView.heading("#0", text=_("Label")) view.treeView.column("sequence", width=40, anchor="e", stretch=False) view.treeView.heading("sequence", text=_("Seq")) view.treeView.column("contextID", width=100, anchor="w", stretch=False) view.treeView.heading("contextID", text="contextRef") view.treeView.column("unitID", width=75, anchor="w", stretch=False) view.unitDisplayID = False # start displaying measures view.treeView.heading("unitID", text="Unit") view.treeView.column("decimals", width=50, anchor="center", stretch=False) view.treeView.heading("decimals", text=_("Dec")) view.treeView.column("precision", width=50, anchor="w", stretch=False) view.treeView.heading("precision", text=_("Prec")) view.treeView.column("language", width=36, anchor="w", stretch=False) view.treeView.heading("language", text=_("Lang")) view.treeView.column("footnoted", width=18, anchor="center", stretch=False) view.treeView.heading("footnoted", text=_("Fn")) view.treeView.column("value", width=200, anchor="w", stretch=False) view.treeView.heading("value", text=_("Value")) view.treeView["displaycolumns"] = ("sequence", "contextID", "unitID", "decimals", "precision", \ "language", "footnoted", "value") view.footnotesRelationshipSet = ModelRelationshipSet( modelXbrl, "XBRL-footnotes") view.blockSelectEvent = 1 view.blockViewModelObject = 0 view.view() view.treeView.bind("<<TreeviewSelect>>", view.treeviewSelect, '+') view.treeView.bind("<Enter>", view.treeviewEnter, '+') view.treeView.bind("<Leave>", view.treeviewLeave, '+') # intercept menu click before pops up to set the viewable tuple (if tuple clicked) view.treeView.bind(view.modelXbrl.modelManager.cntlr.contextMenuClick, view.setViewTupleChildMenuItem, '+') menu = view.contextMenu() if menu is not None: view.menu.insert_cascade(0, label=_("View Tuple Children"), underline=0, command=view.viewTuplesGrid) view.menu.entryconfigure(0, state='disabled') view.menuAddExpandCollapse() view.menuAddClipboard() view.menuAddLangs() view.menuAddLabelRoles(includeConceptName=True) view.menuAddUnitDisplay()
def validateSetup(val, parameters=None): val.validateEBA = val.validateDisclosureSystem and getattr(val.disclosureSystem, "EBA", False) val.validateEIOPA = val.validateDisclosureSystem and getattr(val.disclosureSystem, "EIOPA", False) if not (val.validateEBA or val.validateEIOPA): return val.validateUTR = False # do not use default UTR validation, it's at error level and not streamable val.utrValidator = ValidateUtr(val.modelXbrl, "WARNING", # EBA specifies SHOULD on UTR validation "EBA.2.23") # override utre error-severity message code val.prefixNamespace = {} val.namespacePrefix = {} val.idObjects = {} val.typedDomainQnames = set() val.typedDomainElements = set() for modelConcept in val.modelXbrl.qnameConcepts.values(): if modelConcept.isTypedDimension: typedDomainElement = modelConcept.typedDomainElement if isinstance(typedDomainElement, ModelConcept): val.typedDomainQnames.add(typedDomainElement.qname) val.typedDomainElements.add(typedDomainElement) val.filingIndicators = {} val.numFilingIndicatorTuples = 0 val.cntxEntities = set() val.cntxDates = defaultdict(set) val.unusedCntxIDs = set() val.unusedUnitIDs = set() val.currenciesUsed = {} val.namespacePrefixesUsed = defaultdict(set) val.prefixesUnused = set(val.modelXbrl.modelDocument.xmlRootElement.nsmap.keys()).copy() val.firstFactObjectIndex = sys.maxsize val.firstFact = None val.footnotesRelationshipSet = ModelRelationshipSet(val.modelXbrl, "XBRL-footnotes")
def validateTestcase(self, testcase): self.modelXbrl.info("info", "Testcase", modelDocument=testcase) self.modelXbrl.viewModelObject(testcase.objectId()) if testcase.type in (Type.TESTCASESINDEX, Type.REGISTRY): for doc in sorted(testcase.referencesDocument.keys(), key=lambda doc: doc.uri): self.validateTestcase(doc) # testcases doc's are sorted by their uri (file names), e.g., for formula elif hasattr(testcase, "testcaseVariations"): for modelTestcaseVariation in testcaseVariationsByTarget(testcase.testcaseVariations): # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) # is this a versioning report? resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance resultIsTaxonomyPackage = modelTestcaseVariation.resultIsTaxonomyPackage formulaOutputInstance = None inputDTSes = defaultdict(list) baseForElement = testcase.baseForElement(modelTestcaseVariation) # try to load instance document self.modelXbrl.info("info", _("Variation %(id)s%(name)s%(target)s: %(expected)s - %(description)s"), modelObject=modelTestcaseVariation, id=modelTestcaseVariation.id, name=(" {}".format(modelTestcaseVariation.name) if modelTestcaseVariation.name else ""), target=(" target {}".format(modelTestcaseVariation.ixdsTarget) if modelTestcaseVariation.ixdsTarget else ""), expected=modelTestcaseVariation.expected, description=modelTestcaseVariation.description) if self.modelXbrl.modelManager.formulaOptions.testcaseResultsCaptureWarnings: errorCaptureLevel = logging._checkLevel("WARNING") else: errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY parameters = modelTestcaseVariation.parameters.copy() for readMeFirstUri in modelTestcaseVariation.readMeFirstUris: if isinstance(readMeFirstUri,tuple): # dtsName is for formula instances, but is from/to dts if versioning dtsName, readMeFirstUri = readMeFirstUri elif resultIsVersioningReport: if inputDTSes: dtsName = "to" else: dtsName = "from" else: dtsName = None if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document if dtsName in inputDTSes: dtsName = inputDTSes[dtsName] else: modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager, Type.DTSENTRIES, self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement), isEntry=True, errorCaptureLevel=errorCaptureLevel) DTSdoc = modelXbrl.modelDocument DTSdoc.inDTS = True doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement) if doc is not None: DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement) #fake import doc.inDTS = True elif resultIsTaxonomyPackage: from arelle import PackageManager, PrototypeInstanceObject dtsName = readMeFirstUri modelXbrl = PrototypeInstanceObject.XbrlPrototype(self.modelXbrl.modelManager, readMeFirstUri) PackageManager.packageInfo(self.modelXbrl.modelManager.cntlr, readMeFirstUri, reload=True, errors=modelXbrl.errors) else: # not a multi-schemaRef versioning report if self.useFileSource.isArchive: modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, readMeFirstUri, _("validating"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel, ixdsTarget=modelTestcaseVariation.ixdsTarget) else: # need own file source, may need instance discovery filesource = FileSource.openFileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr, base=baseForElement) if filesource and not filesource.selection and filesource.isArchive: try: if filesource.isTaxonomyPackage: _rptPkgIxdsOptions = {} for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ReportPackageIxdsOptions"): pluginXbrlMethod(self, _rptPkgIxdsOptions) filesource.loadTaxonomyPackageMappings() for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ReportPackageIxds"): filesource.select(pluginXbrlMethod(filesource, **_rptPkgIxdsOptions)) else: from arelle.CntlrCmdLine import filesourceEntrypointFiles entrypoints = filesourceEntrypointFiles(filesource) if entrypoints: # resolve an IXDS in entrypoints for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ArchiveIxds"): pluginXbrlMethod(self, filesource,entrypoints) filesource.select(entrypoints[0].get("file", None) ) except Exception as err: self.modelXbrl.error("exception:" + type(err).__name__, _("Testcase variation validation exception: %(error)s, entry URL: %(instance)s"), modelXbrl=self.modelXbrl, instance=readMeFirstUri, error=err) continue # don't try to load this entry URL modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, filesource, _("validating"), base=baseForElement, errorCaptureLevel=errorCaptureLevel, ixdsTarget=modelTestcaseVariation.ixdsTarget) modelXbrl.isTestcaseVariation = True if modelXbrl.modelDocument is None: modelXbrl.info("arelle:notLoaded", _("Variation %(id)s %(name)s readMeFirst document not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) self.determineNotLoadedTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif resultIsVersioningReport or resultIsTaxonomyPackage: inputDTSes[dtsName] = modelXbrl elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl) self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif testcase.type == Type.REGISTRYTESTCASE: self.instValidator.validate(modelXbrl) # required to set up dimensions, etc self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id, modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest) self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) self.instValidator.close() modelXbrl.close() else: inputDTSes[dtsName].append(modelXbrl) # validate except for formulas _hasFormulae = modelXbrl.hasFormulae modelXbrl.hasFormulae = False try: for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"): pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation) self.instValidator.validate(modelXbrl, parameters) for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"): pluginXbrlMethod(self.modelXbrl, modelXbrl) except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase variation validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=(type(err) is not AssertionError)) modelXbrl.hasFormulae = _hasFormulae if resultIsVersioningReport and modelXbrl.modelDocument: versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl( modelTestcaseVariation.versioningReportUri, baseForElement) if os.path.exists(versReportFile): #validate existing modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report")) if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport) self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors) modelVersReport.close() elif len(inputDTSes) == 2: ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes( versReportFile, inputDTSes["from"], inputDTSes["to"]) modelTestcaseVariation.status = "generated" else: modelXbrl.error("arelle:notLoaded", _("Variation %(id)s %(name)s input DTSes not loaded, unable to generate versioning report: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) modelTestcaseVariation.status = "failed" for inputDTS in inputDTSes.values(): inputDTS.close() del inputDTSes # dereference elif resultIsTaxonomyPackage: self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif inputDTSes: # validate schema, linkbase, or instance modelXbrl = inputDTSes[None][0] expectedDataFiles = set(modelXbrl.modelManager.cntlr.webCache.normalizeUrl(uri, baseForElement) for d in modelTestcaseVariation.dataUris.values() for uri in d if not UrlUtil.isAbsolute(uri)) foundDataFiles = set() variationBase = os.path.dirname(baseForElement) for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters if dtsName: # named instance parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes) elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula for _inputDTS in inputDTS: for docUrl, doc in _inputDTS.urlDocs.items(): if docUrl.startswith(variationBase) and not doc.type == Type.INLINEXBRLDOCUMENTSET: if getattr(doc,"loadedFromXbrlFormula", False): # may have been sourced from xf file if docUrl.replace("-formula.xml", ".xf") in expectedDataFiles: docUrl = docUrl.replace("-formula.xml", ".xf") foundDataFiles.add(docUrl) if expectedDataFiles - foundDataFiles: modelXbrl.info("arelle:testcaseDataNotUsed", _("Variation %(id)s %(name)s data files not used: %(missingDataFiles)s"), modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id, missingDataFiles=", ".join(sorted(os.path.basename(f) for f in expectedDataFiles - foundDataFiles))) if foundDataFiles - expectedDataFiles: modelXbrl.info("arelle:testcaseDataUnexpected", _("Variation %(id)s %(name)s files not in variation data: %(unexpectedDataFiles)s"), modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id, unexpectedDataFiles=", ".join(sorted(os.path.basename(f) for f in foundDataFiles - expectedDataFiles))) if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable: try: RenderingEvaluator.init(modelXbrl) except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase RenderingEvaluator.init exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) modelXbrlHasFormulae = modelXbrl.hasFormulae if modelXbrlHasFormulae and self.modelXbrl.modelManager.formulaOptions.formulaAction != "none": try: # validate only formulae self.instValidator.parameters = parameters ValidateFormula.validate(self.instValidator) except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=(type(err) is not AssertionError)) if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset: for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"): pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri) infoset = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultInfosetUri, _("loading result infoset"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) if infoset.modelDocument is None: modelXbrl.error("arelle:notLoaded", _("Variation %(id)s %(name)s result infoset not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstance)) modelTestcaseVariation.status = "result infoset not loadable" else: # check infoset ValidateInfoset.validate(self.instValidator, modelXbrl, infoset) infoset.close() if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset: # diff (or generate) table infoset resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement) if not any(alternativeValidation(modelXbrl, resultTableUri) for alternativeValidation in pluginClassMethods("Validate.TableInfoset")): try: ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase table linkbase validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) self.instValidator.close() extraErrors = [] for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"): pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors) self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual): # if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost) self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters) self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors) if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance formulaOutputInstance = modelXbrl.formulaOutputInstance modelXbrl.formulaOutputInstance = None # prevent it from being closed now self.instValidator.close() compareIxResultInstance = (modelXbrl.modelDocument.type in (Type.INLINEXBRL, Type.INLINEXBRLDOCUMENTSET) and modelTestcaseVariation.resultXbrlInstanceUri is not None) if compareIxResultInstance: formulaOutputInstance = modelXbrl # compare modelXbrl to generated output instance errMsgPrefix = "ix" else: # delete input instances before formula output comparision for inputDTSlist in inputDTSes.values(): for inputDTS in inputDTSlist: inputDTS.close() del inputDTSes # dereference errMsgPrefix = "formula" if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument: _matchExpectedResultIDs = not modelXbrlHasFormulae # formula restuls have inconsistent IDs expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultXbrlInstanceUri, _("loading expected result XBRL instance"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) if expectedInstance.modelDocument is None: self.modelXbrl.error("{}:expectedResultNotLoaded".format(errMsgPrefix), _("Testcase \"%(name)s\" %(id)s expected result instance not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstanceUri), messageCodes=("formula:expectedResultNotLoaded","ix:expectedResultNotLoaded")) modelTestcaseVariation.status = "result not loadable" else: # compare facts for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.ExpectedInstance.Loaded"): pluginXbrlMethod(expectedInstance, formulaOutputInstance) if len(expectedInstance.facts) != len(formulaOutputInstance.facts): formulaOutputInstance.error("{}:resultFactCounts".format(errMsgPrefix), _("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"), modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts), expectedFacts=len(expectedInstance.facts), messageCodes=("formula:resultFactCounts","ix:resultFactCounts")) else: formulaOutputFootnotesRelSet = ModelRelationshipSet(formulaOutputInstance, "XBRL-footnotes") expectedFootnotesRelSet = ModelRelationshipSet(expectedInstance, "XBRL-footnotes") def factFootnotes(fact, footnotesRelSet): footnotes = {} footnoteRels = footnotesRelSet.fromModelObject(fact) if footnoteRels: # most process rels in same order between two instances, use labels to sort for i, footnoteRel in enumerate(sorted(footnoteRels, key=lambda r: (r.fromLabel,r.toLabel))): modelObject = footnoteRel.toModelObject if isinstance(modelObject, ModelResource): xml = collapseWhitespace(modelObject.viewText().strip()) footnotes["Footnote {}".format(i+1)] = xml #re.sub(r'\s+', ' ', collapseWhitespace(modelObject.stringValue)) elif isinstance(modelObject, ModelFact): footnotes["Footnoted fact {}".format(i+1)] = \ "{} context: {} value: {}".format( modelObject.qname, modelObject.contextID, collapseWhitespace(modelObject.value)) return footnotes for expectedInstanceFact in expectedInstance.facts: unmatchedFactsStack = [] formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=_matchExpectedResultIDs, matchLang=False) #formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=True, matchLang=True) if formulaOutputFact is None: if unmatchedFactsStack: # get missing nested tuple fact, if possible missingFact = unmatchedFactsStack[-1] else: missingFact = expectedInstanceFact # is it possible to show value mismatches? expectedFacts = formulaOutputInstance.factsByQname.get(missingFact.qname) if len(expectedFacts) == 1: formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix), _("Output missing expected fact %(fact)s, extracted value \"%(value1)s\", expected value \"%(value2)s\""), modelXbrl=missingFact, fact=missingFact.qname, value1=missingFact.xValue, value2=next(iter(expectedFacts)).xValue, messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing")) else: formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix), _("Output missing expected fact %(fact)s"), modelXbrl=missingFact, fact=missingFact.qname, messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing")) else: # compare footnotes expectedInstanceFactFootnotes = factFootnotes(expectedInstanceFact, expectedFootnotesRelSet) formulaOutputFactFootnotes = factFootnotes(formulaOutputFact, formulaOutputFootnotesRelSet) if (len(expectedInstanceFactFootnotes) != len(formulaOutputFactFootnotes) or set(expectedInstanceFactFootnotes.values()) != set(formulaOutputFactFootnotes.values())): formulaOutputInstance.error("{}:expectedFactFootnoteDifference".format(errMsgPrefix), _("Output expected fact %(fact)s expected footnotes %(footnotes1)s produced footnotes %(footnotes2)s"), modelXbrl=(formulaOutputFact,expectedInstanceFact), fact=expectedInstanceFact.qname, footnotes1=sorted(expectedInstanceFactFootnotes.items()), footnotes2=sorted(formulaOutputFactFootnotes.items()), messageCodes=("formula:expectedFactFootnoteDifference","ix:expectedFactFootnoteDifference")) # for debugging uncomment next line to save generated instance document # formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml") expectedInstance.close() del expectedInstance # dereference self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors) formulaOutputInstance.close() del formulaOutputInstance if compareIxResultInstance: for inputDTSlist in inputDTSes.values(): for inputDTS in inputDTSlist: inputDTS.close() del inputDTSes # dereference # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) _statusCounts = OrderedDict((("pass",0),("fail",0))) for tv in getattr(testcase, "testcaseVariations", ()): _statusCounts[tv.status] = _statusCounts.get(tv.status, 0) + 1 self.modelXbrl.info("arelle:testCaseResults", ", ".join("{}={}".format(k,c) for k, c in _statusCounts.items() if k)) self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
class IXBRLViewerBuilder: def __init__(self, dts): self.nsmap = NamespaceMap() self.roleMap = NamespaceMap() self.dts = dts self.taxonomyData = { "concepts": {}, "languages": {}, "facts": {}, } self.footnoteRelationshipSet = ModelRelationshipSet( dts, "XBRL-footnotes") def lineWrap(self, s, n=80): return "\n".join([s[i:i + n] for i in range(0, len(s), n)]) def dateFormat(self, d): """ Strip the time component from an ISO date if it's zero """ return re.sub("T00:00:00$", "", d) def escapeJSONForScriptTag(self, s): """ JSON encodes XML special characters XML and HTML apply difference escaping rules to content within script tags and we need our output to be valid XML, but treated as HTML by browsers. If we allow XML escaping to occur in a script tag, browsers treating the document as HTML won't unescape it. If we don't escape XML special characters, it won't be valid XML. We avoid this whole mess by escaping XML special characters using JSON string escapes. This is only safe to do because < > and & can't occur outside a string in JSON. It can't safely be used on JS. """ return s.replace("<", "\\u003C").replace(">", "\\u003E").replace("&", "\\u0026") def makeLanguageName(self, langCode): code = re.sub("-.*", "", langCode) try: language = pycountry.languages.lookup(code) match = re.match(r'^[^-]+-(.*)$', langCode) name = language.name if match is not None: name = "%s (%s)" % (name, match.group(1).upper()) except LookupError: name = langCode return name def addLanguage(self, langCode): if langCode not in self.taxonomyData["languages"]: self.taxonomyData["languages"][langCode] = self.makeLanguageName( langCode) def addELR(self, elr): prefix = self.roleMap.getPrefix(elr) if self.taxonomyData.setdefault("roleDefs", {}).get(prefix, None) is None: rt = self.dts.roleTypes[elr] label = elr if len(rt) > 0: label = rt[0].definition self.taxonomyData["roleDefs"].setdefault(prefix, {})["en"] = label def addConcept(self, concept, dimensionType=None): if concept is None: return labelsRelationshipSet = self.dts.relationshipSet( XbrlConst.conceptLabel) labels = labelsRelationshipSet.fromModelObject(concept) conceptName = self.nsmap.qname(concept.qname) if conceptName not in self.taxonomyData["concepts"]: conceptData = {"labels": {}} for lr in labels: l = lr.toModelObject conceptData["labels"].setdefault( self.roleMap.getPrefix(l.role), {})[l.xmlLang.lower()] = l.text self.addLanguage(l.xmlLang.lower()) refData = [] for _refRel in concept.modelXbrl.relationshipSet( XbrlConst.conceptReference).fromModelObject(concept): ref = [] for _refPart in _refRel.toModelObject.iterchildren(): ref.append( [_refPart.localName, _refPart.stringValue.strip()]) refData.append(ref) if len(refData) > 0: conceptData['r'] = refData if dimensionType is not None: conceptData["d"] = dimensionType self.taxonomyData["concepts"][conceptName] = conceptData def treeWalk(self, rels, item, indent=0): for r in rels.fromModelObject(item): if r.toModelObject is not None: self.treeWalk(rels, r.toModelObject, indent + 1) def getRelationships(self): rels = {} for baseSetKey, baseSetModelLinks in self.dts.baseSets.items(): arcrole, ELR, linkqname, arcqname = baseSetKey if arcrole in (XbrlConst.summationItem, WIDER_NARROWER_ARCROLE) and ELR is not None: self.addELR(ELR) rr = dict() relSet = self.dts.relationshipSet(arcrole, ELR) for r in relSet.modelRelationships: if r.fromModelObject is not None and r.toModelObject is not None: fromKey = self.nsmap.qname(r.fromModelObject.qname) rel = { "t": self.nsmap.qname(r.toModelObject.qname), } if r.weight is not None: rel['w'] = r.weight rr.setdefault(fromKey, []).append(rel) self.addConcept(r.toModelObject) self.addConcept(r.fromModelObject) rels.setdefault(self.roleMap.getPrefix(arcrole), {})[self.roleMap.getPrefix(ELR)] = rr return rels def createViewer(self, scriptUrl="js/dist/ixbrlviewer.js"): """ Create an iXBRL file with XBRL data as a JSON blob, and script tags added """ dts = self.dts iv = iXBRLViewer(dts) idGen = 0 self.roleMap.getPrefix(XbrlConst.standardLabel, "std") self.roleMap.getPrefix(XbrlConst.documentationLabel, "doc") self.roleMap.getPrefix(XbrlConst.summationItem, "calc") self.roleMap.getPrefix(XbrlConst.parentChild, "pres") self.roleMap.getPrefix(WIDER_NARROWER_ARCROLE, "w-n") for f in dts.facts: if f.id is None: f.set("id", "ixv-%d" % (idGen)) idGen += 1 conceptName = self.nsmap.qname(f.qname) scheme, ident = f.context.entityIdentifier aspects = { "c": conceptName, "e": self.nsmap.qname( QName(self.nsmap.getPrefix(scheme, "e"), scheme, ident)), } factData = { "v": f.value if not f.isNil else None, "a": aspects, } if f.format is not None: factData["f"] = str(f.format) if f.isNumeric: if f.unit is not None and len(f.unit.measures[0]): # XXX does not support complex units unit = self.nsmap.qname(f.unit.measures[0][0]) aspects["u"] = unit else: # The presence of the unit aspect is used by the viewer to # identify numeric facts. If the fact has no unit (invalid # XBRL, but we want to support it for draft documents), # include the unit aspect with a null value. aspects["u"] = None d = inferredDecimals(f) if d != float("INF") and not math.isnan(d): factData["d"] = d for d, v in f.context.qnameDims.items(): if v.memberQname is not None: aspects[self.nsmap.qname( v.dimensionQname)] = self.nsmap.qname(v.memberQname) self.addConcept(v.member) self.addConcept(v.dimension, dimensionType="e") elif v.typedMember is not None: aspects[self.nsmap.qname( v.dimensionQname)] = v.typedMember.text self.addConcept(v.dimension, dimensionType="t") if f.context.isForeverPeriod: aspects["p"] = "f" elif f.context.isInstantPeriod and f.context.instantDatetime is not None: aspects["p"] = self.dateFormat( f.context.instantDatetime.isoformat()) elif f.context.isStartEndPeriod and f.context.startDatetime is not None and f.context.endDatetime is not None: aspects["p"] = "%s/%s" % ( self.dateFormat(f.context.startDatetime.isoformat()), self.dateFormat(f.context.endDatetime.isoformat())) frels = self.footnoteRelationshipSet.fromModelObject(f) if frels: for frel in frels: if frel.toModelObject is not None: factData.setdefault("fn", []).append(frel.toModelObject.id) self.taxonomyData["facts"][f.id] = factData if f.concept is not None: self.addConcept(f.concept) self.taxonomyData["prefixes"] = self.nsmap.prefixmap self.taxonomyData["roles"] = self.roleMap.prefixmap self.taxonomyData["rels"] = self.getRelationships() dts.info("viewer:info", "Creating iXBRL viewer") if dts.modelDocument.type == Type.INLINEXBRLDOCUMENTSET: # Sort by object index to preserve order in which files were specified. docSet = sorted(dts.modelDocument.referencesDocument.keys(), key=lambda x: x.objectIndex) docSetFiles = list( map(lambda x: os.path.basename(x.filepath), docSet)) self.taxonomyData["docSetFiles"] = docSetFiles for n in range(0, len(docSet)): iv.addFile( iXBRLViewerFile(docSetFiles[n], docSet[n].xmlDocument)) xmlDocument = docSet[0].xmlDocument else: xmlDocument = dts.modelDocument.xmlDocument filename = os.path.basename(dts.modelDocument.filepath) iv.addFile(iXBRLViewerFile(filename, xmlDocument)) taxonomyDataJSON = self.escapeJSONForScriptTag( json.dumps(self.taxonomyData, indent=1, allow_nan=False)) for child in xmlDocument.getroot(): if child.tag == '{http://www.w3.org/1999/xhtml}body': child.append(etree.Comment("BEGIN IXBRL VIEWER EXTENSIONS")) e = etree.fromstring( "<script xmlns='http://www.w3.org/1999/xhtml' src='%s' type='text/javascript' />" % scriptUrl) # Don't self close e.text = '' child.append(e) # Putting this in the header can interfere with character set # auto detection e = etree.fromstring( "<script xmlns='http://www.w3.org/1999/xhtml' type='application/x.ixbrl-viewer+json'></script>" ) e.text = taxonomyDataJSON child.append(e) child.append(etree.Comment("END IXBRL VIEWER EXTENSIONS")) break return iv
def saveLoadableOIM(modelXbrl, oimFile): isJSON = oimFile.endswith(".json") isCSV = oimFile.endswith(".csv") isXL = oimFile.endswith(".xlsx") isCSVorXL = isCSV or isXL if not isJSON and not isCSVorXL: return namespacePrefixes = {nsOim: "oim"} prefixNamespaces = {"oim": nsOim} def compileQname(qname): if qname.namespaceURI not in namespacePrefixes: namespacePrefixes[qname.namespaceURI] = qname.prefix or "" aspectsDefined = {qnOimConceptAspect, qnOimEntityAspect} if isJSON: aspectsDefined.add(qnOimPeriodAspect) elif isCSVorXL: aspectsDefined.add(qnOimPeriodStartAspect) aspectsDefined.add(qnOimPeriodEndAspect) def oimValue(object, decimals=None): if isinstance(object, QName): if object.namespaceURI not in namespacePrefixes: if object.prefix: namespacePrefixes[object.namespaceURI] = object.prefix else: _prefix = "_{}".format( sum(1 for p in namespacePrefixes if p.startswith("_"))) namespacePrefixes[object.namespaceURI] = _prefix return "{}:{}".format(namespacePrefixes[object.namespaceURI], object.localName) if isinstance(object, Decimal): try: if isinf(object): return "-INF" if object < 0 else "INF" elif isnan(num): return "NaN" else: if object == object.to_integral(): object = object.quantize(ONE) # drop any .0 return "{}".format(object) except: return str(object) if isinstance(object, bool): return object if isinstance(object, (DateTime, YearMonthDuration, DayTimeDuration, Time, gYearMonth, gMonthDay, gYear, gMonth, gDay)): return str(object) return object def oimPeriodValue(cntx): if cntx.isForeverPeriod: pass # not supported elif cntx.isStartEndPeriod: s = cntx.startDatetime e = cntx.endDatetime else: # instant s = e = cntx.instantDatetime return { "start": "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format( s.year, s.month, s.day, s.hour, s.minute, s.second), "end": "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format( e.year, e.month, e.day, e.hour, e.minute, e.second) } hasId = False hasTuple = False hasType = True hasLang = False hasUnits = False hasNumeric = False footnotesRelationshipSet = ModelRelationshipSet(modelXbrl, "XBRL-footnotes") factBaseTypes = set() #compile QNames in instance for OIM for fact in modelXbrl.factsInInstance: if (fact.id or fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))): hasId = True concept = fact.concept if concept is not None: if concept.isNumeric: hasNumeric = True if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang: hasLang = True _baseXsdType = concept.baseXsdType if _baseXsdType == "XBRLI_DATEUNION": if getattr(fact.xValue, "dateOnly", False): _baseXsdType = "date" else: _baseXsdType = "dateTime" factBaseTypes.add(baseTypes.get(_baseXsdType, _baseXsdType)) compileQname(fact.qname) if hasattr(fact, "xValue") and isinstance(fact.xValue, QName): compileQname(fact.xValue) unit = fact.unit if unit is not None: hasUnits = True if fact.modelTupleFacts: hasTuple = True entitySchemePrefixes = {} for cntx in modelXbrl.contexts.values(): if cntx.entityIdentifierElement is not None: scheme = cntx.entityIdentifier[0] if scheme not in entitySchemePrefixes: if not entitySchemePrefixes: # first one is just scheme if scheme == "http://www.sec.gov/CIK": _schemePrefix = "cik" elif scheme == "http://standard.iso.org/iso/17442": _schemePrefix = "lei" else: _schemePrefix = "scheme" else: _schemePrefix = "scheme{}".format( len(entitySchemePrefixes) + 1) entitySchemePrefixes[scheme] = _schemePrefix namespacePrefixes[scheme] = _schemePrefix for dim in cntx.qnameDims.values(): compileQname(dim.dimensionQname) aspectsDefined.add(dim.dimensionQname) if dim.isExplicit: compileQname(dim.memberQname) for unit in modelXbrl.units.values(): if unit is not None: for measures in unit.measures: for measure in measures: compileQname(measure) if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[ XbrlConst.xbrli] != "xbrli": namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix namespacePrefixes[XbrlConst.xsd] = "xsd" if hasLang: aspectsDefined.add(qnOimLangAspect) if hasTuple: aspectsDefined.add(qnOimTupleParentAspect) aspectsDefined.add(qnOimTupleOrderAspect) if hasUnits: aspectsDefined.add(qnOimUnitAspect) # compile footnotes and relationships ''' factRelationships = [] factFootnotes = [] for rel in modelXbrl.relationshipSet(modelXbrl, "XBRL-footnotes").modelRelationships: oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole} factRelationships.append(oimRel) oimRel["fromIds"] = [obj.id if obj.id else elementChildSequence(obj) for obj in rel.fromModelObjects] oimRel["toIds"] = [obj.id if obj.id else elementChildSequence(obj) for obj in rel.toModelObjects] _order = rel.arcElement.get("order") if _order is not None: oimRel["order"] = _order for obj in rel.toModelObjects: if isinstance(obj, ModelResource): # footnote oimFootnote = {"role": obj.role, "id": obj.id if obj.id else elementChildSequence(obj), # value needs work for html elements and for inline footnotes "value": xmlstring(obj, stripXmlns=True)} if obj.xmlLang: oimFootnote["lang"] = obj.xmlLang factFootnotes.append(oimFootnote) oimFootnote ''' dtsReferences = [{ "type": "schema" if doc.type == ModelDocument.Type.SCHEMA else "linkbase" if doc.type == ModelDocument.Type.LINKBASE else "other", "href": doc.uri } for doc, ref in modelXbrl.modelDocument.referencesDocument.items() if ref.referringModelObject.qname in SCHEMA_LB_REFS] ''' roleTypes = [ {"type": "role" if ref.referringModelObject.localName == "roleRef" else "arcroleRef", "href": ref.referringModelObject["href"]} for doc,ref in modelXbrl.modelDocument.referencesDocument.items() if ref.referringModelObject.qname in ROLE_REFS] ''' def factFootnotes(fact): footnotes = [] for footnoteRel in footnotesRelationshipSet.fromModelObject(fact): footnote = OrderedDict((("group", footnoteRel.linkrole), ("footnoteType", footnoteRel.arcrole))) footnotes.append(footnote) if isCSVorXL: footnote["factId"] = fact.id if fact.id else "f{}".format( fact.objectIndex) toObj = footnoteRel.toModelObject if isinstance(toObj, ModelFact): footnote["factRef"] = toObj.id if toObj.id else "f{}".format( toObj.objectIndex) else: footnote["footnote"] = xmlstring(toObj, stripXmlns=True, contentsOnly=True, includeText=True) if toObj.xmlLang: footnote["language"] = toObj.xmlLang return footnotes def factAspects(fact): aspects = OrderedDict() if hasId and fact.id: aspects["id"] = fact.id elif (fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))): aspects["id"] = "f{}".format(fact.objectIndex) parent = fact.getparent() concept = fact.concept _csvType = "Value" if not fact.isTuple: if concept is not None: _baseXsdType = concept.baseXsdType if _baseXsdType == "XBRLI_DATEUNION": if getattr(fact.xValue, "dateOnly", False): _baseXsdType = "date" else: _baseXsdType = "dateTime" aspects["baseType"] = "xsd:{}".format(_baseXsdType) _csvType = baseTypes.get(_baseXsdType, _baseXsdType) + "Value" if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang: aspects[qnOimLangAspect] = fact.xmlLang if fact.isItem: if fact.isNil: _value = None _strValue = "nil" else: _inferredDecimals = inferredDecimals(fact) _value = oimValue(fact.xValue, _inferredDecimals) _strValue = str(_value) if not isCSVorXL: aspects["value"] = _strValue if fact.concept is not None and fact.concept.isNumeric: _numValue = fact.xValue if isinstance(_numValue, Decimal) and not isinf( _numValue) and not isnan(_numValue): if _numValue == _numValue.to_integral(): _numValue = int(_numValue) else: _numValue = float(_numValue) if isCSVorXL: aspects[_csvType] = _numValue else: aspects["numericValue"] = _numValue if not fact.isNil: if isinf(_inferredDecimals): if isJSON: _accuracy = "infinity" elif isCSVorXL: _accuracy = "INF" else: _accuracy = _inferredDecimals aspects["accuracy"] = _accuracy elif isinstance(_value, bool): aspects["booleanValue"] = _value elif isCSVorXL: aspects[_csvType] = _strValue aspects[qnOimConceptAspect] = oimValue(fact.qname) cntx = fact.context if cntx is not None: if cntx.entityIdentifierElement is not None: aspects[qnOimEntityAspect] = oimValue( qname(*cntx.entityIdentifier)) if cntx.period is not None: if isJSON: aspects[qnOimPeriodAspect] = oimPeriodValue(cntx) elif isCSVorXL: _periodValue = oimPeriodValue(cntx) aspects[qnOimPeriodStartAspect] = _periodValue["start"] aspects[qnOimPeriodEndAspect] = _periodValue["end"] for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]): aspects[dim.dimensionQname] = ( oimValue(dim.memberQname) if dim.isExplicit else None if dim.typedMember.get( "{http://www.w3.org/2001/XMLSchema-instance}nil") in ( "true", "1") else dim.typedMember.stringValue) unit = fact.unit if unit is not None: _mMul, _mDiv = unit.measures _sMul = '*'.join( oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m))) if _mDiv: _sDiv = '*'.join( oimValue(m) for m in sorted(_mDiv, key=lambda m: oimValue(m))) if len(_mDiv) > 1: if len(_mMul) > 1: _sUnit = "({})/({})".format(_sMul, _sDiv) else: _sUnit = "{}/({})".format(_sMul, _sDiv) else: if len(_mMul) > 1: _sUnit = "({})/{}".format(_sMul, _sDiv) else: _sUnit = "{}/{}".format(_sMul, _sDiv) else: _sUnit = _sMul aspects[qnOimUnitAspect] = _sUnit if parent.qname != XbrlConst.qnXbrliXbrl: aspects[ qnOimTupleParentAspect] = parent.id if parent.id else "f{}".format( parent.objectIndex) aspects[qnOimTupleOrderAspect] = elementIndex(fact) if isJSON: _footnotes = factFootnotes(fact) if _footnotes: aspects["footnotes"] = _footnotes return aspects if isJSON: # save JSON oimReport = OrderedDict() # top level of oim json output oimFacts = [] oimReport["prefixes"] = OrderedDict( (p, ns) for ns, p in sorted(namespacePrefixes.items(), key=lambda item: item[1])) oimReport["dtsReferences"] = dtsReferences oimReport["facts"] = oimFacts def saveJsonFacts(facts, oimFacts, parentFact): for fact in facts: oimFact = factAspects(fact) oimFacts.append( OrderedDict((str(k), v) for k, v in oimFact.items())) if fact.modelTupleFacts: saveJsonFacts(fact.modelTupleFacts, oimFacts, fact) saveJsonFacts(modelXbrl.facts, oimFacts, None) with open(oimFile, "w", encoding="utf-8") as fh: fh.write(json.dumps(oimReport, indent=1)) elif isCSVorXL: # save CSV aspectQnCol = {} aspectsHeader = [] factsColumns = [] def addAspectQnCol(aspectQn): aspectQnCol[aspectQn] = len(aspectsHeader) _colName = oimValue(aspectQn) aspectsHeader.append(_colName) _colDataType = { "id": "Name", "baseType": "Name", "oim:concept": "QName", "oim:periodStart": "dateTime", "oim:periodEnd": "dateTime", "oim:tupleOrder": "integer", "numericValue": "decimal", "accuracy": "decimal", "booleanValue": "boolean", "oim:entity": "QName", "oim:unit": "string" }.get(_colName) if _colDataType is None: if isCSVorXL and _colName.endswith("Value"): _colDataType = _colName[:-5] else: _colDataType = "string" factsColumns.append( OrderedDict((("name", _colName), ("datatype", _colDataType)))) # pre-ordered aspect columns if hasId: addAspectQnCol("id") addAspectQnCol(qnOimConceptAspect) if hasType: addAspectQnCol("baseType") if isCSVorXL: for _baseType in sorted(factBaseTypes): addAspectQnCol(_baseType + "Value") else: addAspectQnCol("stringValue") addAspectQnCol("numericValue") addAspectQnCol("booleanValue") if hasNumeric: addAspectQnCol("accuracy") if hasTuple: addAspectQnCol(qnOimTupleParentAspect) addAspectQnCol(qnOimTupleOrderAspect) if qnOimEntityAspect in aspectsDefined: addAspectQnCol(qnOimEntityAspect) if qnOimPeriodStartAspect in aspectsDefined: addAspectQnCol(qnOimPeriodStartAspect) addAspectQnCol(qnOimPeriodEndAspect) if qnOimUnitAspect in aspectsDefined: addAspectQnCol(qnOimUnitAspect) for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)): if aspectQn.namespaceURI != nsOim: addAspectQnCol(aspectQn) def aspectCols(fact): cols = [None for i in range(len(aspectsHeader))] _factAspects = factAspects(fact) for aspectQn, aspectValue in _factAspects.items(): if aspectQn in aspectQnCol: cols[aspectQnCol[aspectQn]] = aspectValue return cols # metadata csvTables = [] csvMetadata = OrderedDict( (("@context", ["http://www.w3.org/ns/csvw", { "@base": "./" }]), ("tables", csvTables))) _open = _writerow = _close = None _tableinfo = {} if isCSV: if oimFile.endswith( "-facts.csv" ): # strip -facts.csv if a prior -facts.csv file was chosen _baseURL = oimFile[:-10] elif oimFile.endswith(".csv"): _baseURL = oimFile[:-4] else: _baseURL = oimFile _csvinfo = {} # open file, writer def _open(filesuffix, tabname): _filename = _tableinfo["url"] = _baseURL + filesuffix _csvinfo["file"] = open(_filename, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig') _csvinfo["writer"] = csv.writer(_csvinfo["file"], dialect="excel") def _writerow(row, header=False): _csvinfo["writer"].writerow(row) def _close(): _csvinfo["file"].close() _csvinfo.clear() elif isXL: headerWidths = { "href": 100, "oim:concept": 70, "accuracy": 8, "baseType": 10, "language": 9, "URI": 80, "stringValue": 60, "decimalValue": 12, "booleanValue": 12, "dateValue": 12, "dateTimeValue": 20, "group": 60, "footnoteType": 40, "footnote": 70, "column": 60 } from openpyxl import Workbook from openpyxl.writer.write_only import WriteOnlyCell from openpyxl.styles import Font, PatternFill, Border, Alignment, Color, fills, Side from openpyxl.worksheet.dimensions import ColumnDimension hdrCellFill = PatternFill( patternType=fills.FILL_SOLID, fgColor=Color( "00FFBF5F")) # Excel's light orange fill color = 00FF990 workbook = Workbook(encoding="utf-8", write_only=True) _xlinfo = {} # open file, writer def _open(filesuffix, tabname): _tableinfo["url"] = tabname _xlinfo["ws"] = workbook.create_sheet(title=tabname) def _writerow(rowvalues, header=False): row = [] _ws = _xlinfo["ws"] for i, v in enumerate(rowvalues): cell = WriteOnlyCell(_ws, value=v) if header: cell.fill = hdrCellFill cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True) colLetter = chr(ord('A') + i) _ws.column_dimensions[colLetter] = ColumnDimension( _ws, customWidth=True) _ws.column_dimensions[ colLetter].width = headerWidths.get(v, 20) else: cell.alignment = Alignment( horizontal="right" if isinstance(v, _NUM_TYPES) else "center" if isinstance(v, bool) else "left", vertical="top", wrap_text=isinstance(v, str)) row.append(cell) _ws.append(row) def _close(): _xlinfo.clear() # save facts _open("-facts.csv", "facts") _writerow(aspectsHeader, header=True) def saveCSVfacts(facts): for fact in facts: _writerow(aspectCols(fact)) saveCSVfacts(fact.modelTupleFacts) saveCSVfacts(modelXbrl.facts) _close() factsTableSchema = OrderedDict((("columns", factsColumns), )) csvTables.append( OrderedDict((("url", _tableinfo["url"]), ("tableSchema", factsTableSchema)))) # save namespaces _open("-prefixes.csv", "prefixes") _writerow(("prefix", "URI"), header=True) for _URI, prefix in sorted(namespacePrefixes.items(), key=lambda item: item[1]): _writerow((prefix, _URI)) _close() nsTableSchema = OrderedDict((("columns", [ OrderedDict((("name", "prefix"), ("datatype", "Name"))), OrderedDict((("name", "URI"), ("datatype", "anyURI"))) ]), )) csvTables.append( OrderedDict( (("url", _tableinfo["url"]), ("tableSchema", nsTableSchema)))) # save dts references _open("-dtsReferences.csv", "dtsReferences") _writerow(("type", "href"), header=True) for oimRef in dtsReferences: _writerow((oimRef["type"], oimRef["href"])) _close() dtsRefTableSchema = OrderedDict((("columns", [ OrderedDict((("name", "type"), ("datatype", "Name"))), OrderedDict((("name", "href"), ("datatype", "anyURI"))) ]), )) csvTables.append( OrderedDict((("url", _tableinfo["url"]), ("tableSchema", dtsRefTableSchema)))) # save footnotes if footnotesRelationshipSet.modelRelationships: _open("-footnotes.csv", "footnotes") cols = ("group", "footnoteType", "factId", "factRef", "footnote", "language") _writerow(cols, header=True) def saveCSVfootnotes(facts): for fact in facts: for _footnote in factFootnotes(fact): _writerow( tuple((_footnote.get(col, "") for col in cols))) saveCSVfootnotes(fact.modelTupleFacts) saveCSVfootnotes(modelXbrl.facts) _close() footnoteTableSchema = OrderedDict((("columns", [ OrderedDict((("name", "group"), ("datatype", "anyURI"))), OrderedDict((("name", "footnoteType"), ("datatype", "Name"))), OrderedDict((("name", "factId"), ("datatype", "Name"))), OrderedDict((("name", "factRef"), ("datatype", "Name"))), OrderedDict((("name", "footnote"), ("datatype", "string"))), OrderedDict((("name", "language"), ("datatype", "language"))) ]), )) csvTables.append( OrderedDict((("url", _tableinfo["url"]), ("tableSchema", footnoteTableSchema)))) # save metadata if isCSV: with open(_baseURL + "-metadata.csv", "w", encoding="utf-8") as fh: fh.write( json.dumps(csvMetadata, ensure_ascii=False, indent=1, sort_keys=False)) elif isXL: _open(None, "metadata") _writerow(("table", "column", "datatype"), header=True) for table in csvTables: tablename = table["url"] for column in table["tableSchema"]["columns"]: _writerow((tablename, column["name"], column["datatype"])) _close() if isXL: workbook.save(oimFile)
def saveLoadableOIM(modelXbrl, oimFile, outputZip=None): isJSON = oimFile.endswith(".json") isCSV = oimFile.endswith(".csv") isXL = oimFile.endswith(".xlsx") isCSVorXL = isCSV or isXL if not isJSON and not isCSVorXL: return namespacePrefixes = {nsOim: "xbrl"} prefixNamespaces = {"xbrl": nsOim} def compileQname(qname): if qname.namespaceURI not in namespacePrefixes: namespacePrefixes[qname.namespaceURI] = qname.prefix or "" aspectsDefined = { qnOimConceptAspect, qnOimEntityAspect, qnOimPeriodStartAspect, qnOimPeriodEndAspect} def oimValue(object, decimals=None): if isinstance(object, QName): if object.namespaceURI not in namespacePrefixes: if object.prefix: namespacePrefixes[object.namespaceURI] = object.prefix else: _prefix = "_{}".format(sum(1 for p in namespacePrefixes if p.startswith("_"))) namespacePrefixes[object.namespaceURI] = _prefix return "{}:{}".format(namespacePrefixes[object.namespaceURI], object.localName) if isinstance(object, Decimal): try: if isinf(object): return "-INF" if object < 0 else "INF" elif isnan(num): return "NaN" else: if object == object.to_integral(): object = object.quantize(ONE) # drop any .0 return "{}".format(object) except: return str(object) if isinstance(object, bool): return "true" if object else "false" if isinstance(object, (DateTime, YearMonthDuration, DayTimeDuration, Time, gYearMonth, gMonthDay, gYear, gMonth, gDay, IsoDuration)): return str(object) return object def oimPeriodValue(cntx): if cntx.isForeverPeriod: return OrderedDict() # defaulted elif cntx.isStartEndPeriod: s = cntx.startDatetime e = cntx.endDatetime else: # instant s = e = cntx.instantDatetime if isJSON: return({str(qnOimPeriodAspect): ("{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}/".format( s.year, s.month, s.day, s.hour, s.minute, s.second) if cntx.isStartEndPeriod else "") + "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format( e.year, e.month, e.day, e.hour, e.minute, e.second) }) # CSV, XL return OrderedDict(((str(qnOimPeriodStartAspect), "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format( s.year, s.month, s.day, s.hour, s.minute, s.second)), (str(qnOimPeriodEndAspect), "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format( e.year, e.month, e.day, e.hour, e.minute, e.second)))) hasId = False hasTuple = False hasType = True hasLang = False hasUnits = False hasNumeric = False footnotesRelationshipSet = ModelRelationshipSet(modelXbrl, "XBRL-footnotes") factBaseTypes = set() #compile QNames in instance for OIM for fact in modelXbrl.factsInInstance: if (fact.id or fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))): hasId = True concept = fact.concept if concept is not None: if concept.isNumeric: hasNumeric = True if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang: hasLang = True _baseXsdType = concept.baseXsdType if _baseXsdType == "XBRLI_DATEUNION": if getattr(fact.xValue, "dateOnly", False): _baseXsdType = "date" else: _baseXsdType = "dateTime" factBaseTypes.add(baseTypes.get(_baseXsdType,_baseXsdType)) compileQname(fact.qname) if hasattr(fact, "xValue") and isinstance(fact.xValue, QName): compileQname(fact.xValue) unit = fact.unit if unit is not None: hasUnits = True if fact.modelTupleFacts: hasTuple = True if hasTuple: modelXbrl.error("arelleOIMsaver:tuplesNotAllowed", "Tuples are not allowed in an OIM document", modelObject=modelXbrl) return entitySchemePrefixes = {} for cntx in modelXbrl.contexts.values(): if cntx.entityIdentifierElement is not None: scheme = cntx.entityIdentifier[0] if scheme not in entitySchemePrefixes: if not entitySchemePrefixes: # first one is just scheme if scheme == "http://www.sec.gov/CIK": _schemePrefix = "cik" elif scheme == "http://standard.iso.org/iso/17442": _schemePrefix = "lei" else: _schemePrefix = "scheme" else: _schemePrefix = "scheme{}".format(len(entitySchemePrefixes) + 1) entitySchemePrefixes[scheme] = _schemePrefix namespacePrefixes[scheme] = _schemePrefix for dim in cntx.qnameDims.values(): compileQname(dim.dimensionQname) aspectsDefined.add(dim.dimensionQname) if dim.isExplicit: compileQname(dim.memberQname) for unit in modelXbrl.units.values(): if unit is not None: for measures in unit.measures: for measure in measures: compileQname(measure) if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[XbrlConst.xbrli] != "xbrli": namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix namespacePrefixes[XbrlConst.xsd] = "xsd" if hasLang: aspectsDefined.add(qnOimLangAspect) if hasTuple: aspectsDefined.add(qnOimTupleParentAspect) aspectsDefined.add(qnOimTupleOrderAspect) if hasUnits: aspectsDefined.add(qnOimUnitAspect) for footnoteRel in footnotesRelationshipSet.modelRelationships: typePrefix = "ftTyp_" + os.path.basename(footnoteRel.arcrole) if footnoteRel.linkrole == XbrlConst.defaultLinkRole: groupPrefix = "ftGrp_default" else: groupPrefix = "ftGrp_" + os.path.basename(footnoteRel.linkrole) if typePrefix not in namespacePrefixes: namespacePrefixes[footnoteRel.arcrole] = typePrefix if groupPrefix not in namespacePrefixes: namespacePrefixes[footnoteRel.linkrole] = groupPrefix # compile footnotes and relationships ''' factRelationships = [] factFootnotes = [] for rel in modelXbrl.relationshipSet(modelXbrl, "XBRL-footnotes").modelRelationships: oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole} factRelationships.append(oimRel) oimRel["fromIds"] = [obj.id if obj.id else elementChildSequence(obj) for obj in rel.fromModelObjects] oimRel["toIds"] = [obj.id if obj.id else elementChildSequence(obj) for obj in rel.toModelObjects] _order = rel.arcElement.get("order") if _order is not None: oimRel["order"] = _order for obj in rel.toModelObjects: if isinstance(obj, ModelResource): # footnote oimFootnote = {"role": obj.role, "id": obj.id if obj.id else elementChildSequence(obj), # value needs work for html elements and for inline footnotes "value": xmlstring(obj, stripXmlns=True)} if obj.xmlLang: oimFootnote["lang"] = obj.xmlLang factFootnotes.append(oimFootnote) oimFootnote ''' dtsReferences = set() baseUrl = modelXbrl.modelDocument.uri.partition("#")[0] for doc,ref in sorted(modelXbrl.modelDocument.referencesDocument.items(), key=lambda _item:_item[0].uri): if ref.referringModelObject.qname in SCHEMA_LB_REFS: dtsReferences.add(relativeUri(baseUrl,doc.uri)) for refType in ("role", "arcrole"): for refElt in sorted(modelXbrl.modelDocument.xmlRootElement.iterchildren( "{{http://www.xbrl.org/2003/linkbase}}{}Ref".format(refType)), key=lambda elt:elt.get(refType+"URI") ): dtsReferences.add(refElt.get("{http://www.w3.org/1999/xlink}href").partition("#")[0]) dtsReferences = sorted(dtsReferences) # turn into list footnoteFacts = set() def factFootnotes(fact, oimFact=None): footnotes = [] oimLinks = {} for footnoteRel in footnotesRelationshipSet.fromModelObject(fact): toObj = footnoteRel.toModelObject # json typePrefix = namespacePrefixes[footnoteRel.arcrole] groupPrefix = namespacePrefixes[footnoteRel.linkrole] oimLinks.setdefault(typePrefix,{}).setdefault(groupPrefix,[]).append( toObj.id if toObj.id else "f{}".format(toObj.objectIndex)) # csv/XL footnote = OrderedDict((("group", footnoteRel.linkrole), ("footnoteType", footnoteRel.arcrole))) footnotes.append(footnote) if isCSVorXL: footnote["factId"] = fact.id if fact.id else "f{}".format(fact.objectIndex) if isinstance(toObj, ModelFact): footnote["factRef"] = toObj.id if toObj.id else "f{}".format(toObj.objectIndex) else: footnote["footnote"] = toObj.viewText() if toObj.xmlLang: footnote["language"] = toObj.xmlLang footnoteFacts.add(toObj) if isJSON and oimLinks: oimFact["links"] = OrderedDict(( (typePrefix, OrderedDict(( (groupPrefix, idList) for groupPrefix, idList in sorted(groups.items()) ))) for typePrefix, groups in sorted(oimLinks.items()) )) footnotes.sort(key=lambda f:(f["group"],f.get("factId",f.get("factRef")),f.get("language"))) return footnotes def factAspects(fact): oimFact = OrderedDict() aspects = OrderedDict() if hasId and fact.id: if fact.isTuple: oimFact["tupleId"] = fact.id else: oimFact["id"] = fact.id elif (fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))): oimFact["id"] = "f{}".format(fact.objectIndex) parent = fact.getparent() concept = fact.concept aspects[str(qnOimConceptAspect)] = oimValue(concept.qname) _csvType = "Value" if not fact.isTuple: if concept is not None: _baseXsdType = concept.baseXsdType if _baseXsdType == "XBRLI_DATEUNION": if getattr(fact.xValue, "dateOnly", False): _baseXsdType = "date" else: _baseXsdType = "dateTime" _csvType = baseTypes.get(_baseXsdType,_baseXsdType) + "Value" if concept.baseXbrliType in ("stringItemType", "normalizedStringItemType") and fact.xmlLang: aspects[str(qnOimLangAspect)] = fact.xmlLang if fact.isItem: if fact.isNil: _value = None else: _inferredDecimals = inferredDecimals(fact) _value = oimValue(fact.xValue, _inferredDecimals) oimFact["value"] = _value if fact.concept is not None and fact.concept.isNumeric: _numValue = fact.xValue if isinstance(_numValue, Decimal) and not isinf(_numValue) and not isnan(_numValue): if _numValue == _numValue.to_integral(): _numValue = int(_numValue) else: _numValue = float(_numValue) if not fact.isNil: if not isinf(_inferredDecimals): # accuracy omitted if infinite oimFact["decimals"] = _inferredDecimals oimFact["dimensions"] = aspects cntx = fact.context if cntx is not None: if cntx.entityIdentifierElement is not None: aspects[str(qnOimEntityAspect)] = oimValue(qname(*cntx.entityIdentifier)) if cntx.period is not None: aspects.update(oimPeriodValue(cntx)) for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]): if dim.isExplicit: dimVal = oimValue(dim.memberQname) else: # typed if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1"): dimVal = None else: dimVal = dim.typedMember.stringValue aspects[str(dim.dimensionQname)] = dimVal unit = fact.unit if unit is not None: _mMul, _mDiv = unit.measures _sMul = '*'.join(oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m))) if _mDiv: _sDiv = '*'.join(oimValue(m) for m in sorted(_mDiv, key=lambda m: oimValue(m))) if len(_mDiv) > 1: if len(_mMul) > 1: _sUnit = "({})/({})".format(_sMul,_sDiv) else: _sUnit = "{}/({})".format(_sMul,_sDiv) else: if len(_mMul) > 1: _sUnit = "({})/{}".format(_sMul,_sDiv) else: _sUnit = "{}/{}".format(_sMul,_sDiv) else: _sUnit = _sMul aspects[str(qnOimUnitAspect)] = _sUnit # Tuples removed from xBRL-JSON #if parent.qname != XbrlConst.qnXbrliXbrl: # aspects[str(qnOimTupleParentAspect)] = parent.id if parent.id else "f{}".format(parent.objectIndex) # aspects[str(qnOimTupleOrderAspect)] = elementIndex(fact) if isJSON: factFootnotes(fact, oimFact) return oimFact prefixes = OrderedDict((p,ns) for ns, p in sorted(namespacePrefixes.items(), key=lambda item: item[1])) if isJSON: # save JSON oimReport = OrderedDict() # top level of oim json output oimReport["documentInfo"] = oimDocInfo = OrderedDict() oimReport["facts"] = oimFacts = OrderedDict() oimDocInfo["documentType"] = nsOim + "/xbrl-json" oimDocInfo["features"] = oimFeatures = OrderedDict() oimDocInfo["prefixes"] = prefixes oimDocInfo["taxonomy"] = dtsReferences oimFeatures["xbrl:canonicalValues"] = True def saveJsonFacts(facts, oimFacts, parentFact): for fact in facts: oimFact = factAspects(fact) id = fact.id if fact.id else "f{}".format(fact.objectIndex) oimFacts[id] = oimFact if fact.modelTupleFacts: saveJsonFacts(fact.modelTupleFacts, oimFacts, fact) saveJsonFacts(modelXbrl.facts, oimFacts, None) # add footnotes as pseudo facts for ftObj in footnoteFacts: ftId = ftObj.id if ftObj.id else "f{}".format(ftObj.objectIndex) oimFacts[ftId] = oimFact = OrderedDict() oimFact["value"] = ftObj.viewText() oimFact["dimensions"] = OrderedDict((("concept", "xbrl:note"), ("noteId", ftId))) if ftObj.xmlLang: oimFact["dimensions"]["language"] = ftObj.xmlLang.lower() if outputZip: fh = io.StringIO() else: fh = open(oimFile, "w", encoding="utf-8") fh.write(json.dumps(oimReport, indent=1)) if outputZip: fh.seek(0) outputZip.writestr(os.path.basename(oimFile),fh.read()) fh.close() elif isCSVorXL: # save CSV aspectQnCol = {} aspectsHeader = [] factsColumns = [] def addAspectQnCol(aspectQn): aspectQnCol[str(aspectQn)] = len(aspectsHeader) _aspectQn = oimValue(aspectQn) aspectsHeader.append(_aspectQn) _colName = _aspectQn.replace("xbrl:", "") _colDataType = {"id": "Name", "concept": "QName", "value": "string", "decimals": "decimal", "entity": "QName", "periodStart": "dateTime", "periodEnd": "dateTime", "unit": "string", "tupleId": "Name", "tupleParent": "Name", "tupleOrder": "integer" }.get(_colName, "string") col = OrderedDict((("name", _colName), ("datatype", _colDataType))) if _aspectQn == "value": col["http://xbrl.org/YYYY/model#simpleFactAspects"] = {} elif _aspectQn == "tupleId": col["http://xbrl.org/YYYY/model#tupleFactAspects"] = {} col["http://xbrl.org/YYYY/model#tupleReferenceId"] = "true" else: col["http://xbrl.org/YYYY/model#columnAspect"] = _aspectQn factsColumns.append(col) # pre-ordered aspect columns #if hasId: # addAspectQnCol("id") addAspectQnCol(qnOimConceptAspect) addAspectQnCol("value") if hasNumeric: addAspectQnCol("decimals") if hasTuple: addAspectQnCol("tupleId") addAspectQnCol(qnOimTupleParentAspect) addAspectQnCol(qnOimTupleOrderAspect) if qnOimEntityAspect in aspectsDefined: addAspectQnCol(qnOimEntityAspect) if qnOimPeriodStartAspect in aspectsDefined: addAspectQnCol(qnOimPeriodStartAspect) addAspectQnCol(qnOimPeriodEndAspect) if qnOimUnitAspect in aspectsDefined: addAspectQnCol(qnOimUnitAspect) for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)): if aspectQn.namespaceURI != nsOim: addAspectQnCol(aspectQn) def aspectCols(fact): cols = [None for i in range(len(aspectsHeader))] def setColValues(aspects): for aspectQn, aspectValue in aspects.items(): if isinstance(aspectValue, dict): setColValues(aspectValue) elif aspectQn in aspectQnCol: if aspectValue is None: _aspectValue = "#nil" elif aspectValue == "": _aspectValue = "#empty" elif isinstance(aspectValue, str) and aspectValue.startswith("#"): _aspectValue = "#" + aspectValue else: _aspectValue = aspectValue cols[aspectQnCol[aspectQn]] = _aspectValue setColValues(factAspects(fact)) return cols # metadata csvTables = [] csvMetadata = OrderedDict((("@context", "http://www.w3.org/ns/csvw"), ("http://xbrl.org/YYYY/model#metadata", OrderedDict((("documentType", "http://xbrl.org/YYYY/xbrl-csv"), ("dtsReferences", dtsReferences), ("prefixes", prefixes)))), ("tables", csvTables))) _open = _writerow = _close = None _tableinfo = {} if isCSV: if oimFile.endswith("-facts.csv"): # strip -facts.csv if a prior -facts.csv file was chosen _baseURL = oimFile[:-10] elif oimFile.endswith(".csv"): _baseURL = oimFile[:-4] else: _baseURL = oimFile _csvinfo = {} # open file, writer def _open(filesuffix, tabname): _filename = _tableinfo["url"] = _baseURL + filesuffix _csvinfo["file"] = open(_filename, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig') _csvinfo["writer"] = csv.writer(_csvinfo["file"], dialect="excel") def _writerow(row, header=False): _csvinfo["writer"].writerow(row) def _close(): _csvinfo["file"].close() _csvinfo.clear() elif isXL: headerWidths = {"href": 100, "xbrl:concept": 70, "decimals": 8, "language": 9, "URI": 80, "value": 60, "group": 60, "footnoteType": 40, "footnote": 70, "column": 20, 'conceptAspect': 40, 'tuple': 20, 'simpleFact': 20} from openpyxl import Workbook from openpyxl.writer.write_only import WriteOnlyCell from openpyxl.styles import Font, PatternFill, Border, Alignment, Color, fills, Side from openpyxl.worksheet.dimensions import ColumnDimension hdrCellFill = PatternFill(patternType=fills.FILL_SOLID, fgColor=Color("00FFBF5F")) # Excel's light orange fill color = 00FF990 workbook = Workbook() # remove pre-existing worksheets while len(workbook.worksheets)>0: workbook.remove_sheet(workbook.worksheets[0]) _xlinfo = {} # open file, writer def _open(filesuffix, tabname): _tableinfo["url"] = tabname _xlinfo["ws"] = workbook.create_sheet(title=tabname) def _writerow(rowvalues, header=False): row = [] _ws = _xlinfo["ws"] for i, v in enumerate(rowvalues): cell = WriteOnlyCell(_ws, value=v) if header: cell.fill = hdrCellFill cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True) colLetter = chr(ord('A') + i) _ws.column_dimensions[colLetter] = ColumnDimension(_ws, customWidth=True) _ws.column_dimensions[colLetter].width = headerWidths.get(v, 20) else: cell.alignment = Alignment(horizontal="right" if isinstance(v, _NUM_TYPES) else "center" if isinstance(v, bool) else "left", vertical="top", wrap_text=isinstance(v, str)) row.append(cell) _ws.append(row) def _close(): _xlinfo.clear() # save facts _open("-facts.csv", "facts") _writerow(aspectsHeader, header=True) def saveCSVfacts(facts): for fact in facts: _writerow(aspectCols(fact)) saveCSVfacts(fact.modelTupleFacts) saveCSVfacts(modelXbrl.facts) _close() factsTableSchema = OrderedDict((("columns",factsColumns),)) csvTables.append(OrderedDict((("url",_tableinfo["url"]), ("http://xbrl.org/YYYY/model#tableType", "fact"), ("tableSchema",factsTableSchema)))) # save footnotes if footnotesRelationshipSet.modelRelationships: _open("-footnotes.csv", "footnotes") cols = ("group", "footnoteType", "factId", "factRef", "footnote", "language") _writerow(cols, header=True) def saveCSVfootnotes(facts): for fact in facts: for _footnote in factFootnotes(fact): _writerow(tuple((_footnote.get(col,"") for col in cols))) saveCSVfootnotes(fact.modelTupleFacts) saveCSVfootnotes(modelXbrl.facts) _close() footnoteTableSchema = OrderedDict((("columns",[OrderedDict((("name","group"),("datatype","anyURI"))), OrderedDict((("name","footnoteType"),("datatype","Name"))), OrderedDict((("name","factId"),("datatype","Name"))), OrderedDict((("name","factRef"),("datatype","Name"))), OrderedDict((("name","footnote"),("datatype","string"))), OrderedDict((("name","language"),("datatype","language")))]),)) csvTables.append(OrderedDict((("url",_tableinfo["url"]), ("http://xbrl.org/YYYY/model#tableType", "footnote"), ("tableSchema",footnoteTableSchema)))) # save metadata if isCSV: with open(_baseURL + "-metadata.json", "w", encoding="utf-8") as fh: fh.write(json.dumps(csvMetadata, ensure_ascii=False, indent=1, sort_keys=False)) elif isXL: _open(None, "metadata") hasColumnAspect = hasSimpleFact = hasTupleFact = False for table in csvTables: tablename = table["url"] for column in table["tableSchema"]["columns"]: if "http://xbrl.org/YYYY/model#columnAspect" in column: hasColumnAspect = True if "http://xbrl.org/YYYY/model#simpleFactAspects" in column: hasSimpleFact = True if "http://xbrl.org/YYYY/model#tupleAspects" in column: hasTupleFact = True metadataCols = ["table", "column", "datatype"] if hasColumnAspect: metadataCols.append("columnAspect") if hasSimpleFact: metadataCols.append("simpleFact") if hasTupleFact: metadataCols.append("tuple") _writerow(metadataCols, header=True) for table in csvTables: tablename = table["url"] for column in table["tableSchema"]["columns"]: row = [tablename, column["name"], column["datatype"]] if hasColumnAspect: colAspect = column.get("http://xbrl.org/YYYY/model#columnAspect") if isinstance(colAspect, str): row.append(colAspect) elif isinstance(colAspect, dict): row.append("\n".join("{} [{}]".format(k, ", ".join(_v for _v in v)) for k, v in dict.items())) else: row.append(None) if hasSimpleFact: row.append("\u221a" if "http://xbrl.org/YYYY/model#simpleFactAspects" in column else None) if hasTupleFact: row.append("\u221a" if "http://xbrl.org/YYYY/model#tupleAspects" in column else None) _writerow(row) _close() if isXL: workbook.save(oimFile)
def saveLoadableOIM(modelXbrl, oimFile): isJSON = oimFile.endswith(".json") isCSV = oimFile.endswith(".csv") isXL = oimFile.endswith(".xlsx") isCSVorXL = isCSV or isXL if not isJSON and not isCSVorXL: return namespacePrefixes = {} prefixNamespaces = {} def compileQname(qname): if qname.namespaceURI not in namespacePrefixes: namespacePrefixes[qname.namespaceURI] = qname.prefix or "" aspectsDefined = { qnOimConceptAspect, qnOimEntityAspect} if isJSON: aspectsDefined.add(qnOimPeriodAspect) elif isCSVorXL: aspectsDefined.add(qnOimPeriodStartAspect) aspectsDefined.add(qnOimPeriodEndAspect) def oimValue(object, decimals=None): if isinstance(object, QName): if object.namespaceURI not in namespacePrefixes: if object.prefix: namespacePrefixes[object.namespaceURI] = object.prefix else: _prefix = "_{}".format(sum(1 for p in namespacePrefixes if p.startswith("_"))) namespacePrefixes[object.namespaceURI] = _prefix return "{}:{}".format(namespacePrefixes[object.namespaceURI], object.localName) if isinstance(object, Decimal): try: if isinf(object): return "-INF" if object < 0 else "INF" elif isnan(num): return "NaN" else: if object == object.to_integral(): object = object.quantize(ONE) # drop any .0 return "{}".format(object) except: return str(object) if isinstance(object, bool): return object if isinstance(object, (DateTime, YearMonthDuration, DayTimeDuration, Time, gYearMonth, gMonthDay, gYear, gMonth, gDay)): return str(object) return object def oimPeriodValue(cntx): if cntx.isForeverPeriod: pass # not supported elif cntx.isStartEndPeriod: s = cntx.startDatetime e = cntx.endDatetime else: # instant s = e = cntx.instantDatetime return {"start": "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format( s.year, s.month, s.day, s.hour, s.minute, s.second), "end": "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format( e.year, e.month, e.day, e.hour, e.minute, e.second)} hasId = False hasTuple = False hasType = True hasLang = False hasUnits = False footnotesRelationshipSet = ModelRelationshipSet(modelXbrl, "XBRL-footnotes") factBaseTypes = set() #compile QNames in instance for OIM for fact in modelXbrl.factsInInstance: if (fact.id or fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))): hasId = True concept = fact.concept if concept is not None: if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang: hasLang = True _baseXsdType = concept.baseXsdType if _baseXsdType == "XBRLI_DATEUNION": if getattr(fact.xValue, "dateOnly", False): _baseXsdType = "date" else: _baseXsdType = "dateTime" factBaseTypes.add(baseTypes.get(_baseXsdType,_baseXsdType)) compileQname(fact.qname) if hasattr(fact, "xValue") and isinstance(fact.xValue, QName): compileQname(fact.xValue) unit = fact.unit if unit is not None: hasUnits = True if fact.modelTupleFacts: hasTuple = True entitySchemePrefixes = {} for cntx in modelXbrl.contexts.values(): if cntx.entityIdentifierElement is not None: scheme = cntx.entityIdentifier[0] if scheme not in entitySchemePrefixes: if not entitySchemePrefixes: # first one is just scheme if scheme == "http://www.sec.gov/CIK": _schemePrefix = "cik" elif scheme == "http://standard.iso.org/iso/17442": _schemePrefix = "lei" else: _schemePrefix = "scheme" else: _schemePrefix = "scheme{}".format(len(entitySchemePrefixes) + 1) entitySchemePrefixes[scheme] = _schemePrefix namespacePrefixes[scheme] = _schemePrefix for dim in cntx.qnameDims.values(): compileQname(dim.dimensionQname) aspectsDefined.add(dim.dimensionQname) if dim.isExplicit: compileQname(dim.memberQname) for unit in modelXbrl.units.values(): if unit is not None: for measures in unit.measures: for measure in measures: compileQname(measure) if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[XbrlConst.xbrli] != "xbrli": namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix namespacePrefixes[XbrlConst.xsd] = "xsd" if hasLang: aspectsDefined.add(qnOimLangAspect) if hasTuple: aspectsDefined.add(qnOimTupleParentAspect) aspectsDefined.add(qnOimTupleOrderAspect) if hasUnits: aspectsDefined.add(qnOimUnitAspect) # compile footnotes and relationships ''' factRelationships = [] factFootnotes = [] for rel in modelXbrl.relationshipSet(modelXbrl, "XBRL-footnotes").modelRelationships: oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole} factRelationships.append(oimRel) oimRel["fromIds"] = [obj.id if obj.id else elementChildSequence(obj) for obj in rel.fromModelObjects] oimRel["toIds"] = [obj.id if obj.id else elementChildSequence(obj) for obj in rel.toModelObjects] _order = rel.arcElement.get("order") if _order is not None: oimRel["order"] = _order for obj in rel.toModelObjects: if isinstance(obj, ModelResource): # footnote oimFootnote = {"role": obj.role, "id": obj.id if obj.id else elementChildSequence(obj), # value needs work for html elements and for inline footnotes "value": xmlstring(obj, stripXmlns=True)} if obj.xmlLang: oimFootnote["lang"] = obj.xmlLang factFootnotes.append(oimFootnote) oimFootnote ''' dtsReferences = [ {"type": "schema" if doc.type == ModelDocument.Type.SCHEMA else "linkbase" if doc.type == ModelDocument.Type.LINKBASE else "other", "href": doc.uri} for doc,ref in modelXbrl.modelDocument.referencesDocument.items() if ref.referringModelObject.qname in SCHEMA_LB_REFS] ''' roleTypes = [ {"type": "role" if ref.referringModelObject.localName == "roleRef" else "arcroleRef", "href": ref.referringModelObject["href"]} for doc,ref in modelXbrl.modelDocument.referencesDocument.items() if ref.referringModelObject.qname in ROLE_REFS] ''' def factFootnotes(fact): footnotes = [] for footnoteRel in footnotesRelationshipSet.fromModelObject(fact): footnote = OrderedDict((("group", footnoteRel.linkrole), ("footnoteType", footnoteRel.arcrole))) footnotes.append(footnote) if isCSVorXL: footnote["factId"] = fact.id if fact.id else "f{}".format(fact.objectIndex) toObj = footnoteRel.toModelObject if isinstance(toObj, ModelFact): footnote["factRef"] = toObj.id if toObj.id else "f{}".format(toObj.objectIndex) else: footnote["footnote"] = xmlstring(toObj, stripXmlns=True, contentsOnly=True, includeText=True) if toObj.xmlLang: footnote["language"] = toObj.xmlLang return footnotes def factAspects(fact): aspects = OrderedDict() if hasId and fact.id: aspects["id"] = fact.id elif (fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))): aspects["id"] = "f{}".format(fact.objectIndex) parent = fact.getparent() concept = fact.concept _csvType = "Value" if not fact.isTuple: if concept is not None: _baseXsdType = concept.baseXsdType if _baseXsdType == "XBRLI_DATEUNION": if getattr(fact.xValue, "dateOnly", False): _baseXsdType = "date" else: _baseXsdType = "dateTime" aspects["baseType"] = "xs:{}".format(_baseXsdType) _csvType = baseTypes.get(_baseXsdType,_baseXsdType) + "Value" if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang: aspects[qnOimLangAspect] = fact.xmlLang if fact.isItem: if fact.isNil: _value = None _strValue = "nil" else: _inferredDecimals = inferredDecimals(fact) _value = oimValue(fact.xValue, _inferredDecimals) _strValue = str(_value) if not isCSVorXL: aspects["value"] = _strValue if fact.concept is not None and fact.concept.isNumeric: _numValue = fact.xValue if isinstance(_numValue, Decimal) and not isinf(_numValue) and not isnan(_numValue): if _numValue == _numValue.to_integral(): _numValue = int(_numValue) else: _numValue = float(_numValue) if isCSVorXL: aspects[_csvType] = _numValue else: aspects["numericValue"] = _numValue if not fact.isNil: if isinf(_inferredDecimals): if isJSON: _accuracy = "infinity" elif isCSVorXL: _accuracy = "INF" else: _accuracy = _inferredDecimals aspects["accuracy"] = _accuracy elif isinstance(_value, bool): aspects["booleanValue"] = _value elif isCSVorXL: aspects[_csvType] = _strValue aspects[qnOimConceptAspect] = oimValue(fact.qname) cntx = fact.context if cntx is not None: if cntx.entityIdentifierElement is not None: aspects[qnOimEntityAspect] = oimValue(qname(*cntx.entityIdentifier)) if cntx.period is not None: if isJSON: aspects[qnOimPeriodAspect] = oimPeriodValue(cntx) elif isCSVorXL: _periodValue = oimPeriodValue(cntx) aspects[qnOimPeriodStartAspect] = _periodValue["start"] aspects[qnOimPeriodEndAspect] = _periodValue["end"] for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]): aspects[dim.dimensionQname] = (oimValue(dim.memberQname) if dim.isExplicit else None if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1") else dim.typedMember.stringValue) unit = fact.unit if unit is not None: _mMul, _mDiv = unit.measures _sMul = '*'.join(oimValue(m) for m in sorted(_mMul, key=lambda m: str(m))) if _mDiv: _sDiv = '*'.join(oimValue(m) for m in sorted(_mDiv, key=lambda m: str(m))) if len(mDiv) > 1: if len(mMul) > 1: _sUnit = "({})/({})".format(_sMul,_sDiv) else: _sUnit = "{}/({})".format(_sMul,_sDiv) else: if len(mMul) > 1: _sUnit = "({})/{}".format(_sMul,_sDiv) else: _sUnit = "{}/{}".format(_sMul,_sDiv) else: _sUnit = _sMul aspects[qnOimUnitAspect] = _sUnit if parent.qname != XbrlConst.qnXbrliXbrl: aspects[qnOimTupleParentAspect] = parent.id if parent.id else "f{}".format(parent.objectIndex) aspects[qnOimTupleOrderAspect] = elementIndex(fact) if isJSON: _footnotes = factFootnotes(fact) if _footnotes: aspects["footnotes"] = _footnotes return aspects if isJSON: # save JSON oimReport = OrderedDict() # top level of oim json output oimFacts = [] oimReport["prefixes"] = OrderedDict((p,ns) for ns, p in sorted(namespacePrefixes.items(), key=lambda item: item[1])) oimReport["dtsReferences"] = dtsReferences oimReport["facts"] = oimFacts def saveJsonFacts(facts, oimFacts, parentFact): for fact in facts: oimFact = factAspects(fact) oimFacts.append(OrderedDict((str(k),v) for k,v in oimFact.items())) if fact.modelTupleFacts: saveJsonFacts(fact.modelTupleFacts, oimFacts, fact) saveJsonFacts(modelXbrl.facts, oimFacts, None) with open(oimFile, "w", encoding="utf-8") as fh: fh.write(json.dumps(oimReport, ensure_ascii=False, indent=1, sort_keys=False)) elif isCSVorXL: # save CSV aspectQnCol = {} aspectsHeader = [] factsColumns = [] def addAspectQnCol(aspectQn): aspectQnCol[aspectQn] = len(aspectsHeader) _colName = oimValue(aspectQn) aspectsHeader.append(_colName) _colDataType = {"id": "Name", "baseType": "Name", "oim:concept": "QName", "oim:periodStart": "dateTime", "oim:periodEnd": "dateTime", "oim:tupleOrder": "integer", "numericValue": "decimal", "accuracy": "decimal", "booleanValue": "boolean", "oim:entity": "QName", "oim:unit": "string" }.get(_colName) if _colDataType is None: if isCSVorXL and _colName.endswith("Value"): _colDataType = _colName[:-5] else: _colDataType = "string" factsColumns.append(OrderedDict((("name", _colName), ("datatype", _colDataType)))) # pre-ordered aspect columns if hasId: addAspectQnCol("id") addAspectQnCol(qnOimConceptAspect) if hasType: addAspectQnCol("baseType") if isCSVorXL: for _baseType in sorted(factBaseTypes): addAspectQnCol(_baseType + "Value") else: addAspectQnCol("stringValue") addAspectQnCol("numericValue") addAspectQnCol("booleanValue") addAspectQnCol("accuracy") if hasTuple: addAspectQnCol(qnOimTupleParentAspect) addAspectQnCol(qnOimTupleOrderAspect) if qnOimEntityAspect in aspectsDefined: addAspectQnCol(qnOimEntityAspect) if qnOimPeriodStartAspect in aspectsDefined: addAspectQnCol(qnOimPeriodStartAspect) addAspectQnCol(qnOimPeriodEndAspect) if qnOimUnitAspect in aspectsDefined: addAspectQnCol(qnOimUnitAspect) for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)): if aspectQn.namespaceURI != nsOim: addAspectQnCol(aspectQn) def aspectCols(fact): cols = [None for i in range(len(aspectsHeader))] _factAspects = factAspects(fact) for aspectQn, aspectValue in _factAspects.items(): if aspectQn in aspectQnCol: cols[aspectQnCol[aspectQn]] = aspectValue return cols # metadata csvTables = [] csvMetadata = OrderedDict((("@context",[ "http://www.w3.org/ns/csvw", { "@base": "./" }]), ("tables", csvTables))) _open = _writerow = _close = None _tableinfo = {} if isCSV: if oimFile.endswith("-facts.csv"): # strip -facts.csv if a prior -facts.csv file was chosen _baseURL = oimFile[:-10] elif oimFile.endswith(".csv"): _baseURL = oimFile[:-4] else: _baseURL = oimFile _csvinfo = {} # open file, writer def _open(filesuffix, tabname): _filename = _tableinfo["url"] = _baseURL + filesuffix _csvinfo["file"] = open(_filename, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig') _csvinfo["writer"] = csv.writer(csvFile, dialect="excel") def _writerow(row, header=False): _csvinfo["writer"].writerow(row) def _close(): _csvinfo["writer"].close() _csvinfo.clear() elif isXL: headerWidths = {"href": 100, "oim:concept": 70, "accuracy": 8, "baseType": 10, "language": 9, "URI": 80, "stringValue": 60, "decimalValue": 12, "booleanValue": 12, "dateValue": 12, "dateTimeValue": 20, "group": 60, "footnoteType": 40, "footnote": 70, "column": 60} from openpyxl import Workbook from openpyxl.writer.write_only import WriteOnlyCell from openpyxl.styles import Font, PatternFill, Border, Alignment, Color, fills, Side from openpyxl.worksheet.dimensions import ColumnDimension hdrCellFill = PatternFill(patternType=fills.FILL_SOLID, fgColor=Color("00FFBF5F")) # Excel's light orange fill color = 00FF990 workbook = Workbook(encoding="utf-8", write_only=True) _xlinfo = {} # open file, writer def _open(filesuffix, tabname): _tableinfo["url"] = tabname _xlinfo["ws"] = workbook.create_sheet(title=tabname) def _writerow(rowvalues, header=False): row = [] _ws = _xlinfo["ws"] for i, v in enumerate(rowvalues): cell = WriteOnlyCell(_ws, value=v) if header: cell.fill = hdrCellFill cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True) colLetter = chr(ord('A') + i) _ws.column_dimensions[colLetter] = ColumnDimension(_ws, customWidth=True) _ws.column_dimensions[colLetter].width = headerWidths.get(v, 20) else: cell.alignment = Alignment(horizontal="right" if isinstance(v, _NUM_TYPES) else "center" if isinstance(v, bool) else "left", vertical="top", wrap_text=isinstance(v, str)) row.append(cell) _ws.append(row) def _close(): _xlinfo.clear() # save facts _open("-facts.csv", "facts") _writerow(aspectsHeader, header=True) def saveCSVfacts(facts): for fact in facts: _writerow(aspectCols(fact)) saveCSVfacts(fact.modelTupleFacts) saveCSVfacts(modelXbrl.facts) _close() factsTableSchema = OrderedDict((("columns",factsColumns),)) csvTables.append(OrderedDict((("url",_tableinfo["url"]), ("tableSchema",factsTableSchema)))) # save namespaces _open("-prefixes.csv", "prefixes") _writerow(("prefix", "URI"), header=True) for _URI, prefix in sorted(namespacePrefixes.items(), key=lambda item: item[1]): _writerow((prefix, _URI)) _close() nsTableSchema = OrderedDict((("columns",[OrderedDict((("name","prefix"),("datatype","Name"))), OrderedDict((("name","URI"),("datatype","anyURI")))]),)) csvTables.append(OrderedDict((("url",_tableinfo["url"]), ("tableSchema",nsTableSchema)))) # save dts references _open("-dtsReferences.csv", "dtsReferences") _writerow(("type", "href"), header=True) for oimRef in dtsReferences: _writerow((oimRef["type"], oimRef["href"])) _close() dtsRefTableSchema = OrderedDict((("columns",[OrderedDict((("name","type"),("datatype","Name"))), OrderedDict((("name","href"),("datatype","anyURI")))]),)) csvTables.append(OrderedDict((("url",_tableinfo["url"]), ("tableSchema",dtsRefTableSchema)))) # save footnotes if footnotesRelationshipSet.modelRelationships: _open("-footnotes.csv", "footnotes") cols = ("group", "footnoteType", "factId", "factRef", "footnote", "language") _writerow(cols, header=True) def saveCSVfootnotes(facts): for fact in facts: for _footnote in factFootnotes(fact): _writerow(tuple((_footnote.get(col,"") for col in cols))) saveCSVfootnotes(fact.modelTupleFacts) saveCSVfootnotes(modelXbrl.facts) _close() footnoteTableSchema = OrderedDict((("columns",[OrderedDict((("name","group"),("datatype","anyURI"))), OrderedDict((("name","footnoteType"),("datatype","Name"))), OrderedDict((("name","factId"),("datatype","Name"))), OrderedDict((("name","factRef"),("datatype","Name"))), OrderedDict((("name","footnote"),("datatype","string"))), OrderedDict((("name","language"),("datatype","language")))]),)) csvTables.append(OrderedDict((("url",_tableinfo["url"]), ("tableSchema",footnoteTableSchema)))) # save metadata if isCSV: with open(_baseURL + "-metadata.csv", "w", encoding="utf-8") as fh: fh.write(json.dumps(csvMetadata, ensure_ascii=False, indent=1, sort_keys=False)) elif isXL: _open(None, "metadata") cols = ("group", "footnoteType", "factId", "factRef", "footnote", "language") _writerow(("table", "column", "datatype"), header=True) for table in csvTables: tablename = table["url"] for column in table["tableSchema"]["columns"]: _writerow((tablename, column["name"], column["datatype"])) _close() if isXL: workbook.save(oimFile)
def validateSetup(val, parameters=None, *args, **kwargs): val.validateEBA = val.validateDisclosureSystem and getattr(val.disclosureSystem, "EBA", False) val.validateEIOPA = val.validateDisclosureSystem and getattr(val.disclosureSystem, "EIOPA", False) if not (val.validateEBA or val.validateEIOPA): return val.validateUTR = False # do not use default UTR validation, it's at error level and not streamable val.utrValidator = ValidateUtr(val.modelXbrl, "WARNING", # EBA specifies SHOULD on UTR validation "EBA.2.23") # override utre error-severity message code val.isEIOPAfullVersion = val.isEIOPA_2_0_1 = False modelDocument = val.modelXbrl.modelDocument if modelDocument.type == ModelDocument.Type.INSTANCE: for doc, docRef in modelDocument.referencesDocument.items(): if docRef.referenceType == "href": if docRef.referringModelObject.localName == "schemaRef": _match = schemaRefDatePattern.match(doc.uri) if _match: val.isEIOPAfullVersion = _match.group(1) > "2015-02-28" val.isEIOPA_2_0_1 = _match.group(1) >= "2015-10-21" break else: val.modelXbrl.error("EIOPA.S.1.5.a/EIOPA.S.1.5.b", _('The link:schemaRef element in submitted instances MUST resolve to the full published entry point URL, this schemaRef is missing date portion: %(schemaRef)s.'), modelObject=modelDocument, schemaRef=doc.uri) val.qnDimAF = val.qnDimOC = val.qnCAx1 = None _nsmap = val.modelXbrl.modelDocument.xmlRootElement.nsmap if val.isEIOPA_2_0_1: _hasPiInstanceGenerator = False for pi in modelDocument.processingInstructions: if pi.target == "instance-generator": _hasPiInstanceGenerator = True if not all(pi.get(attr) for attr in ("id", "version", "creationdate")): val.modelXbrl.warning("EIOPA.S.2.23", _('The processing instruction instance-generator SHOULD contain attributes "id", "version" and "creationdate".'), modelObject=modelDocument) if not _hasPiInstanceGenerator: val.modelXbrl.warning("EIOPA.S.2.23", _('The instance SHOULD include a processing instruction "instance-generator".'), modelObject=modelDocument) val.qnDimAF = qname("s2c_dim:AF", _nsmap) val.qnDimOC = qname("s2c_dim:OC", _nsmap) val.qnCAx1 = qname("s2c_CA:x1", _nsmap) val.prefixNamespace = {} val.namespacePrefix = {} val.idObjects = {} val.typedDomainQnames = set() val.typedDomainElements = set() for modelConcept in val.modelXbrl.qnameConcepts.values(): if modelConcept.isTypedDimension: typedDomainElement = modelConcept.typedDomainElement if isinstance(typedDomainElement, ModelConcept): val.typedDomainQnames.add(typedDomainElement.qname) val.typedDomainElements.add(typedDomainElement) val.filingIndicators = {} val.numFilingIndicatorTuples = 0 val.cntxEntities = set() val.cntxDates = defaultdict(set) val.unusedCntxIDs = set() val.unusedUnitIDs = set() val.currenciesUsed = {} val.reportingCurrency = None val.namespacePrefixesUsed = defaultdict(set) val.prefixesUnused = set() for prefix, ns in _nsmap.items(): val.prefixesUnused.add(prefix) val.namespacePrefixesUsed[ns].add(prefix) val.firstFactObjectIndex = sys.maxsize val.firstFact = None val.footnotesRelationshipSet = ModelRelationshipSet(val.modelXbrl, "XBRL-footnotes")
def saveLoadableOIM(modelXbrl, oimFile, outputZip=None): isJSON = oimFile.endswith(".json") isCSV = oimFile.endswith(".csv") isXL = oimFile.endswith(".xlsx") isCSVorXL = isCSV or isXL if not isJSON and not isCSVorXL: return namespacePrefixes = {nsOim: "xbrl"} prefixNamespaces = {"xbrl": nsOim} def compileQname(qname): if qname.namespaceURI not in namespacePrefixes: namespacePrefixes[qname.namespaceURI] = qname.prefix or "" aspectsDefined = { qnOimConceptAspect, qnOimEntityAspect, qnOimPeriodStartAspect, qnOimPeriodEndAspect} def oimValue(object, decimals=None): if isinstance(object, QName): if object.namespaceURI not in namespacePrefixes: if object.prefix: namespacePrefixes[object.namespaceURI] = object.prefix else: _prefix = "_{}".format(sum(1 for p in namespacePrefixes if p.startswith("_"))) namespacePrefixes[object.namespaceURI] = _prefix return "{}:{}".format(namespacePrefixes[object.namespaceURI], object.localName) if isinstance(object, Decimal): try: if isinf(object): return "-INF" if object < 0 else "INF" elif isnan(num): return "NaN" else: if object == object.to_integral(): object = object.quantize(ONE) # drop any .0 return "{}".format(object) except: return str(object) if isinstance(object, bool): return "true" if object else "false" if isinstance(object, (DateTime, YearMonthDuration, DayTimeDuration, Time, gYearMonth, gMonthDay, gYear, gMonth, gDay, IsoDuration)): return str(object) return object def oimPeriodValue(cntx): if cntx.isForeverPeriod: return OrderedDict() # defaulted elif cntx.isStartEndPeriod: s = cntx.startDatetime e = cntx.endDatetime else: # instant s = e = cntx.instantDatetime if isJSON: return({str(qnOimPeriodAspect): ("{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}/".format( s.year, s.month, s.day, s.hour, s.minute, s.second) if cntx.isStartEndPeriod else "") + "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format( e.year, e.month, e.day, e.hour, e.minute, e.second) }) # CSV, XL return OrderedDict(((str(qnOimPeriodStartAspect), "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format( s.year, s.month, s.day, s.hour, s.minute, s.second)), (str(qnOimPeriodEndAspect), "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format( e.year, e.month, e.day, e.hour, e.minute, e.second)))) hasId = False hasTuple = False hasType = True hasLang = False hasUnits = False hasNumeric = False footnotesRelationshipSet = ModelRelationshipSet(modelXbrl, "XBRL-footnotes") factBaseTypes = set() #compile QNames in instance for OIM for fact in modelXbrl.factsInInstance: if (fact.id or fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))): hasId = True concept = fact.concept if concept is not None: if concept.isNumeric: hasNumeric = True if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang: hasLang = True _baseXsdType = concept.baseXsdType if _baseXsdType == "XBRLI_DATEUNION": if getattr(fact.xValue, "dateOnly", False): _baseXsdType = "date" else: _baseXsdType = "dateTime" factBaseTypes.add(baseTypes.get(_baseXsdType,_baseXsdType)) compileQname(fact.qname) if hasattr(fact, "xValue") and isinstance(fact.xValue, QName): compileQname(fact.xValue) unit = fact.unit if unit is not None: hasUnits = True if fact.modelTupleFacts: hasTuple = True if hasTuple: modelXbrl.error("arelleOIMsaver:tuplesNotAllowed", "Tuples are not allowed in an OIM document", modelObject=modelXbrl) return entitySchemePrefixes = {} for cntx in modelXbrl.contexts.values(): if cntx.entityIdentifierElement is not None: scheme = cntx.entityIdentifier[0] if scheme not in entitySchemePrefixes: if not entitySchemePrefixes: # first one is just scheme if scheme == "http://www.sec.gov/CIK": _schemePrefix = "cik" elif scheme == "http://standard.iso.org/iso/17442": _schemePrefix = "lei" else: _schemePrefix = "scheme" else: _schemePrefix = "scheme{}".format(len(entitySchemePrefixes) + 1) entitySchemePrefixes[scheme] = _schemePrefix namespacePrefixes[scheme] = _schemePrefix for dim in cntx.qnameDims.values(): compileQname(dim.dimensionQname) aspectsDefined.add(dim.dimensionQname) if dim.isExplicit: compileQname(dim.memberQname) for unit in modelXbrl.units.values(): if unit is not None: for measures in unit.measures: for measure in measures: compileQname(measure) if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[XbrlConst.xbrli] != "xbrli": namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix namespacePrefixes[XbrlConst.xsd] = "xsd" if hasLang: aspectsDefined.add(qnOimLangAspect) if hasTuple: aspectsDefined.add(qnOimTupleParentAspect) aspectsDefined.add(qnOimTupleOrderAspect) if hasUnits: aspectsDefined.add(qnOimUnitAspect) for footnoteRel in footnotesRelationshipSet.modelRelationships: typePrefix = "ftTyp_" + os.path.basename(footnoteRel.arcrole) if footnoteRel.linkrole == XbrlConst.defaultLinkRole: groupPrefix = "ftGrp_default" else: groupPrefix = "ftGrp_" + os.path.basename(footnoteRel.linkrole) if typePrefix not in namespacePrefixes: namespacePrefixes[footnoteRel.arcrole] = typePrefix if groupPrefix not in namespacePrefixes: namespacePrefixes[footnoteRel.linkrole] = groupPrefix # compile footnotes and relationships ''' factRelationships = [] factFootnotes = [] for rel in modelXbrl.relationshipSet(modelXbrl, "XBRL-footnotes").modelRelationships: oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole} factRelationships.append(oimRel) oimRel["fromIds"] = [obj.id if obj.id else elementChildSequence(obj) for obj in rel.fromModelObjects] oimRel["toIds"] = [obj.id if obj.id else elementChildSequence(obj) for obj in rel.toModelObjects] _order = rel.arcElement.get("order") if _order is not None: oimRel["order"] = _order for obj in rel.toModelObjects: if isinstance(obj, ModelResource): # footnote oimFootnote = {"role": obj.role, "id": obj.id if obj.id else elementChildSequence(obj), # value needs work for html elements and for inline footnotes "value": xmlstring(obj, stripXmlns=True)} if obj.xmlLang: oimFootnote["lang"] = obj.xmlLang factFootnotes.append(oimFootnote) oimFootnote ''' dtsReferences = set() baseUrl = modelXbrl.modelDocument.uri.partition("#")[0] for doc,ref in sorted(modelXbrl.modelDocument.referencesDocument.items(), key=lambda _item:_item[0].uri): if ref.referringModelObject.qname in SCHEMA_LB_REFS: dtsReferences.add(relativeUri(baseUrl,doc.uri)) for refType in ("role", "arcrole"): for refElt in sorted(modelXbrl.modelDocument.xmlRootElement.iterchildren( "{{http://www.xbrl.org/2003/linkbase}}{}Ref".format(refType)), key=lambda elt:elt.get(refType+"URI") ): dtsReferences.add(refElt.get("{http://www.w3.org/1999/xlink}href").partition("#")[0]) dtsReferences = sorted(dtsReferences) # turn into list footnoteFacts = set() def factFootnotes(fact, oimFact=None): footnotes = [] oimLinks = {} for footnoteRel in footnotesRelationshipSet.fromModelObject(fact): toObj = footnoteRel.toModelObject # json typePrefix = namespacePrefixes[footnoteRel.arcrole] groupPrefix = namespacePrefixes[footnoteRel.linkrole] oimLinks.setdefault(typePrefix,{}).setdefault(groupPrefix,[]).append( toObj.id if toObj.id else "f{}".format(toObj.objectIndex)) # csv/XL footnote = OrderedDict((("group", footnoteRel.linkrole), ("footnoteType", footnoteRel.arcrole))) footnotes.append(footnote) if isCSVorXL: footnote["factId"] = fact.id if fact.id else "f{}".format(fact.objectIndex) if isinstance(toObj, ModelFact): footnote["factRef"] = toObj.id if toObj.id else "f{}".format(toObj.objectIndex) else: footnote["footnote"] = toObj.viewText() if toObj.xmlLang: footnote["language"] = toObj.xmlLang footnoteFacts.add(toObj) if isJSON and oimLinks: oimFact["links"] = OrderedDict(( (typePrefix, OrderedDict(( (groupPrefix, idList) for groupPrefix, idList in sorted(groups.items()) ))) for typePrefix, groups in sorted(oimLinks.items()) )) footnotes.sort(key=lambda f:(f["group"],f.get("factId",f.get("factRef")),f.get("language"))) return footnotes def factAspects(fact): oimFact = OrderedDict() aspects = OrderedDict() if hasId and fact.id: if fact.isTuple: oimFact["tupleId"] = fact.id else: oimFact["id"] = fact.id elif (fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))): oimFact["id"] = "f{}".format(fact.objectIndex) parent = fact.getparent() concept = fact.concept aspects[str(qnOimConceptAspect)] = oimValue(concept.qname) _csvType = "Value" if not fact.isTuple: if concept is not None: _baseXsdType = concept.baseXsdType if _baseXsdType == "XBRLI_DATEUNION": if getattr(fact.xValue, "dateOnly", False): _baseXsdType = "date" else: _baseXsdType = "dateTime" _csvType = baseTypes.get(_baseXsdType,_baseXsdType) + "Value" if concept.baseXbrliType in ("stringItemType", "normalizedStringItemType") and fact.xmlLang: aspects[str(qnOimLangAspect)] = fact.xmlLang if fact.isItem: if fact.isNil: _value = None else: _inferredDecimals = inferredDecimals(fact) _value = oimValue(fact.xValue, _inferredDecimals) oimFact["value"] = _value if fact.concept is not None and fact.concept.isNumeric: _numValue = fact.xValue if isinstance(_numValue, Decimal) and not isinf(_numValue) and not isnan(_numValue): if _numValue == _numValue.to_integral(): _numValue = int(_numValue) else: _numValue = float(_numValue) if not fact.isNil: if isinf(_inferredDecimals): if isJSON: _accuracy = "infinity" elif isCSVorXL: _accuracy = "INF" else: _accuracy = _inferredDecimals oimFact["accuracy"] = _accuracy oimFact["aspects"] = aspects cntx = fact.context if cntx is not None: if cntx.entityIdentifierElement is not None: aspects[str(qnOimEntityAspect)] = oimValue(qname(*cntx.entityIdentifier)) if cntx.period is not None: aspects.update(oimPeriodValue(cntx)) for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]): if dim.isExplicit: dimVal = oimValue(dim.memberQname) else: # typed if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1"): dimVal = None else: dimVal = dim.typedMember.stringValue aspects[str(dim.dimensionQname)] = dimVal unit = fact.unit if unit is not None: _mMul, _mDiv = unit.measures _sMul = '*'.join(oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m))) if _mDiv: _sDiv = '*'.join(oimValue(m) for m in sorted(_mDiv, key=lambda m: oimValue(m))) if len(_mDiv) > 1: if len(_mMul) > 1: _sUnit = "({})/({})".format(_sMul,_sDiv) else: _sUnit = "{}/({})".format(_sMul,_sDiv) else: if len(_mMul) > 1: _sUnit = "({})/{}".format(_sMul,_sDiv) else: _sUnit = "{}/{}".format(_sMul,_sDiv) else: _sUnit = _sMul aspects[str(qnOimUnitAspect)] = _sUnit # Tuples removed from xBRL-JSON #if parent.qname != XbrlConst.qnXbrliXbrl: # aspects[str(qnOimTupleParentAspect)] = parent.id if parent.id else "f{}".format(parent.objectIndex) # aspects[str(qnOimTupleOrderAspect)] = elementIndex(fact) if isJSON: factFootnotes(fact, oimFact) return oimFact prefixes = OrderedDict((p,ns) for ns, p in sorted(namespacePrefixes.items(), key=lambda item: item[1])) if isJSON: # save JSON oimReport = OrderedDict() # top level of oim json output oimReport["documentInfo"] = oimDocInfo = OrderedDict() oimReport["facts"] = oimFacts = OrderedDict() oimDocInfo["documentType"] = nsOim + "/xbrl-json" oimDocInfo["features"] = oimFeatures = OrderedDict() oimDocInfo["prefixes"] = prefixes oimDocInfo["taxonomy"] = dtsReferences oimFeatures["xbrl:canonicalValues"] = True def saveJsonFacts(facts, oimFacts, parentFact): for fact in facts: oimFact = factAspects(fact) id = fact.id if fact.id else "f{}".format(fact.objectIndex) oimFacts[id] = oimFact if fact.modelTupleFacts: saveJsonFacts(fact.modelTupleFacts, oimFacts, fact) saveJsonFacts(modelXbrl.facts, oimFacts, None) # add footnotes as pseudo facts for ftObj in footnoteFacts: ftId = ftObj.id if ftObj.id else "f{}".format(ftObj.objectIndex) oimFacts[ftId] = oimFact = OrderedDict() oimFact["value"] = ftObj.viewText() oimFact["aspects"] = OrderedDict((("concept", "xbrl:note"), ("noteId", ftId))) if ftObj.xmlLang: oimFact["aspects"]["language"] = ftObj.xmlLang.lower() if outputZip: fh = io.StringIO() else: fh = open(oimFile, "w", encoding="utf-8") fh.write(json.dumps(oimReport, indent=1)) if outputZip: fh.seek(0) outputZip.writestr(os.path.basename(oimFile),fh.read()) fh.close() elif isCSVorXL: # save CSV aspectQnCol = {} aspectsHeader = [] factsColumns = [] def addAspectQnCol(aspectQn): aspectQnCol[str(aspectQn)] = len(aspectsHeader) _aspectQn = oimValue(aspectQn) aspectsHeader.append(_aspectQn) _colName = _aspectQn.replace("xbrl:", "") _colDataType = {"id": "Name", "concept": "QName", "value": "string", "accuracy": "decimal", "entity": "QName", "periodStart": "dateTime", "periodEnd": "dateTime", "unit": "string", "tupleId": "Name", "tupleParent": "Name", "tupleOrder": "integer" }.get(_colName, "string") col = OrderedDict((("name", _colName), ("datatype", _colDataType))) if _aspectQn == "value": col["http://xbrl.org/YYYY/model#simpleFactAspects"] = {} elif _aspectQn == "tupleId": col["http://xbrl.org/YYYY/model#tupleFactAspects"] = {} col["http://xbrl.org/YYYY/model#tupleReferenceId"] = "true" else: col["http://xbrl.org/YYYY/model#columnAspect"] = _aspectQn factsColumns.append(col) # pre-ordered aspect columns #if hasId: # addAspectQnCol("id") addAspectQnCol(qnOimConceptAspect) addAspectQnCol("value") if hasNumeric: addAspectQnCol("accuracy") if hasTuple: addAspectQnCol("tupleId") addAspectQnCol(qnOimTupleParentAspect) addAspectQnCol(qnOimTupleOrderAspect) if qnOimEntityAspect in aspectsDefined: addAspectQnCol(qnOimEntityAspect) if qnOimPeriodStartAspect in aspectsDefined: addAspectQnCol(qnOimPeriodStartAspect) addAspectQnCol(qnOimPeriodEndAspect) if qnOimUnitAspect in aspectsDefined: addAspectQnCol(qnOimUnitAspect) for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)): if aspectQn.namespaceURI != nsOim: addAspectQnCol(aspectQn) def aspectCols(fact): cols = [None for i in range(len(aspectsHeader))] def setColValues(aspects): for aspectQn, aspectValue in aspects.items(): if isinstance(aspectValue, dict): setColValues(aspectValue) elif aspectQn in aspectQnCol: if aspectValue is None: _aspectValue = "#nil" elif aspectValue == "": _aspectValue = "#empty" elif isinstance(aspectValue, str) and aspectValue.startswith("#"): _aspectValue = "#" + aspectValue else: _aspectValue = aspectValue cols[aspectQnCol[aspectQn]] = _aspectValue setColValues(factAspects(fact)) return cols # metadata csvTables = [] csvMetadata = OrderedDict((("@context", "http://www.w3.org/ns/csvw"), ("http://xbrl.org/YYYY/model#metadata", OrderedDict((("documentType", "http://xbrl.org/YYYY/xbrl-csv"), ("dtsReferences", dtsReferences), ("prefixes", prefixes)))), ("tables", csvTables))) _open = _writerow = _close = None _tableinfo = {} if isCSV: if oimFile.endswith("-facts.csv"): # strip -facts.csv if a prior -facts.csv file was chosen _baseURL = oimFile[:-10] elif oimFile.endswith(".csv"): _baseURL = oimFile[:-4] else: _baseURL = oimFile _csvinfo = {} # open file, writer def _open(filesuffix, tabname): _filename = _tableinfo["url"] = _baseURL + filesuffix _csvinfo["file"] = open(_filename, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig') _csvinfo["writer"] = csv.writer(_csvinfo["file"], dialect="excel") def _writerow(row, header=False): _csvinfo["writer"].writerow(row) def _close(): _csvinfo["file"].close() _csvinfo.clear() elif isXL: headerWidths = {"href": 100, "xbrl:concept": 70, "accuracy": 8, "language": 9, "URI": 80, "value": 60, "group": 60, "footnoteType": 40, "footnote": 70, "column": 20, 'conceptAspect': 40, 'tuple': 20, 'simpleFact': 20} from openpyxl import Workbook from openpyxl.writer.write_only import WriteOnlyCell from openpyxl.styles import Font, PatternFill, Border, Alignment, Color, fills, Side from openpyxl.worksheet.dimensions import ColumnDimension hdrCellFill = PatternFill(patternType=fills.FILL_SOLID, fgColor=Color("00FFBF5F")) # Excel's light orange fill color = 00FF990 workbook = Workbook() # remove pre-existing worksheets while len(workbook.worksheets)>0: workbook.remove_sheet(workbook.worksheets[0]) _xlinfo = {} # open file, writer def _open(filesuffix, tabname): _tableinfo["url"] = tabname _xlinfo["ws"] = workbook.create_sheet(title=tabname) def _writerow(rowvalues, header=False): row = [] _ws = _xlinfo["ws"] for i, v in enumerate(rowvalues): cell = WriteOnlyCell(_ws, value=v) if header: cell.fill = hdrCellFill cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True) colLetter = chr(ord('A') + i) _ws.column_dimensions[colLetter] = ColumnDimension(_ws, customWidth=True) _ws.column_dimensions[colLetter].width = headerWidths.get(v, 20) else: cell.alignment = Alignment(horizontal="right" if isinstance(v, _NUM_TYPES) else "center" if isinstance(v, bool) else "left", vertical="top", wrap_text=isinstance(v, str)) row.append(cell) _ws.append(row) def _close(): _xlinfo.clear() # save facts _open("-facts.csv", "facts") _writerow(aspectsHeader, header=True) def saveCSVfacts(facts): for fact in facts: _writerow(aspectCols(fact)) saveCSVfacts(fact.modelTupleFacts) saveCSVfacts(modelXbrl.facts) _close() factsTableSchema = OrderedDict((("columns",factsColumns),)) csvTables.append(OrderedDict((("url",_tableinfo["url"]), ("http://xbrl.org/YYYY/model#tableType", "fact"), ("tableSchema",factsTableSchema)))) # save footnotes if footnotesRelationshipSet.modelRelationships: _open("-footnotes.csv", "footnotes") cols = ("group", "footnoteType", "factId", "factRef", "footnote", "language") _writerow(cols, header=True) def saveCSVfootnotes(facts): for fact in facts: for _footnote in factFootnotes(fact): _writerow(tuple((_footnote.get(col,"") for col in cols))) saveCSVfootnotes(fact.modelTupleFacts) saveCSVfootnotes(modelXbrl.facts) _close() footnoteTableSchema = OrderedDict((("columns",[OrderedDict((("name","group"),("datatype","anyURI"))), OrderedDict((("name","footnoteType"),("datatype","Name"))), OrderedDict((("name","factId"),("datatype","Name"))), OrderedDict((("name","factRef"),("datatype","Name"))), OrderedDict((("name","footnote"),("datatype","string"))), OrderedDict((("name","language"),("datatype","language")))]),)) csvTables.append(OrderedDict((("url",_tableinfo["url"]), ("http://xbrl.org/YYYY/model#tableType", "footnote"), ("tableSchema",footnoteTableSchema)))) # save metadata if isCSV: with open(_baseURL + "-metadata.json", "w", encoding="utf-8") as fh: fh.write(json.dumps(csvMetadata, ensure_ascii=False, indent=1, sort_keys=False)) elif isXL: _open(None, "metadata") hasColumnAspect = hasSimpleFact = hasTupleFact = False for table in csvTables: tablename = table["url"] for column in table["tableSchema"]["columns"]: if "http://xbrl.org/YYYY/model#columnAspect" in column: hasColumnAspect = True if "http://xbrl.org/YYYY/model#simpleFactAspects" in column: hasSimpleFact = True if "http://xbrl.org/YYYY/model#tupleAspects" in column: hasTupleFact = True metadataCols = ["table", "column", "datatype"] if hasColumnAspect: metadataCols.append("columnAspect") if hasSimpleFact: metadataCols.append("simpleFact") if hasTupleFact: metadataCols.append("tuple") _writerow(metadataCols, header=True) for table in csvTables: tablename = table["url"] for column in table["tableSchema"]["columns"]: row = [tablename, column["name"], column["datatype"]] if hasColumnAspect: colAspect = column.get("http://xbrl.org/YYYY/model#columnAspect") if isinstance(colAspect, str): row.append(colAspect) elif isinstance(colAspect, dict): row.append("\n".join("{} [{}]".format(k, ", ".join(_v for _v in v)) for k, v in dict.items())) else: row.append(None) if hasSimpleFact: row.append("\u221a" if "http://xbrl.org/YYYY/model#simpleFactAspects" in column else None) if hasTupleFact: row.append("\u221a" if "http://xbrl.org/YYYY/model#tupleAspects" in column else None) _writerow(row) _close() if isXL: workbook.save(oimFile)
def saveLoadableOIM(modelXbrl, oimFile, outputZip=None): isJSON = oimFile.endswith(".json") isCSV = oimFile.endswith(".csv") isXL = oimFile.endswith(".xlsx") isCSVorXL = isCSV or isXL if not isJSON and not isCSVorXL: return namespacePrefixes = {nsOim: "xbrl"} prefixNamespaces = {} linkTypeAliases = {} groupAliases = {} linkTypePrefixes = {} linkTypeUris = {} linkGroupPrefixes = {} linkGroupUris = {} def compileQname(qname): if qname.namespaceURI not in namespacePrefixes: namespacePrefixes[qname.namespaceURI] = qname.prefix or "" aspectsDefined = {qnOimConceptAspect, qnOimEntityAspect, qnOimPeriodAspect} def oimValue(object, decimals=None): if isinstance(object, QName): if object.namespaceURI not in namespacePrefixes: if object.prefix: namespacePrefixes[object.namespaceURI] = object.prefix else: _prefix = "_{}".format( sum(1 for p in namespacePrefixes if p.startswith("_"))) namespacePrefixes[object.namespaceURI] = _prefix return "{}:{}".format(namespacePrefixes[object.namespaceURI], object.localName) if isinstance(object, (float, Decimal)): try: if isinf(object): return "-INF" if object < 0 else "INF" elif isnan(object): return "NaN" else: if isinstance(object, Decimal) and object == object.to_integral(): object = object.quantize(ONE) # drop any .0 return "{}".format(object) except: return str(object) if isinstance(object, bool): return "true" if object else "false" if isinstance( object, (DateTime, YearMonthDuration, DayTimeDuration, Time, gYearMonth, gMonthDay, gYear, gMonth, gDay, IsoDuration, int)): return str(object) return object def oimPeriodValue(cntx): if cntx.isStartEndPeriod: s = cntx.startDatetime e = cntx.endDatetime else: # instant s = e = cntx.instantDatetime return ({ str(qnOimPeriodAspect): ("{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}/".format( s.year, s.month, s.day, s.hour, s.minute, s.second) if cntx.isStartEndPeriod else "") + "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format( e.year, e.month, e.day, e.hour, e.minute, e.second) }) hasId = False hasTuple = False hasType = True hasLang = False hasUnits = False hasNumeric = False footnotesRelationshipSet = ModelRelationshipSet(modelXbrl, "XBRL-footnotes") #compile QNames in instance for OIM for fact in modelXbrl.factsInInstance: if (fact.id or fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))): hasId = True concept = fact.concept if concept is not None: if concept.isNumeric: hasNumeric = True if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang: hasLang = True compileQname(fact.qname) if hasattr(fact, "xValue") and isinstance(fact.xValue, QName): compileQname(fact.xValue) unit = fact.unit if unit is not None: hasUnits = True if fact.modelTupleFacts: hasTuple = True if hasTuple: modelXbrl.error("arelleOIMsaver:tuplesNotAllowed", "Tuples are not allowed in an OIM document", modelObject=modelXbrl) return entitySchemePrefixes = {} for cntx in modelXbrl.contexts.values(): if cntx.entityIdentifierElement is not None: scheme = cntx.entityIdentifier[0] if scheme not in entitySchemePrefixes: if not entitySchemePrefixes: # first one is just scheme if scheme == "http://www.sec.gov/CIK": _schemePrefix = "cik" elif scheme == "http://standard.iso.org/iso/17442": _schemePrefix = "lei" else: _schemePrefix = "scheme" else: _schemePrefix = "scheme{}".format( len(entitySchemePrefixes) + 1) entitySchemePrefixes[scheme] = _schemePrefix namespacePrefixes[scheme] = _schemePrefix for dim in cntx.qnameDims.values(): compileQname(dim.dimensionQname) aspectsDefined.add(dim.dimensionQname) if dim.isExplicit: compileQname(dim.memberQname) for unit in modelXbrl.units.values(): if unit is not None: for measures in unit.measures: for measure in measures: compileQname(measure) if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[ XbrlConst.xbrli] != "xbrli": namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix if hasLang: aspectsDefined.add(qnOimLangAspect) if hasUnits: aspectsDefined.add(qnOimUnitAspect) for footnoteRel in footnotesRelationshipSet.modelRelationships: typePrefix = "ftTyp_" + os.path.basename(footnoteRel.arcrole) if footnoteRel.linkrole == XbrlConst.defaultLinkRole: groupPrefix = "ftGrp_default" else: groupPrefix = "ftGrp_" + os.path.basename(footnoteRel.linkrole) if footnoteRel.arcrole not in linkTypeAliases: linkTypeAliases[footnoteRel.arcrole] = typePrefix if groupPrefix not in groupAliases: groupAliases[footnoteRel.linkrole] = groupPrefix dtsReferences = set() baseUrl = modelXbrl.modelDocument.uri.partition("#")[0] for doc, ref in sorted(modelXbrl.modelDocument.referencesDocument.items(), key=lambda _item: _item[0].uri): if ref.referringModelObject.qname in SCHEMA_LB_REFS: dtsReferences.add(relativeUri(baseUrl, doc.uri)) for refType in ("role", "arcrole"): for refElt in sorted( modelXbrl.modelDocument.xmlRootElement.iterchildren( "{{http://www.xbrl.org/2003/linkbase}}{}Ref".format( refType)), key=lambda elt: elt.get(refType + "URI")): dtsReferences.add( refElt.get("{http://www.w3.org/1999/xlink}href").partition("#") [0]) dtsReferences = sorted(dtsReferences) # turn into list footnoteFacts = set() def factFootnotes(fact, oimFact=None, csvLinks=None): footnotes = [] if isJSON: oimLinks = {} elif isCSVorXL: oimLinks = csvLinks for footnoteRel in footnotesRelationshipSet.fromModelObject(fact): srcId = fact.id if fact.id else "f{}".format(fact.objectIndex) toObj = footnoteRel.toModelObject # json typePrefix = linkTypeAliases[footnoteRel.arcrole] groupPrefix = groupAliases[footnoteRel.linkrole] if typePrefix not in oimLinks: oimLinks[typePrefix] = OrderedDict() _link = oimLinks[typePrefix] if groupPrefix not in _link: if isJSON: _link[groupPrefix] = [] elif isCSVorXL: _link[groupPrefix] = OrderedDict() if isJSON: tgtIdList = _link[groupPrefix] elif isCSVorXL: tgtIdList = _link[groupPrefix].setdefault(srcId, []) tgtId = toObj.id if toObj.id else "f{}".format(toObj.objectIndex) tgtIdList.append(tgtId) footnote = OrderedDict((("group", footnoteRel.linkrole), ("footnoteType", footnoteRel.arcrole))) if isinstance(toObj, ModelFact): footnote["factRef"] = tgtId else: # text footnotes footnote["id"] = tgtId footnote["footnote"] = toObj.viewText() if toObj.xmlLang: footnote["language"] = toObj.xmlLang footnoteFacts.add(toObj) footnotes.append(footnote) if oimLinks: _links = OrderedDict( ((typePrefix, OrderedDict( ((groupPrefix, idList) for groupPrefix, idList in sorted(groups.items())))) for typePrefix, groups in sorted(oimLinks.items()))) if isJSON: oimFact["links"] = _links return footnotes def factAspects(fact): oimFact = OrderedDict() aspects = OrderedDict() if isCSVorXL: oimFact["id"] = fact.id or "f{}".format(fact.objectIndex) parent = fact.getparent() concept = fact.concept aspects[str(qnOimConceptAspect)] = oimValue(concept.qname) if concept is not None: if concept.type.isOimTextFactType and fact.xmlLang: aspects[str(qnOimLangAspect)] = fact.xmlLang if fact.isItem: if fact.isNil: _value = None else: _inferredDecimals = inferredDecimals(fact) _value = oimValue(fact.xValue, _inferredDecimals) oimFact["value"] = _value if fact.concept is not None and fact.concept.isNumeric: _numValue = fact.xValue if isinstance(_numValue, Decimal) and not isinf( _numValue) and not isnan(_numValue): if _numValue == _numValue.to_integral(): _numValue = int(_numValue) else: _numValue = float(_numValue) if not fact.isNil: if not isinf( _inferredDecimals): # accuracy omitted if infinite oimFact["decimals"] = _inferredDecimals oimFact["dimensions"] = aspects cntx = fact.context if cntx is not None: if cntx.entityIdentifierElement is not None and cntx.entityIdentifier != ENTITY_NA_QNAME: aspects[str(qnOimEntityAspect)] = oimValue( qname(*cntx.entityIdentifier)) if cntx.period is not None and not cntx.isForeverPeriod: aspects.update(oimPeriodValue(cntx)) for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]): if dim.isExplicit: dimVal = oimValue(dim.memberQname) else: # typed if dim.typedMember.get( "{http://www.w3.org/2001/XMLSchema-instance}nil" ) in ("true", "1"): dimVal = None else: dimVal = dim.typedMember.stringValue aspects[str(dim.dimensionQname)] = dimVal unit = fact.unit if unit is not None: _mMul, _mDiv = unit.measures _sMul = '*'.join( oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m))) if _mDiv: _sDiv = '*'.join( oimValue(m) for m in sorted(_mDiv, key=lambda m: oimValue(m))) if len(_mDiv) > 1: if len(_mMul) > 1: _sUnit = "({})/({})".format(_sMul, _sDiv) else: _sUnit = "{}/({})".format(_sMul, _sDiv) else: if len(_mMul) > 1: _sUnit = "({})/{}".format(_sMul, _sDiv) else: _sUnit = "{}/{}".format(_sMul, _sDiv) else: _sUnit = _sMul if _sUnit != "xbrli:pure": aspects[str(qnOimUnitAspect)] = _sUnit # Tuples removed from xBRL-JSON #if parent.qname != XbrlConst.qnXbrliXbrl: # aspects[str(qnOimTupleParentAspect)] = parent.id if parent.id else "f{}".format(parent.objectIndex) # aspects[str(qnOimTupleOrderAspect)] = elementIndex(fact) if isJSON: factFootnotes(fact, oimFact=oimFact) return oimFact namespaces = OrderedDict((p, ns) for ns, p in sorted(namespacePrefixes.items(), key=lambda item: item[1])) # common metadata oimReport = OrderedDict() # top level of oim json output oimReport["documentInfo"] = oimDocInfo = OrderedDict() oimDocInfo["documentType"] = nsOim + ("/xbrl-json" if isJSON else "/xbrl-csv") if isJSON: oimDocInfo["features"] = oimFeatures = OrderedDict() oimDocInfo["namespaces"] = namespaces if linkTypeAliases: oimDocInfo["linkTypes"] = OrderedDict( (a, u) for u, a in sorted(linkTypeAliases.items(), key=lambda item: item[1])) if linkTypeAliases: oimDocInfo["linkGroups"] = OrderedDict( (a, u) for u, a in sorted(groupAliases.items(), key=lambda item: item[1])) oimDocInfo["taxonomy"] = dtsReferences if isJSON: oimFeatures["xbrl:canonicalValues"] = True if isJSON: # save JSON oimReport["facts"] = oimFacts = OrderedDict() def saveJsonFacts(facts, oimFacts, parentFact): for fact in facts: oimFact = factAspects(fact) id = fact.id if fact.id else "f{}".format(fact.objectIndex) oimFacts[id] = oimFact if fact.modelTupleFacts: saveJsonFacts(fact.modelTupleFacts, oimFacts, fact) saveJsonFacts(modelXbrl.facts, oimFacts, None) # add footnotes as pseudo facts for ftObj in footnoteFacts: ftId = ftObj.id if ftObj.id else "f{}".format(ftObj.objectIndex) oimFacts[ftId] = oimFact = OrderedDict() oimFact["value"] = ftObj.viewText() oimFact["dimensions"] = OrderedDict( (("concept", "xbrl:note"), ("noteId", ftId))) if ftObj.xmlLang: oimFact["dimensions"]["language"] = ftObj.xmlLang.lower() if outputZip: fh = io.StringIO() else: fh = open(oimFile, "w", encoding="utf-8") fh.write(json.dumps(oimReport, indent=1)) if outputZip: fh.seek(0) outputZip.writestr(os.path.basename(oimFile), fh.read()) fh.close() elif isCSVorXL: # save CSV oimReport["tables"] = oimTables = OrderedDict() oimReport["tableTemplates"] = csvTableTemplates = OrderedDict() oimTables["facts"] = csvTable = OrderedDict() csvTable["template"] = "facts" csvTable["url"] = "tbd" csvTableTemplates["facts"] = csvTableTemplate = OrderedDict() csvTableTemplate["rowIdColumn"] = "id" csvTableTemplate["dimensions"] = csvTableDimensions = OrderedDict() csvTableTemplate["columns"] = csvFactColumns = OrderedDict() if footnotesRelationshipSet.modelRelationships: csvLinks = OrderedDict() oimReport["links"] = csvLinks aspectQnCol = {} aspectsHeader = [] factsColumns = [] def addAspectQnCol(aspectQn): colQName = str(aspectQn).replace("xbrl:", "") colNCName = colQName.replace(":", "_") if colNCName not in aspectQnCol: aspectQnCol[colNCName] = len(aspectsHeader) aspectsHeader.append(colNCName) if colNCName == "value": csvFactColumns[colNCName] = col = OrderedDict() col["dimensions"] = OrderedDict() elif colNCName == "id": csvFactColumns[colNCName] = {} # empty object elif colNCName == "decimals": csvFactColumns[colNCName] = {} # empty object csvTableTemplate[colQName] = "$" + colNCName else: csvFactColumns[colNCName] = {} # empty object csvTableDimensions[colQName] = "$" + colNCName # pre-ordered aspect columns #if hasId: addAspectQnCol("id") addAspectQnCol(qnOimConceptAspect) if hasNumeric: addAspectQnCol("decimals") if qnOimEntityAspect in aspectsDefined: addAspectQnCol(qnOimEntityAspect) if qnOimPeriodAspect in aspectsDefined: addAspectQnCol(qnOimPeriodAspect) if qnOimUnitAspect in aspectsDefined: addAspectQnCol(qnOimUnitAspect) for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)): if aspectQn.namespaceURI != nsOim: addAspectQnCol(aspectQn) addAspectQnCol("value") def aspectCols(fact): cols = [None for i in range(len(aspectsHeader))] def setColValues(aspects): for aspectQn, aspectValue in aspects.items(): colQName = str(aspectQn).replace("xbrl:", "") colNCName = colQName.replace(":", "_") if isinstance(aspectValue, dict): setColValues(aspectValue) elif colNCName in aspectQnCol: if aspectValue is None: _aspectValue = "#nil" elif aspectValue == "": _aspectValue = "#empty" elif isinstance(aspectValue, str) and aspectValue.startswith("#"): _aspectValue = "#" + aspectValue else: _aspectValue = aspectValue cols[aspectQnCol[colNCName]] = _aspectValue setColValues(factAspects(fact)) return cols # metadata _open = _writerow = _close = None if isCSV: if oimFile.endswith( "-facts.csv" ): # strip -facts.csv if a prior -facts.csv file was chosen _baseURL = oimFile[:-10] elif oimFile.endswith(".csv"): _baseURL = oimFile[:-4] else: _baseURL = oimFile _csvinfo = {} # open file, writer def _open(filesuffix, tabname, csvTable=None): _filename = _baseURL + filesuffix if csvTable is not None: csvTable["url"] = os.path.basename( _filename) # located in same directory with metadata _csvinfo["file"] = open(_filename, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig') _csvinfo["writer"] = csv.writer(_csvinfo["file"], dialect="excel") def _writerow(row, header=False): _csvinfo["writer"].writerow(row) def _close(): _csvinfo["file"].close() _csvinfo.clear() elif isXL: headerWidths = { "concept": 40, "decimals": 8, "language": 9, "value": 50, "entity": 20, "period": 20, "unit": 20, "metadata": 100 } from openpyxl import Workbook from openpyxl.cell.cell import WriteOnlyCell from openpyxl.styles import Font, PatternFill, Border, Alignment, Color, fills, Side from openpyxl.worksheet.dimensions import ColumnDimension hdrCellFill = PatternFill( patternType=fills.FILL_SOLID, fgColor=Color( "00FFBF5F")) # Excel's light orange fill color = 00FF990 workbook = Workbook() # remove pre-existing worksheets while len(workbook.worksheets) > 0: workbook.remove(workbook.worksheets[0]) _xlinfo = {} # open file, writer def _open(filesuffix, tabname, csvTable=None): if csvTable is not None: csvTable["url"] = tabname + "!" _xlinfo["ws"] = workbook.create_sheet(title=tabname) def _writerow(rowvalues, header=False): row = [] _ws = _xlinfo["ws"] for i, v in enumerate(rowvalues): cell = WriteOnlyCell(_ws, value=v) if header: cell.fill = hdrCellFill cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True) colLetter = chr(ord('A') + i) _ws.column_dimensions[colLetter] = ColumnDimension( _ws, customWidth=True) _ws.column_dimensions[ colLetter].width = headerWidths.get(v, 40) else: cell.alignment = Alignment( horizontal="right" if isinstance(v, _NUM_TYPES) else "center" if isinstance(v, bool) else "left", vertical="top", wrap_text=isinstance(v, str)) row.append(cell) _ws.append(row) def _close(): _xlinfo.clear() # save facts _open("-facts.csv", "facts", csvTable) _writerow(aspectsHeader, header=True) def saveCSVfacts(facts): for fact in facts: _writerow(aspectCols(fact)) saveCSVfacts(fact.modelTupleFacts) saveCSVfacts(modelXbrl.facts) _close() # save footnotes if footnotesRelationshipSet.modelRelationships: footnotes = sorted( (footnote for fact in modelXbrl.facts for footnote in factFootnotes(fact, csvLinks=csvLinks)), key=lambda footnote: footnote["id"]) if footnotes: # text footnotes oimTables["footnotes"] = csvFtTable = OrderedDict() csvFtTable["url"] = "tbd" csvFtTable[ "tableDimensions"] = csvFtTableDimensions = OrderedDict() csvFtTable["factColumns"] = csvFtFactColumns = OrderedDict() csvFtFactColumns["footnote"] = csvFtValCol = OrderedDict() csvFtValCol["id"] = "$id" csvFtValCol["noteId"] = "$id" csvFtValCol["concept"] = "xbrl:note" csvFtValCol["language"] = "$language" _open("-footnotes.csv", "footnotes", csvFtTable) cols = ("id", "footnote", "language") _writerow(cols, header=True) for footnote in footnotes: _writerow(tuple((footnote.get(col, "") for col in cols))) _close() # save metadata if isCSV: with open(_baseURL + "-metadata.json", "w", encoding="utf-8") as fh: fh.write( json.dumps(oimReport, ensure_ascii=False, indent=2, sort_keys=False)) elif isXL: _open(None, "metadata") _writerow(["metadata"], header=True) _writerow([ json.dumps(oimReport, ensure_ascii=False, indent=1, sort_keys=False) ]) _close() if isXL: workbook.save(oimFile)
def validateTestcase(self, testcase): self.modelXbrl.info("info", "Testcase", modelDocument=testcase) self.modelXbrl.viewModelObject(testcase.objectId()) if hasattr(testcase, "testcaseVariations"): for modelTestcaseVariation in testcase.testcaseVariations: # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) # is this a versioning report? resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance resultIsTaxonomyPackage = modelTestcaseVariation.resultIsTaxonomyPackage formulaOutputInstance = None inputDTSes = defaultdict(list) baseForElement = testcase.baseForElement(modelTestcaseVariation) # try to load instance document self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s - %(description)s"), modelObject=modelTestcaseVariation, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, expected=modelTestcaseVariation.expected, description=modelTestcaseVariation.description) errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY parameters = modelTestcaseVariation.parameters.copy() for readMeFirstUri in modelTestcaseVariation.readMeFirstUris: if isinstance(readMeFirstUri,tuple): # dtsName is for formula instances, but is from/to dts if versioning dtsName, readMeFirstUri = readMeFirstUri elif resultIsVersioningReport: if inputDTSes: dtsName = "to" else: dtsName = "from" else: dtsName = None if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document if dtsName in inputDTSes: dtsName = inputDTSes[dtsName] else: modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager, Type.DTSENTRIES, self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement), isEntry=True, errorCaptureLevel=errorCaptureLevel) DTSdoc = modelXbrl.modelDocument DTSdoc.inDTS = True doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement) if doc is not None: DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement) #fake import doc.inDTS = True elif resultIsTaxonomyPackage: from arelle import PackageManager, PrototypeInstanceObject dtsName = readMeFirstUri modelXbrl = PrototypeInstanceObject.XbrlPrototype(self.modelXbrl.modelManager, readMeFirstUri) PackageManager.packageInfo(self.modelXbrl.modelManager.cntlr, readMeFirstUri, reload=True, errors=modelXbrl.errors) else: # not a multi-schemaRef versioning report if self.useFileSource.isArchive: modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, readMeFirstUri, _("validating"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) else: # need own file source, may need instance discovery filesource = FileSource.FileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr) if filesource and not filesource.selection and filesource.isArchive: for _archiveFile in filesource.dir: # find instance document in archive filesource.select(_archiveFile) if ModelDocument.Type.identify(filesource, filesource.url) in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL): break # use this selection modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, filesource, _("validating"), base=baseForElement, errorCaptureLevel=errorCaptureLevel) modelXbrl.isTestcaseVariation = True if modelXbrl.modelDocument is None: modelXbrl.error("arelle:notLoaded", _("Testcase %(id)s %(name)s document not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) self.determineNotLoadedTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif resultIsVersioningReport or resultIsTaxonomyPackage: inputDTSes[dtsName] = modelXbrl elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl) self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif testcase.type == Type.REGISTRYTESTCASE: self.instValidator.validate(modelXbrl) # required to set up dimensions, etc self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id, modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest) self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) self.instValidator.close() modelXbrl.close() else: inputDTSes[dtsName].append(modelXbrl) # validate except for formulas _hasFormulae = modelXbrl.hasFormulae modelXbrl.hasFormulae = False try: for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"): pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation) self.instValidator.validate(modelXbrl, parameters) for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"): pluginXbrlMethod(self.modelXbrl, modelXbrl) except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase variation validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) modelXbrl.hasFormulae = _hasFormulae if resultIsVersioningReport and modelXbrl.modelDocument: versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl( modelTestcaseVariation.versioningReportUri, baseForElement) if os.path.exists(versReportFile): #validate existing modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report")) if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport) self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors) modelVersReport.close() elif len(inputDTSes) == 2: ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes( versReportFile, inputDTSes["from"], inputDTSes["to"]) modelTestcaseVariation.status = "generated" else: modelXbrl.error("arelle:notLoaded", _("Testcase %(id)s %(name)s DTSes not loaded, unable to generate versioning report: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) modelTestcaseVariation.status = "failed" for inputDTS in inputDTSes.values(): inputDTS.close() del inputDTSes # dereference elif resultIsTaxonomyPackage: self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif inputDTSes: # validate schema, linkbase, or instance modelXbrl = inputDTSes[None][0] for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters if dtsName: # named instance parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes) elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable: RenderingEvaluator.init(modelXbrl) if modelXbrl.hasFormulae: try: # validate only formulae self.instValidator.parameters = parameters ValidateFormula.validate(self.instValidator) except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset: for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"): pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri) infoset = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultInfosetUri, _("loading result infoset"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) if infoset.modelDocument is None: modelXbrl.error("arelle:notLoaded", _("Testcase %(id)s %(name)s result infoset not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstance)) modelTestcaseVariation.status = "result infoset not loadable" else: # check infoset ValidateInfoset.validate(self.instValidator, modelXbrl, infoset) infoset.close() if modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset: # diff (or generate) table infoset resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement) if not any(alternativeValidation(modelXbrl, resultTableUri) for alternativeValidation in pluginClassMethods("Validate.TableInfoset")): ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files self.instValidator.close() extraErrors = [] for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"): pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors) self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual): # if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost) self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters) self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors) if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance formulaOutputInstance = modelXbrl.formulaOutputInstance modelXbrl.formulaOutputInstance = None # prevent it from being closed now self.instValidator.close() compareIxResultInstance = getattr(modelXbrl, "extractedInlineInstance", False) and modelTestcaseVariation.resultXbrlInstanceUri if compareIxResultInstance: formulaOutputInstance = modelXbrl # compare modelXbrl to generated output instance errMsgPrefix = "ix" else: # delete input instances before formula output comparision for inputDTSlist in inputDTSes.values(): for inputDTS in inputDTSlist: inputDTS.close() del inputDTSes # dereference errMsgPrefix = "formula" if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument: expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultXbrlInstanceUri, _("loading expected result XBRL instance"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) if expectedInstance.modelDocument is None: self.modelXbrl.error("{}:expectedResultNotLoaded".format(errMsgPrefix), _("Testcase %(id)s %(name)s expected result instance not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstanceUri), messageCodes=("formula:expectedResultNotLoaded","ix:expectedResultNotLoaded")) modelTestcaseVariation.status = "result not loadable" else: # compare facts if len(expectedInstance.facts) != len(formulaOutputInstance.facts): formulaOutputInstance.error("{}:resultFactCounts".format(errMsgPrefix), _("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"), modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts), expectedFacts=len(expectedInstance.facts), messageCodes=("formula:resultFactCounts","ix:resultFactCounts")) else: formulaOutputFootnotesRelSet = ModelRelationshipSet(formulaOutputInstance, "XBRL-footnotes") expectedFootnotesRelSet = ModelRelationshipSet(expectedInstance, "XBRL-footnotes") def factFootnotes(fact, footnotesRelSet): footnotes = [] footnoteRels = footnotesRelSet.fromModelObject(fact) if footnoteRels: # most process rels in same order between two instances, use labels to sort for i, footnoteRel in enumerate(sorted(footnoteRels, key=lambda r: (r.fromLabel,r.toLabel))): modelObject = footnoteRel.toModelObject if isinstance(modelObject, ModelResource): xml = modelObject.viewText().strip() footnotes.append("Footnote {}: {}".format( i+1, # compare footnote with HTML serialized xml, #re.sub(r'\s+', ' ', collapseWhitespace(modelObject.stringValue)) )) elif isinstance(modelObject, ModelFact): footnotes.append("Footnoted fact {}: {} context: {} value: {}".format( i+1, modelObject.qname, modelObject.contextID, collapseWhitespace(modelObject.value))) return footnotes for expectedInstanceFact in expectedInstance.facts: unmatchedFactsStack = [] formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True) if formulaOutputFact is None: if unmatchedFactsStack: # get missing nested tuple fact, if possible missingFact = unmatchedFactsStack[-1] else: missingFact = expectedInstanceFact formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix), _("Output missing expected fact %(fact)s"), modelXbrl=missingFact, fact=missingFact.qname, messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing")) else: # compare footnotes expectedInstanceFactFootnotes = factFootnotes(expectedInstanceFact, expectedFootnotesRelSet) formulaOutputFactFootnotes = factFootnotes(formulaOutputFact, formulaOutputFootnotesRelSet) if expectedInstanceFactFootnotes != formulaOutputFactFootnotes: formulaOutputInstance.error("{}:expectedFactFootnoteDifference".format(errMsgPrefix), _("Output expected fact %(fact)s expected footnotes %(footnotes1)s produced footnotes %(footnotes2)s"), modelXbrl=(formulaOutputFact,expectedInstanceFact), fact=expectedInstanceFact.qname, footnotes1=expectedInstanceFactFootnotes, footnotes2=formulaOutputFactFootnotes, messageCodes=("formula:expectedFactFootnoteDifference","ix:expectedFactFootnoteDifference")) # for debugging uncomment next line to save generated instance document # formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml") expectedInstance.close() del expectedInstance # dereference self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors) formulaOutputInstance.close() del formulaOutputInstance if compareIxResultInstance: for inputDTSlist in inputDTSes.values(): for inputDTS in inputDTSlist: inputDTS.close() del inputDTSes # dereference # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def saveLoadableOIM(modelXbrl, oimFile): isJSON = oimFile.endswith(".json") isCSV = oimFile.endswith(".csv") namespacePrefixes = {} def compileQname(qname): if qname.namespaceURI not in namespacePrefixes: namespacePrefixes[qname.namespaceURI] = qname.prefix or "" aspectsDefined = { qnOimConceptAspect, qnOimEntityAspect, qnOimTypeAspect} if isJSON: aspectsDefined.add(qnOimPeriodAspect) elif isCSV: aspectsDefined.add(qnOimPeriodStartAspect) aspectsDefined.add(qnOimPeriodDurationAspect) def oimValue(object, decimals=None): if isinstance(object, QName): if object.namespaceURI not in namespacePrefixes: if object.prefix: namespacePrefixes[object.namespaceURI] = object.prefix else: _prefix = "_{}".format(sum(1 for p in namespacePrefixes if p.startswith("_"))) namespacePrefixes[object.namespaceURI] = _prefix return "{}:{}".format(namespacePrefixes[object.namespaceURI], object.localName) if isinstance(object, Decimal): try: if isinf(object): return "-INF" if object < 0 else "INF" elif isnan(num): return "NaN" else: if object == object.to_integral(): object = object.quantize(ONE) # drop any .0 return "{}".format(object) except: return str(object) if isinstance(object, bool): return object if isinstance(object, (DateTime, YearMonthDuration, DayTimeDuration, Time, gYearMonth, gMonthDay, gYear, gMonth, gDay)): return str(object) return object def oimPeriodValue(cntx): if cntx.isForeverPeriod: if isCSV: return "0000-01-01T00:00:00/P9999Y" return "forever" elif cntx.isStartEndPeriod: d = cntx.startDatetime duration = yearMonthDayTimeDuration(cntx.startDatetime, cntx.endDatetime) else: # instant d = cntx.instantDatetime duration = "PT0S" return "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}/{6}".format( d.year, d.month, d.day, d.hour, d.minute, d.second, duration) hasId = False hasTuple = False hasType = True hasLang = False hasUnits = False hasUnitMulMeasures = False hasUnitDivMeasures = False footnotesRelationshipSet = ModelRelationshipSet(modelXbrl, "XBRL-footnotes") #compile QNames in instance for OIM for fact in modelXbrl.factsInInstance: if (fact.id or fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSV and footnotesRelationshipSet.fromModelObject(fact))): hasId = True concept = fact.concept if concept is not None: if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang: hasLang = True compileQname(fact.qname) if hasattr(fact, "xValue") and isinstance(fact.xValue, QName): compileQname(fact.xValue) unit = fact.unit if unit is not None: hasUnits = True if unit.measures[0]: hasUnitMulMeasures = True if unit.measures[1]: hasUnitDivMeasures = True if fact.modelTupleFacts: hasTuple = True entitySchemePrefixes = {} for cntx in modelXbrl.contexts.values(): if cntx.entityIdentifierElement is not None: scheme = cntx.entityIdentifier[0] if scheme not in entitySchemePrefixes: if not entitySchemePrefixes: # first one is just scheme if scheme == "http://www.sec.gov/CIK": _schemePrefix = "cik" elif scheme == "http://standard.iso.org/iso/17442": _schemePrefix = "lei" else: _schemePrefix = "scheme" else: _schemePrefix = "scheme{}".format(len(entitySchemePrefixes) + 1) entitySchemePrefixes[scheme] = _schemePrefix namespacePrefixes[scheme] = _schemePrefix for dim in cntx.qnameDims.values(): compileQname(dim.dimensionQname) aspectsDefined.add(dim.dimensionQname) if dim.isExplicit: compileQname(dim.memberQname) for unit in modelXbrl.units.values(): if unit is not None: for measures in unit.measures: for measure in measures: compileQname(measure) if unit.measures[0]: aspectsDefined.add(qnOimUnitNumeratorsAspect) if unit.measures[1]: aspectsDefined.add(qnOimUnitDenominatorsAspect) if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[XbrlConst.xbrli] != "xbrli": namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix namespacePrefixes[XbrlConst.xsd] = "xsd" if hasLang: aspectsDefined.add(qnOimLangAspect) if hasTuple: aspectsDefined.add(qnOimTupleParentAspect) aspectsDefined.add(qnOimTupleOrderAspect) if hasUnits: aspectsDefined.add(qnOimUnitAspect) # compile footnotes and relationships ''' factRelationships = [] factFootnotes = [] for rel in modelXbrl.relationshipSet(modelXbrl, "XBRL-footnotes").modelRelationships: oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole} factRelationships.append(oimRel) oimRel["fromIds"] = [obj.id if obj.id else elementChildSequence(obj) for obj in rel.fromModelObjects] oimRel["toIds"] = [obj.id if obj.id else elementChildSequence(obj) for obj in rel.toModelObjects] _order = rel.arcElement.get("order") if _order is not None: oimRel["order"] = _order for obj in rel.toModelObjects: if isinstance(obj, ModelResource): # footnote oimFootnote = {"role": obj.role, "id": obj.id if obj.id else elementChildSequence(obj), # value needs work for html elements and for inline footnotes "value": xmlstring(obj, stripXmlns=True)} if obj.xmlLang: oimFootnote["lang"] = obj.xmlLang factFootnotes.append(oimFootnote) oimFootnote ''' dtsReferences = [ {"type": "schema" if doc.type == ModelDocument.Type.SCHEMA else "linkbase" if doc.type == ModelDocument.Type.LINKBASE else "other", "href": doc.uri} for doc,ref in modelXbrl.modelDocument.referencesDocument.items() if ref.referringModelObject.qname in SCHEMA_LB_REFS] ''' roleTypes = [ {"type": "role" if ref.referringModelObject.localName == "roleRef" else "arcroleRef", "href": ref.referringModelObject["href"]} for doc,ref in modelXbrl.modelDocument.referencesDocument.items() if ref.referringModelObject.qname in ROLE_REFS] ''' def factFootnotes(fact): footnotes = [] for footnoteRel in footnotesRelationshipSet.fromModelObject(fact): footnote = OrderedDict((("group", footnoteRel.arcrole),)) footnotes.append(footnote) if isCSV: footnote["factId"] = fact.id if fact.id else "f{}".format(fact.objectIndex) toObj = footnoteRel.toModelObject if isinstance(toObj, ModelFact): footnote["factRef"] = toObj.id if toObj.id else "f{}".format(toObj.objectIndex) else: footnote["footnoteType"] = toObj.role footnote["footnote"] = xmlstring(toObj, stripXmlns=True, contentsOnly=True, includeText=True) if toObj.xmlLang: footnote["language"] = toObj.xmlLang return footnotes def factAspects(fact): aspects = OrderedDict() if hasId and fact.id: aspects["id"] = fact.id elif (fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or (isCSV and footnotesRelationshipSet.fromModelObject(fact))): aspects["id"] = "f{}".format(fact.objectIndex) parent = fact.getparent() concept = fact.concept if not fact.isTuple: if concept is not None: _baseXsdType = concept.baseXsdType if _baseXsdType == "XBRLI_DATEUNION": if getattr(fact.xValue, "dateOnly", False): _baseXsdType = "date" else: _baseXsdType = "dateTime" aspects["baseType"] = "xs:{}".format(_baseXsdType) if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang: aspects[qnOimLangAspect] = fact.xmlLang aspects[qnOimTypeAspect] = concept.baseXbrliType if fact.isItem: if fact.isNil: _value = None _strValue = "nil" else: _inferredDecimals = inferredDecimals(fact) _value = oimValue(fact.xValue, _inferredDecimals) _strValue = str(_value) if not isCSV: aspects["value"] = _strValue if fact.concept is not None and fact.concept.isNumeric: _numValue = fact.xValue if isinstance(_numValue, Decimal) and not isinf(_numValue) and not isnan(_numValue): if _numValue == _numValue.to_integral(): _numValue = int(_numValue) else: _numValue = float(_numValue) aspects["numericValue"] = _numValue if not fact.isNil: if isinf(_inferredDecimals): if isJSON: _accuracy = "infinity" elif isCSV: _accuracy = "INF" else: _accuracy = _inferredDecimals aspects["accuracy"] = _inferredDecimals elif isinstance(_value, bool): aspects["booleanValue"] = _value elif isCSV: aspects["stringValue"] = _strValue aspects[qnOimConceptAspect] = oimValue(fact.qname) cntx = fact.context if cntx is not None: if cntx.entityIdentifierElement is not None: aspects[qnOimEntityAspect] = oimValue(qname(*cntx.entityIdentifier)) if cntx.period is not None: if isJSON: aspects[qnOimPeriodAspect] = oimPeriodValue(cntx) elif isCSV: _periodValue = oimPeriodValue(cntx).split("/") + ["", ""] # default blank if no value aspects[qnOimPeriodStartAspect] = _periodValue[0] aspects[qnOimPeriodDurationAspect] = _periodValue[1] for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]): aspects[dim.dimensionQname] = (oimValue(dim.memberQname) if dim.isExplicit else None if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1") else dim.typedMember.stringValue) unit = fact.unit if unit is not None: _mMul, _mDiv = unit.measures if isJSON: aspects[qnOimUnitAspect] = OrderedDict( # use tuple instead of list for hashability (("numerators", tuple(oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m)))),) ) if _mDiv: aspects[qnOimUnitAspect]["denominators"] = tuple(oimValue(m) for m in sorted(_mDiv, key=lambda m: oimValue(m))) else: # CSV if _mMul: aspects[qnOimUnitNumeratorsAspect] = " ".join(oimValue(m) for m in sorted(_mMul, key=lambda m: str(m))) if _mDiv: aspects[qnOimUnitDenominatorsAspect] = " ".join(oimValue(m) for m in sorted(_mDiv, key=lambda m: str(m))) if parent.qname != XbrlConst.qnXbrliXbrl: aspects[qnOimTupleParentAspect] = parent.id if parent.id else "f{}".format(parent.objectIndex) aspects[qnOimTupleOrderAspect] = elementIndex(fact) if isJSON: _footnotes = factFootnotes(fact) if _footnotes: aspects["footnotes"] = _footnotes return aspects if isJSON: # save JSON oimReport = OrderedDict() # top level of oim json output oimFacts = [] oimReport["prefixes"] = OrderedDict((p,ns) for ns, p in sorted(namespacePrefixes.items(), key=lambda item: item[1])) oimReport["dtsReferences"] = dtsReferences oimReport["facts"] = oimFacts def saveJsonFacts(facts, oimFacts, parentFact): for fact in facts: oimFact = factAspects(fact) oimFacts.append(OrderedDict((str(k),v) for k,v in oimFact.items())) if fact.modelTupleFacts: saveJsonFacts(fact.modelTupleFacts, oimFacts, fact) saveJsonFacts(modelXbrl.facts, oimFacts, None) with open(oimFile, "w", encoding="utf-8") as fh: fh.write(json.dumps(oimReport, ensure_ascii=False, indent=1, sort_keys=False)) elif isCSV: # save CSV aspectQnCol = {} aspectsHeader = [] factsColumns = [] def addAspectQnCol(aspectQn): aspectQnCol[aspectQn] = len(aspectsHeader) _colName = oimValue(aspectQn) aspectsHeader.append(_colName) _colDataType = {"id": "Name", "baseType": "Name", "oim:concept": "Name", "oim:periodStart": "dateTime", # forever is 0000-01-01T00:00:00 "oim:periodDuration": "duration", # forever is P9999Y "oim:tupleOrder": "integer", "numericValue": "decimal", "accuracy": "decimal", "booleanValue": "boolean", "oim:unitNumerators": OrderedDict((("base","Name"), ("separator"," "))), "oim:unitDenominators": OrderedDict((("base","Name"), ("separator"," "))), }.get(_colName, "string") factsColumns.append(OrderedDict((("name", _colName), ("datatype", _colDataType)))) # pre-ordered aspect columns if hasId: addAspectQnCol("id") if hasType: addAspectQnCol("baseType") addAspectQnCol("stringValue") addAspectQnCol("numericValue") addAspectQnCol("accuracy") addAspectQnCol("booleanValue") if hasTuple: addAspectQnCol(qnOimTupleParentAspect) addAspectQnCol(qnOimTupleOrderAspect) addAspectQnCol(qnOimConceptAspect) if qnOimEntityAspect in aspectsDefined: addAspectQnCol(qnOimEntityAspect) if qnOimPeriodStartAspect in aspectsDefined: addAspectQnCol(qnOimPeriodStartAspect) addAspectQnCol(qnOimPeriodDurationAspect) if qnOimUnitNumeratorsAspect in aspectsDefined: addAspectQnCol(qnOimUnitNumeratorsAspect) if qnOimUnitDenominatorsAspect in aspectsDefined: addAspectQnCol(qnOimUnitDenominatorsAspect) for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)): if aspectQn.namespaceURI != nsOim: addAspectQnCol(aspectQn) def aspectCols(fact): cols = [None for i in range(len(aspectsHeader))] _factAspects = factAspects(fact) for aspectQn, aspectValue in _factAspects.items(): if aspectQn in aspectQnCol: cols[aspectQnCol[aspectQn]] = aspectValue return cols # metadata csvTables = [] csvMetadata = OrderedDict((("@context",[ "http://www.w3.org/ns/csvw", { "@base": "./" }]), ("tables", csvTables))) if oimFile.endswith("-facts.csv"): # strip -facts.csv if a prior -facts.csv file was chosen _baseURL = oimFile[:-10] elif oimFile.endswith(".csv"): _baseURL = oimFile[:-4] else: _baseURL = oimFile # save facts _factsFile = _baseURL + "-facts.csv" csvFile = open(_factsFile, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig') csvWriter = csv.writer(csvFile, dialect="excel") csvWriter.writerow(aspectsHeader) def saveCSVfacts(facts): for fact in facts: csvWriter.writerow(aspectCols(fact)) saveCSVfacts(fact.modelTupleFacts) saveCSVfacts(modelXbrl.facts) csvFile.close() factsTableSchema = OrderedDict((("columns",factsColumns),)) csvTables.append(OrderedDict((("url",os.path.basename(_factsFile)), ("tableSchema",factsTableSchema)))) # save namespaces _nsFile = _baseURL + "-prefixes.csv" csvFile = open(_nsFile, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig') csvWriter = csv.writer(csvFile, dialect="excel") csvWriter.writerow(("prefix", "URI")) for _URI, prefix in sorted(namespacePrefixes.items(), key=lambda item: item[1]): csvWriter.writerow((prefix, _URI)) csvFile.close() nsTableSchema = OrderedDict((("columns",[OrderedDict((("prefix","string"), ("URI","anyURI")))]),)) csvTables.append(OrderedDict((("url",os.path.basename(_nsFile)), ("tableSchema",nsTableSchema)))) # save dts references _dtsRefFile = _baseURL + "-dtsReferences.csv" csvFile = open(_dtsRefFile, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig') csvWriter = csv.writer(csvFile, dialect="excel") csvWriter.writerow(("type", "href")) for oimRef in dtsReferences: csvWriter.writerow((oimRef["type"], oimRef["href"])) csvFile.close() dtsRefTableSchema = OrderedDict((("columns",[OrderedDict((("type","string"), ("href","anyURI")))]),)) csvTables.append(OrderedDict((("url",os.path.basename(_dtsRefFile)), ("tableSchema",dtsRefTableSchema)))) # save footnotes if footnotesRelationshipSet.modelRelationships: _footnoteFile = oimFile.replace(".csv", "-footnotes.csv") csvFile = open(_footnoteFile, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig') csvWriter = csv.writer(csvFile, dialect="excel") cols = ("group", "factId", "factRef", "footnoteType", "footnote", "language") csvWriter.writerow(cols) def saveCSVfootnotes(facts): for fact in facts: for _footnote in factFootnotes(fact): csvWriter.writerow(tuple((_footnote.get(col,"") for col in cols))) saveCSVfootnotes(fact.modelTupleFacts) saveCSVfootnotes(modelXbrl.facts) csvFile.close() footnoteTableSchema = OrderedDict((("columns",[OrderedDict((("group","anyURI"), ("factId","Name"), ("factRef","Name"), ("footnoteType","Name"), ("footnote","string"), ("language","language")))]),)) csvTables.append(OrderedDict((("url",os.path.basename(_footnoteFile)), ("tableSchema",footnoteTableSchema)))) # save metadata with open(_baseURL + "-metadata.csv", "w", encoding="utf-8") as fh: fh.write(json.dumps(csvMetadata, ensure_ascii=False, indent=1, sort_keys=False))