def __init__(self, dts):
self.nsmap = NamespaceMap()
self.roleMap = NamespaceMap()
self.dts = dts
self.taxonomyData = {
"concepts": {},
"languages": {},
"facts": {},
}
self.footnoteRelationshipSet = ModelRelationshipSet(dts, "XBRL-footnotes")
def base_dimension_sets(dts):
"""Get the Xule base dimension sets.
This is like the baseSets dictionary of a model. The base dimension set is a dictionary keyed by the drs role and hypercube. The drs role is the role of the initial 'all' relationship or the target role of the initial
'all' relationship if ther eis a target role. The value of the diction is a set of the 'all' relationships.
"""
_imports()
if not hasattr(dts, 'xuleBaseDimensionSets'):
dts.xuleBaseDimensionSets = collections.defaultdict(set)
for base_set in dts.baseSets:
if (base_set[XuleProperties.NETWORK_ARCROLE] == 'http://xbrl.org/int/dim/arcrole/all' and
base_set[XuleProperties.NETWORK_ROLE] is not None and
base_set[XuleProperties.NETWORK_LINK] is not None and
base_set[XuleProperties.NETWORK_ARC] is not None):
# This is an 'all' dimension base set find the hypercubes
relationship_set =dts.relationshipSets.get(base_set,
ModelRelationshipSet(dts,
base_set[XuleProperties.NETWORK_ARCROLE],
base_set[XuleProperties.NETWORK_ROLE],
base_set[XuleProperties.NETWORK_LINK],
base_set[XuleProperties.NETWORK_ARC]))
for rel in relationship_set.modelRelationships:
drs_role = rel.targetRole or base_set[XuleProperties.NETWORK_ROLE]
hypercube = rel.toModelObject
dts.xuleBaseDimensionSets[(drs_role, hypercube)].add(rel)
return dts.xuleBaseDimensionSets
def relationship_set(dts, relationship_set_info):
_imports()
return (dts.relationshipSets[relationship_set_info]
if relationship_set_info in dts.relationshipSets
else ModelRelationshipSet(dts,
relationship_set_info[XuleProperties.NETWORK_ARCROLE],
relationship_set_info[XuleProperties.NETWORK_ROLE],
relationship_set_info[XuleProperties.NETWORK_LINK],
relationship_set_info[XuleProperties.NETWORK_ARC]))
def validateSetup(val, parameters=None):
val.validateEBA = val.validateDisclosureSystem and getattr(
val.disclosureSystem, "EBA", False)
val.validateEIOPA = val.validateDisclosureSystem and getattr(
val.disclosureSystem, "EIOPA", False)
if not (val.validateEBA or val.validateEIOPA):
return
val.validateUTR = False # do not use default UTR validation, it's at error level and not streamable
val.utrValidator = ValidateUtr(
val.modelXbrl,
"WARNING", # EBA specifies SHOULD on UTR validation
"EBA.2.23") # override utre error-severity message code
val.isEIOPAfullVersion = False
modelDocument = val.modelXbrl.modelDocument
if modelDocument.type == ModelDocument.Type.INSTANCE:
for doc, docRef in modelDocument.referencesDocument.items():
if docRef.referenceType == "href":
if docRef.referringModelObject.localName == "schemaRef":
_match = schemaRefDatePattern.match(doc.uri)
if _match:
val.isEIOPAfullVersion = _match.group(1) > "2015-02-28"
break
val.prefixNamespace = {}
val.namespacePrefix = {}
val.idObjects = {}
val.typedDomainQnames = set()
val.typedDomainElements = set()
for modelConcept in val.modelXbrl.qnameConcepts.values():
if modelConcept.isTypedDimension:
typedDomainElement = modelConcept.typedDomainElement
if isinstance(typedDomainElement, ModelConcept):
val.typedDomainQnames.add(typedDomainElement.qname)
val.typedDomainElements.add(typedDomainElement)
val.filingIndicators = {}
val.numFilingIndicatorTuples = 0
val.cntxEntities = set()
val.cntxDates = defaultdict(set)
val.unusedCntxIDs = set()
val.unusedUnitIDs = set()
val.currenciesUsed = {}
val.namespacePrefixesUsed = defaultdict(set)
val.prefixesUnused = set(
val.modelXbrl.modelDocument.xmlRootElement.nsmap.keys()).copy()
val.firstFactObjectIndex = sys.maxsize
val.firstFact = None
val.footnotesRelationshipSet = ModelRelationshipSet(
val.modelXbrl, "XBRL-footnotes")
def groupRelationshipSet(modelXbrl, arcrole, linkrole, linkqname, arcqname):
if isinstance(arcrole, (list,tuple)): # (group-name, [arcroles])
arcroles = arcrole[1]
relationshipSet = ModelRelationshipSet(modelXbrl, arcroles[0], linkrole, linkqname, arcqname)
for arcrole in arcroles[1:]:
if arcrole != XbrlConst.arcroleGroupDetect:
rels = modelXbrl.relationshipSet(arcrole, linkrole, linkqname, arcqname)
if rels:
relationshipSet.modelRelationships.extend(rels.modelRelationships)
relationshipSet.modelRelationships.sort(key=lambda rel: rel.order)
else:
relationshipSet = modelXbrl.relationshipSet(arcrole, linkrole, linkqname, arcqname)
return relationshipSet
def network(node,
sphinxContext,
args,
linkqname=None,
linkrole=None,
arcqname=None,
arcrole=None):
dts = dtsArg(node, sphinxContext, args)
return ModelRelationshipSet(dts,
arcrole,
linkrole=linkrole,
linkqname=linkqname,
arcqname=arcqname)
def viewFacts(modelXbrl, tabWin, lang=None):
modelXbrl.modelManager.showStatus(_("viewing facts"))
view = ViewFactList(modelXbrl, tabWin, lang)
view.treeView["columns"] = ("sequence", "contextID", "unitID", "decimals",
"precision", "language", "footnoted", "value")
view.treeView.column("#0", width=200, anchor="w")
view.treeView.heading("#0", text=_("Label"))
view.treeView.column("sequence", width=40, anchor="e", stretch=False)
view.treeView.heading("sequence", text=_("Seq"))
view.treeView.column("contextID", width=100, anchor="w", stretch=False)
view.treeView.heading("contextID", text="contextRef")
view.treeView.column("unitID", width=75, anchor="w", stretch=False)
view.unitDisplayID = False # start displaying measures
view.treeView.heading("unitID", text="Unit")
view.treeView.column("decimals", width=50, anchor="center", stretch=False)
view.treeView.heading("decimals", text=_("Dec"))
view.treeView.column("precision", width=50, anchor="w", stretch=False)
view.treeView.heading("precision", text=_("Prec"))
view.treeView.column("language", width=36, anchor="w", stretch=False)
view.treeView.heading("language", text=_("Lang"))
view.treeView.column("footnoted", width=18, anchor="center", stretch=False)
view.treeView.heading("footnoted", text=_("Fn"))
view.treeView.column("value", width=200, anchor="w", stretch=False)
view.treeView.heading("value", text=_("Value"))
view.treeView["displaycolumns"] = ("sequence", "contextID", "unitID", "decimals", "precision", \
"language", "footnoted", "value")
view.footnotesRelationshipSet = ModelRelationshipSet(
modelXbrl, "XBRL-footnotes")
view.blockSelectEvent = 1
view.blockViewModelObject = 0
view.view()
view.treeView.bind("<<TreeviewSelect>>", view.treeviewSelect, '+')
view.treeView.bind("<Enter>", view.treeviewEnter, '+')
view.treeView.bind("<Leave>", view.treeviewLeave, '+')
# intercept menu click before pops up to set the viewable tuple (if tuple clicked)
view.treeView.bind(view.modelXbrl.modelManager.cntlr.contextMenuClick,
view.setViewTupleChildMenuItem, '+')
menu = view.contextMenu()
if menu is not None:
view.menu.insert_cascade(0,
label=_("View Tuple Children"),
underline=0,
command=view.viewTuplesGrid)
view.menu.entryconfigure(0, state='disabled')
view.menuAddExpandCollapse()
view.menuAddClipboard()
view.menuAddLangs()
view.menuAddLabelRoles(includeConceptName=True)
view.menuAddUnitDisplay()
def validateSetup(val, parameters=None):
val.validateEBA = val.validateDisclosureSystem and getattr(val.disclosureSystem, "EBA", False)
val.validateEIOPA = val.validateDisclosureSystem and getattr(val.disclosureSystem, "EIOPA", False)
if not (val.validateEBA or val.validateEIOPA):
return
val.validateUTR = False # do not use default UTR validation, it's at error level and not streamable
val.utrValidator = ValidateUtr(val.modelXbrl,
"WARNING", # EBA specifies SHOULD on UTR validation
"EBA.2.23") # override utre error-severity message code
val.prefixNamespace = {}
val.namespacePrefix = {}
val.idObjects = {}
val.typedDomainQnames = set()
val.typedDomainElements = set()
for modelConcept in val.modelXbrl.qnameConcepts.values():
if modelConcept.isTypedDimension:
typedDomainElement = modelConcept.typedDomainElement
if isinstance(typedDomainElement, ModelConcept):
val.typedDomainQnames.add(typedDomainElement.qname)
val.typedDomainElements.add(typedDomainElement)
val.filingIndicators = {}
val.numFilingIndicatorTuples = 0
val.cntxEntities = set()
val.cntxDates = defaultdict(set)
val.unusedCntxIDs = set()
val.unusedUnitIDs = set()
val.currenciesUsed = {}
val.namespacePrefixesUsed = defaultdict(set)
val.prefixesUnused = set(val.modelXbrl.modelDocument.xmlRootElement.nsmap.keys()).copy()
val.firstFactObjectIndex = sys.maxsize
val.firstFact = None
val.footnotesRelationshipSet = ModelRelationshipSet(val.modelXbrl, "XBRL-footnotes")
def validateTestcase(self, testcase):
self.modelXbrl.info("info", "Testcase", modelDocument=testcase)
self.modelXbrl.viewModelObject(testcase.objectId())
if testcase.type in (Type.TESTCASESINDEX, Type.REGISTRY):
for doc in sorted(testcase.referencesDocument.keys(), key=lambda doc: doc.uri):
self.validateTestcase(doc) # testcases doc's are sorted by their uri (file names), e.g., for formula
elif hasattr(testcase, "testcaseVariations"):
for modelTestcaseVariation in testcaseVariationsByTarget(testcase.testcaseVariations):
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
# is this a versioning report?
resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
resultIsTaxonomyPackage = modelTestcaseVariation.resultIsTaxonomyPackage
formulaOutputInstance = None
inputDTSes = defaultdict(list)
baseForElement = testcase.baseForElement(modelTestcaseVariation)
# try to load instance document
self.modelXbrl.info("info", _("Variation %(id)s%(name)s%(target)s: %(expected)s - %(description)s"),
modelObject=modelTestcaseVariation,
id=modelTestcaseVariation.id,
name=(" {}".format(modelTestcaseVariation.name) if modelTestcaseVariation.name else ""),
target=(" target {}".format(modelTestcaseVariation.ixdsTarget) if modelTestcaseVariation.ixdsTarget else ""),
expected=modelTestcaseVariation.expected,
description=modelTestcaseVariation.description)
if self.modelXbrl.modelManager.formulaOptions.testcaseResultsCaptureWarnings:
errorCaptureLevel = logging._checkLevel("WARNING")
else:
errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY
parameters = modelTestcaseVariation.parameters.copy()
for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
if isinstance(readMeFirstUri,tuple):
# dtsName is for formula instances, but is from/to dts if versioning
dtsName, readMeFirstUri = readMeFirstUri
elif resultIsVersioningReport:
if inputDTSes: dtsName = "to"
else: dtsName = "from"
else:
dtsName = None
if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
if dtsName in inputDTSes:
dtsName = inputDTSes[dtsName]
else:
modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager,
Type.DTSENTRIES,
self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
isEntry=True,
errorCaptureLevel=errorCaptureLevel)
DTSdoc = modelXbrl.modelDocument
DTSdoc.inDTS = True
doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement)
if doc is not None:
DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement) #fake import
doc.inDTS = True
elif resultIsTaxonomyPackage:
from arelle import PackageManager, PrototypeInstanceObject
dtsName = readMeFirstUri
modelXbrl = PrototypeInstanceObject.XbrlPrototype(self.modelXbrl.modelManager, readMeFirstUri)
PackageManager.packageInfo(self.modelXbrl.modelManager.cntlr, readMeFirstUri, reload=True, errors=modelXbrl.errors)
else: # not a multi-schemaRef versioning report
if self.useFileSource.isArchive:
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
readMeFirstUri,
_("validating"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel,
ixdsTarget=modelTestcaseVariation.ixdsTarget)
else: # need own file source, may need instance discovery
filesource = FileSource.openFileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr, base=baseForElement)
if filesource and not filesource.selection and filesource.isArchive:
try:
if filesource.isTaxonomyPackage:
_rptPkgIxdsOptions = {}
for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ReportPackageIxdsOptions"):
pluginXbrlMethod(self, _rptPkgIxdsOptions)
filesource.loadTaxonomyPackageMappings()
for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ReportPackageIxds"):
filesource.select(pluginXbrlMethod(filesource, **_rptPkgIxdsOptions))
else:
from arelle.CntlrCmdLine import filesourceEntrypointFiles
entrypoints = filesourceEntrypointFiles(filesource)
if entrypoints:
# resolve an IXDS in entrypoints
for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ArchiveIxds"):
pluginXbrlMethod(self, filesource,entrypoints)
filesource.select(entrypoints[0].get("file", None) )
except Exception as err:
self.modelXbrl.error("exception:" + type(err).__name__,
_("Testcase variation validation exception: %(error)s, entry URL: %(instance)s"),
modelXbrl=self.modelXbrl, instance=readMeFirstUri, error=err)
continue # don't try to load this entry URL
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
filesource,
_("validating"),
base=baseForElement,
errorCaptureLevel=errorCaptureLevel,
ixdsTarget=modelTestcaseVariation.ixdsTarget)
modelXbrl.isTestcaseVariation = True
if modelXbrl.modelDocument is None:
modelXbrl.info("arelle:notLoaded",
_("Variation %(id)s %(name)s readMeFirst document not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
self.determineNotLoadedTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif resultIsVersioningReport or resultIsTaxonomyPackage:
inputDTSes[dtsName] = modelXbrl
elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif testcase.type == Type.REGISTRYTESTCASE:
self.instValidator.validate(modelXbrl) # required to set up dimensions, etc
self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id,
modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
self.instValidator.close()
modelXbrl.close()
else:
inputDTSes[dtsName].append(modelXbrl)
# validate except for formulas
_hasFormulae = modelXbrl.hasFormulae
modelXbrl.hasFormulae = False
try:
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation)
self.instValidator.validate(modelXbrl, parameters)
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=(type(err) is not AssertionError))
modelXbrl.hasFormulae = _hasFormulae
if resultIsVersioningReport and modelXbrl.modelDocument:
versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
modelTestcaseVariation.versioningReportUri, baseForElement)
if os.path.exists(versReportFile): #validate existing
modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors)
modelVersReport.close()
elif len(inputDTSes) == 2:
ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
versReportFile, inputDTSes["from"], inputDTSes["to"])
modelTestcaseVariation.status = "generated"
else:
modelXbrl.error("arelle:notLoaded",
_("Variation %(id)s %(name)s input DTSes not loaded, unable to generate versioning report: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
modelTestcaseVariation.status = "failed"
for inputDTS in inputDTSes.values():
inputDTS.close()
del inputDTSes # dereference
elif resultIsTaxonomyPackage:
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif inputDTSes:
# validate schema, linkbase, or instance
modelXbrl = inputDTSes[None][0]
expectedDataFiles = set(modelXbrl.modelManager.cntlr.webCache.normalizeUrl(uri, baseForElement)
for d in modelTestcaseVariation.dataUris.values() for uri in d
if not UrlUtil.isAbsolute(uri))
foundDataFiles = set()
variationBase = os.path.dirname(baseForElement)
for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters
if dtsName: # named instance
parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes)
elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents
parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula
for _inputDTS in inputDTS:
for docUrl, doc in _inputDTS.urlDocs.items():
if docUrl.startswith(variationBase) and not doc.type == Type.INLINEXBRLDOCUMENTSET:
if getattr(doc,"loadedFromXbrlFormula", False): # may have been sourced from xf file
if docUrl.replace("-formula.xml", ".xf") in expectedDataFiles:
docUrl = docUrl.replace("-formula.xml", ".xf")
foundDataFiles.add(docUrl)
if expectedDataFiles - foundDataFiles:
modelXbrl.info("arelle:testcaseDataNotUsed",
_("Variation %(id)s %(name)s data files not used: %(missingDataFiles)s"),
modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id,
missingDataFiles=", ".join(sorted(os.path.basename(f) for f in expectedDataFiles - foundDataFiles)))
if foundDataFiles - expectedDataFiles:
modelXbrl.info("arelle:testcaseDataUnexpected",
_("Variation %(id)s %(name)s files not in variation data: %(unexpectedDataFiles)s"),
modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id,
unexpectedDataFiles=", ".join(sorted(os.path.basename(f) for f in foundDataFiles - expectedDataFiles)))
if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable:
try:
RenderingEvaluator.init(modelXbrl)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase RenderingEvaluator.init exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
modelXbrlHasFormulae = modelXbrl.hasFormulae
if modelXbrlHasFormulae and self.modelXbrl.modelManager.formulaOptions.formulaAction != "none":
try:
# validate only formulae
self.instValidator.parameters = parameters
ValidateFormula.validate(self.instValidator)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=(type(err) is not AssertionError))
if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset:
for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"):
pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri)
infoset = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultInfosetUri,
_("loading result infoset"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if infoset.modelDocument is None:
modelXbrl.error("arelle:notLoaded",
_("Variation %(id)s %(name)s result infoset not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result infoset not loadable"
else: # check infoset
ValidateInfoset.validate(self.instValidator, modelXbrl, infoset)
infoset.close()
if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset:
# diff (or generate) table infoset
resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement)
if not any(alternativeValidation(modelXbrl, resultTableUri)
for alternativeValidation in pluginClassMethods("Validate.TableInfoset")):
try:
ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase table linkbase validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
self.instValidator.close()
extraErrors = []
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors)
self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status
if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual):
# if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it
modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost)
self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors)
if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance
formulaOutputInstance = modelXbrl.formulaOutputInstance
modelXbrl.formulaOutputInstance = None # prevent it from being closed now
self.instValidator.close()
compareIxResultInstance = (modelXbrl.modelDocument.type in (Type.INLINEXBRL, Type.INLINEXBRLDOCUMENTSET) and
modelTestcaseVariation.resultXbrlInstanceUri is not None)
if compareIxResultInstance:
formulaOutputInstance = modelXbrl # compare modelXbrl to generated output instance
errMsgPrefix = "ix"
else: # delete input instances before formula output comparision
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
errMsgPrefix = "formula"
if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
_matchExpectedResultIDs = not modelXbrlHasFormulae # formula restuls have inconsistent IDs
expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultXbrlInstanceUri,
_("loading expected result XBRL instance"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if expectedInstance.modelDocument is None:
self.modelXbrl.error("{}:expectedResultNotLoaded".format(errMsgPrefix),
_("Testcase \"%(name)s\" %(id)s expected result instance not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstanceUri),
messageCodes=("formula:expectedResultNotLoaded","ix:expectedResultNotLoaded"))
modelTestcaseVariation.status = "result not loadable"
else: # compare facts
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.ExpectedInstance.Loaded"):
pluginXbrlMethod(expectedInstance, formulaOutputInstance)
if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
formulaOutputInstance.error("{}:resultFactCounts".format(errMsgPrefix),
_("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"),
modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts),
expectedFacts=len(expectedInstance.facts),
messageCodes=("formula:resultFactCounts","ix:resultFactCounts"))
else:
formulaOutputFootnotesRelSet = ModelRelationshipSet(formulaOutputInstance, "XBRL-footnotes")
expectedFootnotesRelSet = ModelRelationshipSet(expectedInstance, "XBRL-footnotes")
def factFootnotes(fact, footnotesRelSet):
footnotes = {}
footnoteRels = footnotesRelSet.fromModelObject(fact)
if footnoteRels:
# most process rels in same order between two instances, use labels to sort
for i, footnoteRel in enumerate(sorted(footnoteRels,
key=lambda r: (r.fromLabel,r.toLabel))):
modelObject = footnoteRel.toModelObject
if isinstance(modelObject, ModelResource):
xml = collapseWhitespace(modelObject.viewText().strip())
footnotes["Footnote {}".format(i+1)] = xml #re.sub(r'\s+', ' ', collapseWhitespace(modelObject.stringValue))
elif isinstance(modelObject, ModelFact):
footnotes["Footnoted fact {}".format(i+1)] = \
"{} context: {} value: {}".format(
modelObject.qname,
modelObject.contextID,
collapseWhitespace(modelObject.value))
return footnotes
for expectedInstanceFact in expectedInstance.facts:
unmatchedFactsStack = []
formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=_matchExpectedResultIDs, matchLang=False)
#formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=True, matchLang=True)
if formulaOutputFact is None:
if unmatchedFactsStack: # get missing nested tuple fact, if possible
missingFact = unmatchedFactsStack[-1]
else:
missingFact = expectedInstanceFact
# is it possible to show value mismatches?
expectedFacts = formulaOutputInstance.factsByQname.get(missingFact.qname)
if len(expectedFacts) == 1:
formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix),
_("Output missing expected fact %(fact)s, extracted value \"%(value1)s\", expected value \"%(value2)s\""),
modelXbrl=missingFact, fact=missingFact.qname, value1=missingFact.xValue, value2=next(iter(expectedFacts)).xValue,
messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing"))
else:
formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix),
_("Output missing expected fact %(fact)s"),
modelXbrl=missingFact, fact=missingFact.qname,
messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing"))
else: # compare footnotes
expectedInstanceFactFootnotes = factFootnotes(expectedInstanceFact, expectedFootnotesRelSet)
formulaOutputFactFootnotes = factFootnotes(formulaOutputFact, formulaOutputFootnotesRelSet)
if (len(expectedInstanceFactFootnotes) != len(formulaOutputFactFootnotes) or
set(expectedInstanceFactFootnotes.values()) != set(formulaOutputFactFootnotes.values())):
formulaOutputInstance.error("{}:expectedFactFootnoteDifference".format(errMsgPrefix),
_("Output expected fact %(fact)s expected footnotes %(footnotes1)s produced footnotes %(footnotes2)s"),
modelXbrl=(formulaOutputFact,expectedInstanceFact), fact=expectedInstanceFact.qname, footnotes1=sorted(expectedInstanceFactFootnotes.items()), footnotes2=sorted(formulaOutputFactFootnotes.items()),
messageCodes=("formula:expectedFactFootnoteDifference","ix:expectedFactFootnoteDifference"))
# for debugging uncomment next line to save generated instance document
# formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml")
expectedInstance.close()
del expectedInstance # dereference
self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors)
formulaOutputInstance.close()
del formulaOutputInstance
if compareIxResultInstance:
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
_statusCounts = OrderedDict((("pass",0),("fail",0)))
for tv in getattr(testcase, "testcaseVariations", ()):
_statusCounts[tv.status] = _statusCounts.get(tv.status, 0) + 1
self.modelXbrl.info("arelle:testCaseResults", ", ".join("{}={}".format(k,c) for k, c in _statusCounts.items() if k))
self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
class IXBRLViewerBuilder:
def __init__(self, dts):
self.nsmap = NamespaceMap()
self.roleMap = NamespaceMap()
self.dts = dts
self.taxonomyData = {
"concepts": {},
"languages": {},
"facts": {},
}
self.footnoteRelationshipSet = ModelRelationshipSet(
dts, "XBRL-footnotes")
def lineWrap(self, s, n=80):
return "\n".join([s[i:i + n] for i in range(0, len(s), n)])
def dateFormat(self, d):
"""
Strip the time component from an ISO date if it's zero
"""
return re.sub("T00:00:00$", "", d)
def escapeJSONForScriptTag(self, s):
"""
JSON encodes XML special characters XML and HTML apply difference escaping rules to content
within script tags and we need our output to be valid XML, but treated as HTML by browsers.
If we allow XML escaping to occur in a script tag, browsers treating
the document as HTML won't unescape it. If we don't escape XML special
characters, it won't be valid XML.
We avoid this whole mess by escaping XML special characters using JSON
string escapes. This is only safe to do because < > and & can't occur
outside a string in JSON. It can't safely be used on JS.
"""
return s.replace("<",
"\\u003C").replace(">",
"\\u003E").replace("&", "\\u0026")
def makeLanguageName(self, langCode):
code = re.sub("-.*", "", langCode)
try:
language = pycountry.languages.lookup(code)
match = re.match(r'^[^-]+-(.*)$', langCode)
name = language.name
if match is not None:
name = "%s (%s)" % (name, match.group(1).upper())
except LookupError:
name = langCode
return name
def addLanguage(self, langCode):
if langCode not in self.taxonomyData["languages"]:
self.taxonomyData["languages"][langCode] = self.makeLanguageName(
langCode)
def addELR(self, elr):
prefix = self.roleMap.getPrefix(elr)
if self.taxonomyData.setdefault("roleDefs", {}).get(prefix,
None) is None:
rt = self.dts.roleTypes[elr]
label = elr
if len(rt) > 0:
label = rt[0].definition
self.taxonomyData["roleDefs"].setdefault(prefix, {})["en"] = label
def addConcept(self, concept, dimensionType=None):
if concept is None:
return
labelsRelationshipSet = self.dts.relationshipSet(
XbrlConst.conceptLabel)
labels = labelsRelationshipSet.fromModelObject(concept)
conceptName = self.nsmap.qname(concept.qname)
if conceptName not in self.taxonomyData["concepts"]:
conceptData = {"labels": {}}
for lr in labels:
l = lr.toModelObject
conceptData["labels"].setdefault(
self.roleMap.getPrefix(l.role),
{})[l.xmlLang.lower()] = l.text
self.addLanguage(l.xmlLang.lower())
refData = []
for _refRel in concept.modelXbrl.relationshipSet(
XbrlConst.conceptReference).fromModelObject(concept):
ref = []
for _refPart in _refRel.toModelObject.iterchildren():
ref.append(
[_refPart.localName,
_refPart.stringValue.strip()])
refData.append(ref)
if len(refData) > 0:
conceptData['r'] = refData
if dimensionType is not None:
conceptData["d"] = dimensionType
self.taxonomyData["concepts"][conceptName] = conceptData
def treeWalk(self, rels, item, indent=0):
for r in rels.fromModelObject(item):
if r.toModelObject is not None:
self.treeWalk(rels, r.toModelObject, indent + 1)
def getRelationships(self):
rels = {}
for baseSetKey, baseSetModelLinks in self.dts.baseSets.items():
arcrole, ELR, linkqname, arcqname = baseSetKey
if arcrole in (XbrlConst.summationItem,
WIDER_NARROWER_ARCROLE) and ELR is not None:
self.addELR(ELR)
rr = dict()
relSet = self.dts.relationshipSet(arcrole, ELR)
for r in relSet.modelRelationships:
if r.fromModelObject is not None and r.toModelObject is not None:
fromKey = self.nsmap.qname(r.fromModelObject.qname)
rel = {
"t": self.nsmap.qname(r.toModelObject.qname),
}
if r.weight is not None:
rel['w'] = r.weight
rr.setdefault(fromKey, []).append(rel)
self.addConcept(r.toModelObject)
self.addConcept(r.fromModelObject)
rels.setdefault(self.roleMap.getPrefix(arcrole),
{})[self.roleMap.getPrefix(ELR)] = rr
return rels
def createViewer(self, scriptUrl="js/dist/ixbrlviewer.js"):
"""
Create an iXBRL file with XBRL data as a JSON blob, and script tags added
"""
dts = self.dts
iv = iXBRLViewer(dts)
idGen = 0
self.roleMap.getPrefix(XbrlConst.standardLabel, "std")
self.roleMap.getPrefix(XbrlConst.documentationLabel, "doc")
self.roleMap.getPrefix(XbrlConst.summationItem, "calc")
self.roleMap.getPrefix(XbrlConst.parentChild, "pres")
self.roleMap.getPrefix(WIDER_NARROWER_ARCROLE, "w-n")
for f in dts.facts:
if f.id is None:
f.set("id", "ixv-%d" % (idGen))
idGen += 1
conceptName = self.nsmap.qname(f.qname)
scheme, ident = f.context.entityIdentifier
aspects = {
"c":
conceptName,
"e":
self.nsmap.qname(
QName(self.nsmap.getPrefix(scheme, "e"), scheme, ident)),
}
factData = {
"v": f.value if not f.isNil else None,
"a": aspects,
}
if f.format is not None:
factData["f"] = str(f.format)
if f.isNumeric:
if f.unit is not None and len(f.unit.measures[0]):
# XXX does not support complex units
unit = self.nsmap.qname(f.unit.measures[0][0])
aspects["u"] = unit
else:
# The presence of the unit aspect is used by the viewer to
# identify numeric facts. If the fact has no unit (invalid
# XBRL, but we want to support it for draft documents),
# include the unit aspect with a null value.
aspects["u"] = None
d = inferredDecimals(f)
if d != float("INF") and not math.isnan(d):
factData["d"] = d
for d, v in f.context.qnameDims.items():
if v.memberQname is not None:
aspects[self.nsmap.qname(
v.dimensionQname)] = self.nsmap.qname(v.memberQname)
self.addConcept(v.member)
self.addConcept(v.dimension, dimensionType="e")
elif v.typedMember is not None:
aspects[self.nsmap.qname(
v.dimensionQname)] = v.typedMember.text
self.addConcept(v.dimension, dimensionType="t")
if f.context.isForeverPeriod:
aspects["p"] = "f"
elif f.context.isInstantPeriod and f.context.instantDatetime is not None:
aspects["p"] = self.dateFormat(
f.context.instantDatetime.isoformat())
elif f.context.isStartEndPeriod and f.context.startDatetime is not None and f.context.endDatetime is not None:
aspects["p"] = "%s/%s" % (
self.dateFormat(f.context.startDatetime.isoformat()),
self.dateFormat(f.context.endDatetime.isoformat()))
frels = self.footnoteRelationshipSet.fromModelObject(f)
if frels:
for frel in frels:
if frel.toModelObject is not None:
factData.setdefault("fn",
[]).append(frel.toModelObject.id)
self.taxonomyData["facts"][f.id] = factData
if f.concept is not None:
self.addConcept(f.concept)
self.taxonomyData["prefixes"] = self.nsmap.prefixmap
self.taxonomyData["roles"] = self.roleMap.prefixmap
self.taxonomyData["rels"] = self.getRelationships()
dts.info("viewer:info", "Creating iXBRL viewer")
if dts.modelDocument.type == Type.INLINEXBRLDOCUMENTSET:
# Sort by object index to preserve order in which files were specified.
docSet = sorted(dts.modelDocument.referencesDocument.keys(),
key=lambda x: x.objectIndex)
docSetFiles = list(
map(lambda x: os.path.basename(x.filepath), docSet))
self.taxonomyData["docSetFiles"] = docSetFiles
for n in range(0, len(docSet)):
iv.addFile(
iXBRLViewerFile(docSetFiles[n], docSet[n].xmlDocument))
xmlDocument = docSet[0].xmlDocument
else:
xmlDocument = dts.modelDocument.xmlDocument
filename = os.path.basename(dts.modelDocument.filepath)
iv.addFile(iXBRLViewerFile(filename, xmlDocument))
taxonomyDataJSON = self.escapeJSONForScriptTag(
json.dumps(self.taxonomyData, indent=1, allow_nan=False))
for child in xmlDocument.getroot():
if child.tag == '{http://www.w3.org/1999/xhtml}body':
child.append(etree.Comment("BEGIN IXBRL VIEWER EXTENSIONS"))
e = etree.fromstring(
"<script xmlns='http://www.w3.org/1999/xhtml' src='%s' type='text/javascript' />"
% scriptUrl)
# Don't self close
e.text = ''
child.append(e)
# Putting this in the header can interfere with character set
# auto detection
e = etree.fromstring(
"<script xmlns='http://www.w3.org/1999/xhtml' type='application/x.ixbrl-viewer+json'></script>"
)
e.text = taxonomyDataJSON
child.append(e)
child.append(etree.Comment("END IXBRL VIEWER EXTENSIONS"))
break
return iv
def saveLoadableOIM(modelXbrl, oimFile):
isJSON = oimFile.endswith(".json")
isCSV = oimFile.endswith(".csv")
isXL = oimFile.endswith(".xlsx")
isCSVorXL = isCSV or isXL
if not isJSON and not isCSVorXL:
return
namespacePrefixes = {nsOim: "oim"}
prefixNamespaces = {"oim": nsOim}
def compileQname(qname):
if qname.namespaceURI not in namespacePrefixes:
namespacePrefixes[qname.namespaceURI] = qname.prefix or ""
aspectsDefined = {qnOimConceptAspect, qnOimEntityAspect}
if isJSON:
aspectsDefined.add(qnOimPeriodAspect)
elif isCSVorXL:
aspectsDefined.add(qnOimPeriodStartAspect)
aspectsDefined.add(qnOimPeriodEndAspect)
def oimValue(object, decimals=None):
if isinstance(object, QName):
if object.namespaceURI not in namespacePrefixes:
if object.prefix:
namespacePrefixes[object.namespaceURI] = object.prefix
else:
_prefix = "_{}".format(
sum(1 for p in namespacePrefixes if p.startswith("_")))
namespacePrefixes[object.namespaceURI] = _prefix
return "{}:{}".format(namespacePrefixes[object.namespaceURI],
object.localName)
if isinstance(object, Decimal):
try:
if isinf(object):
return "-INF" if object < 0 else "INF"
elif isnan(num):
return "NaN"
else:
if object == object.to_integral():
object = object.quantize(ONE) # drop any .0
return "{}".format(object)
except:
return str(object)
if isinstance(object, bool):
return object
if isinstance(object,
(DateTime, YearMonthDuration, DayTimeDuration, Time,
gYearMonth, gMonthDay, gYear, gMonth, gDay)):
return str(object)
return object
def oimPeriodValue(cntx):
if cntx.isForeverPeriod:
pass # not supported
elif cntx.isStartEndPeriod:
s = cntx.startDatetime
e = cntx.endDatetime
else: # instant
s = e = cntx.instantDatetime
return {
"start":
"{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format(
s.year, s.month, s.day, s.hour, s.minute, s.second),
"end":
"{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format(
e.year, e.month, e.day, e.hour, e.minute, e.second)
}
hasId = False
hasTuple = False
hasType = True
hasLang = False
hasUnits = False
hasNumeric = False
footnotesRelationshipSet = ModelRelationshipSet(modelXbrl,
"XBRL-footnotes")
factBaseTypes = set()
#compile QNames in instance for OIM
for fact in modelXbrl.factsInInstance:
if (fact.id or fact.isTuple
or footnotesRelationshipSet.toModelObject(fact) or
(isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))):
hasId = True
concept = fact.concept
if concept is not None:
if concept.isNumeric:
hasNumeric = True
if concept.baseXbrliType in ("string", "normalizedString",
"token") and fact.xmlLang:
hasLang = True
_baseXsdType = concept.baseXsdType
if _baseXsdType == "XBRLI_DATEUNION":
if getattr(fact.xValue, "dateOnly", False):
_baseXsdType = "date"
else:
_baseXsdType = "dateTime"
factBaseTypes.add(baseTypes.get(_baseXsdType, _baseXsdType))
compileQname(fact.qname)
if hasattr(fact, "xValue") and isinstance(fact.xValue, QName):
compileQname(fact.xValue)
unit = fact.unit
if unit is not None:
hasUnits = True
if fact.modelTupleFacts:
hasTuple = True
entitySchemePrefixes = {}
for cntx in modelXbrl.contexts.values():
if cntx.entityIdentifierElement is not None:
scheme = cntx.entityIdentifier[0]
if scheme not in entitySchemePrefixes:
if not entitySchemePrefixes: # first one is just scheme
if scheme == "http://www.sec.gov/CIK":
_schemePrefix = "cik"
elif scheme == "http://standard.iso.org/iso/17442":
_schemePrefix = "lei"
else:
_schemePrefix = "scheme"
else:
_schemePrefix = "scheme{}".format(
len(entitySchemePrefixes) + 1)
entitySchemePrefixes[scheme] = _schemePrefix
namespacePrefixes[scheme] = _schemePrefix
for dim in cntx.qnameDims.values():
compileQname(dim.dimensionQname)
aspectsDefined.add(dim.dimensionQname)
if dim.isExplicit:
compileQname(dim.memberQname)
for unit in modelXbrl.units.values():
if unit is not None:
for measures in unit.measures:
for measure in measures:
compileQname(measure)
if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[
XbrlConst.xbrli] != "xbrli":
namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix
namespacePrefixes[XbrlConst.xsd] = "xsd"
if hasLang: aspectsDefined.add(qnOimLangAspect)
if hasTuple:
aspectsDefined.add(qnOimTupleParentAspect)
aspectsDefined.add(qnOimTupleOrderAspect)
if hasUnits: aspectsDefined.add(qnOimUnitAspect)
# compile footnotes and relationships
'''
factRelationships = []
factFootnotes = []
for rel in modelXbrl.relationshipSet(modelXbrl, "XBRL-footnotes").modelRelationships:
oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole}
factRelationships.append(oimRel)
oimRel["fromIds"] = [obj.id if obj.id
else elementChildSequence(obj)
for obj in rel.fromModelObjects]
oimRel["toIds"] = [obj.id if obj.id
else elementChildSequence(obj)
for obj in rel.toModelObjects]
_order = rel.arcElement.get("order")
if _order is not None:
oimRel["order"] = _order
for obj in rel.toModelObjects:
if isinstance(obj, ModelResource): # footnote
oimFootnote = {"role": obj.role,
"id": obj.id if obj.id
else elementChildSequence(obj),
# value needs work for html elements and for inline footnotes
"value": xmlstring(obj, stripXmlns=True)}
if obj.xmlLang:
oimFootnote["lang"] = obj.xmlLang
factFootnotes.append(oimFootnote)
oimFootnote
'''
dtsReferences = [{
"type":
"schema" if doc.type == ModelDocument.Type.SCHEMA else
"linkbase" if doc.type == ModelDocument.Type.LINKBASE else "other",
"href":
doc.uri
} for doc, ref in modelXbrl.modelDocument.referencesDocument.items()
if ref.referringModelObject.qname in SCHEMA_LB_REFS]
'''
roleTypes = [
{"type": "role" if ref.referringModelObject.localName == "roleRef" else "arcroleRef",
"href": ref.referringModelObject["href"]}
for doc,ref in modelXbrl.modelDocument.referencesDocument.items()
if ref.referringModelObject.qname in ROLE_REFS]
'''
def factFootnotes(fact):
footnotes = []
for footnoteRel in footnotesRelationshipSet.fromModelObject(fact):
footnote = OrderedDict((("group", footnoteRel.linkrole),
("footnoteType", footnoteRel.arcrole)))
footnotes.append(footnote)
if isCSVorXL:
footnote["factId"] = fact.id if fact.id else "f{}".format(
fact.objectIndex)
toObj = footnoteRel.toModelObject
if isinstance(toObj, ModelFact):
footnote["factRef"] = toObj.id if toObj.id else "f{}".format(
toObj.objectIndex)
else:
footnote["footnote"] = xmlstring(toObj,
stripXmlns=True,
contentsOnly=True,
includeText=True)
if toObj.xmlLang:
footnote["language"] = toObj.xmlLang
return footnotes
def factAspects(fact):
aspects = OrderedDict()
if hasId and fact.id:
aspects["id"] = fact.id
elif (fact.isTuple or footnotesRelationshipSet.toModelObject(fact) or
(isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))):
aspects["id"] = "f{}".format(fact.objectIndex)
parent = fact.getparent()
concept = fact.concept
_csvType = "Value"
if not fact.isTuple:
if concept is not None:
_baseXsdType = concept.baseXsdType
if _baseXsdType == "XBRLI_DATEUNION":
if getattr(fact.xValue, "dateOnly", False):
_baseXsdType = "date"
else:
_baseXsdType = "dateTime"
aspects["baseType"] = "xsd:{}".format(_baseXsdType)
_csvType = baseTypes.get(_baseXsdType, _baseXsdType) + "Value"
if concept.baseXbrliType in ("string", "normalizedString",
"token") and fact.xmlLang:
aspects[qnOimLangAspect] = fact.xmlLang
if fact.isItem:
if fact.isNil:
_value = None
_strValue = "nil"
else:
_inferredDecimals = inferredDecimals(fact)
_value = oimValue(fact.xValue, _inferredDecimals)
_strValue = str(_value)
if not isCSVorXL:
aspects["value"] = _strValue
if fact.concept is not None and fact.concept.isNumeric:
_numValue = fact.xValue
if isinstance(_numValue, Decimal) and not isinf(
_numValue) and not isnan(_numValue):
if _numValue == _numValue.to_integral():
_numValue = int(_numValue)
else:
_numValue = float(_numValue)
if isCSVorXL:
aspects[_csvType] = _numValue
else:
aspects["numericValue"] = _numValue
if not fact.isNil:
if isinf(_inferredDecimals):
if isJSON: _accuracy = "infinity"
elif isCSVorXL: _accuracy = "INF"
else:
_accuracy = _inferredDecimals
aspects["accuracy"] = _accuracy
elif isinstance(_value, bool):
aspects["booleanValue"] = _value
elif isCSVorXL:
aspects[_csvType] = _strValue
aspects[qnOimConceptAspect] = oimValue(fact.qname)
cntx = fact.context
if cntx is not None:
if cntx.entityIdentifierElement is not None:
aspects[qnOimEntityAspect] = oimValue(
qname(*cntx.entityIdentifier))
if cntx.period is not None:
if isJSON:
aspects[qnOimPeriodAspect] = oimPeriodValue(cntx)
elif isCSVorXL:
_periodValue = oimPeriodValue(cntx)
aspects[qnOimPeriodStartAspect] = _periodValue["start"]
aspects[qnOimPeriodEndAspect] = _periodValue["end"]
for _qn, dim in sorted(cntx.qnameDims.items(),
key=lambda item: item[0]):
aspects[dim.dimensionQname] = (
oimValue(dim.memberQname)
if dim.isExplicit else None if dim.typedMember.get(
"{http://www.w3.org/2001/XMLSchema-instance}nil") in (
"true", "1") else dim.typedMember.stringValue)
unit = fact.unit
if unit is not None:
_mMul, _mDiv = unit.measures
_sMul = '*'.join(
oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m)))
if _mDiv:
_sDiv = '*'.join(
oimValue(m)
for m in sorted(_mDiv, key=lambda m: oimValue(m)))
if len(_mDiv) > 1:
if len(_mMul) > 1:
_sUnit = "({})/({})".format(_sMul, _sDiv)
else:
_sUnit = "{}/({})".format(_sMul, _sDiv)
else:
if len(_mMul) > 1:
_sUnit = "({})/{}".format(_sMul, _sDiv)
else:
_sUnit = "{}/{}".format(_sMul, _sDiv)
else:
_sUnit = _sMul
aspects[qnOimUnitAspect] = _sUnit
if parent.qname != XbrlConst.qnXbrliXbrl:
aspects[
qnOimTupleParentAspect] = parent.id if parent.id else "f{}".format(
parent.objectIndex)
aspects[qnOimTupleOrderAspect] = elementIndex(fact)
if isJSON:
_footnotes = factFootnotes(fact)
if _footnotes:
aspects["footnotes"] = _footnotes
return aspects
if isJSON:
# save JSON
oimReport = OrderedDict() # top level of oim json output
oimFacts = []
oimReport["prefixes"] = OrderedDict(
(p, ns) for ns, p in sorted(namespacePrefixes.items(),
key=lambda item: item[1]))
oimReport["dtsReferences"] = dtsReferences
oimReport["facts"] = oimFacts
def saveJsonFacts(facts, oimFacts, parentFact):
for fact in facts:
oimFact = factAspects(fact)
oimFacts.append(
OrderedDict((str(k), v) for k, v in oimFact.items()))
if fact.modelTupleFacts:
saveJsonFacts(fact.modelTupleFacts, oimFacts, fact)
saveJsonFacts(modelXbrl.facts, oimFacts, None)
with open(oimFile, "w", encoding="utf-8") as fh:
fh.write(json.dumps(oimReport, indent=1))
elif isCSVorXL:
# save CSV
aspectQnCol = {}
aspectsHeader = []
factsColumns = []
def addAspectQnCol(aspectQn):
aspectQnCol[aspectQn] = len(aspectsHeader)
_colName = oimValue(aspectQn)
aspectsHeader.append(_colName)
_colDataType = {
"id": "Name",
"baseType": "Name",
"oim:concept": "QName",
"oim:periodStart": "dateTime",
"oim:periodEnd": "dateTime",
"oim:tupleOrder": "integer",
"numericValue": "decimal",
"accuracy": "decimal",
"booleanValue": "boolean",
"oim:entity": "QName",
"oim:unit": "string"
}.get(_colName)
if _colDataType is None:
if isCSVorXL and _colName.endswith("Value"):
_colDataType = _colName[:-5]
else:
_colDataType = "string"
factsColumns.append(
OrderedDict((("name", _colName), ("datatype", _colDataType))))
# pre-ordered aspect columns
if hasId:
addAspectQnCol("id")
addAspectQnCol(qnOimConceptAspect)
if hasType:
addAspectQnCol("baseType")
if isCSVorXL:
for _baseType in sorted(factBaseTypes):
addAspectQnCol(_baseType + "Value")
else:
addAspectQnCol("stringValue")
addAspectQnCol("numericValue")
addAspectQnCol("booleanValue")
if hasNumeric:
addAspectQnCol("accuracy")
if hasTuple:
addAspectQnCol(qnOimTupleParentAspect)
addAspectQnCol(qnOimTupleOrderAspect)
if qnOimEntityAspect in aspectsDefined:
addAspectQnCol(qnOimEntityAspect)
if qnOimPeriodStartAspect in aspectsDefined:
addAspectQnCol(qnOimPeriodStartAspect)
addAspectQnCol(qnOimPeriodEndAspect)
if qnOimUnitAspect in aspectsDefined:
addAspectQnCol(qnOimUnitAspect)
for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)):
if aspectQn.namespaceURI != nsOim:
addAspectQnCol(aspectQn)
def aspectCols(fact):
cols = [None for i in range(len(aspectsHeader))]
_factAspects = factAspects(fact)
for aspectQn, aspectValue in _factAspects.items():
if aspectQn in aspectQnCol:
cols[aspectQnCol[aspectQn]] = aspectValue
return cols
# metadata
csvTables = []
csvMetadata = OrderedDict(
(("@context", ["http://www.w3.org/ns/csvw", {
"@base": "./"
}]), ("tables", csvTables)))
_open = _writerow = _close = None
_tableinfo = {}
if isCSV:
if oimFile.endswith(
"-facts.csv"
): # strip -facts.csv if a prior -facts.csv file was chosen
_baseURL = oimFile[:-10]
elif oimFile.endswith(".csv"):
_baseURL = oimFile[:-4]
else:
_baseURL = oimFile
_csvinfo = {} # open file, writer
def _open(filesuffix, tabname):
_filename = _tableinfo["url"] = _baseURL + filesuffix
_csvinfo["file"] = open(_filename,
csvOpenMode,
newline=csvOpenNewline,
encoding='utf-8-sig')
_csvinfo["writer"] = csv.writer(_csvinfo["file"],
dialect="excel")
def _writerow(row, header=False):
_csvinfo["writer"].writerow(row)
def _close():
_csvinfo["file"].close()
_csvinfo.clear()
elif isXL:
headerWidths = {
"href": 100,
"oim:concept": 70,
"accuracy": 8,
"baseType": 10,
"language": 9,
"URI": 80,
"stringValue": 60,
"decimalValue": 12,
"booleanValue": 12,
"dateValue": 12,
"dateTimeValue": 20,
"group": 60,
"footnoteType": 40,
"footnote": 70,
"column": 60
}
from openpyxl import Workbook
from openpyxl.writer.write_only import WriteOnlyCell
from openpyxl.styles import Font, PatternFill, Border, Alignment, Color, fills, Side
from openpyxl.worksheet.dimensions import ColumnDimension
hdrCellFill = PatternFill(
patternType=fills.FILL_SOLID, fgColor=Color(
"00FFBF5F")) # Excel's light orange fill color = 00FF990
workbook = Workbook(encoding="utf-8", write_only=True)
_xlinfo = {} # open file, writer
def _open(filesuffix, tabname):
_tableinfo["url"] = tabname
_xlinfo["ws"] = workbook.create_sheet(title=tabname)
def _writerow(rowvalues, header=False):
row = []
_ws = _xlinfo["ws"]
for i, v in enumerate(rowvalues):
cell = WriteOnlyCell(_ws, value=v)
if header:
cell.fill = hdrCellFill
cell.alignment = Alignment(horizontal="center",
vertical="center",
wrap_text=True)
colLetter = chr(ord('A') + i)
_ws.column_dimensions[colLetter] = ColumnDimension(
_ws, customWidth=True)
_ws.column_dimensions[
colLetter].width = headerWidths.get(v, 20)
else:
cell.alignment = Alignment(
horizontal="right" if isinstance(v, _NUM_TYPES)
else "center" if isinstance(v, bool) else "left",
vertical="top",
wrap_text=isinstance(v, str))
row.append(cell)
_ws.append(row)
def _close():
_xlinfo.clear()
# save facts
_open("-facts.csv", "facts")
_writerow(aspectsHeader, header=True)
def saveCSVfacts(facts):
for fact in facts:
_writerow(aspectCols(fact))
saveCSVfacts(fact.modelTupleFacts)
saveCSVfacts(modelXbrl.facts)
_close()
factsTableSchema = OrderedDict((("columns", factsColumns), ))
csvTables.append(
OrderedDict((("url", _tableinfo["url"]), ("tableSchema",
factsTableSchema))))
# save namespaces
_open("-prefixes.csv", "prefixes")
_writerow(("prefix", "URI"), header=True)
for _URI, prefix in sorted(namespacePrefixes.items(),
key=lambda item: item[1]):
_writerow((prefix, _URI))
_close()
nsTableSchema = OrderedDict((("columns", [
OrderedDict((("name", "prefix"), ("datatype", "Name"))),
OrderedDict((("name", "URI"), ("datatype", "anyURI")))
]), ))
csvTables.append(
OrderedDict(
(("url", _tableinfo["url"]), ("tableSchema", nsTableSchema))))
# save dts references
_open("-dtsReferences.csv", "dtsReferences")
_writerow(("type", "href"), header=True)
for oimRef in dtsReferences:
_writerow((oimRef["type"], oimRef["href"]))
_close()
dtsRefTableSchema = OrderedDict((("columns", [
OrderedDict((("name", "type"), ("datatype", "Name"))),
OrderedDict((("name", "href"), ("datatype", "anyURI")))
]), ))
csvTables.append(
OrderedDict((("url", _tableinfo["url"]), ("tableSchema",
dtsRefTableSchema))))
# save footnotes
if footnotesRelationshipSet.modelRelationships:
_open("-footnotes.csv", "footnotes")
cols = ("group", "footnoteType", "factId", "factRef", "footnote",
"language")
_writerow(cols, header=True)
def saveCSVfootnotes(facts):
for fact in facts:
for _footnote in factFootnotes(fact):
_writerow(
tuple((_footnote.get(col, "") for col in cols)))
saveCSVfootnotes(fact.modelTupleFacts)
saveCSVfootnotes(modelXbrl.facts)
_close()
footnoteTableSchema = OrderedDict((("columns", [
OrderedDict((("name", "group"), ("datatype", "anyURI"))),
OrderedDict((("name", "footnoteType"), ("datatype", "Name"))),
OrderedDict((("name", "factId"), ("datatype", "Name"))),
OrderedDict((("name", "factRef"), ("datatype", "Name"))),
OrderedDict((("name", "footnote"), ("datatype", "string"))),
OrderedDict((("name", "language"), ("datatype", "language")))
]), ))
csvTables.append(
OrderedDict((("url", _tableinfo["url"]),
("tableSchema", footnoteTableSchema))))
# save metadata
if isCSV:
with open(_baseURL + "-metadata.csv", "w", encoding="utf-8") as fh:
fh.write(
json.dumps(csvMetadata,
ensure_ascii=False,
indent=1,
sort_keys=False))
elif isXL:
_open(None, "metadata")
_writerow(("table", "column", "datatype"), header=True)
for table in csvTables:
tablename = table["url"]
for column in table["tableSchema"]["columns"]:
_writerow((tablename, column["name"], column["datatype"]))
_close()
if isXL:
workbook.save(oimFile)
def saveLoadableOIM(modelXbrl, oimFile, outputZip=None):
isJSON = oimFile.endswith(".json")
isCSV = oimFile.endswith(".csv")
isXL = oimFile.endswith(".xlsx")
isCSVorXL = isCSV or isXL
if not isJSON and not isCSVorXL:
return
namespacePrefixes = {nsOim: "xbrl"}
prefixNamespaces = {"xbrl": nsOim}
def compileQname(qname):
if qname.namespaceURI not in namespacePrefixes:
namespacePrefixes[qname.namespaceURI] = qname.prefix or ""
aspectsDefined = {
qnOimConceptAspect,
qnOimEntityAspect,
qnOimPeriodStartAspect,
qnOimPeriodEndAspect}
def oimValue(object, decimals=None):
if isinstance(object, QName):
if object.namespaceURI not in namespacePrefixes:
if object.prefix:
namespacePrefixes[object.namespaceURI] = object.prefix
else:
_prefix = "_{}".format(sum(1 for p in namespacePrefixes if p.startswith("_")))
namespacePrefixes[object.namespaceURI] = _prefix
return "{}:{}".format(namespacePrefixes[object.namespaceURI], object.localName)
if isinstance(object, Decimal):
try:
if isinf(object):
return "-INF" if object < 0 else "INF"
elif isnan(num):
return "NaN"
else:
if object == object.to_integral():
object = object.quantize(ONE) # drop any .0
return "{}".format(object)
except:
return str(object)
if isinstance(object, bool):
return "true" if object else "false"
if isinstance(object, (DateTime, YearMonthDuration, DayTimeDuration, Time,
gYearMonth, gMonthDay, gYear, gMonth, gDay,
IsoDuration)):
return str(object)
return object
def oimPeriodValue(cntx):
if cntx.isForeverPeriod:
return OrderedDict() # defaulted
elif cntx.isStartEndPeriod:
s = cntx.startDatetime
e = cntx.endDatetime
else: # instant
s = e = cntx.instantDatetime
if isJSON:
return({str(qnOimPeriodAspect):
("{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}/".format(
s.year, s.month, s.day, s.hour, s.minute, s.second)
if cntx.isStartEndPeriod else "") +
"{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format(
e.year, e.month, e.day, e.hour, e.minute, e.second)
})
# CSV, XL
return OrderedDict(((str(qnOimPeriodStartAspect), "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format(
s.year, s.month, s.day, s.hour, s.minute, s.second)),
(str(qnOimPeriodEndAspect), "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format(
e.year, e.month, e.day, e.hour, e.minute, e.second))))
hasId = False
hasTuple = False
hasType = True
hasLang = False
hasUnits = False
hasNumeric = False
footnotesRelationshipSet = ModelRelationshipSet(modelXbrl, "XBRL-footnotes")
factBaseTypes = set()
#compile QNames in instance for OIM
for fact in modelXbrl.factsInInstance:
if (fact.id or fact.isTuple or
footnotesRelationshipSet.toModelObject(fact) or
(isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))):
hasId = True
concept = fact.concept
if concept is not None:
if concept.isNumeric:
hasNumeric = True
if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang:
hasLang = True
_baseXsdType = concept.baseXsdType
if _baseXsdType == "XBRLI_DATEUNION":
if getattr(fact.xValue, "dateOnly", False):
_baseXsdType = "date"
else:
_baseXsdType = "dateTime"
factBaseTypes.add(baseTypes.get(_baseXsdType,_baseXsdType))
compileQname(fact.qname)
if hasattr(fact, "xValue") and isinstance(fact.xValue, QName):
compileQname(fact.xValue)
unit = fact.unit
if unit is not None:
hasUnits = True
if fact.modelTupleFacts:
hasTuple = True
if hasTuple:
modelXbrl.error("arelleOIMsaver:tuplesNotAllowed",
"Tuples are not allowed in an OIM document",
modelObject=modelXbrl)
return
entitySchemePrefixes = {}
for cntx in modelXbrl.contexts.values():
if cntx.entityIdentifierElement is not None:
scheme = cntx.entityIdentifier[0]
if scheme not in entitySchemePrefixes:
if not entitySchemePrefixes: # first one is just scheme
if scheme == "http://www.sec.gov/CIK":
_schemePrefix = "cik"
elif scheme == "http://standard.iso.org/iso/17442":
_schemePrefix = "lei"
else:
_schemePrefix = "scheme"
else:
_schemePrefix = "scheme{}".format(len(entitySchemePrefixes) + 1)
entitySchemePrefixes[scheme] = _schemePrefix
namespacePrefixes[scheme] = _schemePrefix
for dim in cntx.qnameDims.values():
compileQname(dim.dimensionQname)
aspectsDefined.add(dim.dimensionQname)
if dim.isExplicit:
compileQname(dim.memberQname)
for unit in modelXbrl.units.values():
if unit is not None:
for measures in unit.measures:
for measure in measures:
compileQname(measure)
if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[XbrlConst.xbrli] != "xbrli":
namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix
namespacePrefixes[XbrlConst.xsd] = "xsd"
if hasLang: aspectsDefined.add(qnOimLangAspect)
if hasTuple:
aspectsDefined.add(qnOimTupleParentAspect)
aspectsDefined.add(qnOimTupleOrderAspect)
if hasUnits: aspectsDefined.add(qnOimUnitAspect)
for footnoteRel in footnotesRelationshipSet.modelRelationships:
typePrefix = "ftTyp_" + os.path.basename(footnoteRel.arcrole)
if footnoteRel.linkrole == XbrlConst.defaultLinkRole:
groupPrefix = "ftGrp_default"
else:
groupPrefix = "ftGrp_" + os.path.basename(footnoteRel.linkrole)
if typePrefix not in namespacePrefixes:
namespacePrefixes[footnoteRel.arcrole] = typePrefix
if groupPrefix not in namespacePrefixes:
namespacePrefixes[footnoteRel.linkrole] = groupPrefix
# compile footnotes and relationships
'''
factRelationships = []
factFootnotes = []
for rel in modelXbrl.relationshipSet(modelXbrl, "XBRL-footnotes").modelRelationships:
oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole}
factRelationships.append(oimRel)
oimRel["fromIds"] = [obj.id if obj.id
else elementChildSequence(obj)
for obj in rel.fromModelObjects]
oimRel["toIds"] = [obj.id if obj.id
else elementChildSequence(obj)
for obj in rel.toModelObjects]
_order = rel.arcElement.get("order")
if _order is not None:
oimRel["order"] = _order
for obj in rel.toModelObjects:
if isinstance(obj, ModelResource): # footnote
oimFootnote = {"role": obj.role,
"id": obj.id if obj.id
else elementChildSequence(obj),
# value needs work for html elements and for inline footnotes
"value": xmlstring(obj, stripXmlns=True)}
if obj.xmlLang:
oimFootnote["lang"] = obj.xmlLang
factFootnotes.append(oimFootnote)
oimFootnote
'''
dtsReferences = set()
baseUrl = modelXbrl.modelDocument.uri.partition("#")[0]
for doc,ref in sorted(modelXbrl.modelDocument.referencesDocument.items(),
key=lambda _item:_item[0].uri):
if ref.referringModelObject.qname in SCHEMA_LB_REFS:
dtsReferences.add(relativeUri(baseUrl,doc.uri))
for refType in ("role", "arcrole"):
for refElt in sorted(modelXbrl.modelDocument.xmlRootElement.iterchildren(
"{{http://www.xbrl.org/2003/linkbase}}{}Ref".format(refType)),
key=lambda elt:elt.get(refType+"URI")
):
dtsReferences.add(refElt.get("{http://www.w3.org/1999/xlink}href").partition("#")[0])
dtsReferences = sorted(dtsReferences) # turn into list
footnoteFacts = set()
def factFootnotes(fact, oimFact=None):
footnotes = []
oimLinks = {}
for footnoteRel in footnotesRelationshipSet.fromModelObject(fact):
toObj = footnoteRel.toModelObject
# json
typePrefix = namespacePrefixes[footnoteRel.arcrole]
groupPrefix = namespacePrefixes[footnoteRel.linkrole]
oimLinks.setdefault(typePrefix,{}).setdefault(groupPrefix,[]).append(
toObj.id if toObj.id else "f{}".format(toObj.objectIndex))
# csv/XL
footnote = OrderedDict((("group", footnoteRel.linkrole),
("footnoteType", footnoteRel.arcrole)))
footnotes.append(footnote)
if isCSVorXL:
footnote["factId"] = fact.id if fact.id else "f{}".format(fact.objectIndex)
if isinstance(toObj, ModelFact):
footnote["factRef"] = toObj.id if toObj.id else "f{}".format(toObj.objectIndex)
else:
footnote["footnote"] = toObj.viewText()
if toObj.xmlLang:
footnote["language"] = toObj.xmlLang
footnoteFacts.add(toObj)
if isJSON and oimLinks:
oimFact["links"] = OrderedDict((
(typePrefix, OrderedDict((
(groupPrefix, idList)
for groupPrefix, idList in sorted(groups.items())
)))
for typePrefix, groups in sorted(oimLinks.items())
))
footnotes.sort(key=lambda f:(f["group"],f.get("factId",f.get("factRef")),f.get("language")))
return footnotes
def factAspects(fact):
oimFact = OrderedDict()
aspects = OrderedDict()
if hasId and fact.id:
if fact.isTuple:
oimFact["tupleId"] = fact.id
else:
oimFact["id"] = fact.id
elif (fact.isTuple or
footnotesRelationshipSet.toModelObject(fact) or
(isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))):
oimFact["id"] = "f{}".format(fact.objectIndex)
parent = fact.getparent()
concept = fact.concept
aspects[str(qnOimConceptAspect)] = oimValue(concept.qname)
_csvType = "Value"
if not fact.isTuple:
if concept is not None:
_baseXsdType = concept.baseXsdType
if _baseXsdType == "XBRLI_DATEUNION":
if getattr(fact.xValue, "dateOnly", False):
_baseXsdType = "date"
else:
_baseXsdType = "dateTime"
_csvType = baseTypes.get(_baseXsdType,_baseXsdType) + "Value"
if concept.baseXbrliType in ("stringItemType", "normalizedStringItemType") and fact.xmlLang:
aspects[str(qnOimLangAspect)] = fact.xmlLang
if fact.isItem:
if fact.isNil:
_value = None
else:
_inferredDecimals = inferredDecimals(fact)
_value = oimValue(fact.xValue, _inferredDecimals)
oimFact["value"] = _value
if fact.concept is not None and fact.concept.isNumeric:
_numValue = fact.xValue
if isinstance(_numValue, Decimal) and not isinf(_numValue) and not isnan(_numValue):
if _numValue == _numValue.to_integral():
_numValue = int(_numValue)
else:
_numValue = float(_numValue)
if not fact.isNil:
if not isinf(_inferredDecimals): # accuracy omitted if infinite
oimFact["decimals"] = _inferredDecimals
oimFact["dimensions"] = aspects
cntx = fact.context
if cntx is not None:
if cntx.entityIdentifierElement is not None:
aspects[str(qnOimEntityAspect)] = oimValue(qname(*cntx.entityIdentifier))
if cntx.period is not None:
aspects.update(oimPeriodValue(cntx))
for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]):
if dim.isExplicit:
dimVal = oimValue(dim.memberQname)
else: # typed
if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1"):
dimVal = None
else:
dimVal = dim.typedMember.stringValue
aspects[str(dim.dimensionQname)] = dimVal
unit = fact.unit
if unit is not None:
_mMul, _mDiv = unit.measures
_sMul = '*'.join(oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m)))
if _mDiv:
_sDiv = '*'.join(oimValue(m) for m in sorted(_mDiv, key=lambda m: oimValue(m)))
if len(_mDiv) > 1:
if len(_mMul) > 1:
_sUnit = "({})/({})".format(_sMul,_sDiv)
else:
_sUnit = "{}/({})".format(_sMul,_sDiv)
else:
if len(_mMul) > 1:
_sUnit = "({})/{}".format(_sMul,_sDiv)
else:
_sUnit = "{}/{}".format(_sMul,_sDiv)
else:
_sUnit = _sMul
aspects[str(qnOimUnitAspect)] = _sUnit
# Tuples removed from xBRL-JSON
#if parent.qname != XbrlConst.qnXbrliXbrl:
# aspects[str(qnOimTupleParentAspect)] = parent.id if parent.id else "f{}".format(parent.objectIndex)
# aspects[str(qnOimTupleOrderAspect)] = elementIndex(fact)
if isJSON:
factFootnotes(fact, oimFact)
return oimFact
prefixes = OrderedDict((p,ns) for ns, p in sorted(namespacePrefixes.items(),
key=lambda item: item[1]))
if isJSON:
# save JSON
oimReport = OrderedDict() # top level of oim json output
oimReport["documentInfo"] = oimDocInfo = OrderedDict()
oimReport["facts"] = oimFacts = OrderedDict()
oimDocInfo["documentType"] = nsOim + "/xbrl-json"
oimDocInfo["features"] = oimFeatures = OrderedDict()
oimDocInfo["prefixes"] = prefixes
oimDocInfo["taxonomy"] = dtsReferences
oimFeatures["xbrl:canonicalValues"] = True
def saveJsonFacts(facts, oimFacts, parentFact):
for fact in facts:
oimFact = factAspects(fact)
id = fact.id if fact.id else "f{}".format(fact.objectIndex)
oimFacts[id] = oimFact
if fact.modelTupleFacts:
saveJsonFacts(fact.modelTupleFacts, oimFacts, fact)
saveJsonFacts(modelXbrl.facts, oimFacts, None)
# add footnotes as pseudo facts
for ftObj in footnoteFacts:
ftId = ftObj.id if ftObj.id else "f{}".format(ftObj.objectIndex)
oimFacts[ftId] = oimFact = OrderedDict()
oimFact["value"] = ftObj.viewText()
oimFact["dimensions"] = OrderedDict((("concept", "xbrl:note"),
("noteId", ftId)))
if ftObj.xmlLang:
oimFact["dimensions"]["language"] = ftObj.xmlLang.lower()
if outputZip:
fh = io.StringIO()
else:
fh = open(oimFile, "w", encoding="utf-8")
fh.write(json.dumps(oimReport, indent=1))
if outputZip:
fh.seek(0)
outputZip.writestr(os.path.basename(oimFile),fh.read())
fh.close()
elif isCSVorXL:
# save CSV
aspectQnCol = {}
aspectsHeader = []
factsColumns = []
def addAspectQnCol(aspectQn):
aspectQnCol[str(aspectQn)] = len(aspectsHeader)
_aspectQn = oimValue(aspectQn)
aspectsHeader.append(_aspectQn)
_colName = _aspectQn.replace("xbrl:", "")
_colDataType = {"id": "Name",
"concept": "QName",
"value": "string",
"decimals": "decimal",
"entity": "QName",
"periodStart": "dateTime",
"periodEnd": "dateTime",
"unit": "string",
"tupleId": "Name",
"tupleParent": "Name",
"tupleOrder": "integer"
}.get(_colName, "string")
col = OrderedDict((("name", _colName),
("datatype", _colDataType)))
if _aspectQn == "value":
col["http://xbrl.org/YYYY/model#simpleFactAspects"] = {}
elif _aspectQn == "tupleId":
col["http://xbrl.org/YYYY/model#tupleFactAspects"] = {}
col["http://xbrl.org/YYYY/model#tupleReferenceId"] = "true"
else:
col["http://xbrl.org/YYYY/model#columnAspect"] = _aspectQn
factsColumns.append(col)
# pre-ordered aspect columns
#if hasId:
# addAspectQnCol("id")
addAspectQnCol(qnOimConceptAspect)
addAspectQnCol("value")
if hasNumeric:
addAspectQnCol("decimals")
if hasTuple:
addAspectQnCol("tupleId")
addAspectQnCol(qnOimTupleParentAspect)
addAspectQnCol(qnOimTupleOrderAspect)
if qnOimEntityAspect in aspectsDefined:
addAspectQnCol(qnOimEntityAspect)
if qnOimPeriodStartAspect in aspectsDefined:
addAspectQnCol(qnOimPeriodStartAspect)
addAspectQnCol(qnOimPeriodEndAspect)
if qnOimUnitAspect in aspectsDefined:
addAspectQnCol(qnOimUnitAspect)
for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)):
if aspectQn.namespaceURI != nsOim:
addAspectQnCol(aspectQn)
def aspectCols(fact):
cols = [None for i in range(len(aspectsHeader))]
def setColValues(aspects):
for aspectQn, aspectValue in aspects.items():
if isinstance(aspectValue, dict):
setColValues(aspectValue)
elif aspectQn in aspectQnCol:
if aspectValue is None:
_aspectValue = "#nil"
elif aspectValue == "":
_aspectValue = "#empty"
elif isinstance(aspectValue, str) and aspectValue.startswith("#"):
_aspectValue = "#" + aspectValue
else:
_aspectValue = aspectValue
cols[aspectQnCol[aspectQn]] = _aspectValue
setColValues(factAspects(fact))
return cols
# metadata
csvTables = []
csvMetadata = OrderedDict((("@context", "http://www.w3.org/ns/csvw"),
("http://xbrl.org/YYYY/model#metadata",
OrderedDict((("documentType", "http://xbrl.org/YYYY/xbrl-csv"),
("dtsReferences", dtsReferences),
("prefixes", prefixes)))),
("tables", csvTables)))
_open = _writerow = _close = None
_tableinfo = {}
if isCSV:
if oimFile.endswith("-facts.csv"): # strip -facts.csv if a prior -facts.csv file was chosen
_baseURL = oimFile[:-10]
elif oimFile.endswith(".csv"):
_baseURL = oimFile[:-4]
else:
_baseURL = oimFile
_csvinfo = {} # open file, writer
def _open(filesuffix, tabname):
_filename = _tableinfo["url"] = _baseURL + filesuffix
_csvinfo["file"] = open(_filename, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
_csvinfo["writer"] = csv.writer(_csvinfo["file"], dialect="excel")
def _writerow(row, header=False):
_csvinfo["writer"].writerow(row)
def _close():
_csvinfo["file"].close()
_csvinfo.clear()
elif isXL:
headerWidths = {"href": 100, "xbrl:concept": 70, "decimals": 8, "language": 9, "URI": 80,
"value": 60,
"group": 60, "footnoteType": 40, "footnote": 70, "column": 20,
'conceptAspect': 40, 'tuple': 20, 'simpleFact': 20}
from openpyxl import Workbook
from openpyxl.writer.write_only import WriteOnlyCell
from openpyxl.styles import Font, PatternFill, Border, Alignment, Color, fills, Side
from openpyxl.worksheet.dimensions import ColumnDimension
hdrCellFill = PatternFill(patternType=fills.FILL_SOLID, fgColor=Color("00FFBF5F")) # Excel's light orange fill color = 00FF990
workbook = Workbook()
# remove pre-existing worksheets
while len(workbook.worksheets)>0:
workbook.remove_sheet(workbook.worksheets[0])
_xlinfo = {} # open file, writer
def _open(filesuffix, tabname):
_tableinfo["url"] = tabname
_xlinfo["ws"] = workbook.create_sheet(title=tabname)
def _writerow(rowvalues, header=False):
row = []
_ws = _xlinfo["ws"]
for i, v in enumerate(rowvalues):
cell = WriteOnlyCell(_ws, value=v)
if header:
cell.fill = hdrCellFill
cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True)
colLetter = chr(ord('A') + i)
_ws.column_dimensions[colLetter] = ColumnDimension(_ws, customWidth=True)
_ws.column_dimensions[colLetter].width = headerWidths.get(v, 20)
else:
cell.alignment = Alignment(horizontal="right" if isinstance(v, _NUM_TYPES)
else "center" if isinstance(v, bool)
else "left",
vertical="top",
wrap_text=isinstance(v, str))
row.append(cell)
_ws.append(row)
def _close():
_xlinfo.clear()
# save facts
_open("-facts.csv", "facts")
_writerow(aspectsHeader, header=True)
def saveCSVfacts(facts):
for fact in facts:
_writerow(aspectCols(fact))
saveCSVfacts(fact.modelTupleFacts)
saveCSVfacts(modelXbrl.facts)
_close()
factsTableSchema = OrderedDict((("columns",factsColumns),))
csvTables.append(OrderedDict((("url",_tableinfo["url"]),
("http://xbrl.org/YYYY/model#tableType", "fact"),
("tableSchema",factsTableSchema))))
# save footnotes
if footnotesRelationshipSet.modelRelationships:
_open("-footnotes.csv", "footnotes")
cols = ("group", "footnoteType", "factId", "factRef", "footnote", "language")
_writerow(cols, header=True)
def saveCSVfootnotes(facts):
for fact in facts:
for _footnote in factFootnotes(fact):
_writerow(tuple((_footnote.get(col,"") for col in cols)))
saveCSVfootnotes(fact.modelTupleFacts)
saveCSVfootnotes(modelXbrl.facts)
_close()
footnoteTableSchema = OrderedDict((("columns",[OrderedDict((("name","group"),("datatype","anyURI"))),
OrderedDict((("name","footnoteType"),("datatype","Name"))),
OrderedDict((("name","factId"),("datatype","Name"))),
OrderedDict((("name","factRef"),("datatype","Name"))),
OrderedDict((("name","footnote"),("datatype","string"))),
OrderedDict((("name","language"),("datatype","language")))]),))
csvTables.append(OrderedDict((("url",_tableinfo["url"]),
("http://xbrl.org/YYYY/model#tableType", "footnote"),
("tableSchema",footnoteTableSchema))))
# save metadata
if isCSV:
with open(_baseURL + "-metadata.json", "w", encoding="utf-8") as fh:
fh.write(json.dumps(csvMetadata, ensure_ascii=False, indent=1, sort_keys=False))
elif isXL:
_open(None, "metadata")
hasColumnAspect = hasSimpleFact = hasTupleFact = False
for table in csvTables:
tablename = table["url"]
for column in table["tableSchema"]["columns"]:
if "http://xbrl.org/YYYY/model#columnAspect" in column:
hasColumnAspect = True
if "http://xbrl.org/YYYY/model#simpleFactAspects" in column:
hasSimpleFact = True
if "http://xbrl.org/YYYY/model#tupleAspects" in column:
hasTupleFact = True
metadataCols = ["table", "column", "datatype"]
if hasColumnAspect:
metadataCols.append("columnAspect")
if hasSimpleFact:
metadataCols.append("simpleFact")
if hasTupleFact:
metadataCols.append("tuple")
_writerow(metadataCols, header=True)
for table in csvTables:
tablename = table["url"]
for column in table["tableSchema"]["columns"]:
row = [tablename, column["name"], column["datatype"]]
if hasColumnAspect:
colAspect = column.get("http://xbrl.org/YYYY/model#columnAspect")
if isinstance(colAspect, str):
row.append(colAspect)
elif isinstance(colAspect, dict):
row.append("\n".join("{} [{}]".format(k, ", ".join(_v for _v in v))
for k, v in dict.items()))
else:
row.append(None)
if hasSimpleFact:
row.append("\u221a" if "http://xbrl.org/YYYY/model#simpleFactAspects" in column else None)
if hasTupleFact:
row.append("\u221a" if "http://xbrl.org/YYYY/model#tupleAspects" in column else None)
_writerow(row)
_close()
if isXL:
workbook.save(oimFile)
def saveLoadableOIM(modelXbrl, oimFile):
isJSON = oimFile.endswith(".json")
isCSV = oimFile.endswith(".csv")
isXL = oimFile.endswith(".xlsx")
isCSVorXL = isCSV or isXL
if not isJSON and not isCSVorXL:
return
namespacePrefixes = {}
prefixNamespaces = {}
def compileQname(qname):
if qname.namespaceURI not in namespacePrefixes:
namespacePrefixes[qname.namespaceURI] = qname.prefix or ""
aspectsDefined = {
qnOimConceptAspect,
qnOimEntityAspect}
if isJSON:
aspectsDefined.add(qnOimPeriodAspect)
elif isCSVorXL:
aspectsDefined.add(qnOimPeriodStartAspect)
aspectsDefined.add(qnOimPeriodEndAspect)
def oimValue(object, decimals=None):
if isinstance(object, QName):
if object.namespaceURI not in namespacePrefixes:
if object.prefix:
namespacePrefixes[object.namespaceURI] = object.prefix
else:
_prefix = "_{}".format(sum(1 for p in namespacePrefixes if p.startswith("_")))
namespacePrefixes[object.namespaceURI] = _prefix
return "{}:{}".format(namespacePrefixes[object.namespaceURI], object.localName)
if isinstance(object, Decimal):
try:
if isinf(object):
return "-INF" if object < 0 else "INF"
elif isnan(num):
return "NaN"
else:
if object == object.to_integral():
object = object.quantize(ONE) # drop any .0
return "{}".format(object)
except:
return str(object)
if isinstance(object, bool):
return object
if isinstance(object, (DateTime, YearMonthDuration, DayTimeDuration, Time,
gYearMonth, gMonthDay, gYear, gMonth, gDay)):
return str(object)
return object
def oimPeriodValue(cntx):
if cntx.isForeverPeriod:
pass # not supported
elif cntx.isStartEndPeriod:
s = cntx.startDatetime
e = cntx.endDatetime
else: # instant
s = e = cntx.instantDatetime
return {"start": "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format(
s.year, s.month, s.day, s.hour, s.minute, s.second),
"end": "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format(
e.year, e.month, e.day, e.hour, e.minute, e.second)}
hasId = False
hasTuple = False
hasType = True
hasLang = False
hasUnits = False
footnotesRelationshipSet = ModelRelationshipSet(modelXbrl, "XBRL-footnotes")
factBaseTypes = set()
#compile QNames in instance for OIM
for fact in modelXbrl.factsInInstance:
if (fact.id or fact.isTuple or
footnotesRelationshipSet.toModelObject(fact) or
(isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))):
hasId = True
concept = fact.concept
if concept is not None:
if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang:
hasLang = True
_baseXsdType = concept.baseXsdType
if _baseXsdType == "XBRLI_DATEUNION":
if getattr(fact.xValue, "dateOnly", False):
_baseXsdType = "date"
else:
_baseXsdType = "dateTime"
factBaseTypes.add(baseTypes.get(_baseXsdType,_baseXsdType))
compileQname(fact.qname)
if hasattr(fact, "xValue") and isinstance(fact.xValue, QName):
compileQname(fact.xValue)
unit = fact.unit
if unit is not None:
hasUnits = True
if fact.modelTupleFacts:
hasTuple = True
entitySchemePrefixes = {}
for cntx in modelXbrl.contexts.values():
if cntx.entityIdentifierElement is not None:
scheme = cntx.entityIdentifier[0]
if scheme not in entitySchemePrefixes:
if not entitySchemePrefixes: # first one is just scheme
if scheme == "http://www.sec.gov/CIK":
_schemePrefix = "cik"
elif scheme == "http://standard.iso.org/iso/17442":
_schemePrefix = "lei"
else:
_schemePrefix = "scheme"
else:
_schemePrefix = "scheme{}".format(len(entitySchemePrefixes) + 1)
entitySchemePrefixes[scheme] = _schemePrefix
namespacePrefixes[scheme] = _schemePrefix
for dim in cntx.qnameDims.values():
compileQname(dim.dimensionQname)
aspectsDefined.add(dim.dimensionQname)
if dim.isExplicit:
compileQname(dim.memberQname)
for unit in modelXbrl.units.values():
if unit is not None:
for measures in unit.measures:
for measure in measures:
compileQname(measure)
if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[XbrlConst.xbrli] != "xbrli":
namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix
namespacePrefixes[XbrlConst.xsd] = "xsd"
if hasLang: aspectsDefined.add(qnOimLangAspect)
if hasTuple:
aspectsDefined.add(qnOimTupleParentAspect)
aspectsDefined.add(qnOimTupleOrderAspect)
if hasUnits: aspectsDefined.add(qnOimUnitAspect)
# compile footnotes and relationships
'''
factRelationships = []
factFootnotes = []
for rel in modelXbrl.relationshipSet(modelXbrl, "XBRL-footnotes").modelRelationships:
oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole}
factRelationships.append(oimRel)
oimRel["fromIds"] = [obj.id if obj.id
else elementChildSequence(obj)
for obj in rel.fromModelObjects]
oimRel["toIds"] = [obj.id if obj.id
else elementChildSequence(obj)
for obj in rel.toModelObjects]
_order = rel.arcElement.get("order")
if _order is not None:
oimRel["order"] = _order
for obj in rel.toModelObjects:
if isinstance(obj, ModelResource): # footnote
oimFootnote = {"role": obj.role,
"id": obj.id if obj.id
else elementChildSequence(obj),
# value needs work for html elements and for inline footnotes
"value": xmlstring(obj, stripXmlns=True)}
if obj.xmlLang:
oimFootnote["lang"] = obj.xmlLang
factFootnotes.append(oimFootnote)
oimFootnote
'''
dtsReferences = [
{"type": "schema" if doc.type == ModelDocument.Type.SCHEMA
else "linkbase" if doc.type == ModelDocument.Type.LINKBASE
else "other",
"href": doc.uri}
for doc,ref in modelXbrl.modelDocument.referencesDocument.items()
if ref.referringModelObject.qname in SCHEMA_LB_REFS]
'''
roleTypes = [
{"type": "role" if ref.referringModelObject.localName == "roleRef" else "arcroleRef",
"href": ref.referringModelObject["href"]}
for doc,ref in modelXbrl.modelDocument.referencesDocument.items()
if ref.referringModelObject.qname in ROLE_REFS]
'''
def factFootnotes(fact):
footnotes = []
for footnoteRel in footnotesRelationshipSet.fromModelObject(fact):
footnote = OrderedDict((("group", footnoteRel.linkrole),
("footnoteType", footnoteRel.arcrole)))
footnotes.append(footnote)
if isCSVorXL:
footnote["factId"] = fact.id if fact.id else "f{}".format(fact.objectIndex)
toObj = footnoteRel.toModelObject
if isinstance(toObj, ModelFact):
footnote["factRef"] = toObj.id if toObj.id else "f{}".format(toObj.objectIndex)
else:
footnote["footnote"] = xmlstring(toObj, stripXmlns=True, contentsOnly=True, includeText=True)
if toObj.xmlLang:
footnote["language"] = toObj.xmlLang
return footnotes
def factAspects(fact):
aspects = OrderedDict()
if hasId and fact.id:
aspects["id"] = fact.id
elif (fact.isTuple or
footnotesRelationshipSet.toModelObject(fact) or
(isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))):
aspects["id"] = "f{}".format(fact.objectIndex)
parent = fact.getparent()
concept = fact.concept
_csvType = "Value"
if not fact.isTuple:
if concept is not None:
_baseXsdType = concept.baseXsdType
if _baseXsdType == "XBRLI_DATEUNION":
if getattr(fact.xValue, "dateOnly", False):
_baseXsdType = "date"
else:
_baseXsdType = "dateTime"
aspects["baseType"] = "xs:{}".format(_baseXsdType)
_csvType = baseTypes.get(_baseXsdType,_baseXsdType) + "Value"
if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang:
aspects[qnOimLangAspect] = fact.xmlLang
if fact.isItem:
if fact.isNil:
_value = None
_strValue = "nil"
else:
_inferredDecimals = inferredDecimals(fact)
_value = oimValue(fact.xValue, _inferredDecimals)
_strValue = str(_value)
if not isCSVorXL:
aspects["value"] = _strValue
if fact.concept is not None and fact.concept.isNumeric:
_numValue = fact.xValue
if isinstance(_numValue, Decimal) and not isinf(_numValue) and not isnan(_numValue):
if _numValue == _numValue.to_integral():
_numValue = int(_numValue)
else:
_numValue = float(_numValue)
if isCSVorXL:
aspects[_csvType] = _numValue
else:
aspects["numericValue"] = _numValue
if not fact.isNil:
if isinf(_inferredDecimals):
if isJSON: _accuracy = "infinity"
elif isCSVorXL: _accuracy = "INF"
else:
_accuracy = _inferredDecimals
aspects["accuracy"] = _accuracy
elif isinstance(_value, bool):
aspects["booleanValue"] = _value
elif isCSVorXL:
aspects[_csvType] = _strValue
aspects[qnOimConceptAspect] = oimValue(fact.qname)
cntx = fact.context
if cntx is not None:
if cntx.entityIdentifierElement is not None:
aspects[qnOimEntityAspect] = oimValue(qname(*cntx.entityIdentifier))
if cntx.period is not None:
if isJSON:
aspects[qnOimPeriodAspect] = oimPeriodValue(cntx)
elif isCSVorXL:
_periodValue = oimPeriodValue(cntx)
aspects[qnOimPeriodStartAspect] = _periodValue["start"]
aspects[qnOimPeriodEndAspect] = _periodValue["end"]
for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]):
aspects[dim.dimensionQname] = (oimValue(dim.memberQname) if dim.isExplicit
else None if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1")
else dim.typedMember.stringValue)
unit = fact.unit
if unit is not None:
_mMul, _mDiv = unit.measures
_sMul = '*'.join(oimValue(m) for m in sorted(_mMul, key=lambda m: str(m)))
if _mDiv:
_sDiv = '*'.join(oimValue(m) for m in sorted(_mDiv, key=lambda m: str(m)))
if len(mDiv) > 1:
if len(mMul) > 1:
_sUnit = "({})/({})".format(_sMul,_sDiv)
else:
_sUnit = "{}/({})".format(_sMul,_sDiv)
else:
if len(mMul) > 1:
_sUnit = "({})/{}".format(_sMul,_sDiv)
else:
_sUnit = "{}/{}".format(_sMul,_sDiv)
else:
_sUnit = _sMul
aspects[qnOimUnitAspect] = _sUnit
if parent.qname != XbrlConst.qnXbrliXbrl:
aspects[qnOimTupleParentAspect] = parent.id if parent.id else "f{}".format(parent.objectIndex)
aspects[qnOimTupleOrderAspect] = elementIndex(fact)
if isJSON:
_footnotes = factFootnotes(fact)
if _footnotes:
aspects["footnotes"] = _footnotes
return aspects
if isJSON:
# save JSON
oimReport = OrderedDict() # top level of oim json output
oimFacts = []
oimReport["prefixes"] = OrderedDict((p,ns) for ns, p in sorted(namespacePrefixes.items(),
key=lambda item: item[1]))
oimReport["dtsReferences"] = dtsReferences
oimReport["facts"] = oimFacts
def saveJsonFacts(facts, oimFacts, parentFact):
for fact in facts:
oimFact = factAspects(fact)
oimFacts.append(OrderedDict((str(k),v) for k,v in oimFact.items()))
if fact.modelTupleFacts:
saveJsonFacts(fact.modelTupleFacts, oimFacts, fact)
saveJsonFacts(modelXbrl.facts, oimFacts, None)
with open(oimFile, "w", encoding="utf-8") as fh:
fh.write(json.dumps(oimReport, ensure_ascii=False, indent=1, sort_keys=False))
elif isCSVorXL:
# save CSV
aspectQnCol = {}
aspectsHeader = []
factsColumns = []
def addAspectQnCol(aspectQn):
aspectQnCol[aspectQn] = len(aspectsHeader)
_colName = oimValue(aspectQn)
aspectsHeader.append(_colName)
_colDataType = {"id": "Name",
"baseType": "Name",
"oim:concept": "QName",
"oim:periodStart": "dateTime",
"oim:periodEnd": "dateTime",
"oim:tupleOrder": "integer",
"numericValue": "decimal",
"accuracy": "decimal",
"booleanValue": "boolean",
"oim:entity": "QName",
"oim:unit": "string"
}.get(_colName)
if _colDataType is None:
if isCSVorXL and _colName.endswith("Value"):
_colDataType = _colName[:-5]
else:
_colDataType = "string"
factsColumns.append(OrderedDict((("name", _colName),
("datatype", _colDataType))))
# pre-ordered aspect columns
if hasId:
addAspectQnCol("id")
addAspectQnCol(qnOimConceptAspect)
if hasType:
addAspectQnCol("baseType")
if isCSVorXL:
for _baseType in sorted(factBaseTypes):
addAspectQnCol(_baseType + "Value")
else:
addAspectQnCol("stringValue")
addAspectQnCol("numericValue")
addAspectQnCol("booleanValue")
addAspectQnCol("accuracy")
if hasTuple:
addAspectQnCol(qnOimTupleParentAspect)
addAspectQnCol(qnOimTupleOrderAspect)
if qnOimEntityAspect in aspectsDefined:
addAspectQnCol(qnOimEntityAspect)
if qnOimPeriodStartAspect in aspectsDefined:
addAspectQnCol(qnOimPeriodStartAspect)
addAspectQnCol(qnOimPeriodEndAspect)
if qnOimUnitAspect in aspectsDefined:
addAspectQnCol(qnOimUnitAspect)
for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)):
if aspectQn.namespaceURI != nsOim:
addAspectQnCol(aspectQn)
def aspectCols(fact):
cols = [None for i in range(len(aspectsHeader))]
_factAspects = factAspects(fact)
for aspectQn, aspectValue in _factAspects.items():
if aspectQn in aspectQnCol:
cols[aspectQnCol[aspectQn]] = aspectValue
return cols
# metadata
csvTables = []
csvMetadata = OrderedDict((("@context",[ "http://www.w3.org/ns/csvw", { "@base": "./" }]),
("tables", csvTables)))
_open = _writerow = _close = None
_tableinfo = {}
if isCSV:
if oimFile.endswith("-facts.csv"): # strip -facts.csv if a prior -facts.csv file was chosen
_baseURL = oimFile[:-10]
elif oimFile.endswith(".csv"):
_baseURL = oimFile[:-4]
else:
_baseURL = oimFile
_csvinfo = {} # open file, writer
def _open(filesuffix, tabname):
_filename = _tableinfo["url"] = _baseURL + filesuffix
_csvinfo["file"] = open(_filename, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
_csvinfo["writer"] = csv.writer(csvFile, dialect="excel")
def _writerow(row, header=False):
_csvinfo["writer"].writerow(row)
def _close():
_csvinfo["writer"].close()
_csvinfo.clear()
elif isXL:
headerWidths = {"href": 100, "oim:concept": 70, "accuracy": 8, "baseType": 10, "language": 9, "URI": 80,
"stringValue": 60, "decimalValue": 12, "booleanValue": 12, "dateValue": 12, "dateTimeValue": 20,
"group": 60, "footnoteType": 40, "footnote": 70, "column": 60}
from openpyxl import Workbook
from openpyxl.writer.write_only import WriteOnlyCell
from openpyxl.styles import Font, PatternFill, Border, Alignment, Color, fills, Side
from openpyxl.worksheet.dimensions import ColumnDimension
hdrCellFill = PatternFill(patternType=fills.FILL_SOLID, fgColor=Color("00FFBF5F")) # Excel's light orange fill color = 00FF990
workbook = Workbook(encoding="utf-8", write_only=True)
_xlinfo = {} # open file, writer
def _open(filesuffix, tabname):
_tableinfo["url"] = tabname
_xlinfo["ws"] = workbook.create_sheet(title=tabname)
def _writerow(rowvalues, header=False):
row = []
_ws = _xlinfo["ws"]
for i, v in enumerate(rowvalues):
cell = WriteOnlyCell(_ws, value=v)
if header:
cell.fill = hdrCellFill
cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True)
colLetter = chr(ord('A') + i)
_ws.column_dimensions[colLetter] = ColumnDimension(_ws, customWidth=True)
_ws.column_dimensions[colLetter].width = headerWidths.get(v, 20)
else:
cell.alignment = Alignment(horizontal="right" if isinstance(v, _NUM_TYPES)
else "center" if isinstance(v, bool)
else "left",
vertical="top",
wrap_text=isinstance(v, str))
row.append(cell)
_ws.append(row)
def _close():
_xlinfo.clear()
# save facts
_open("-facts.csv", "facts")
_writerow(aspectsHeader, header=True)
def saveCSVfacts(facts):
for fact in facts:
_writerow(aspectCols(fact))
saveCSVfacts(fact.modelTupleFacts)
saveCSVfacts(modelXbrl.facts)
_close()
factsTableSchema = OrderedDict((("columns",factsColumns),))
csvTables.append(OrderedDict((("url",_tableinfo["url"]),
("tableSchema",factsTableSchema))))
# save namespaces
_open("-prefixes.csv", "prefixes")
_writerow(("prefix", "URI"), header=True)
for _URI, prefix in sorted(namespacePrefixes.items(), key=lambda item: item[1]):
_writerow((prefix, _URI))
_close()
nsTableSchema = OrderedDict((("columns",[OrderedDict((("name","prefix"),("datatype","Name"))),
OrderedDict((("name","URI"),("datatype","anyURI")))]),))
csvTables.append(OrderedDict((("url",_tableinfo["url"]),
("tableSchema",nsTableSchema))))
# save dts references
_open("-dtsReferences.csv", "dtsReferences")
_writerow(("type", "href"), header=True)
for oimRef in dtsReferences:
_writerow((oimRef["type"], oimRef["href"]))
_close()
dtsRefTableSchema = OrderedDict((("columns",[OrderedDict((("name","type"),("datatype","Name"))),
OrderedDict((("name","href"),("datatype","anyURI")))]),))
csvTables.append(OrderedDict((("url",_tableinfo["url"]),
("tableSchema",dtsRefTableSchema))))
# save footnotes
if footnotesRelationshipSet.modelRelationships:
_open("-footnotes.csv", "footnotes")
cols = ("group", "footnoteType", "factId", "factRef", "footnote", "language")
_writerow(cols, header=True)
def saveCSVfootnotes(facts):
for fact in facts:
for _footnote in factFootnotes(fact):
_writerow(tuple((_footnote.get(col,"") for col in cols)))
saveCSVfootnotes(fact.modelTupleFacts)
saveCSVfootnotes(modelXbrl.facts)
_close()
footnoteTableSchema = OrderedDict((("columns",[OrderedDict((("name","group"),("datatype","anyURI"))),
OrderedDict((("name","footnoteType"),("datatype","Name"))),
OrderedDict((("name","factId"),("datatype","Name"))),
OrderedDict((("name","factRef"),("datatype","Name"))),
OrderedDict((("name","footnote"),("datatype","string"))),
OrderedDict((("name","language"),("datatype","language")))]),))
csvTables.append(OrderedDict((("url",_tableinfo["url"]),
("tableSchema",footnoteTableSchema))))
# save metadata
if isCSV:
with open(_baseURL + "-metadata.csv", "w", encoding="utf-8") as fh:
fh.write(json.dumps(csvMetadata, ensure_ascii=False, indent=1, sort_keys=False))
elif isXL:
_open(None, "metadata")
cols = ("group", "footnoteType", "factId", "factRef", "footnote", "language")
_writerow(("table", "column", "datatype"), header=True)
for table in csvTables:
tablename = table["url"]
for column in table["tableSchema"]["columns"]:
_writerow((tablename, column["name"], column["datatype"]))
_close()
if isXL:
workbook.save(oimFile)
def validateSetup(val, parameters=None, *args, **kwargs):
val.validateEBA = val.validateDisclosureSystem and getattr(val.disclosureSystem, "EBA", False)
val.validateEIOPA = val.validateDisclosureSystem and getattr(val.disclosureSystem, "EIOPA", False)
if not (val.validateEBA or val.validateEIOPA):
return
val.validateUTR = False # do not use default UTR validation, it's at error level and not streamable
val.utrValidator = ValidateUtr(val.modelXbrl,
"WARNING", # EBA specifies SHOULD on UTR validation
"EBA.2.23") # override utre error-severity message code
val.isEIOPAfullVersion = val.isEIOPA_2_0_1 = False
modelDocument = val.modelXbrl.modelDocument
if modelDocument.type == ModelDocument.Type.INSTANCE:
for doc, docRef in modelDocument.referencesDocument.items():
if docRef.referenceType == "href":
if docRef.referringModelObject.localName == "schemaRef":
_match = schemaRefDatePattern.match(doc.uri)
if _match:
val.isEIOPAfullVersion = _match.group(1) > "2015-02-28"
val.isEIOPA_2_0_1 = _match.group(1) >= "2015-10-21"
break
else:
val.modelXbrl.error("EIOPA.S.1.5.a/EIOPA.S.1.5.b",
_('The link:schemaRef element in submitted instances MUST resolve to the full published entry point URL, this schemaRef is missing date portion: %(schemaRef)s.'),
modelObject=modelDocument, schemaRef=doc.uri)
val.qnDimAF = val.qnDimOC = val.qnCAx1 = None
_nsmap = val.modelXbrl.modelDocument.xmlRootElement.nsmap
if val.isEIOPA_2_0_1:
_hasPiInstanceGenerator = False
for pi in modelDocument.processingInstructions:
if pi.target == "instance-generator":
_hasPiInstanceGenerator = True
if not all(pi.get(attr) for attr in ("id", "version", "creationdate")):
val.modelXbrl.warning("EIOPA.S.2.23",
_('The processing instruction instance-generator SHOULD contain attributes "id", "version" and "creationdate".'),
modelObject=modelDocument)
if not _hasPiInstanceGenerator:
val.modelXbrl.warning("EIOPA.S.2.23",
_('The instance SHOULD include a processing instruction "instance-generator".'),
modelObject=modelDocument)
val.qnDimAF = qname("s2c_dim:AF", _nsmap)
val.qnDimOC = qname("s2c_dim:OC", _nsmap)
val.qnCAx1 = qname("s2c_CA:x1", _nsmap)
val.prefixNamespace = {}
val.namespacePrefix = {}
val.idObjects = {}
val.typedDomainQnames = set()
val.typedDomainElements = set()
for modelConcept in val.modelXbrl.qnameConcepts.values():
if modelConcept.isTypedDimension:
typedDomainElement = modelConcept.typedDomainElement
if isinstance(typedDomainElement, ModelConcept):
val.typedDomainQnames.add(typedDomainElement.qname)
val.typedDomainElements.add(typedDomainElement)
val.filingIndicators = {}
val.numFilingIndicatorTuples = 0
val.cntxEntities = set()
val.cntxDates = defaultdict(set)
val.unusedCntxIDs = set()
val.unusedUnitIDs = set()
val.currenciesUsed = {}
val.reportingCurrency = None
val.namespacePrefixesUsed = defaultdict(set)
val.prefixesUnused = set()
for prefix, ns in _nsmap.items():
val.prefixesUnused.add(prefix)
val.namespacePrefixesUsed[ns].add(prefix)
val.firstFactObjectIndex = sys.maxsize
val.firstFact = None
val.footnotesRelationshipSet = ModelRelationshipSet(val.modelXbrl, "XBRL-footnotes")
def saveLoadableOIM(modelXbrl, oimFile, outputZip=None):
isJSON = oimFile.endswith(".json")
isCSV = oimFile.endswith(".csv")
isXL = oimFile.endswith(".xlsx")
isCSVorXL = isCSV or isXL
if not isJSON and not isCSVorXL:
return
namespacePrefixes = {nsOim: "xbrl"}
prefixNamespaces = {"xbrl": nsOim}
def compileQname(qname):
if qname.namespaceURI not in namespacePrefixes:
namespacePrefixes[qname.namespaceURI] = qname.prefix or ""
aspectsDefined = {
qnOimConceptAspect,
qnOimEntityAspect,
qnOimPeriodStartAspect,
qnOimPeriodEndAspect}
def oimValue(object, decimals=None):
if isinstance(object, QName):
if object.namespaceURI not in namespacePrefixes:
if object.prefix:
namespacePrefixes[object.namespaceURI] = object.prefix
else:
_prefix = "_{}".format(sum(1 for p in namespacePrefixes if p.startswith("_")))
namespacePrefixes[object.namespaceURI] = _prefix
return "{}:{}".format(namespacePrefixes[object.namespaceURI], object.localName)
if isinstance(object, Decimal):
try:
if isinf(object):
return "-INF" if object < 0 else "INF"
elif isnan(num):
return "NaN"
else:
if object == object.to_integral():
object = object.quantize(ONE) # drop any .0
return "{}".format(object)
except:
return str(object)
if isinstance(object, bool):
return "true" if object else "false"
if isinstance(object, (DateTime, YearMonthDuration, DayTimeDuration, Time,
gYearMonth, gMonthDay, gYear, gMonth, gDay,
IsoDuration)):
return str(object)
return object
def oimPeriodValue(cntx):
if cntx.isForeverPeriod:
return OrderedDict() # defaulted
elif cntx.isStartEndPeriod:
s = cntx.startDatetime
e = cntx.endDatetime
else: # instant
s = e = cntx.instantDatetime
if isJSON:
return({str(qnOimPeriodAspect):
("{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}/".format(
s.year, s.month, s.day, s.hour, s.minute, s.second)
if cntx.isStartEndPeriod else "") +
"{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format(
e.year, e.month, e.day, e.hour, e.minute, e.second)
})
# CSV, XL
return OrderedDict(((str(qnOimPeriodStartAspect), "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format(
s.year, s.month, s.day, s.hour, s.minute, s.second)),
(str(qnOimPeriodEndAspect), "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format(
e.year, e.month, e.day, e.hour, e.minute, e.second))))
hasId = False
hasTuple = False
hasType = True
hasLang = False
hasUnits = False
hasNumeric = False
footnotesRelationshipSet = ModelRelationshipSet(modelXbrl, "XBRL-footnotes")
factBaseTypes = set()
#compile QNames in instance for OIM
for fact in modelXbrl.factsInInstance:
if (fact.id or fact.isTuple or
footnotesRelationshipSet.toModelObject(fact) or
(isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))):
hasId = True
concept = fact.concept
if concept is not None:
if concept.isNumeric:
hasNumeric = True
if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang:
hasLang = True
_baseXsdType = concept.baseXsdType
if _baseXsdType == "XBRLI_DATEUNION":
if getattr(fact.xValue, "dateOnly", False):
_baseXsdType = "date"
else:
_baseXsdType = "dateTime"
factBaseTypes.add(baseTypes.get(_baseXsdType,_baseXsdType))
compileQname(fact.qname)
if hasattr(fact, "xValue") and isinstance(fact.xValue, QName):
compileQname(fact.xValue)
unit = fact.unit
if unit is not None:
hasUnits = True
if fact.modelTupleFacts:
hasTuple = True
if hasTuple:
modelXbrl.error("arelleOIMsaver:tuplesNotAllowed",
"Tuples are not allowed in an OIM document",
modelObject=modelXbrl)
return
entitySchemePrefixes = {}
for cntx in modelXbrl.contexts.values():
if cntx.entityIdentifierElement is not None:
scheme = cntx.entityIdentifier[0]
if scheme not in entitySchemePrefixes:
if not entitySchemePrefixes: # first one is just scheme
if scheme == "http://www.sec.gov/CIK":
_schemePrefix = "cik"
elif scheme == "http://standard.iso.org/iso/17442":
_schemePrefix = "lei"
else:
_schemePrefix = "scheme"
else:
_schemePrefix = "scheme{}".format(len(entitySchemePrefixes) + 1)
entitySchemePrefixes[scheme] = _schemePrefix
namespacePrefixes[scheme] = _schemePrefix
for dim in cntx.qnameDims.values():
compileQname(dim.dimensionQname)
aspectsDefined.add(dim.dimensionQname)
if dim.isExplicit:
compileQname(dim.memberQname)
for unit in modelXbrl.units.values():
if unit is not None:
for measures in unit.measures:
for measure in measures:
compileQname(measure)
if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[XbrlConst.xbrli] != "xbrli":
namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix
namespacePrefixes[XbrlConst.xsd] = "xsd"
if hasLang: aspectsDefined.add(qnOimLangAspect)
if hasTuple:
aspectsDefined.add(qnOimTupleParentAspect)
aspectsDefined.add(qnOimTupleOrderAspect)
if hasUnits: aspectsDefined.add(qnOimUnitAspect)
for footnoteRel in footnotesRelationshipSet.modelRelationships:
typePrefix = "ftTyp_" + os.path.basename(footnoteRel.arcrole)
if footnoteRel.linkrole == XbrlConst.defaultLinkRole:
groupPrefix = "ftGrp_default"
else:
groupPrefix = "ftGrp_" + os.path.basename(footnoteRel.linkrole)
if typePrefix not in namespacePrefixes:
namespacePrefixes[footnoteRel.arcrole] = typePrefix
if groupPrefix not in namespacePrefixes:
namespacePrefixes[footnoteRel.linkrole] = groupPrefix
# compile footnotes and relationships
'''
factRelationships = []
factFootnotes = []
for rel in modelXbrl.relationshipSet(modelXbrl, "XBRL-footnotes").modelRelationships:
oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole}
factRelationships.append(oimRel)
oimRel["fromIds"] = [obj.id if obj.id
else elementChildSequence(obj)
for obj in rel.fromModelObjects]
oimRel["toIds"] = [obj.id if obj.id
else elementChildSequence(obj)
for obj in rel.toModelObjects]
_order = rel.arcElement.get("order")
if _order is not None:
oimRel["order"] = _order
for obj in rel.toModelObjects:
if isinstance(obj, ModelResource): # footnote
oimFootnote = {"role": obj.role,
"id": obj.id if obj.id
else elementChildSequence(obj),
# value needs work for html elements and for inline footnotes
"value": xmlstring(obj, stripXmlns=True)}
if obj.xmlLang:
oimFootnote["lang"] = obj.xmlLang
factFootnotes.append(oimFootnote)
oimFootnote
'''
dtsReferences = set()
baseUrl = modelXbrl.modelDocument.uri.partition("#")[0]
for doc,ref in sorted(modelXbrl.modelDocument.referencesDocument.items(),
key=lambda _item:_item[0].uri):
if ref.referringModelObject.qname in SCHEMA_LB_REFS:
dtsReferences.add(relativeUri(baseUrl,doc.uri))
for refType in ("role", "arcrole"):
for refElt in sorted(modelXbrl.modelDocument.xmlRootElement.iterchildren(
"{{http://www.xbrl.org/2003/linkbase}}{}Ref".format(refType)),
key=lambda elt:elt.get(refType+"URI")
):
dtsReferences.add(refElt.get("{http://www.w3.org/1999/xlink}href").partition("#")[0])
dtsReferences = sorted(dtsReferences) # turn into list
footnoteFacts = set()
def factFootnotes(fact, oimFact=None):
footnotes = []
oimLinks = {}
for footnoteRel in footnotesRelationshipSet.fromModelObject(fact):
toObj = footnoteRel.toModelObject
# json
typePrefix = namespacePrefixes[footnoteRel.arcrole]
groupPrefix = namespacePrefixes[footnoteRel.linkrole]
oimLinks.setdefault(typePrefix,{}).setdefault(groupPrefix,[]).append(
toObj.id if toObj.id else "f{}".format(toObj.objectIndex))
# csv/XL
footnote = OrderedDict((("group", footnoteRel.linkrole),
("footnoteType", footnoteRel.arcrole)))
footnotes.append(footnote)
if isCSVorXL:
footnote["factId"] = fact.id if fact.id else "f{}".format(fact.objectIndex)
if isinstance(toObj, ModelFact):
footnote["factRef"] = toObj.id if toObj.id else "f{}".format(toObj.objectIndex)
else:
footnote["footnote"] = toObj.viewText()
if toObj.xmlLang:
footnote["language"] = toObj.xmlLang
footnoteFacts.add(toObj)
if isJSON and oimLinks:
oimFact["links"] = OrderedDict((
(typePrefix, OrderedDict((
(groupPrefix, idList)
for groupPrefix, idList in sorted(groups.items())
)))
for typePrefix, groups in sorted(oimLinks.items())
))
footnotes.sort(key=lambda f:(f["group"],f.get("factId",f.get("factRef")),f.get("language")))
return footnotes
def factAspects(fact):
oimFact = OrderedDict()
aspects = OrderedDict()
if hasId and fact.id:
if fact.isTuple:
oimFact["tupleId"] = fact.id
else:
oimFact["id"] = fact.id
elif (fact.isTuple or
footnotesRelationshipSet.toModelObject(fact) or
(isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))):
oimFact["id"] = "f{}".format(fact.objectIndex)
parent = fact.getparent()
concept = fact.concept
aspects[str(qnOimConceptAspect)] = oimValue(concept.qname)
_csvType = "Value"
if not fact.isTuple:
if concept is not None:
_baseXsdType = concept.baseXsdType
if _baseXsdType == "XBRLI_DATEUNION":
if getattr(fact.xValue, "dateOnly", False):
_baseXsdType = "date"
else:
_baseXsdType = "dateTime"
_csvType = baseTypes.get(_baseXsdType,_baseXsdType) + "Value"
if concept.baseXbrliType in ("stringItemType", "normalizedStringItemType") and fact.xmlLang:
aspects[str(qnOimLangAspect)] = fact.xmlLang
if fact.isItem:
if fact.isNil:
_value = None
else:
_inferredDecimals = inferredDecimals(fact)
_value = oimValue(fact.xValue, _inferredDecimals)
oimFact["value"] = _value
if fact.concept is not None and fact.concept.isNumeric:
_numValue = fact.xValue
if isinstance(_numValue, Decimal) and not isinf(_numValue) and not isnan(_numValue):
if _numValue == _numValue.to_integral():
_numValue = int(_numValue)
else:
_numValue = float(_numValue)
if not fact.isNil:
if isinf(_inferredDecimals):
if isJSON: _accuracy = "infinity"
elif isCSVorXL: _accuracy = "INF"
else:
_accuracy = _inferredDecimals
oimFact["accuracy"] = _accuracy
oimFact["aspects"] = aspects
cntx = fact.context
if cntx is not None:
if cntx.entityIdentifierElement is not None:
aspects[str(qnOimEntityAspect)] = oimValue(qname(*cntx.entityIdentifier))
if cntx.period is not None:
aspects.update(oimPeriodValue(cntx))
for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]):
if dim.isExplicit:
dimVal = oimValue(dim.memberQname)
else: # typed
if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1"):
dimVal = None
else:
dimVal = dim.typedMember.stringValue
aspects[str(dim.dimensionQname)] = dimVal
unit = fact.unit
if unit is not None:
_mMul, _mDiv = unit.measures
_sMul = '*'.join(oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m)))
if _mDiv:
_sDiv = '*'.join(oimValue(m) for m in sorted(_mDiv, key=lambda m: oimValue(m)))
if len(_mDiv) > 1:
if len(_mMul) > 1:
_sUnit = "({})/({})".format(_sMul,_sDiv)
else:
_sUnit = "{}/({})".format(_sMul,_sDiv)
else:
if len(_mMul) > 1:
_sUnit = "({})/{}".format(_sMul,_sDiv)
else:
_sUnit = "{}/{}".format(_sMul,_sDiv)
else:
_sUnit = _sMul
aspects[str(qnOimUnitAspect)] = _sUnit
# Tuples removed from xBRL-JSON
#if parent.qname != XbrlConst.qnXbrliXbrl:
# aspects[str(qnOimTupleParentAspect)] = parent.id if parent.id else "f{}".format(parent.objectIndex)
# aspects[str(qnOimTupleOrderAspect)] = elementIndex(fact)
if isJSON:
factFootnotes(fact, oimFact)
return oimFact
prefixes = OrderedDict((p,ns) for ns, p in sorted(namespacePrefixes.items(),
key=lambda item: item[1]))
if isJSON:
# save JSON
oimReport = OrderedDict() # top level of oim json output
oimReport["documentInfo"] = oimDocInfo = OrderedDict()
oimReport["facts"] = oimFacts = OrderedDict()
oimDocInfo["documentType"] = nsOim + "/xbrl-json"
oimDocInfo["features"] = oimFeatures = OrderedDict()
oimDocInfo["prefixes"] = prefixes
oimDocInfo["taxonomy"] = dtsReferences
oimFeatures["xbrl:canonicalValues"] = True
def saveJsonFacts(facts, oimFacts, parentFact):
for fact in facts:
oimFact = factAspects(fact)
id = fact.id if fact.id else "f{}".format(fact.objectIndex)
oimFacts[id] = oimFact
if fact.modelTupleFacts:
saveJsonFacts(fact.modelTupleFacts, oimFacts, fact)
saveJsonFacts(modelXbrl.facts, oimFacts, None)
# add footnotes as pseudo facts
for ftObj in footnoteFacts:
ftId = ftObj.id if ftObj.id else "f{}".format(ftObj.objectIndex)
oimFacts[ftId] = oimFact = OrderedDict()
oimFact["value"] = ftObj.viewText()
oimFact["aspects"] = OrderedDict((("concept", "xbrl:note"),
("noteId", ftId)))
if ftObj.xmlLang:
oimFact["aspects"]["language"] = ftObj.xmlLang.lower()
if outputZip:
fh = io.StringIO()
else:
fh = open(oimFile, "w", encoding="utf-8")
fh.write(json.dumps(oimReport, indent=1))
if outputZip:
fh.seek(0)
outputZip.writestr(os.path.basename(oimFile),fh.read())
fh.close()
elif isCSVorXL:
# save CSV
aspectQnCol = {}
aspectsHeader = []
factsColumns = []
def addAspectQnCol(aspectQn):
aspectQnCol[str(aspectQn)] = len(aspectsHeader)
_aspectQn = oimValue(aspectQn)
aspectsHeader.append(_aspectQn)
_colName = _aspectQn.replace("xbrl:", "")
_colDataType = {"id": "Name",
"concept": "QName",
"value": "string",
"accuracy": "decimal",
"entity": "QName",
"periodStart": "dateTime",
"periodEnd": "dateTime",
"unit": "string",
"tupleId": "Name",
"tupleParent": "Name",
"tupleOrder": "integer"
}.get(_colName, "string")
col = OrderedDict((("name", _colName),
("datatype", _colDataType)))
if _aspectQn == "value":
col["http://xbrl.org/YYYY/model#simpleFactAspects"] = {}
elif _aspectQn == "tupleId":
col["http://xbrl.org/YYYY/model#tupleFactAspects"] = {}
col["http://xbrl.org/YYYY/model#tupleReferenceId"] = "true"
else:
col["http://xbrl.org/YYYY/model#columnAspect"] = _aspectQn
factsColumns.append(col)
# pre-ordered aspect columns
#if hasId:
# addAspectQnCol("id")
addAspectQnCol(qnOimConceptAspect)
addAspectQnCol("value")
if hasNumeric:
addAspectQnCol("accuracy")
if hasTuple:
addAspectQnCol("tupleId")
addAspectQnCol(qnOimTupleParentAspect)
addAspectQnCol(qnOimTupleOrderAspect)
if qnOimEntityAspect in aspectsDefined:
addAspectQnCol(qnOimEntityAspect)
if qnOimPeriodStartAspect in aspectsDefined:
addAspectQnCol(qnOimPeriodStartAspect)
addAspectQnCol(qnOimPeriodEndAspect)
if qnOimUnitAspect in aspectsDefined:
addAspectQnCol(qnOimUnitAspect)
for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)):
if aspectQn.namespaceURI != nsOim:
addAspectQnCol(aspectQn)
def aspectCols(fact):
cols = [None for i in range(len(aspectsHeader))]
def setColValues(aspects):
for aspectQn, aspectValue in aspects.items():
if isinstance(aspectValue, dict):
setColValues(aspectValue)
elif aspectQn in aspectQnCol:
if aspectValue is None:
_aspectValue = "#nil"
elif aspectValue == "":
_aspectValue = "#empty"
elif isinstance(aspectValue, str) and aspectValue.startswith("#"):
_aspectValue = "#" + aspectValue
else:
_aspectValue = aspectValue
cols[aspectQnCol[aspectQn]] = _aspectValue
setColValues(factAspects(fact))
return cols
# metadata
csvTables = []
csvMetadata = OrderedDict((("@context", "http://www.w3.org/ns/csvw"),
("http://xbrl.org/YYYY/model#metadata",
OrderedDict((("documentType", "http://xbrl.org/YYYY/xbrl-csv"),
("dtsReferences", dtsReferences),
("prefixes", prefixes)))),
("tables", csvTables)))
_open = _writerow = _close = None
_tableinfo = {}
if isCSV:
if oimFile.endswith("-facts.csv"): # strip -facts.csv if a prior -facts.csv file was chosen
_baseURL = oimFile[:-10]
elif oimFile.endswith(".csv"):
_baseURL = oimFile[:-4]
else:
_baseURL = oimFile
_csvinfo = {} # open file, writer
def _open(filesuffix, tabname):
_filename = _tableinfo["url"] = _baseURL + filesuffix
_csvinfo["file"] = open(_filename, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
_csvinfo["writer"] = csv.writer(_csvinfo["file"], dialect="excel")
def _writerow(row, header=False):
_csvinfo["writer"].writerow(row)
def _close():
_csvinfo["file"].close()
_csvinfo.clear()
elif isXL:
headerWidths = {"href": 100, "xbrl:concept": 70, "accuracy": 8, "language": 9, "URI": 80,
"value": 60,
"group": 60, "footnoteType": 40, "footnote": 70, "column": 20,
'conceptAspect': 40, 'tuple': 20, 'simpleFact': 20}
from openpyxl import Workbook
from openpyxl.writer.write_only import WriteOnlyCell
from openpyxl.styles import Font, PatternFill, Border, Alignment, Color, fills, Side
from openpyxl.worksheet.dimensions import ColumnDimension
hdrCellFill = PatternFill(patternType=fills.FILL_SOLID, fgColor=Color("00FFBF5F")) # Excel's light orange fill color = 00FF990
workbook = Workbook()
# remove pre-existing worksheets
while len(workbook.worksheets)>0:
workbook.remove_sheet(workbook.worksheets[0])
_xlinfo = {} # open file, writer
def _open(filesuffix, tabname):
_tableinfo["url"] = tabname
_xlinfo["ws"] = workbook.create_sheet(title=tabname)
def _writerow(rowvalues, header=False):
row = []
_ws = _xlinfo["ws"]
for i, v in enumerate(rowvalues):
cell = WriteOnlyCell(_ws, value=v)
if header:
cell.fill = hdrCellFill
cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True)
colLetter = chr(ord('A') + i)
_ws.column_dimensions[colLetter] = ColumnDimension(_ws, customWidth=True)
_ws.column_dimensions[colLetter].width = headerWidths.get(v, 20)
else:
cell.alignment = Alignment(horizontal="right" if isinstance(v, _NUM_TYPES)
else "center" if isinstance(v, bool)
else "left",
vertical="top",
wrap_text=isinstance(v, str))
row.append(cell)
_ws.append(row)
def _close():
_xlinfo.clear()
# save facts
_open("-facts.csv", "facts")
_writerow(aspectsHeader, header=True)
def saveCSVfacts(facts):
for fact in facts:
_writerow(aspectCols(fact))
saveCSVfacts(fact.modelTupleFacts)
saveCSVfacts(modelXbrl.facts)
_close()
factsTableSchema = OrderedDict((("columns",factsColumns),))
csvTables.append(OrderedDict((("url",_tableinfo["url"]),
("http://xbrl.org/YYYY/model#tableType", "fact"),
("tableSchema",factsTableSchema))))
# save footnotes
if footnotesRelationshipSet.modelRelationships:
_open("-footnotes.csv", "footnotes")
cols = ("group", "footnoteType", "factId", "factRef", "footnote", "language")
_writerow(cols, header=True)
def saveCSVfootnotes(facts):
for fact in facts:
for _footnote in factFootnotes(fact):
_writerow(tuple((_footnote.get(col,"") for col in cols)))
saveCSVfootnotes(fact.modelTupleFacts)
saveCSVfootnotes(modelXbrl.facts)
_close()
footnoteTableSchema = OrderedDict((("columns",[OrderedDict((("name","group"),("datatype","anyURI"))),
OrderedDict((("name","footnoteType"),("datatype","Name"))),
OrderedDict((("name","factId"),("datatype","Name"))),
OrderedDict((("name","factRef"),("datatype","Name"))),
OrderedDict((("name","footnote"),("datatype","string"))),
OrderedDict((("name","language"),("datatype","language")))]),))
csvTables.append(OrderedDict((("url",_tableinfo["url"]),
("http://xbrl.org/YYYY/model#tableType", "footnote"),
("tableSchema",footnoteTableSchema))))
# save metadata
if isCSV:
with open(_baseURL + "-metadata.json", "w", encoding="utf-8") as fh:
fh.write(json.dumps(csvMetadata, ensure_ascii=False, indent=1, sort_keys=False))
elif isXL:
_open(None, "metadata")
hasColumnAspect = hasSimpleFact = hasTupleFact = False
for table in csvTables:
tablename = table["url"]
for column in table["tableSchema"]["columns"]:
if "http://xbrl.org/YYYY/model#columnAspect" in column:
hasColumnAspect = True
if "http://xbrl.org/YYYY/model#simpleFactAspects" in column:
hasSimpleFact = True
if "http://xbrl.org/YYYY/model#tupleAspects" in column:
hasTupleFact = True
metadataCols = ["table", "column", "datatype"]
if hasColumnAspect:
metadataCols.append("columnAspect")
if hasSimpleFact:
metadataCols.append("simpleFact")
if hasTupleFact:
metadataCols.append("tuple")
_writerow(metadataCols, header=True)
for table in csvTables:
tablename = table["url"]
for column in table["tableSchema"]["columns"]:
row = [tablename, column["name"], column["datatype"]]
if hasColumnAspect:
colAspect = column.get("http://xbrl.org/YYYY/model#columnAspect")
if isinstance(colAspect, str):
row.append(colAspect)
elif isinstance(colAspect, dict):
row.append("\n".join("{} [{}]".format(k, ", ".join(_v for _v in v))
for k, v in dict.items()))
else:
row.append(None)
if hasSimpleFact:
row.append("\u221a" if "http://xbrl.org/YYYY/model#simpleFactAspects" in column else None)
if hasTupleFact:
row.append("\u221a" if "http://xbrl.org/YYYY/model#tupleAspects" in column else None)
_writerow(row)
_close()
if isXL:
workbook.save(oimFile)
def saveLoadableOIM(modelXbrl, oimFile, outputZip=None):
isJSON = oimFile.endswith(".json")
isCSV = oimFile.endswith(".csv")
isXL = oimFile.endswith(".xlsx")
isCSVorXL = isCSV or isXL
if not isJSON and not isCSVorXL:
return
namespacePrefixes = {nsOim: "xbrl"}
prefixNamespaces = {}
linkTypeAliases = {}
groupAliases = {}
linkTypePrefixes = {}
linkTypeUris = {}
linkGroupPrefixes = {}
linkGroupUris = {}
def compileQname(qname):
if qname.namespaceURI not in namespacePrefixes:
namespacePrefixes[qname.namespaceURI] = qname.prefix or ""
aspectsDefined = {qnOimConceptAspect, qnOimEntityAspect, qnOimPeriodAspect}
def oimValue(object, decimals=None):
if isinstance(object, QName):
if object.namespaceURI not in namespacePrefixes:
if object.prefix:
namespacePrefixes[object.namespaceURI] = object.prefix
else:
_prefix = "_{}".format(
sum(1 for p in namespacePrefixes if p.startswith("_")))
namespacePrefixes[object.namespaceURI] = _prefix
return "{}:{}".format(namespacePrefixes[object.namespaceURI],
object.localName)
if isinstance(object, (float, Decimal)):
try:
if isinf(object):
return "-INF" if object < 0 else "INF"
elif isnan(object):
return "NaN"
else:
if isinstance(object,
Decimal) and object == object.to_integral():
object = object.quantize(ONE) # drop any .0
return "{}".format(object)
except:
return str(object)
if isinstance(object, bool):
return "true" if object else "false"
if isinstance(
object,
(DateTime, YearMonthDuration, DayTimeDuration, Time, gYearMonth,
gMonthDay, gYear, gMonth, gDay, IsoDuration, int)):
return str(object)
return object
def oimPeriodValue(cntx):
if cntx.isStartEndPeriod:
s = cntx.startDatetime
e = cntx.endDatetime
else: # instant
s = e = cntx.instantDatetime
return ({
str(qnOimPeriodAspect):
("{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}/".format(
s.year, s.month, s.day, s.hour, s.minute, s.second)
if cntx.isStartEndPeriod else "") +
"{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}".format(
e.year, e.month, e.day, e.hour, e.minute, e.second)
})
hasId = False
hasTuple = False
hasType = True
hasLang = False
hasUnits = False
hasNumeric = False
footnotesRelationshipSet = ModelRelationshipSet(modelXbrl,
"XBRL-footnotes")
#compile QNames in instance for OIM
for fact in modelXbrl.factsInInstance:
if (fact.id or fact.isTuple
or footnotesRelationshipSet.toModelObject(fact) or
(isCSVorXL and footnotesRelationshipSet.fromModelObject(fact))):
hasId = True
concept = fact.concept
if concept is not None:
if concept.isNumeric:
hasNumeric = True
if concept.baseXbrliType in ("string", "normalizedString",
"token") and fact.xmlLang:
hasLang = True
compileQname(fact.qname)
if hasattr(fact, "xValue") and isinstance(fact.xValue, QName):
compileQname(fact.xValue)
unit = fact.unit
if unit is not None:
hasUnits = True
if fact.modelTupleFacts:
hasTuple = True
if hasTuple:
modelXbrl.error("arelleOIMsaver:tuplesNotAllowed",
"Tuples are not allowed in an OIM document",
modelObject=modelXbrl)
return
entitySchemePrefixes = {}
for cntx in modelXbrl.contexts.values():
if cntx.entityIdentifierElement is not None:
scheme = cntx.entityIdentifier[0]
if scheme not in entitySchemePrefixes:
if not entitySchemePrefixes: # first one is just scheme
if scheme == "http://www.sec.gov/CIK":
_schemePrefix = "cik"
elif scheme == "http://standard.iso.org/iso/17442":
_schemePrefix = "lei"
else:
_schemePrefix = "scheme"
else:
_schemePrefix = "scheme{}".format(
len(entitySchemePrefixes) + 1)
entitySchemePrefixes[scheme] = _schemePrefix
namespacePrefixes[scheme] = _schemePrefix
for dim in cntx.qnameDims.values():
compileQname(dim.dimensionQname)
aspectsDefined.add(dim.dimensionQname)
if dim.isExplicit:
compileQname(dim.memberQname)
for unit in modelXbrl.units.values():
if unit is not None:
for measures in unit.measures:
for measure in measures:
compileQname(measure)
if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[
XbrlConst.xbrli] != "xbrli":
namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix
if hasLang: aspectsDefined.add(qnOimLangAspect)
if hasUnits: aspectsDefined.add(qnOimUnitAspect)
for footnoteRel in footnotesRelationshipSet.modelRelationships:
typePrefix = "ftTyp_" + os.path.basename(footnoteRel.arcrole)
if footnoteRel.linkrole == XbrlConst.defaultLinkRole:
groupPrefix = "ftGrp_default"
else:
groupPrefix = "ftGrp_" + os.path.basename(footnoteRel.linkrole)
if footnoteRel.arcrole not in linkTypeAliases:
linkTypeAliases[footnoteRel.arcrole] = typePrefix
if groupPrefix not in groupAliases:
groupAliases[footnoteRel.linkrole] = groupPrefix
dtsReferences = set()
baseUrl = modelXbrl.modelDocument.uri.partition("#")[0]
for doc, ref in sorted(modelXbrl.modelDocument.referencesDocument.items(),
key=lambda _item: _item[0].uri):
if ref.referringModelObject.qname in SCHEMA_LB_REFS:
dtsReferences.add(relativeUri(baseUrl, doc.uri))
for refType in ("role", "arcrole"):
for refElt in sorted(
modelXbrl.modelDocument.xmlRootElement.iterchildren(
"{{http://www.xbrl.org/2003/linkbase}}{}Ref".format(
refType)),
key=lambda elt: elt.get(refType + "URI")):
dtsReferences.add(
refElt.get("{http://www.w3.org/1999/xlink}href").partition("#")
[0])
dtsReferences = sorted(dtsReferences) # turn into list
footnoteFacts = set()
def factFootnotes(fact, oimFact=None, csvLinks=None):
footnotes = []
if isJSON:
oimLinks = {}
elif isCSVorXL:
oimLinks = csvLinks
for footnoteRel in footnotesRelationshipSet.fromModelObject(fact):
srcId = fact.id if fact.id else "f{}".format(fact.objectIndex)
toObj = footnoteRel.toModelObject
# json
typePrefix = linkTypeAliases[footnoteRel.arcrole]
groupPrefix = groupAliases[footnoteRel.linkrole]
if typePrefix not in oimLinks:
oimLinks[typePrefix] = OrderedDict()
_link = oimLinks[typePrefix]
if groupPrefix not in _link:
if isJSON:
_link[groupPrefix] = []
elif isCSVorXL:
_link[groupPrefix] = OrderedDict()
if isJSON:
tgtIdList = _link[groupPrefix]
elif isCSVorXL:
tgtIdList = _link[groupPrefix].setdefault(srcId, [])
tgtId = toObj.id if toObj.id else "f{}".format(toObj.objectIndex)
tgtIdList.append(tgtId)
footnote = OrderedDict((("group", footnoteRel.linkrole),
("footnoteType", footnoteRel.arcrole)))
if isinstance(toObj, ModelFact):
footnote["factRef"] = tgtId
else: # text footnotes
footnote["id"] = tgtId
footnote["footnote"] = toObj.viewText()
if toObj.xmlLang:
footnote["language"] = toObj.xmlLang
footnoteFacts.add(toObj)
footnotes.append(footnote)
if oimLinks:
_links = OrderedDict(
((typePrefix,
OrderedDict(
((groupPrefix, idList)
for groupPrefix, idList in sorted(groups.items()))))
for typePrefix, groups in sorted(oimLinks.items())))
if isJSON:
oimFact["links"] = _links
return footnotes
def factAspects(fact):
oimFact = OrderedDict()
aspects = OrderedDict()
if isCSVorXL:
oimFact["id"] = fact.id or "f{}".format(fact.objectIndex)
parent = fact.getparent()
concept = fact.concept
aspects[str(qnOimConceptAspect)] = oimValue(concept.qname)
if concept is not None:
if concept.type.isOimTextFactType and fact.xmlLang:
aspects[str(qnOimLangAspect)] = fact.xmlLang
if fact.isItem:
if fact.isNil:
_value = None
else:
_inferredDecimals = inferredDecimals(fact)
_value = oimValue(fact.xValue, _inferredDecimals)
oimFact["value"] = _value
if fact.concept is not None and fact.concept.isNumeric:
_numValue = fact.xValue
if isinstance(_numValue, Decimal) and not isinf(
_numValue) and not isnan(_numValue):
if _numValue == _numValue.to_integral():
_numValue = int(_numValue)
else:
_numValue = float(_numValue)
if not fact.isNil:
if not isinf(
_inferredDecimals): # accuracy omitted if infinite
oimFact["decimals"] = _inferredDecimals
oimFact["dimensions"] = aspects
cntx = fact.context
if cntx is not None:
if cntx.entityIdentifierElement is not None and cntx.entityIdentifier != ENTITY_NA_QNAME:
aspects[str(qnOimEntityAspect)] = oimValue(
qname(*cntx.entityIdentifier))
if cntx.period is not None and not cntx.isForeverPeriod:
aspects.update(oimPeriodValue(cntx))
for _qn, dim in sorted(cntx.qnameDims.items(),
key=lambda item: item[0]):
if dim.isExplicit:
dimVal = oimValue(dim.memberQname)
else: # typed
if dim.typedMember.get(
"{http://www.w3.org/2001/XMLSchema-instance}nil"
) in ("true", "1"):
dimVal = None
else:
dimVal = dim.typedMember.stringValue
aspects[str(dim.dimensionQname)] = dimVal
unit = fact.unit
if unit is not None:
_mMul, _mDiv = unit.measures
_sMul = '*'.join(
oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m)))
if _mDiv:
_sDiv = '*'.join(
oimValue(m)
for m in sorted(_mDiv, key=lambda m: oimValue(m)))
if len(_mDiv) > 1:
if len(_mMul) > 1:
_sUnit = "({})/({})".format(_sMul, _sDiv)
else:
_sUnit = "{}/({})".format(_sMul, _sDiv)
else:
if len(_mMul) > 1:
_sUnit = "({})/{}".format(_sMul, _sDiv)
else:
_sUnit = "{}/{}".format(_sMul, _sDiv)
else:
_sUnit = _sMul
if _sUnit != "xbrli:pure":
aspects[str(qnOimUnitAspect)] = _sUnit
# Tuples removed from xBRL-JSON
#if parent.qname != XbrlConst.qnXbrliXbrl:
# aspects[str(qnOimTupleParentAspect)] = parent.id if parent.id else "f{}".format(parent.objectIndex)
# aspects[str(qnOimTupleOrderAspect)] = elementIndex(fact)
if isJSON:
factFootnotes(fact, oimFact=oimFact)
return oimFact
namespaces = OrderedDict((p, ns)
for ns, p in sorted(namespacePrefixes.items(),
key=lambda item: item[1]))
# common metadata
oimReport = OrderedDict() # top level of oim json output
oimReport["documentInfo"] = oimDocInfo = OrderedDict()
oimDocInfo["documentType"] = nsOim + ("/xbrl-json"
if isJSON else "/xbrl-csv")
if isJSON:
oimDocInfo["features"] = oimFeatures = OrderedDict()
oimDocInfo["namespaces"] = namespaces
if linkTypeAliases:
oimDocInfo["linkTypes"] = OrderedDict(
(a, u) for u, a in sorted(linkTypeAliases.items(),
key=lambda item: item[1]))
if linkTypeAliases:
oimDocInfo["linkGroups"] = OrderedDict(
(a, u)
for u, a in sorted(groupAliases.items(), key=lambda item: item[1]))
oimDocInfo["taxonomy"] = dtsReferences
if isJSON:
oimFeatures["xbrl:canonicalValues"] = True
if isJSON:
# save JSON
oimReport["facts"] = oimFacts = OrderedDict()
def saveJsonFacts(facts, oimFacts, parentFact):
for fact in facts:
oimFact = factAspects(fact)
id = fact.id if fact.id else "f{}".format(fact.objectIndex)
oimFacts[id] = oimFact
if fact.modelTupleFacts:
saveJsonFacts(fact.modelTupleFacts, oimFacts, fact)
saveJsonFacts(modelXbrl.facts, oimFacts, None)
# add footnotes as pseudo facts
for ftObj in footnoteFacts:
ftId = ftObj.id if ftObj.id else "f{}".format(ftObj.objectIndex)
oimFacts[ftId] = oimFact = OrderedDict()
oimFact["value"] = ftObj.viewText()
oimFact["dimensions"] = OrderedDict(
(("concept", "xbrl:note"), ("noteId", ftId)))
if ftObj.xmlLang:
oimFact["dimensions"]["language"] = ftObj.xmlLang.lower()
if outputZip:
fh = io.StringIO()
else:
fh = open(oimFile, "w", encoding="utf-8")
fh.write(json.dumps(oimReport, indent=1))
if outputZip:
fh.seek(0)
outputZip.writestr(os.path.basename(oimFile), fh.read())
fh.close()
elif isCSVorXL:
# save CSV
oimReport["tables"] = oimTables = OrderedDict()
oimReport["tableTemplates"] = csvTableTemplates = OrderedDict()
oimTables["facts"] = csvTable = OrderedDict()
csvTable["template"] = "facts"
csvTable["url"] = "tbd"
csvTableTemplates["facts"] = csvTableTemplate = OrderedDict()
csvTableTemplate["rowIdColumn"] = "id"
csvTableTemplate["dimensions"] = csvTableDimensions = OrderedDict()
csvTableTemplate["columns"] = csvFactColumns = OrderedDict()
if footnotesRelationshipSet.modelRelationships:
csvLinks = OrderedDict()
oimReport["links"] = csvLinks
aspectQnCol = {}
aspectsHeader = []
factsColumns = []
def addAspectQnCol(aspectQn):
colQName = str(aspectQn).replace("xbrl:", "")
colNCName = colQName.replace(":", "_")
if colNCName not in aspectQnCol:
aspectQnCol[colNCName] = len(aspectsHeader)
aspectsHeader.append(colNCName)
if colNCName == "value":
csvFactColumns[colNCName] = col = OrderedDict()
col["dimensions"] = OrderedDict()
elif colNCName == "id":
csvFactColumns[colNCName] = {} # empty object
elif colNCName == "decimals":
csvFactColumns[colNCName] = {} # empty object
csvTableTemplate[colQName] = "$" + colNCName
else:
csvFactColumns[colNCName] = {} # empty object
csvTableDimensions[colQName] = "$" + colNCName
# pre-ordered aspect columns
#if hasId:
addAspectQnCol("id")
addAspectQnCol(qnOimConceptAspect)
if hasNumeric:
addAspectQnCol("decimals")
if qnOimEntityAspect in aspectsDefined:
addAspectQnCol(qnOimEntityAspect)
if qnOimPeriodAspect in aspectsDefined:
addAspectQnCol(qnOimPeriodAspect)
if qnOimUnitAspect in aspectsDefined:
addAspectQnCol(qnOimUnitAspect)
for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)):
if aspectQn.namespaceURI != nsOim:
addAspectQnCol(aspectQn)
addAspectQnCol("value")
def aspectCols(fact):
cols = [None for i in range(len(aspectsHeader))]
def setColValues(aspects):
for aspectQn, aspectValue in aspects.items():
colQName = str(aspectQn).replace("xbrl:", "")
colNCName = colQName.replace(":", "_")
if isinstance(aspectValue, dict):
setColValues(aspectValue)
elif colNCName in aspectQnCol:
if aspectValue is None:
_aspectValue = "#nil"
elif aspectValue == "":
_aspectValue = "#empty"
elif isinstance(aspectValue,
str) and aspectValue.startswith("#"):
_aspectValue = "#" + aspectValue
else:
_aspectValue = aspectValue
cols[aspectQnCol[colNCName]] = _aspectValue
setColValues(factAspects(fact))
return cols
# metadata
_open = _writerow = _close = None
if isCSV:
if oimFile.endswith(
"-facts.csv"
): # strip -facts.csv if a prior -facts.csv file was chosen
_baseURL = oimFile[:-10]
elif oimFile.endswith(".csv"):
_baseURL = oimFile[:-4]
else:
_baseURL = oimFile
_csvinfo = {} # open file, writer
def _open(filesuffix, tabname, csvTable=None):
_filename = _baseURL + filesuffix
if csvTable is not None:
csvTable["url"] = os.path.basename(
_filename) # located in same directory with metadata
_csvinfo["file"] = open(_filename,
csvOpenMode,
newline=csvOpenNewline,
encoding='utf-8-sig')
_csvinfo["writer"] = csv.writer(_csvinfo["file"],
dialect="excel")
def _writerow(row, header=False):
_csvinfo["writer"].writerow(row)
def _close():
_csvinfo["file"].close()
_csvinfo.clear()
elif isXL:
headerWidths = {
"concept": 40,
"decimals": 8,
"language": 9,
"value": 50,
"entity": 20,
"period": 20,
"unit": 20,
"metadata": 100
}
from openpyxl import Workbook
from openpyxl.cell.cell import WriteOnlyCell
from openpyxl.styles import Font, PatternFill, Border, Alignment, Color, fills, Side
from openpyxl.worksheet.dimensions import ColumnDimension
hdrCellFill = PatternFill(
patternType=fills.FILL_SOLID, fgColor=Color(
"00FFBF5F")) # Excel's light orange fill color = 00FF990
workbook = Workbook()
# remove pre-existing worksheets
while len(workbook.worksheets) > 0:
workbook.remove(workbook.worksheets[0])
_xlinfo = {} # open file, writer
def _open(filesuffix, tabname, csvTable=None):
if csvTable is not None:
csvTable["url"] = tabname + "!"
_xlinfo["ws"] = workbook.create_sheet(title=tabname)
def _writerow(rowvalues, header=False):
row = []
_ws = _xlinfo["ws"]
for i, v in enumerate(rowvalues):
cell = WriteOnlyCell(_ws, value=v)
if header:
cell.fill = hdrCellFill
cell.alignment = Alignment(horizontal="center",
vertical="center",
wrap_text=True)
colLetter = chr(ord('A') + i)
_ws.column_dimensions[colLetter] = ColumnDimension(
_ws, customWidth=True)
_ws.column_dimensions[
colLetter].width = headerWidths.get(v, 40)
else:
cell.alignment = Alignment(
horizontal="right" if isinstance(v, _NUM_TYPES)
else "center" if isinstance(v, bool) else "left",
vertical="top",
wrap_text=isinstance(v, str))
row.append(cell)
_ws.append(row)
def _close():
_xlinfo.clear()
# save facts
_open("-facts.csv", "facts", csvTable)
_writerow(aspectsHeader, header=True)
def saveCSVfacts(facts):
for fact in facts:
_writerow(aspectCols(fact))
saveCSVfacts(fact.modelTupleFacts)
saveCSVfacts(modelXbrl.facts)
_close()
# save footnotes
if footnotesRelationshipSet.modelRelationships:
footnotes = sorted(
(footnote for fact in modelXbrl.facts
for footnote in factFootnotes(fact, csvLinks=csvLinks)),
key=lambda footnote: footnote["id"])
if footnotes: # text footnotes
oimTables["footnotes"] = csvFtTable = OrderedDict()
csvFtTable["url"] = "tbd"
csvFtTable[
"tableDimensions"] = csvFtTableDimensions = OrderedDict()
csvFtTable["factColumns"] = csvFtFactColumns = OrderedDict()
csvFtFactColumns["footnote"] = csvFtValCol = OrderedDict()
csvFtValCol["id"] = "$id"
csvFtValCol["noteId"] = "$id"
csvFtValCol["concept"] = "xbrl:note"
csvFtValCol["language"] = "$language"
_open("-footnotes.csv", "footnotes", csvFtTable)
cols = ("id", "footnote", "language")
_writerow(cols, header=True)
for footnote in footnotes:
_writerow(tuple((footnote.get(col, "") for col in cols)))
_close()
# save metadata
if isCSV:
with open(_baseURL + "-metadata.json", "w",
encoding="utf-8") as fh:
fh.write(
json.dumps(oimReport,
ensure_ascii=False,
indent=2,
sort_keys=False))
elif isXL:
_open(None, "metadata")
_writerow(["metadata"], header=True)
_writerow([
json.dumps(oimReport,
ensure_ascii=False,
indent=1,
sort_keys=False)
])
_close()
if isXL:
workbook.save(oimFile)
def validateTestcase(self, testcase):
self.modelXbrl.info("info", "Testcase", modelDocument=testcase)
self.modelXbrl.viewModelObject(testcase.objectId())
if hasattr(testcase, "testcaseVariations"):
for modelTestcaseVariation in testcase.testcaseVariations:
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
# is this a versioning report?
resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
resultIsTaxonomyPackage = modelTestcaseVariation.resultIsTaxonomyPackage
formulaOutputInstance = None
inputDTSes = defaultdict(list)
baseForElement = testcase.baseForElement(modelTestcaseVariation)
# try to load instance document
self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s - %(description)s"),
modelObject=modelTestcaseVariation,
id=modelTestcaseVariation.id,
name=modelTestcaseVariation.name,
expected=modelTestcaseVariation.expected,
description=modelTestcaseVariation.description)
errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY
parameters = modelTestcaseVariation.parameters.copy()
for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
if isinstance(readMeFirstUri,tuple):
# dtsName is for formula instances, but is from/to dts if versioning
dtsName, readMeFirstUri = readMeFirstUri
elif resultIsVersioningReport:
if inputDTSes: dtsName = "to"
else: dtsName = "from"
else:
dtsName = None
if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
if dtsName in inputDTSes:
dtsName = inputDTSes[dtsName]
else:
modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager,
Type.DTSENTRIES,
self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
isEntry=True,
errorCaptureLevel=errorCaptureLevel)
DTSdoc = modelXbrl.modelDocument
DTSdoc.inDTS = True
doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement)
if doc is not None:
DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement) #fake import
doc.inDTS = True
elif resultIsTaxonomyPackage:
from arelle import PackageManager, PrototypeInstanceObject
dtsName = readMeFirstUri
modelXbrl = PrototypeInstanceObject.XbrlPrototype(self.modelXbrl.modelManager, readMeFirstUri)
PackageManager.packageInfo(self.modelXbrl.modelManager.cntlr, readMeFirstUri, reload=True, errors=modelXbrl.errors)
else: # not a multi-schemaRef versioning report
if self.useFileSource.isArchive:
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
readMeFirstUri,
_("validating"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
else: # need own file source, may need instance discovery
filesource = FileSource.FileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr)
if filesource and not filesource.selection and filesource.isArchive:
for _archiveFile in filesource.dir: # find instance document in archive
filesource.select(_archiveFile)
if ModelDocument.Type.identify(filesource, filesource.url) in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL):
break # use this selection
modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager,
filesource,
_("validating"),
base=baseForElement,
errorCaptureLevel=errorCaptureLevel)
modelXbrl.isTestcaseVariation = True
if modelXbrl.modelDocument is None:
modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s document not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
self.determineNotLoadedTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif resultIsVersioningReport or resultIsTaxonomyPackage:
inputDTSes[dtsName] = modelXbrl
elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif testcase.type == Type.REGISTRYTESTCASE:
self.instValidator.validate(modelXbrl) # required to set up dimensions, etc
self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id,
modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
self.instValidator.close()
modelXbrl.close()
else:
inputDTSes[dtsName].append(modelXbrl)
# validate except for formulas
_hasFormulae = modelXbrl.hasFormulae
modelXbrl.hasFormulae = False
try:
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation)
self.instValidator.validate(modelXbrl, parameters)
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
modelXbrl.hasFormulae = _hasFormulae
if resultIsVersioningReport and modelXbrl.modelDocument:
versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
modelTestcaseVariation.versioningReportUri, baseForElement)
if os.path.exists(versReportFile): #validate existing
modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT:
ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors)
modelVersReport.close()
elif len(inputDTSes) == 2:
ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
versReportFile, inputDTSes["from"], inputDTSes["to"])
modelTestcaseVariation.status = "generated"
else:
modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s DTSes not loaded, unable to generate versioning report: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
modelTestcaseVariation.status = "failed"
for inputDTS in inputDTSes.values():
inputDTS.close()
del inputDTSes # dereference
elif resultIsTaxonomyPackage:
self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
modelXbrl.close()
elif inputDTSes:
# validate schema, linkbase, or instance
modelXbrl = inputDTSes[None][0]
for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters
if dtsName: # named instance
parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes)
elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents
parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula
if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable:
RenderingEvaluator.init(modelXbrl)
if modelXbrl.hasFormulae:
try:
# validate only formulae
self.instValidator.parameters = parameters
ValidateFormula.validate(self.instValidator)
except Exception as err:
modelXbrl.error("exception:" + type(err).__name__,
_("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"),
modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset:
for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"):
pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri)
infoset = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultInfosetUri,
_("loading result infoset"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if infoset.modelDocument is None:
modelXbrl.error("arelle:notLoaded",
_("Testcase %(id)s %(name)s result infoset not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
modelTestcaseVariation.status = "result infoset not loadable"
else: # check infoset
ValidateInfoset.validate(self.instValidator, modelXbrl, infoset)
infoset.close()
if modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset:
# diff (or generate) table infoset
resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement)
if not any(alternativeValidation(modelXbrl, resultTableUri)
for alternativeValidation in pluginClassMethods("Validate.TableInfoset")):
ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files
self.instValidator.close()
extraErrors = []
for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"):
pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors)
self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status
if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual):
# if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it
modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost)
self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors)
if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance
formulaOutputInstance = modelXbrl.formulaOutputInstance
modelXbrl.formulaOutputInstance = None # prevent it from being closed now
self.instValidator.close()
compareIxResultInstance = getattr(modelXbrl, "extractedInlineInstance", False) and modelTestcaseVariation.resultXbrlInstanceUri
if compareIxResultInstance:
formulaOutputInstance = modelXbrl # compare modelXbrl to generated output instance
errMsgPrefix = "ix"
else: # delete input instances before formula output comparision
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
errMsgPrefix = "formula"
if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager,
modelTestcaseVariation.resultXbrlInstanceUri,
_("loading expected result XBRL instance"),
base=baseForElement,
useFileSource=self.useFileSource,
errorCaptureLevel=errorCaptureLevel)
if expectedInstance.modelDocument is None:
self.modelXbrl.error("{}:expectedResultNotLoaded".format(errMsgPrefix),
_("Testcase %(id)s %(name)s expected result instance not loaded: %(file)s"),
modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name,
file=os.path.basename(modelTestcaseVariation.resultXbrlInstanceUri),
messageCodes=("formula:expectedResultNotLoaded","ix:expectedResultNotLoaded"))
modelTestcaseVariation.status = "result not loadable"
else: # compare facts
if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
formulaOutputInstance.error("{}:resultFactCounts".format(errMsgPrefix),
_("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"),
modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts),
expectedFacts=len(expectedInstance.facts),
messageCodes=("formula:resultFactCounts","ix:resultFactCounts"))
else:
formulaOutputFootnotesRelSet = ModelRelationshipSet(formulaOutputInstance, "XBRL-footnotes")
expectedFootnotesRelSet = ModelRelationshipSet(expectedInstance, "XBRL-footnotes")
def factFootnotes(fact, footnotesRelSet):
footnotes = []
footnoteRels = footnotesRelSet.fromModelObject(fact)
if footnoteRels:
# most process rels in same order between two instances, use labels to sort
for i, footnoteRel in enumerate(sorted(footnoteRels,
key=lambda r: (r.fromLabel,r.toLabel))):
modelObject = footnoteRel.toModelObject
if isinstance(modelObject, ModelResource):
xml = modelObject.viewText().strip()
footnotes.append("Footnote {}: {}".format(
i+1, # compare footnote with HTML serialized
xml,
#re.sub(r'\s+', ' ', collapseWhitespace(modelObject.stringValue))
))
elif isinstance(modelObject, ModelFact):
footnotes.append("Footnoted fact {}: {} context: {} value: {}".format(
i+1,
modelObject.qname,
modelObject.contextID,
collapseWhitespace(modelObject.value)))
return footnotes
for expectedInstanceFact in expectedInstance.facts:
unmatchedFactsStack = []
formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True)
if formulaOutputFact is None:
if unmatchedFactsStack: # get missing nested tuple fact, if possible
missingFact = unmatchedFactsStack[-1]
else:
missingFact = expectedInstanceFact
formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix),
_("Output missing expected fact %(fact)s"),
modelXbrl=missingFact, fact=missingFact.qname,
messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing"))
else: # compare footnotes
expectedInstanceFactFootnotes = factFootnotes(expectedInstanceFact, expectedFootnotesRelSet)
formulaOutputFactFootnotes = factFootnotes(formulaOutputFact, formulaOutputFootnotesRelSet)
if expectedInstanceFactFootnotes != formulaOutputFactFootnotes:
formulaOutputInstance.error("{}:expectedFactFootnoteDifference".format(errMsgPrefix),
_("Output expected fact %(fact)s expected footnotes %(footnotes1)s produced footnotes %(footnotes2)s"),
modelXbrl=(formulaOutputFact,expectedInstanceFact), fact=expectedInstanceFact.qname, footnotes1=expectedInstanceFactFootnotes, footnotes2=formulaOutputFactFootnotes,
messageCodes=("formula:expectedFactFootnoteDifference","ix:expectedFactFootnoteDifference"))
# for debugging uncomment next line to save generated instance document
# formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml")
expectedInstance.close()
del expectedInstance # dereference
self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors)
formulaOutputInstance.close()
del formulaOutputInstance
if compareIxResultInstance:
for inputDTSlist in inputDTSes.values():
for inputDTS in inputDTSlist:
inputDTS.close()
del inputDTSes # dereference
# update ui thread via modelManager (running in background here)
self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def saveLoadableOIM(modelXbrl, oimFile):
isJSON = oimFile.endswith(".json")
isCSV = oimFile.endswith(".csv")
namespacePrefixes = {}
def compileQname(qname):
if qname.namespaceURI not in namespacePrefixes:
namespacePrefixes[qname.namespaceURI] = qname.prefix or ""
aspectsDefined = {
qnOimConceptAspect,
qnOimEntityAspect,
qnOimTypeAspect}
if isJSON:
aspectsDefined.add(qnOimPeriodAspect)
elif isCSV:
aspectsDefined.add(qnOimPeriodStartAspect)
aspectsDefined.add(qnOimPeriodDurationAspect)
def oimValue(object, decimals=None):
if isinstance(object, QName):
if object.namespaceURI not in namespacePrefixes:
if object.prefix:
namespacePrefixes[object.namespaceURI] = object.prefix
else:
_prefix = "_{}".format(sum(1 for p in namespacePrefixes if p.startswith("_")))
namespacePrefixes[object.namespaceURI] = _prefix
return "{}:{}".format(namespacePrefixes[object.namespaceURI], object.localName)
if isinstance(object, Decimal):
try:
if isinf(object):
return "-INF" if object < 0 else "INF"
elif isnan(num):
return "NaN"
else:
if object == object.to_integral():
object = object.quantize(ONE) # drop any .0
return "{}".format(object)
except:
return str(object)
if isinstance(object, bool):
return object
if isinstance(object, (DateTime, YearMonthDuration, DayTimeDuration, Time,
gYearMonth, gMonthDay, gYear, gMonth, gDay)):
return str(object)
return object
def oimPeriodValue(cntx):
if cntx.isForeverPeriod:
if isCSV:
return "0000-01-01T00:00:00/P9999Y"
return "forever"
elif cntx.isStartEndPeriod:
d = cntx.startDatetime
duration = yearMonthDayTimeDuration(cntx.startDatetime, cntx.endDatetime)
else: # instant
d = cntx.instantDatetime
duration = "PT0S"
return "{0:04n}-{1:02n}-{2:02n}T{3:02n}:{4:02n}:{5:02n}/{6}".format(
d.year, d.month, d.day, d.hour, d.minute, d.second,
duration)
hasId = False
hasTuple = False
hasType = True
hasLang = False
hasUnits = False
hasUnitMulMeasures = False
hasUnitDivMeasures = False
footnotesRelationshipSet = ModelRelationshipSet(modelXbrl, "XBRL-footnotes")
#compile QNames in instance for OIM
for fact in modelXbrl.factsInInstance:
if (fact.id or fact.isTuple or
footnotesRelationshipSet.toModelObject(fact) or
(isCSV and footnotesRelationshipSet.fromModelObject(fact))):
hasId = True
concept = fact.concept
if concept is not None:
if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang:
hasLang = True
compileQname(fact.qname)
if hasattr(fact, "xValue") and isinstance(fact.xValue, QName):
compileQname(fact.xValue)
unit = fact.unit
if unit is not None:
hasUnits = True
if unit.measures[0]:
hasUnitMulMeasures = True
if unit.measures[1]:
hasUnitDivMeasures = True
if fact.modelTupleFacts:
hasTuple = True
entitySchemePrefixes = {}
for cntx in modelXbrl.contexts.values():
if cntx.entityIdentifierElement is not None:
scheme = cntx.entityIdentifier[0]
if scheme not in entitySchemePrefixes:
if not entitySchemePrefixes: # first one is just scheme
if scheme == "http://www.sec.gov/CIK":
_schemePrefix = "cik"
elif scheme == "http://standard.iso.org/iso/17442":
_schemePrefix = "lei"
else:
_schemePrefix = "scheme"
else:
_schemePrefix = "scheme{}".format(len(entitySchemePrefixes) + 1)
entitySchemePrefixes[scheme] = _schemePrefix
namespacePrefixes[scheme] = _schemePrefix
for dim in cntx.qnameDims.values():
compileQname(dim.dimensionQname)
aspectsDefined.add(dim.dimensionQname)
if dim.isExplicit:
compileQname(dim.memberQname)
for unit in modelXbrl.units.values():
if unit is not None:
for measures in unit.measures:
for measure in measures:
compileQname(measure)
if unit.measures[0]:
aspectsDefined.add(qnOimUnitNumeratorsAspect)
if unit.measures[1]:
aspectsDefined.add(qnOimUnitDenominatorsAspect)
if XbrlConst.xbrli in namespacePrefixes and namespacePrefixes[XbrlConst.xbrli] != "xbrli":
namespacePrefixes[XbrlConst.xbrli] = "xbrli" # normalize xbrli prefix
namespacePrefixes[XbrlConst.xsd] = "xsd"
if hasLang: aspectsDefined.add(qnOimLangAspect)
if hasTuple:
aspectsDefined.add(qnOimTupleParentAspect)
aspectsDefined.add(qnOimTupleOrderAspect)
if hasUnits: aspectsDefined.add(qnOimUnitAspect)
# compile footnotes and relationships
'''
factRelationships = []
factFootnotes = []
for rel in modelXbrl.relationshipSet(modelXbrl, "XBRL-footnotes").modelRelationships:
oimRel = {"linkrole": rel.linkrole, "arcrole": rel.arcrole}
factRelationships.append(oimRel)
oimRel["fromIds"] = [obj.id if obj.id
else elementChildSequence(obj)
for obj in rel.fromModelObjects]
oimRel["toIds"] = [obj.id if obj.id
else elementChildSequence(obj)
for obj in rel.toModelObjects]
_order = rel.arcElement.get("order")
if _order is not None:
oimRel["order"] = _order
for obj in rel.toModelObjects:
if isinstance(obj, ModelResource): # footnote
oimFootnote = {"role": obj.role,
"id": obj.id if obj.id
else elementChildSequence(obj),
# value needs work for html elements and for inline footnotes
"value": xmlstring(obj, stripXmlns=True)}
if obj.xmlLang:
oimFootnote["lang"] = obj.xmlLang
factFootnotes.append(oimFootnote)
oimFootnote
'''
dtsReferences = [
{"type": "schema" if doc.type == ModelDocument.Type.SCHEMA
else "linkbase" if doc.type == ModelDocument.Type.LINKBASE
else "other",
"href": doc.uri}
for doc,ref in modelXbrl.modelDocument.referencesDocument.items()
if ref.referringModelObject.qname in SCHEMA_LB_REFS]
'''
roleTypes = [
{"type": "role" if ref.referringModelObject.localName == "roleRef" else "arcroleRef",
"href": ref.referringModelObject["href"]}
for doc,ref in modelXbrl.modelDocument.referencesDocument.items()
if ref.referringModelObject.qname in ROLE_REFS]
'''
def factFootnotes(fact):
footnotes = []
for footnoteRel in footnotesRelationshipSet.fromModelObject(fact):
footnote = OrderedDict((("group", footnoteRel.arcrole),))
footnotes.append(footnote)
if isCSV:
footnote["factId"] = fact.id if fact.id else "f{}".format(fact.objectIndex)
toObj = footnoteRel.toModelObject
if isinstance(toObj, ModelFact):
footnote["factRef"] = toObj.id if toObj.id else "f{}".format(toObj.objectIndex)
else:
footnote["footnoteType"] = toObj.role
footnote["footnote"] = xmlstring(toObj, stripXmlns=True, contentsOnly=True, includeText=True)
if toObj.xmlLang:
footnote["language"] = toObj.xmlLang
return footnotes
def factAspects(fact):
aspects = OrderedDict()
if hasId and fact.id:
aspects["id"] = fact.id
elif (fact.isTuple or
footnotesRelationshipSet.toModelObject(fact) or
(isCSV and footnotesRelationshipSet.fromModelObject(fact))):
aspects["id"] = "f{}".format(fact.objectIndex)
parent = fact.getparent()
concept = fact.concept
if not fact.isTuple:
if concept is not None:
_baseXsdType = concept.baseXsdType
if _baseXsdType == "XBRLI_DATEUNION":
if getattr(fact.xValue, "dateOnly", False):
_baseXsdType = "date"
else:
_baseXsdType = "dateTime"
aspects["baseType"] = "xs:{}".format(_baseXsdType)
if concept.baseXbrliType in ("string", "normalizedString", "token") and fact.xmlLang:
aspects[qnOimLangAspect] = fact.xmlLang
aspects[qnOimTypeAspect] = concept.baseXbrliType
if fact.isItem:
if fact.isNil:
_value = None
_strValue = "nil"
else:
_inferredDecimals = inferredDecimals(fact)
_value = oimValue(fact.xValue, _inferredDecimals)
_strValue = str(_value)
if not isCSV:
aspects["value"] = _strValue
if fact.concept is not None and fact.concept.isNumeric:
_numValue = fact.xValue
if isinstance(_numValue, Decimal) and not isinf(_numValue) and not isnan(_numValue):
if _numValue == _numValue.to_integral():
_numValue = int(_numValue)
else:
_numValue = float(_numValue)
aspects["numericValue"] = _numValue
if not fact.isNil:
if isinf(_inferredDecimals):
if isJSON: _accuracy = "infinity"
elif isCSV: _accuracy = "INF"
else:
_accuracy = _inferredDecimals
aspects["accuracy"] = _inferredDecimals
elif isinstance(_value, bool):
aspects["booleanValue"] = _value
elif isCSV:
aspects["stringValue"] = _strValue
aspects[qnOimConceptAspect] = oimValue(fact.qname)
cntx = fact.context
if cntx is not None:
if cntx.entityIdentifierElement is not None:
aspects[qnOimEntityAspect] = oimValue(qname(*cntx.entityIdentifier))
if cntx.period is not None:
if isJSON:
aspects[qnOimPeriodAspect] = oimPeriodValue(cntx)
elif isCSV:
_periodValue = oimPeriodValue(cntx).split("/") + ["", ""] # default blank if no value
aspects[qnOimPeriodStartAspect] = _periodValue[0]
aspects[qnOimPeriodDurationAspect] = _periodValue[1]
for _qn, dim in sorted(cntx.qnameDims.items(), key=lambda item: item[0]):
aspects[dim.dimensionQname] = (oimValue(dim.memberQname) if dim.isExplicit
else None if dim.typedMember.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true", "1")
else dim.typedMember.stringValue)
unit = fact.unit
if unit is not None:
_mMul, _mDiv = unit.measures
if isJSON:
aspects[qnOimUnitAspect] = OrderedDict( # use tuple instead of list for hashability
(("numerators", tuple(oimValue(m) for m in sorted(_mMul, key=lambda m: oimValue(m)))),)
)
if _mDiv:
aspects[qnOimUnitAspect]["denominators"] = tuple(oimValue(m) for m in sorted(_mDiv, key=lambda m: oimValue(m)))
else: # CSV
if _mMul:
aspects[qnOimUnitNumeratorsAspect] = " ".join(oimValue(m)
for m in sorted(_mMul, key=lambda m: str(m)))
if _mDiv:
aspects[qnOimUnitDenominatorsAspect] = " ".join(oimValue(m)
for m in sorted(_mDiv, key=lambda m: str(m)))
if parent.qname != XbrlConst.qnXbrliXbrl:
aspects[qnOimTupleParentAspect] = parent.id if parent.id else "f{}".format(parent.objectIndex)
aspects[qnOimTupleOrderAspect] = elementIndex(fact)
if isJSON:
_footnotes = factFootnotes(fact)
if _footnotes:
aspects["footnotes"] = _footnotes
return aspects
if isJSON:
# save JSON
oimReport = OrderedDict() # top level of oim json output
oimFacts = []
oimReport["prefixes"] = OrderedDict((p,ns) for ns, p in sorted(namespacePrefixes.items(),
key=lambda item: item[1]))
oimReport["dtsReferences"] = dtsReferences
oimReport["facts"] = oimFacts
def saveJsonFacts(facts, oimFacts, parentFact):
for fact in facts:
oimFact = factAspects(fact)
oimFacts.append(OrderedDict((str(k),v) for k,v in oimFact.items()))
if fact.modelTupleFacts:
saveJsonFacts(fact.modelTupleFacts, oimFacts, fact)
saveJsonFacts(modelXbrl.facts, oimFacts, None)
with open(oimFile, "w", encoding="utf-8") as fh:
fh.write(json.dumps(oimReport, ensure_ascii=False, indent=1, sort_keys=False))
elif isCSV:
# save CSV
aspectQnCol = {}
aspectsHeader = []
factsColumns = []
def addAspectQnCol(aspectQn):
aspectQnCol[aspectQn] = len(aspectsHeader)
_colName = oimValue(aspectQn)
aspectsHeader.append(_colName)
_colDataType = {"id": "Name",
"baseType": "Name",
"oim:concept": "Name",
"oim:periodStart": "dateTime", # forever is 0000-01-01T00:00:00
"oim:periodDuration": "duration", # forever is P9999Y
"oim:tupleOrder": "integer",
"numericValue": "decimal",
"accuracy": "decimal",
"booleanValue": "boolean",
"oim:unitNumerators": OrderedDict((("base","Name"), ("separator"," "))),
"oim:unitDenominators": OrderedDict((("base","Name"), ("separator"," "))),
}.get(_colName, "string")
factsColumns.append(OrderedDict((("name", _colName),
("datatype", _colDataType))))
# pre-ordered aspect columns
if hasId:
addAspectQnCol("id")
if hasType:
addAspectQnCol("baseType")
addAspectQnCol("stringValue")
addAspectQnCol("numericValue")
addAspectQnCol("accuracy")
addAspectQnCol("booleanValue")
if hasTuple:
addAspectQnCol(qnOimTupleParentAspect)
addAspectQnCol(qnOimTupleOrderAspect)
addAspectQnCol(qnOimConceptAspect)
if qnOimEntityAspect in aspectsDefined:
addAspectQnCol(qnOimEntityAspect)
if qnOimPeriodStartAspect in aspectsDefined:
addAspectQnCol(qnOimPeriodStartAspect)
addAspectQnCol(qnOimPeriodDurationAspect)
if qnOimUnitNumeratorsAspect in aspectsDefined:
addAspectQnCol(qnOimUnitNumeratorsAspect)
if qnOimUnitDenominatorsAspect in aspectsDefined:
addAspectQnCol(qnOimUnitDenominatorsAspect)
for aspectQn in sorted(aspectsDefined, key=lambda qn: str(qn)):
if aspectQn.namespaceURI != nsOim:
addAspectQnCol(aspectQn)
def aspectCols(fact):
cols = [None for i in range(len(aspectsHeader))]
_factAspects = factAspects(fact)
for aspectQn, aspectValue in _factAspects.items():
if aspectQn in aspectQnCol:
cols[aspectQnCol[aspectQn]] = aspectValue
return cols
# metadata
csvTables = []
csvMetadata = OrderedDict((("@context",[ "http://www.w3.org/ns/csvw", { "@base": "./" }]),
("tables", csvTables)))
if oimFile.endswith("-facts.csv"): # strip -facts.csv if a prior -facts.csv file was chosen
_baseURL = oimFile[:-10]
elif oimFile.endswith(".csv"):
_baseURL = oimFile[:-4]
else:
_baseURL = oimFile
# save facts
_factsFile = _baseURL + "-facts.csv"
csvFile = open(_factsFile, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
csvWriter.writerow(aspectsHeader)
def saveCSVfacts(facts):
for fact in facts:
csvWriter.writerow(aspectCols(fact))
saveCSVfacts(fact.modelTupleFacts)
saveCSVfacts(modelXbrl.facts)
csvFile.close()
factsTableSchema = OrderedDict((("columns",factsColumns),))
csvTables.append(OrderedDict((("url",os.path.basename(_factsFile)),
("tableSchema",factsTableSchema))))
# save namespaces
_nsFile = _baseURL + "-prefixes.csv"
csvFile = open(_nsFile, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
csvWriter.writerow(("prefix", "URI"))
for _URI, prefix in sorted(namespacePrefixes.items(), key=lambda item: item[1]):
csvWriter.writerow((prefix, _URI))
csvFile.close()
nsTableSchema = OrderedDict((("columns",[OrderedDict((("prefix","string"),
("URI","anyURI")))]),))
csvTables.append(OrderedDict((("url",os.path.basename(_nsFile)),
("tableSchema",nsTableSchema))))
# save dts references
_dtsRefFile = _baseURL + "-dtsReferences.csv"
csvFile = open(_dtsRefFile, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
csvWriter.writerow(("type", "href"))
for oimRef in dtsReferences:
csvWriter.writerow((oimRef["type"], oimRef["href"]))
csvFile.close()
dtsRefTableSchema = OrderedDict((("columns",[OrderedDict((("type","string"),
("href","anyURI")))]),))
csvTables.append(OrderedDict((("url",os.path.basename(_dtsRefFile)),
("tableSchema",dtsRefTableSchema))))
# save footnotes
if footnotesRelationshipSet.modelRelationships:
_footnoteFile = oimFile.replace(".csv", "-footnotes.csv")
csvFile = open(_footnoteFile, csvOpenMode, newline=csvOpenNewline, encoding='utf-8-sig')
csvWriter = csv.writer(csvFile, dialect="excel")
cols = ("group", "factId", "factRef", "footnoteType", "footnote", "language")
csvWriter.writerow(cols)
def saveCSVfootnotes(facts):
for fact in facts:
for _footnote in factFootnotes(fact):
csvWriter.writerow(tuple((_footnote.get(col,"") for col in cols)))
saveCSVfootnotes(fact.modelTupleFacts)
saveCSVfootnotes(modelXbrl.facts)
csvFile.close()
footnoteTableSchema = OrderedDict((("columns",[OrderedDict((("group","anyURI"),
("factId","Name"),
("factRef","Name"),
("footnoteType","Name"),
("footnote","string"),
("language","language")))]),))
csvTables.append(OrderedDict((("url",os.path.basename(_footnoteFile)),
("tableSchema",footnoteTableSchema))))
# save metadata
with open(_baseURL + "-metadata.csv", "w", encoding="utf-8") as fh:
fh.write(json.dumps(csvMetadata, ensure_ascii=False, indent=1, sort_keys=False))