def __init__(self, savedOptions=None, xbrlInstance=None):
self.entityIdentScheme = ""
self.entityIdentValue = ""
self.startDate = "" # use string values so structure can be json-saved
self.endDate = ""
self.monetaryUnit = ""
self.monetaryDecimals = ""
self.nonMonetaryDecimals = ""
if savedOptions is not None:
self.__dict__.update(savedOptions)
elif xbrlInstance is not None:
for fact in xbrlInstance.facts:
cntx = fact.context
unit = fact.unit
if fact.isItem and cntx is not None:
if not self.entityIdentScheme:
self.entityIdentScheme, self.entityIdentValue = cntx.entityIdentifier
if not self.startDate and cntx.isStartEndPeriod:
self.startDate = XmlUtil.dateunionValue(cntx.startDatetime)
if not self.startDate and (cntx.isStartEndPeriod or cntx.isInstantPeriod):
self.endDate = XmlUtil.dateunionValue(cntx.endDatetime, subtractOneDay=True)
if fact.isNumeric and unit is not None:
if fact.concept.isMonetary:
if not self.monetaryUnit and unit.measures[0] and unit.measures[0][0].namespaceURI == XbrlConst.iso4217:
self.monetaryUnit = unit.measures[0][0].localName
if not self.monetaryDecimals:
self.monetaryDecimals = fact.decimals
elif not self.nonMonetaryDecimals:
self.nonMonetaryDecimals = fact.decimals
if self.entityIdentScheme and self.startDate and self.monetaryUnit and self.monetaryDecimals and self.nonMonetaryDecimals:
break
def factAspectValue(fact, aspect, view=False):
if aspect == Aspect.LOCATION:
parentQname = fact.getparent().qname
if parentQname == XbrlConst.qnXbrliXbrl: # not tuple
return NONE
return parentQname # tuple
elif aspect == Aspect.CONCEPT:
return fact.qname
elif fact.isTuple or fact.context is None:
return NONE #subsequent aspects don't exist for tuples
elif aspect == Aspect.UNIT:
if fact.unit is None:
return NONE
measures = fact.unit.measures
if measures[1]:
return "{0} / {1}".format(' '.join(str(m) for m in measures[0]),
' '.join(str(m) for m in measures[1]))
else:
return ' '.join(str(m) for m in measures[0])
else:
context = fact.context
if aspect == Aspect.PERIOD:
return ("forever" if context.isForeverPeriod else
XmlUtil.dateunionValue(context.instantDatetime, subtractOneDay=True) if context.isInstantPeriod else
XmlUtil.dateunionValue(context.startDatetime) + "-" + XmlUtil.dateunionValue(context.endDatetime, subtractOneDay=True))
elif aspect == Aspect.ENTITY_IDENTIFIER:
if view:
return context.entityIdentifier[1]
else:
return context.entityIdentifier # (scheme, identifier)
elif aspect in (Aspect.COMPLETE_SEGMENT, Aspect.COMPLETE_SCENARIO,
Aspect.NON_XDT_SEGMENT, Aspect.NON_XDT_SCENARIO):
return ''.join(XmlUtil.xmlstring(elt, stripXmlns=True, prettyPrint=True)
for elt in context.nonDimValues(aspect))
def __init__(self, mainWin, options):
self.mainWin = mainWin
parent = mainWin.parent
super(DialogNewFactItemOptions, self).__init__(parent)
self.parent = parent
self.options = options
parentGeometry = re.match("(\d+)x(\d+)[+]?([-]?\d+)[+]?([-]?\d+)", parent.geometry())
dialogX = int(parentGeometry.group(3))
dialogY = int(parentGeometry.group(4))
self.accepted = False
self.transient(self.parent)
self.title(_("New Fact Item Options"))
frame = Frame(self)
label(frame, 1, 1, "Entity scheme:")
self.cellEntityIdentScheme = gridCell(frame, 2, 1, getattr(options,"entityIdentScheme",""), width=50)
ToolTip(self.cellEntityIdentScheme, text=_("Enter the scheme for the context entity identifier"), wraplength=240)
label(frame, 1, 2, "Entity identifier:")
self.cellEntityIdentValue = gridCell(frame, 2, 2, getattr(options,"entityIdentValue",""))
ToolTip(self.cellEntityIdentValue, text=_("Enter the entity identifier value (e.g., stock ticker)"), wraplength=240)
label(frame, 1, 3, "Start date:")
startDate = getattr(options,"startDate",None)
self.cellStartDate = gridCell(frame, 2, 3, XmlUtil.dateunionValue(startDate) if startDate else "")
ToolTip(self.cellStartDate, text=_("Enter the start date for the report period (e.g., 2010-01-01)"), wraplength=240)
label(frame, 1, 4, "End date:")
endDate = getattr(options,"endDate",None)
self.cellEndDate = gridCell(frame, 2, 4, XmlUtil.dateunionValue(endDate, subtractOneDay=True) if endDate else "")
ToolTip(self.cellEndDate, text=_("Enter the end date for the report period (e.g., 2010-12-31)"), wraplength=240)
label(frame, 1, 5, "Monetary unit:")
self.cellMonetaryUnit = gridCombobox(frame, 2, 5, getattr(options,"monetaryUnit",""), values=monetaryUnits)
ToolTip(self.cellMonetaryUnit, text=_("Select a monetary unit (e.g., EUR)"), wraplength=240)
label(frame, 1, 6, "Monetary decimals:")
self.cellMonetaryDecimals = gridCell(frame, 2, 6, getattr(options,"monetaryDecimals","2"))
ToolTip(self.cellMonetaryDecimals, text=_("Enter decimals for monetary items"), wraplength=240)
label(frame, 1, 7, "Non-monetary decimals:")
self.cellNonMonetaryDecimals = gridCell(frame, 2, 7, getattr(options,"nonMonetaryDecimals","0"))
ToolTip(self.cellNonMonetaryDecimals, text=_("Enter decimals for non-monetary items (e.g., stock shares)"), wraplength=240)
cancelButton = Button(frame, text=_("Cancel"), width=8, command=self.close)
ToolTip(cancelButton, text=_("Cancel operation, discarding changes and entries"))
okButton = Button(frame, text=_("OK"), width=8, command=self.ok)
ToolTip(okButton, text=_("Accept the options as entered above"))
cancelButton.grid(row=8, column=1, columnspan=3, sticky=E, pady=3, padx=3)
okButton.grid(row=8, column=1, columnspan=3, sticky=E, pady=3, padx=86)
frame.grid(row=0, column=0, sticky=(N,S,E,W))
frame.columnconfigure(2, weight=1)
window = self.winfo_toplevel()
window.columnconfigure(0, weight=1)
self.geometry("+{0}+{1}".format(dialogX+50,dialogY+100))
#self.bind("<Return>", self.ok)
#self.bind("<Escape>", self.close)
self.protocol("WM_DELETE_WINDOW", self.close)
self.grab_set()
self.wait_window(self)
def __repr__(self):
return ("modelContext[{0}, period: {1}, {2}{3} line {4}]"
.format(self.id,
"forever" if self.isForeverPeriod else
"instant " + XmlUtil.dateunionValue(self.instantDatetime, subtractOneDay=True) if self.isInstantPeriod else
"duration " + XmlUtil.dateunionValue(self.startDatetime) + " - " + XmlUtil.dateunionValue(self.endDatetime, subtractOneDay=True),
"dimensions: ({0}) {1},".format(len(self.qnameDims),
tuple(mem.propertyView for dim,mem in sorted(self.qnameDims.items())))
if self.qnameDims else "",
self.modelDocument.basename, self.sourceline))
def propertyView(self):
scheme, entityId = self.entityIdentifier
return ((("entity", entityId, (("scheme", scheme),)),) +
((("forever", ""),) if self.isForeverPeriod else
(("instant", XmlUtil.dateunionValue(self.instantDatetime, subtractOneDay=True)),) if self.isInstantPeriod else
(("startDate", XmlUtil.dateunionValue(self.startDatetime)),("endDate", XmlUtil.dateunionValue(self.endDatetime, subtractOneDay=True)))) +
(("dimensions", "({0})".format(len(self.qnameDims)),
tuple(mem.propertyView for dim,mem in sorted(self.qnameDims.items())))
if self.qnameDims else (),
))
def insertAccession(self, rssItem):
self.showStatus("insert accession")
# accession graph -> document vertices
new_accession = {'class':'accession',
'is_most_current': True}
if self.modelXbrl.modelDocument.creationSoftwareComment:
new_accession['creation_software'] = self.modelXbrl.modelDocument.creationSoftwareComment
datetimeNow = datetime.datetime.now()
datetimeNowStr = XmlUtil.dateunionValue(datetimeNow)
if rssItem is not None: # sec accession
# set self.
accessionType = "SEC_accession"
# for an RSS Feed entry from SEC, use rss item's accession information
new_accession['accepted_timestamp'] = XmlUtil.dateunionValue(rssItem.acceptanceDatetime)
new_accession['filing_date'] = XmlUtil.dateunionValue(rssItem.filingDate)
new_accession['entity_id'] = rssItem.cikNumber
new_accession['entity_name'] = rssItem.companyName
new_accession['standard_industrial_classification'] = rssItem.assignedSic
new_accession['sec_html_url'] = rssItem.htmlUrl
new_accession['entry_url'] = rssItem.url
new_accession['filing_accession_number'] = filing_accession_number = rssItem.accessionNumber
else:
# not an RSS Feed item, make up our own accession ID (the time in seconds of epoch)
self.accessionId = int(time.time()) # only available if entered from an SEC filing
intNow = int(time.time())
accessionType = "independent_filing"
new_accession['accepted_timestamp'] = datetimeNowStr
new_accession['filing_date'] = datetimeNowStr
new_accession['entry_url'] = self.modelXbrl.fileSource.url
new_accession['filing_accession_number'] = filing_accession_number = str(intNow)
for id in self.execute("Insert accession " + accessionType, """
r = g.v(root_accessions_id)
// check if accession already has a vertex
vIt = g.V('filing_accession_number', new_accession.filing_accession_number)
// use prior vertex, or if none, create new vertex for it
accession = (vIt.hasNext() ? vIt.next() : g.addVertex(new_accession) )
// TBD: modify accession timestamp (last-updated-at, if it already existed)
// check if vertex has edge to root_accessions vertex
vIn = accession.in
// if no edge, add one
vIn.hasNext() && vIn.next() == r ?: g.addEdge(r, accession, accession_type)
accession.id
""",
params={'root_accessions_id': self.root_accessions_id,
'new_accession': new_accession,
'accession_type': accessionType,
'datetime_now': datetimeNowStr,
})["results"]:
self.accession_id = int(id)
# relationshipSets are a dts property
self.relationshipSets = [(arcrole, ELR, linkqname, arcqname)
for arcrole, ELR, linkqname, arcqname in self.modelXbrl.baseSets.keys()
if ELR and linkqname and arcqname and not arcrole.startswith("XBRL-")]
def factAspectValue(fact, aspect, view=False):
if fact is DEFAULT:
return 'none'
elif fact is NONDEFAULT:
return '*'
elif fact is DEFAULTorNONDEFAULT:
return '**'
elif aspect == Aspect.LOCATION:
parentQname = fact.getparent().qname
if parentQname == XbrlConst.qnXbrliXbrl: # not tuple
return NONE
return parentQname # tuple
elif aspect == Aspect.CONCEPT:
return fact.qname
elif fact.isTuple or fact.context is None:
return NONE #subsequent aspects don't exist for tuples
elif aspect == Aspect.UNIT:
if fact.unit is None:
return NONE
measures = fact.unit.measures
if measures[1]:
return "{0} / {1}".format(' '.join(str(m) for m in measures[0]),
' '.join(str(m) for m in measures[1]))
else:
return ' '.join(str(m) for m in measures[0])
else:
context = fact.context
if aspect == Aspect.PERIOD:
return ("forever" if context.isForeverPeriod else
XmlUtil.dateunionValue(context.instantDatetime, subtractOneDay=True) if context.isInstantPeriod else
XmlUtil.dateunionValue(context.startDatetime) + "-" + XmlUtil.dateunionValue(context.endDatetime, subtractOneDay=True))
elif aspect == Aspect.ENTITY_IDENTIFIER:
if view:
return context.entityIdentifier[1]
else:
return context.entityIdentifier # (scheme, identifier)
elif aspect in (Aspect.COMPLETE_SEGMENT, Aspect.COMPLETE_SCENARIO,
Aspect.NON_XDT_SEGMENT, Aspect.NON_XDT_SCENARIO):
return ''.join(XmlUtil.xmlstring(elt, stripXmlns=True, prettyPrint=True)
for elt in context.nonDimValues(aspect))
elif aspect == Aspect.DIMENSIONS:
return context.dimAspects(fact.xpCtx.defaultDimensionAspects)
elif isinstance(aspect, QName):
dimValue = context.dimValue(aspect)
if dimValue is None:
return NONE
else:
if isinstance(dimValue, QName): #default dim
return dimValue
elif dimValue.isExplicit:
return dimValue.memberQname
else: # explicit
return dimValue.typedMember.xValue # typed element value
def insertFiling(self, rssItem, g):
self.showStatus("insert filing")
# accession graph -> document vertices
new_filing = {'documents': []}
if self.modelXbrl.modelDocument.creationSoftwareComment:
new_filing['creation_software'] = self.modelXbrl.modelDocument.creationSoftwareComment
datetimeNow = datetime.datetime.now()
datetimeNowStr = XmlUtil.dateunionValue(datetimeNow)
entryUri = modelObjectDocumentUri(self.modelXbrl)
if rssItem is not None: # sec accession
# set self.
new_filing['filingType'] = "SEC filing"
# for an RSS Feed entry from SEC, use rss item's accession information
new_filing['filingNumber'] = filingNumber = rssItem.accessionNumber
new_filing['acceptedTimestamp'] = XmlUtil.dateunionValue(rssItem.acceptanceDatetime)
new_filing['filingDate'] = XmlUtil.dateunionValue(rssItem.filingDate)
new_filing['entityId'] = rssItem.cikNumber
new_filing['entityName'] = rssItem.companyName
new_filing['SICCode'] = rssItem.assignedSic
new_filing['SECHtmlUrl'] = rssItem.htmlUrl
new_filing['entryUrl'] = rssItem.url
self.filingURI = rssItem.htmlUrl
else:
# not an RSS Feed item, make up our own accession ID (the time in seconds of epoch)
intNow = int(time.time())
new_filing['filingNumber'] = filingNumber = str(intNow)
self.filingId = int(time.time()) # only available if entered from an SEC filing
new_filing['filingType'] = "independent filing"
new_filing['acceptedTimestamp'] = datetimeNowStr
new_filing['filingDate'] = datetimeNowStr
new_filing['entryUrl'] = UrlUtil.ensureUrl(self.modelXbrl.fileSource.url)
self.filingURI = filingNumber
g[FILINGS][self.filingURI] = new_filing
self.filing = new_filing
# for now only one report per filing (but SEC may have multiple in future, such as form SD)
self.reportURI = modelObjectDocumentUri(self.modelXbrl)
self.report = {'filing': self.filingURI,
'aspectProxies': {},
'relationshipSets': {},
'dataPoints': {},
'messages': {}}
new_filing['reports'] = {self.reportURI: self.report}
# relationshipSets are a dts property
self.relationshipSets = [(arcrole, ELR, linkqname, arcqname)
for arcrole, ELR, linkqname, arcqname in self.modelXbrl.baseSets.keys()
if ELR and (arcrole.startswith("XBRL-") or (linkqname and arcqname))]
def __init__(self, cntlr, xfFile):
self.modelXbrl = cntlr.modelManager.modelXbrl
self.xfFile = xfFile
self.xfLines = []
self.xmlns = {}
self.eltTypeCount = {}
self.nextIdDupNbr = {}
cntlr.showStatus(_("Initializing Formula Grammar"))
XPathParser.initializeParser(cntlr.modelManager)
cntlr.showStatus(None)
for cfQnameArity in sorted(
qnameArity for qnameArity in
self.modelXbrl.modelCustomFunctionSignatures.keys()
if isinstance(qnameArity, (tuple, list))):
cfObject = self.modelXbrl.modelCustomFunctionSignatures[
cfQnameArity]
self.doObject(cfObject, None, "", set())
rootObjects = rootFormulaObjects(self) # sets var sets up
# put parameters at root regardless of whether linked to
for qn, param in sorted(self.modelXbrl.qnameParameters.items(),
key=lambda i: i[0]):
self.doObject(param, None, "", set())
for rootObject in sorted(rootObjects, key=formulaObjSortKey):
self.doObject(rootObject, None, "", set())
if self.xmlns:
self.xfLines.insert(0, "")
for prefix, ns in sorted(self.xmlns.items(), reverse=True):
self.xfLines.insert(
0, "namespace {} = \"{}\";".format(prefix, ns))
self.xfLines.insert(0, "")
self.xfLines.insert(
0, "(: Generated from {} by Arelle on {} :)".format(
self.modelXbrl.modelDocument.basename,
XmlUtil.dateunionValue(datetime.datetime.now())))
with open(xfFile, "w", encoding="utf-8") as fh:
fh.write("\n".join(self.xfLines))
self.modelXbrl.info("info", "saved formula file %(file)s", file=xfFile)
def run(self, options, sourceZipStream=None):
"""Process command line arguments or web service request, such as to load and validate an XBRL document, or start web server.
When a web server has been requested, this method may be called multiple times, once for each web service (REST) request that requires processing.
Otherwise (when called for a command line request) this method is called only once for the command line arguments request.
:param options: OptionParser options from parse_args of main argv arguments (when called from command line) or corresponding arguments from web service (REST) request.
:type options: optparse.Values
"""
if options.showOptions: # debug options
for optName, optValue in sorted(options.__dict__.items(), key=lambda optItem: optItem[0]):
self.addToLog("Option {0}={1}".format(optName, optValue), messageCode="info")
self.addToLog("sys.argv {0}".format(sys.argv), messageCode="info")
if options.uiLang: # set current UI Lang (but not config setting)
self.setUiLanguage(options.uiLang)
if options.proxy:
if options.proxy != "show":
proxySettings = proxyTuple(options.proxy)
self.webCache.resetProxies(proxySettings)
self.config["proxySettings"] = proxySettings
self.saveConfig()
self.addToLog(_("Proxy configuration has been set."), messageCode="info")
useOsProxy, urlAddr, urlPort, user, password = self.config.get("proxySettings", proxyTuple("none"))
if useOsProxy:
self.addToLog(_("Proxy configured to use {0}.").format(
_('Microsoft Windows Internet Settings') if sys.platform.startswith("win")
else (_('Mac OS X System Configuration') if sys.platform in ("darwin", "macos")
else _('environment variables'))), messageCode="info")
elif urlAddr:
self.addToLog(_("Proxy setting: http://{0}{1}{2}{3}{4}").format(
user if user else "",
":****" if password else "",
"@" if (user or password) else "",
urlAddr,
":{0}".format(urlPort) if urlPort else ""), messageCode="info")
else:
self.addToLog(_("Proxy is disabled."), messageCode="info")
if options.plugins:
from arelle import PluginManager
resetPlugins = False
savePluginChanges = True
showPluginModules = False
for pluginCmd in options.plugins.split('|'):
cmd = pluginCmd.strip()
if cmd == "show":
showPluginModules = True
elif cmd == "temp":
savePluginChanges = False
elif cmd.startswith("+"):
moduleInfo = PluginManager.addPluginModule(cmd[1:])
if moduleInfo:
self.addToLog(_("Addition of plug-in {0} successful.").format(moduleInfo.get("name")),
messageCode="info", file=moduleInfo.get("moduleURL"))
resetPlugins = True
else:
self.addToLog(_("Unable to load plug-in."), messageCode="info", file=cmd[1:])
elif cmd.startswith("~"):
if PluginManager.reloadPluginModule(cmd[1:]):
self.addToLog(_("Reload of plug-in successful."), messageCode="info", file=cmd[1:])
resetPlugins = True
else:
self.addToLog(_("Unable to reload plug-in."), messageCode="info", file=cmd[1:])
elif cmd.startswith("-"):
if PluginManager.removePluginModule(cmd[1:]):
self.addToLog(_("Deletion of plug-in successful."), messageCode="info", file=cmd[1:])
resetPlugins = True
else:
self.addToLog(_("Unable to delete plug-in."), messageCode="info", file=cmd[1:])
else: # assume it is a module or package
savePluginChanges = False
moduleInfo = PluginManager.addPluginModule(cmd)
if moduleInfo:
self.addToLog(_("Activation of plug-in {0} successful.").format(moduleInfo.get("name")),
messageCode="info", file=moduleInfo.get("moduleURL"))
resetPlugins = True
else:
self.addToLog(_("Unable to load {0} as a plug-in or {0} is not recognized as a command. ").format(cmd), messageCode="info", file=cmd)
if resetPlugins:
PluginManager.reset()
if savePluginChanges:
PluginManager.save(self)
if showPluginModules:
self.addToLog(_("Plug-in modules:"), messageCode="info")
for i, moduleItem in enumerate(sorted(PluginManager.pluginConfig.get("modules", {}).items())):
moduleInfo = moduleItem[1]
self.addToLog(_("Plug-in: {0}; author: {1}; version: {2}; status: {3}; date: {4}; description: {5}; license {6}.").format(
moduleItem[0], moduleInfo.get("author"), moduleInfo.get("version"), moduleInfo.get("status"),
moduleInfo.get("fileDate"), moduleInfo.get("description"), moduleInfo.get("license")),
messageCode="info", file=moduleInfo.get("moduleURL"))
# run utility command line options that don't depend on entrypoint Files
hasUtilityPlugin = False
for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Utility.Run"):
hasUtilityPlugin = True
pluginXbrlMethod(self, options)
# if no entrypointFile is applicable, quit now
if options.proxy or options.plugins or hasUtilityPlugin:
if not options.entrypointFile:
return True # success
self.username = options.username
self.password = options.password
self.entrypointFile = options.entrypointFile
if self.entrypointFile:
filesource = FileSource.openFileSource(self.entrypointFile, self, sourceZipStream)
else:
filesource = None
if options.validateEFM:
if options.disclosureSystemName:
self.addToLog(_("both --efm and --disclosureSystem validation are requested, proceeding with --efm only"),
messageCode="info", file=self.entrypointFile)
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select("efm")
elif options.disclosureSystemName:
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select(options.disclosureSystemName)
elif options.validateHMRC:
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select("hmrc")
else:
self.modelManager.disclosureSystem.select(None) # just load ordinary mappings
self.modelManager.validateDisclosureSystem = False
if options.utrUrl: # override disclosureSystem utrUrl
self.modelManager.disclosureSystem.utrUrl = options.utrUrl
# can be set now because the utr is first loaded at validation time
# disclosure system sets logging filters, override disclosure filters, if specified by command line
if options.logLevelFilter:
self.setLogLevelFilter(options.logLevelFilter)
if options.logCodeFilter:
self.setLogCodeFilter(options.logCodeFilter)
if options.calcDecimals:
if options.calcPrecision:
self.addToLog(_("both --calcDecimals and --calcPrecision validation are requested, proceeding with --calcDecimals only"),
messageCode="info", file=self.entrypointFile)
self.modelManager.validateInferDecimals = True
self.modelManager.validateCalcLB = True
elif options.calcPrecision:
self.modelManager.validateInferDecimals = False
self.modelManager.validateCalcLB = True
if options.utrValidate:
self.modelManager.validateUtr = True
if options.infosetValidate:
self.modelManager.validateInfoset = True
if options.abortOnMajorError:
self.modelManager.abortOnMajorError = True
if options.collectProfileStats:
self.modelManager.collectProfileStats = True
if options.internetConnectivity == "offline":
self.webCache.workOffline = True
elif options.internetConnectivity == "online":
self.webCache.workOffline = False
if options.internetTimeout is not None:
self.webCache.timeout = (options.internetTimeout or None) # use None if zero specified to disable timeout
fo = FormulaOptions()
if options.parameters:
parameterSeparator = (options.parameterSeparator or ',')
fo.parameterValues = dict(((qname(key, noPrefixIsNoNamespace=True),(None,value))
for param in options.parameters.split(parameterSeparator)
for key,sep,value in (param.partition('='),) ) )
if options.formulaParamExprResult:
fo.traceParameterExpressionResult = True
if options.formulaParamInputValue:
fo.traceParameterInputValue = True
if options.formulaCallExprSource:
fo.traceCallExpressionSource = True
if options.formulaCallExprCode:
fo.traceCallExpressionCode = True
if options.formulaCallExprEval:
fo.traceCallExpressionEvaluation = True
if options.formulaCallExprResult:
fo.traceCallExpressionResult = True
if options.formulaVarSetExprEval:
fo.traceVariableSetExpressionEvaluation = True
if options.formulaVarSetExprResult:
fo.traceVariableSetExpressionResult = True
if options.formulaAsserResultCounts:
fo.traceAssertionResultCounts = True
if options.formulaFormulaRules:
fo.traceFormulaRules = True
if options.formulaVarsOrder:
fo.traceVariablesOrder = True
if options.formulaVarExpressionSource:
fo.traceVariableExpressionSource = True
if options.formulaVarExpressionCode:
fo.traceVariableExpressionCode = True
if options.formulaVarExpressionEvaluation:
fo.traceVariableExpressionEvaluation = True
if options.formulaVarExpressionResult:
fo.traceVariableExpressionResult = True
if options.timeVariableSetEvaluation:
fo.timeVariableSetEvaluation = True
if options.formulaVarFilterWinnowing:
fo.traceVariableFilterWinnowing = True
if options.formulaVarFiltersResult:
fo.traceVariableFiltersResult = True
if options.formulaVarFiltersResult:
fo.traceVariableFiltersResult = True
self.modelManager.formulaOptions = fo
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
firstStartedAt = startedAt = time.time()
modelDiffReport = None
success = True
modelXbrl = None
try:
if filesource:
modelXbrl = self.modelManager.load(filesource, _("views loading"))
except ModelDocument.LoadingException:
pass
except Exception as err:
self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format(
err,
traceback.format_tb(sys.exc_info()[2])))
success = False # loading errors, don't attempt to utilize loaded DTS
if modelXbrl and modelXbrl.modelDocument:
loadTime = time.time() - startedAt
modelXbrl.profileStat(_("load"), loadTime)
self.addToLog(format_string(self.modelManager.locale,
_("loaded in %.2f secs at %s"),
(loadTime, timeNow)),
messageCode="info", file=self.entrypointFile)
if options.importFiles:
for importFile in options.importFiles.split("|"):
fileName = importFile.strip()
if sourceZipStream is not None and not (fileName.startswith('http://') or os.path.isabs(fileName)):
fileName = os.path.dirname(modelXbrl.uri) + os.sep + fileName # make relative to sourceZipStream
ModelDocument.load(modelXbrl, fileName)
loadTime = time.time() - startedAt
self.addToLog(format_string(self.modelManager.locale,
_("import in %.2f secs at %s"),
(loadTime, timeNow)),
messageCode="info", file=importFile)
modelXbrl.profileStat(_("import"), loadTime)
if modelXbrl.errors:
success = False # loading errors, don't attempt to utilize loaded DTS
if modelXbrl.modelDocument.type in ModelDocument.Type.TESTCASETYPES:
for pluginXbrlMethod in pluginClassMethods("Testcases.Start"):
pluginXbrlMethod(self, options, modelXbrl)
else: # not a test case, probably instance or DTS
for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Loaded"):
pluginXbrlMethod(self, options, modelXbrl)
else:
success = False
if success and options.diffFile and options.versReportFile:
try:
diffFilesource = FileSource.FileSource(options.diffFile,self)
startedAt = time.time()
modelXbrl2 = self.modelManager.load(diffFilesource, _("views loading"))
if modelXbrl2.errors:
if not options.keepOpen:
modelXbrl2.close()
success = False
else:
loadTime = time.time() - startedAt
modelXbrl.profileStat(_("load"), loadTime)
self.addToLog(format_string(self.modelManager.locale,
_("diff comparison DTS loaded in %.2f secs"),
loadTime),
messageCode="info", file=self.entrypointFile)
startedAt = time.time()
modelDiffReport = self.modelManager.compareDTSes(options.versReportFile)
diffTime = time.time() - startedAt
modelXbrl.profileStat(_("diff"), diffTime)
self.addToLog(format_string(self.modelManager.locale,
_("compared in %.2f secs"),
diffTime),
messageCode="info", file=self.entrypointFile)
except ModelDocument.LoadingException:
success = False
except Exception as err:
success = False
self.addToLog(_("[Exception] Failed to doad diff file: \n{0} \n{1}").format(
err,
traceback.format_tb(sys.exc_info()[2])))
if success:
try:
modelXbrl = self.modelManager.modelXbrl
hasFormulae = modelXbrl.hasFormulae
if options.validate:
startedAt = time.time()
if options.formulaAction: # don't automatically run formulas
modelXbrl.hasFormulae = False
self.modelManager.validate()
if options.formulaAction: # restore setting
modelXbrl.hasFormulae = hasFormulae
self.addToLog(format_string(self.modelManager.locale,
_("validated in %.2f secs"),
time.time() - startedAt),
messageCode="info", file=self.entrypointFile)
if options.formulaAction in ("validate", "run"): # do nothing here if "none"
from arelle import ValidateXbrlDimensions, ValidateFormula
startedAt = time.time()
if not options.validate:
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
# setup fresh parameters from formula optoins
modelXbrl.parameters = fo.typedParameters()
ValidateFormula.validate(modelXbrl, compileOnly=(options.formulaAction != "run"))
self.addToLog(format_string(self.modelManager.locale,
_("formula validation and execution in %.2f secs")
if options.formulaAction == "run"
else _("formula validation only in %.2f secs"),
time.time() - startedAt),
messageCode="info", file=self.entrypointFile)
if options.testReport:
ViewFileTests.viewTests(self.modelManager.modelXbrl, options.testReport, options.testReportCols)
if options.rssReport:
ViewFileRssFeed.viewRssFeed(self.modelManager.modelXbrl, options.rssReport, options.rssReportCols)
if options.DTSFile:
ViewFileDTS.viewDTS(modelXbrl, options.DTSFile)
if options.factsFile:
ViewFileFactList.viewFacts(modelXbrl, options.factsFile, labelrole=options.labelRole, lang=options.labelLang, cols=options.factListCols)
if options.factTableFile:
ViewFileFactTable.viewFacts(modelXbrl, options.factTableFile, labelrole=options.labelRole, lang=options.labelLang)
if options.conceptsFile:
ViewFileConcepts.viewConcepts(modelXbrl, options.conceptsFile, labelrole=options.labelRole, lang=options.labelLang)
if options.preFile:
ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.preFile, "Presentation Linkbase", "http://www.xbrl.org/2003/arcrole/parent-child", labelrole=options.labelRole, lang=options.labelLang)
if options.calFile:
ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.calFile, "Calculation Linkbase", "http://www.xbrl.org/2003/arcrole/summation-item", labelrole=options.labelRole, lang=options.labelLang)
if options.dimFile:
ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.dimFile, "Dimensions", "XBRL-dimensions", labelrole=options.labelRole, lang=options.labelLang)
if options.formulaeFile:
ViewFileFormulae.viewFormulae(modelXbrl, options.formulaeFile, "Formulae", lang=options.labelLang)
if options.viewArcrole and options.viewFile:
ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.viewFile, os.path.basename(options.viewArcrole), options.viewArcrole, labelrole=options.labelRole, lang=options.labelLang)
for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Run"):
pluginXbrlMethod(self, options, modelXbrl)
except (IOError, EnvironmentError) as err:
self.addToLog(_("[IOError] Failed to save output:\n {0}").format(err))
success = False
except Exception as err:
self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format(
err,
traceback.format_tb(sys.exc_info()[2])))
success = False
if modelXbrl:
modelXbrl.profileStat(_("total"), time.time() - firstStartedAt)
if options.collectProfileStats and modelXbrl:
modelXbrl.logProfileStats()
if not options.keepOpen:
if modelDiffReport:
self.modelManager.close(modelDiffReport)
elif modelXbrl:
self.modelManager.close(modelXbrl)
self.username = self.password = None #dereference password
return success
def setArgForFactProperty(param, modelFact, propertyNameParts):
propVal = None
property = propertyNameParts[0]
if property == "value":
if isinstance(modelFact.xValue, Decimal):
propVal = "{:,}".format(modelFact.xValue)
else:
propVal = modelFact.value
elif property == "decimals":
propVal = modelFact.decimals
elif property == "label" and modelFact.concept is not None:
propVal = modelFact.concept.label(
labelrole,
lang=lang,
linkroleHint=XbrlConst.defaultLinkRole)
elif property == "name":
propVal = str(modelFact.qname)
elif property == "localName":
propVal = modelFact.qname.localName
else:
cntx = modelFact.context
unit = modelFact.unit
if cntx is not None:
if property == "period":
if len(propertyNameParts) == 1:
if cntx.isForeverPeriod:
propVal = "forever"
elif cntx.isInstantPeriod:
propVal = XmlUtil.dateunionValue(
cntx.instantDatetime, subtractOneDay=True)
else:
propVal = "{} to {}".format(
XmlUtil.dateunionValue(cntx.startDatetime),
XmlUtil.dateunionValue(
cntx.endDatetime, subtractOneDay=True))
else:
dateSelection = propertyNameParts[1]
if dateSelection == "startDate":
propVal = XmlUtil.dateunionValue(
cntx.startDatetime)
elif dateSelection == "endDate":
propVal = XmlUtil.dateunionValue(
cntx.endDatetime, subtractOneDay=True)
elif dateSelection == "instant":
propVal = XmlUtil.dateunionValue(
cntx.instant, subtractOneDay=True)
elif dateSelection == "durationDays":
propVal = str((cntx.endDatetime -
cntx.startDatetime).days)
elif property == "dimensions":
if cntx.qnameDims:
propVal = "\n".join(
"{} = {}".format(
d.dimensionQname,
d.memberQname if d.isExplicit else XmlUtil.
xmlstring(XmlUtil.child(d),
stripXmlns=True,
prettyPrint=True))
for d in cntx.qnameDims.values())
else:
propVal = "none"
if property == "unit":
if unit is None:
propVal = "none"
else:
measures = unit.measures
if measures[1]:
propVal = 'mul {}\ndiv {} '.format(
', '.join(
measureFormat(m) for m in measures[0]),
', '.join(
measureFormat(m) for m in measures[1]))
else:
propVal = ', '.join(
measureFormat(m) for m in measures[0])
fmtArgs[param] = propVal
def runFromExcel(self, options):
#testGenFileName = options.excelfilename
testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation-index.xls"
testGenDir = os.path.dirname(testGenFileName)
schemaDir = os.path.dirname(testGenDir) + os.sep + "schema"
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
if options.testfiledate:
today = options.testfiledate
else:
today = XmlUtil.dateunionValue(datetime.date.today())
startedAt = time.time()
LogHandler(self) # start logger
self.logMessages = []
logMessagesFile = testGenDir + os.sep + 'log-generation-messages.txt'
modelTestcases = ModelXbrl.create(self.modelManager, url=testGenFileName, isEntry=True)
testcaseIndexBook = xlrd.open_workbook(testGenFileName)
testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0)
self.addToLog(_("[info] xls loaded in {0:.2} secs at {1}").format(time.time() - startedAt, timeNow))
# start index file
indexFiles = [testGenDir + os.sep + 'creation-testcases-index.xml',
testGenDir + os.sep + 'consumption-testcases-index.xml']
indexDocs = []
testcasesElements = []
for purpose in ("Creation","Consumption"):
file = io.StringIO(
#'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- XBRL Versioning 1.0 {0} Tests -->'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>'
'<testcases name="XBRL Versioning 1.0 {0} Tests" '
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">'
'</testcases>'.format(purpose, today)
)
doc = etree.parse(file)
file.close()
indexDocs.append(doc)
testcasesElements.append(doc.getroot())
priorTestcasesDir = None
testcaseFiles = None
testcaseDocs = None
for iRow in range(1, testcaseIndexSheet.nrows):
try:
row = testcaseIndexSheet.row(iRow)
if (row[0].ctype == xlrd.XL_CELL_EMPTY or # must have directory
row[1].ctype == xlrd.XL_CELL_EMPTY or # from
row[2].ctype == xlrd.XL_CELL_EMPTY): # to
continue
testDir = row[0].value
uriFrom = row[1].value
uriTo = row[2].value
overrideReport = row[3].value
description = row[4].value
if description is None or len(description) == 0:
continue # test not ready to run
assignment = row[5].value
expectedEvents = row[6].value # comma space separated if multiple
note = row[7].value
useCase = row[8].value
base = os.path.join(os.path.dirname(testGenFileName),testDir) + os.sep
self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom))
if uriFrom and uriTo and assignment.lower() not in ("n.a.", "error") and expectedEvents != "N.A.":
modelDTSfrom = modelDTSto = None
for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"), True), (uriTo, _("loading to DTS"), False)):
if ',' not in URIs:
modelDTS = ModelXbrl.load(self.modelManager, URIs, msg, base=base)
else:
modelDTS = ModelXbrl.create(self.modelManager,
ModelDocument.Type.DTSENTRIES,
self.webCache.normalizeUrl(URIs.replace(", ","_") + ".dts",
base),
isEntry=True)
DTSdoc = modelDTS.modelDocument
DTSdoc.inDTS = True
for uri in URIs.split(','):
doc = ModelDocument.load(modelDTS, uri.strip(), base=base)
if doc is not None:
DTSdoc.referencesDocument[doc] = "import" #fake import
doc.inDTS = True
if isFrom: modelDTSfrom = modelDTS
else: modelDTSto = modelDTS
if modelDTSfrom is not None and modelDTSto is not None:
# generate differences report
reportUri = uriFrom.partition(',')[0] # first file
reportDir = os.path.dirname(reportUri)
if reportDir: reportDir += os.sep
reportName = os.path.basename(reportUri).replace("from.xsd","report.xml")
reportFile = reportDir + "out" + os.sep + reportName
#reportFile = reportDir + "report" + os.sep + reportName
reportFullPath = self.webCache.normalizeUrl(
reportFile,
base)
testcasesDir = os.path.dirname(os.path.dirname(reportFullPath))
if testcasesDir != priorTestcasesDir:
# close prior report
if priorTestcasesDir:
for i,testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
testcaseName = os.path.basename(testcasesDir)
testcaseFiles = [testcasesDir + os.sep + testcaseName + "-creation-testcase.xml",
testcasesDir + os.sep + testcaseName + "-consumption-testcase.xml"]
for i,testcaseFile in enumerate(testcaseFiles):
etree.SubElement(testcasesElements[i], "testcase",
attrib={"uri":
testcaseFile[len(testGenDir)+1:].replace("\\","/")} )
# start testcase file
testcaseDocs = []
testcaseElements = []
testcaseNumber = testcaseName[0:4]
if testcaseNumber.isnumeric():
testcaseNumberElement = "<number>{0}</number>".format(testcaseNumber)
testcaseName = testcaseName[5:]
else:
testcaseNumberElement = ""
testDirSegments = testDir.split('/')
if len(testDirSegments) >= 2 and '-' in testDirSegments[1]:
testedModule = testDirSegments[1][testDirSegments[1].index('-') + 1:]
else:
testedModule = ''
for purpose in ("Creation","Consumption"):
file = io.StringIO(
#'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>'
'<testcase '
' xmlns="http://xbrl.org/2008/conformance"'
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">'
'<creator>'
'<name>Roland Hommes</name>'
'<email>[email protected]</email>'
'</creator>'
'{0}'
'<name>{1}</name>'
# '<description>{0}</description>'
'<reference>'
'{2}'
'{3}'
'</reference>'
'</testcase>'.format(testcaseNumberElement,
testcaseName,
'<name>{0}</name>'.format(testedModule) if testedModule else '',
'<id>{0}</id>'.format(useCase) if useCase else '')
)
doc = etree.parse(file)
file.close()
testcaseDocs.append(doc)
testcaseElements.append(doc.getroot())
priorTestcasesDir = testcasesDir
variationSeq = 1
try:
os.makedirs(os.path.dirname(reportFullPath))
except WindowsError:
pass # dir already exists
modelVersReport = ModelVersReport.ModelVersReport(modelTestcases)
modelVersReport.diffDTSes(reportFullPath,modelDTSfrom, modelDTSto,
assignment=assignment,
schemaDir=schemaDir)
# check for expected elements
if expectedEvents:
for expectedEvent in expectedEvents.split(","):
if expectedEvent not in ("No change", "N.A."):
prefix, sep, localName = expectedEvent.partition(':')
if sep and len(modelVersReport.xmlDocument.findall(
'//{{{0}}}{1}'.format(
XbrlConst.verPrefixNS.get(prefix),
localName))) == 0:
modelTestcases.warning("warning",
"Generated test case %(reportName)s missing expected event %(event)s",
reportName=reportName,
event=expectedEvent)
modelVersReport.close()
uriFromParts = uriFrom.split('_')
if len(uriFromParts) >= 2:
variationId = uriFromParts[1]
else:
variationId = "_{0:02n}".format(variationSeq)
for i,testcaseElt in enumerate(testcaseElements):
variationElement = etree.SubElement(testcaseElt, "{http://xbrl.org/2008/conformance}variation",
attrib={"id": variationId})
nameElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}description")
nameElement.text = description
''' (removed per RH 2011/10/04
if note:
paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}description")
paramElement.text = "Note: " + note
if useCase:
paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}reference")
paramElement.set("specification", "versioning-requirements")
paramElement.set("useCase", useCase)
'''
dataElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}data")
if i == 0: # result is report
if expectedEvents:
paramElement = etree.SubElement(dataElement, "{http://xbrl.org/2008/conformance}parameter",
attrib={"name":"expectedEvent",
"value":expectedEvents.replace(',',' ')},
nsmap={"conf":"http://xbrl.org/2008/conformance",
None:""})
if assignment:
paramElement = etree.SubElement(dataElement, "{http://xbrl.org/2008/conformance}parameter",
attrib={"name":"assignment",
"value":assignment},
nsmap={"conf":"http://xbrl.org/2008/conformance",
None:""})
for schemaURIs, dtsAttr in ((uriFrom,"from"), (uriTo,"to")):
for schemaURI in schemaURIs.split(","):
schemaElement = etree.SubElement(dataElement, "{http://xbrl.org/2008/conformance}schema")
schemaElement.set("dts",dtsAttr)
if i == 0:
schemaElement.set("readMeFirst","true")
schemaElement.text=os.path.basename(schemaURI.strip())
resultElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}result")
reportElement = etree.SubElement(resultElement if i == 0 else dataElement,
"{http://xbrl.org/2008/conformance}versioningReport")
if i == 1:
reportElement.set("readMeFirst","true")
reportElement.text = "report/" + reportName
variationSeq += 1
except Exception as err:
modelTestcases.error("exception",
_("Exception: %(error)s, Excel row: %(excelRow)s"),
error=err,
excelRow=iRow,
exc_info=True)
# add tests-error-code index files to consumption
for testcaseFile in self.testcaseFiles(testGenDir + os.sep + "tests-error-code"):
etree.SubElement(testcasesElements[1], "testcase",
attrib={"uri":
testcaseFile[len(testGenDir)+1:].replace("\\","/")} )
with open(logMessagesFile, "w") as fh:
fh.writelines(self.logMessages)
if priorTestcasesDir:
for i,testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
for i,indexFile in enumerate(indexFiles):
with open(indexFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def createContext(self,
entityIdentScheme,
entityIdentValue,
periodType,
periodStart,
periodEndInstant,
priItem,
dims,
segOCCs,
scenOCCs,
afterSibling=None,
beforeSibling=None):
xbrlElt = self.modelDocument.xmlRootElement
if afterSibling == AUTO_LOCATE_ELEMENT:
afterSibling = XmlUtil.lastChild(
xbrlElt, XbrlConst.xbrli,
("schemaLocation", "roleType", "arcroleType", "context"))
cntxId = 'c-{0:02n}'.format(len(self.contexts) + 1)
newCntxElt = XmlUtil.addChild(xbrlElt,
XbrlConst.xbrli,
"context",
attributes=("id", cntxId),
afterSibling=afterSibling,
beforeSibling=beforeSibling)
entityElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "entity")
XmlUtil.addChild(entityElt,
XbrlConst.xbrli,
"identifier",
attributes=("scheme", entityIdentScheme),
text=entityIdentValue)
periodElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "period")
if periodType == "forever":
XmlUtil.addChild(periodElt, XbrlConst.xbrli, "forever")
elif periodType == "instant":
XmlUtil.addChild(periodElt,
XbrlConst.xbrli,
"instant",
text=XmlUtil.dateunionValue(periodEndInstant,
subtractOneDay=True))
elif periodType == "duration":
XmlUtil.addChild(periodElt,
XbrlConst.xbrli,
"startDate",
text=XmlUtil.dateunionValue(periodStart))
XmlUtil.addChild(periodElt,
XbrlConst.xbrli,
"endDate",
text=XmlUtil.dateunionValue(periodEndInstant,
subtractOneDay=True))
segmentElt = None
scenarioElt = None
from arelle.ModelInstanceObject import ModelDimensionValue
if dims: # requires primary item to determin ambiguous concepts
''' in theory we have to check full set of dimensions for validity in source or any other
context element, but for shortcut will see if each dimension is already reported in an
unambiguous valid contextElement
'''
from arelle.PrototypeInstanceObject import FactPrototype, ContextPrototype, DimValuePrototype
fp = FactPrototype(self, priItem, dims.items())
# force trying a valid prototype's context Elements
if not isFactDimensionallyValid(
self, fp, setPrototypeContextElements=True):
self.info(
"arelleLinfo",
_("Create context for %(priItem)s, cannot determine valid context elements, no suitable hypercubes"
),
modelObject=self,
priItem=priItem)
fpDims = fp.context.qnameDims
for dimQname in sorted(fpDims.keys()):
dimValue = fpDims[dimQname]
if isinstance(dimValue, DimValuePrototype):
dimMemberQname = dimValue.memberQname # None if typed dimension
contextEltName = dimValue.contextElement
else: # qname for explicit or node for typed
dimMemberQname = None
contextEltName = None
if contextEltName == "segment":
if segmentElt is None:
segmentElt = XmlUtil.addChild(entityElt,
XbrlConst.xbrli,
"segment")
contextElt = segmentElt
elif contextEltName == "scenario":
if scenarioElt is None:
scenarioElt = XmlUtil.addChild(newCntxElt,
XbrlConst.xbrli,
"scenario")
contextElt = scenarioElt
else:
self.info(
"arelleLinfo",
_("Create context, %(dimension)s, cannot determine context element, either no all relationship or validation issue"
),
modelObject=self,
dimension=dimQname),
continue
dimConcept = self.qnameConcepts[dimQname]
dimAttr = ("dimension",
XmlUtil.addQnameValue(xbrlElt, dimConcept.qname))
if dimConcept.isTypedDimension:
dimElt = XmlUtil.addChild(contextElt,
XbrlConst.xbrldi,
"xbrldi:typedMember",
attributes=dimAttr)
if isinstance(dimValue,
(ModelDimensionValue,
DimValuePrototype)) and dimValue.isTyped:
XmlUtil.copyNodes(dimElt, dimValue.typedMember)
elif dimMemberQname:
dimElt = XmlUtil.addChild(contextElt,
XbrlConst.xbrldi,
"xbrldi:explicitMember",
attributes=dimAttr,
text=XmlUtil.addQnameValue(
xbrlElt, dimMemberQname))
if segOCCs:
if segmentElt is None:
segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli,
"segment")
XmlUtil.copyNodes(segmentElt, segOCCs)
if scenOCCs:
if scenarioElt is None:
scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli,
"scenario")
XmlUtil.copyNodes(scenarioElt, scenOCCs)
self.modelDocument.contextDiscover(newCntxElt)
XmlValidate.validate(self, newCntxElt)
return newCntxElt
def final(val):
if not (val.validateEBA or val.validateEIOPA):
return
modelXbrl = val.modelXbrl
modelDocument = modelXbrl.modelDocument
_statusMsg = _("validating {0} filing rules").format(val.disclosureSystem.name)
modelXbrl.profileActivity()
val.showStatus(_statusMsg)
if modelDocument.type == ModelDocument.Type.INSTANCE and (val.validateEBA or val.validateEIOPA):
if not modelDocument.uri.endswith(".xbrl"):
modelXbrl.warning("EBA.1.1",
_('XBRL instance documents SHOULD use the extension ".xbrl" but it is "%(extension)s"'),
modelObject=modelDocument, xmlEncoding=os.path.splitext(modelDocument.basename)[1])
if modelDocument.documentEncoding.lower() not in ("utf-8", "utf-8-sig"):
modelXbrl.error("EBA.1.4",
_('XBRL instance documents MUST use "UTF-8" encoding but is "%(xmlEncoding)s"'),
modelObject=modelDocument, xmlEncoding=modelDocument.documentEncoding)
schemaRefElts = []
schemaRefFileNames = []
for doc, docRef in modelDocument.referencesDocument.items():
if docRef.referenceType == "href":
if docRef.referringModelObject.localName == "schemaRef":
schemaRefElts.append(docRef.referringModelObject)
schemaRefFileNames.append(doc.basename)
if not UrlUtil.isAbsolute(doc.uri):
modelXbrl.error("EBA.2.2",
_('The link:schemaRef element in submitted instances MUST resolve to the full published entry point URL: %(url)s.'),
modelObject=docRef.referringModelObject, url=doc.uri)
elif docRef.referringModelObject.localName == "linkbaseRef":
modelXbrl.error("EBA.2.3",
_('The link:linkbaseRef element is not allowed: %(fileName)s.'),
modelObject=docRef.referringModelObject, fileName=doc.basename)
if len(schemaRefFileNames) > 1:
modelXbrl.error("EBA.1.5",
_('XBRL instance documents MUST reference only one entry point schema but %(numEntryPoints)s were found: %(entryPointNames)s'),
modelObject=modelDocument, numEntryPoints=len(schemaRefFileNames), entryPointNames=', '.join(sorted(schemaRefFileNames)))
### check entry point names appropriate for filing indicator (DPM DB?)
if len(schemaRefElts) != 1:
modelXbrl.error("EBA.2.3",
_('Any reported XBRL instance document MUST contain only one xbrli:xbrl/link:schemaRef node, but %(entryPointCount)s.'),
modelObject=schemaRefElts, entryPointCount=len(schemaRefElts))
filingIndicators = {}
for fIndicator in modelXbrl.factsByQname[qnFilingIndicator]:
_value = (fIndicator.xValue or fIndicator.value) # use validated xValue if DTS else value for skipDTS
if _value in filingIndicators:
modelXbrl.error("EBA.1.6.1",
_('Multiple filing indicators facts for indicator %(filingIndicator)s.'),
modelObject=(fIndicator, filingIndicators[_value]), filingIndicator=_value)
filingIndicators[_value] = fIndicator
if not filingIndicators:
modelXbrl.error("EBA.1.6",
_('Missing filing indicators. Reported XBRL instances MUST include appropriate filing indicator elements'),
modelObject=modelDocument)
numFilingIndicatorTuples = len(modelXbrl.factsByQname[qnFIndicators])
if numFilingIndicatorTuples > 1 and not getattr(modelXbrl, "isStreamingMode", False):
modelXbrl.info("EBA.1.6.2",
_('Multiple filing indicators tuples when not in streaming mode (info).'),
modelObject=modelXbrl.factsByQname[qnFIndicators])
# note EBA 2.1 is in ModelDocument.py
cntxIDs = set()
cntxEntities = set()
cntxDates = defaultdict(list)
timelessDatePattern = re.compile(r"\s*([0-9]{4})-([0-9]{2})-([0-9]{2})\s*$")
for cntx in modelXbrl.contexts.values():
cntxIDs.add(cntx.id)
cntxEntities.add(cntx.entityIdentifier)
dateElts = XmlUtil.descendants(cntx, XbrlConst.xbrli, ("startDate","endDate","instant"))
if any(not timelessDatePattern.match(e.textValue) for e in dateElts):
modelXbrl.error("EBA.2.10",
_('Period dates must be whole dates without time or timezone: %(dates)s.'),
modelObject=cntx, dates=", ".join(e.text for e in dateElts))
if cntx.isForeverPeriod:
modelXbrl.error("EBA.2.11",
_('Forever context period is not allowed.'),
modelObject=cntx)
elif cntx.isStartEndPeriod:
modelXbrl.error("EBA.2.13",
_('Start-End (flow) context period is not allowed.'),
modelObject=cntx)
elif cntx.isInstantPeriod:
cntxDates[cntx.instantDatetime].append(cntx)
if XmlUtil.hasChild(cntx, XbrlConst.xbrli, "segment"):
modelXbrl.error("EBA.2.14",
_("The segment element not allowed in context Id: %(context)s"),
modelObject=cntx, context=cntx.contextID)
for scenElt in XmlUtil.descendants(cntx, XbrlConst.xbrli, "scenario"):
childTags = ", ".join([child.prefixedName for child in scenElt.iterchildren()
if isinstance(child,ModelObject) and
child.tag != "{http://xbrl.org/2006/xbrldi}explicitMember" and
child.tag != "{http://xbrl.org/2006/xbrldi}typedMember"])
if len(childTags) > 0:
modelXbrl.error("EBA.2.15",
_("Scenario of context Id %(context)s has disallowed content: %(content)s"),
modelObject=cntx, context=cntx.id, content=childTags)
if len(cntxDates) > 1:
modelXbrl.error("EBA.2.13",
_('Contexts must have the same date: %(dates)s.'),
modelObject=[_cntx for _cntxs in cntxDates.values() for _cntx in _cntxs],
dates=', '.join(XmlUtil.dateunionValue(_dt, subtractOneDay=True)
for _dt in cntxDates.keys()))
unusedCntxIDs = cntxIDs - {fact.contextID
for fact in modelXbrl.factsInInstance
if fact.contextID} # skip tuples
if unusedCntxIDs:
modelXbrl.warning("EBA.2.7",
_('Unused xbrli:context nodes SHOULD NOT be present in the instance: %(unusedContextIDs)s.'),
modelObject=[modelXbrl.contexts[unusedCntxID] for unusedCntxID in unusedCntxIDs],
unusedContextIDs=", ".join(sorted(unusedCntxIDs)))
if len(cntxEntities) > 1:
modelXbrl.warning("EBA.2.9",
_('All entity identifiers and schemes must be the same, %(count)s found: %(entities)s.'),
modelObject=modelDocument, count=len(cntxEntities),
entities=", ".join(sorted(str(cntxEntity) for cntxEntity in cntxEntities)))
otherFacts = {} # (contextHash, unitHash, xmlLangHash) : fact
nilFacts = []
stringFactsWithoutXmlLang = []
nonMonetaryNonPureFacts = []
unitIDsUsed = set()
currenciesUsed = {}
for qname, facts in modelXbrl.factsByQname.items():
for f in facts:
if modelXbrl.skipDTS:
c = f.qname.localName[0]
isNumeric = c in ('m', 'p', 'i')
isMonetary = c == 'm'
isInteger = c == 'i'
isPercent = c == 'p'
isString = c == 's'
else:
concept = f.concept
if concept is not None:
isNumeric = concept.isNumeric
isMonetary = concept.isMonetary
isInteger = concept.baseXbrliType in integerItemTypes
isPercent = concept.typeQname == qnPercentItemType
isString = concept.baseXbrliType in ("stringItemType", "normalizedStringItemType")
else:
isNumeric = isString = False # error situation
k = (f.getparent().objectIndex,
f.qname,
f.context.contextDimAwareHash if f.context is not None else None,
f.unit.hash if f.unit is not None else None,
hash(f.xmlLang))
if f.qname == qnFIndicators and val.validateEIOPA:
pass
elif k not in otherFacts:
otherFacts[k] = {f}
else:
matches = [o
for o in otherFacts[k]
if (f.getparent().objectIndex == o.getparent().objectIndex and
f.qname == o.qname and
f.context.isEqualTo(o.context) if f.context is not None and o.context is not None else True) and
(f.unit.isEqualTo(o.unit) if f.unit is not None and o.unit is not None else True) and
(f.xmlLang == o.xmlLang)]
if matches:
contexts = [f.contextID] + [o.contextID for o in matches]
modelXbrl.error("EBA.2.16",
_('Facts are duplicates %(fact)s contexts %(contexts)s.'),
modelObject=[f] + matches, fact=f.qname, contexts=', '.join(contexts))
else:
otherFacts[k].add(f)
if isNumeric:
if f.precision:
modelXbrl.error("EBA.2.17",
_("Numeric fact %(fact)s of context %(contextID)s has a precision attribute '%(precision)s'"),
modelObject=f, fact=f.qname, contextID=f.contextID, precision=f.precision)
if f.decimals and f.decimals != "INF":
try:
dec = int(f.decimals)
if isMonetary:
if dec < -3:
modelXbrl.error("EBA.2.17",
_("Monetary fact %(fact)s of context %(contextID)s has a decimal attribute < -3: '%(decimals)s'"),
modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals)
elif isInteger:
if dec != 0:
modelXbrl.error("EBA.2.17",
_("Integer fact %(fact)s of context %(contextID)s has a decimal attribute \u2260 0: '%(decimals)s'"),
modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals)
elif isPercent:
if dec < 4:
modelXbrl.error("EBA.2.17",
_("Percent fact %(fact)s of context %(contextID)s has a decimal attribute < 4: '%(decimals)s'"),
modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals)
except ValueError:
pass # should have been reported as a schema error by loader
'''' (not intended by EBA 2.18, paste here is from EFM)
if not f.isNil and getattr(f,"xValid", 0) == 4:
try:
insignificance = insignificantDigits(f.xValue, decimals=f.decimals)
if insignificance: # if not None, returns (truncatedDigits, insiginficantDigits)
modelXbrl.error(("EFM.6.05.37", "GFM.1.02.26"),
_("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s has nonzero digits in insignificant portion %(insignificantDigits)s."),
modelObject=f1, fact=f1.qname, contextID=f1.contextID, decimals=f1.decimals,
value=f1.xValue, truncatedDigits=insignificance[0], insignificantDigits=insignificance[1])
except (ValueError,TypeError):
modelXbrl.error(("EBA.2.18"),
_("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s causes Value Error exception."),
modelObject=f1, fact=f1.qname, contextID=f1.contextID, decimals=f1.decimals, value=f1.value)
'''
unit = f.unit
if unit is not None:
if isMonetary:
if unit.measures[0]:
currenciesUsed[unit.measures[0][0]] = unit
elif not unit.isSingleMeasure or unit.measures[0][0] != XbrlConst.qnXbrliPure:
nonMonetaryNonPureFacts.append(f)
elif isString:
if not f.xmlLang:
stringFactsWithoutXmlLang.append(f)
if f.unitID is not None:
unitIDsUsed.add(f.unitID)
if f.isNil:
nilFacts.append(f)
if nilFacts:
modelXbrl.error("EBA.2.19",
_('Nil facts MUST NOT be present in the instance: %(nilFacts)s.'),
modelObject=nilFacts, nilFacts=", ".join(str(f.qname) for f in nilFacts))
if stringFactsWithoutXmlLang:
modelXbrl.error("EBA.2.20",
_("String facts need to report xml:lang: '%(langLessFacts)s'"),
modelObject=stringFactsWithoutXmlLang, langLEssFacts=", ".join(str(f.qname) for f in stringFactsWithoutXmlLang))
if nonMonetaryNonPureFacts:
modelXbrl.error("EBA.3.2",
_("Non monetary (numeric) facts MUST use the pure unit: '%(langLessFacts)s'"),
modelObject=nonMonetaryNonPureFacts, langLessFacts=", ".join(str(f.qname) for f in nonMonetaryNonPureFacts))
unusedUnitIDs = modelXbrl.units.keys() - unitIDsUsed
if unusedUnitIDs:
modelXbrl.warning("EBA.2.21",
_('Unused xbrli:unit nodes SHOULD NOT be present in the instance: %(unusedUnitIDs)s.'),
modelObject=[modelXbrl.units[unusedUnitID] for unusedUnitID in unusedUnitIDs],
unusedUnitIDs=", ".join(sorted(unusedUnitIDs)))
unitHashes = {}
for unit in modelXbrl.units.values():
h = hash(unit)
if h in unitHashes and unit.isEqualTo(unitHashes[h]):
modelXbrl.warning("EBA.2.32",
_("Duplicate units SHOULD NOT be reported, units %(unit1)s and %(unit2)s have same measures.'"),
modelObject=(unit, unitHashes[h]), unit1=unit.id, unit2=unitHashes[h].id)
else:
unitHashes[h] = unit
if len(currenciesUsed) > 1:
modelXbrl.error("EBA.3.1",
_("There MUST be only one currency but %(numCurrencies)s were found: %(currencies)s.'"),
modelObject=currenciesUsed.values(), numCurrencies=len(currenciesUsed), currencies=", ".join(str(c) for c in currenciesUsed.keys()))
namespacePrefixesUsed = defaultdict(set)
prefixesUnused = set(modelDocument.xmlRootElement.keys()).copy()
for elt in modelDocument.xmlRootElement.iter():
if isinstance(elt, ModelObject): # skip comments and processing instructions
namespacePrefixesUsed[elt.qname.namespaceURI].add(elt.qname.prefix)
prefixesUnused.discard(elt.qname.prefix)
if prefixesUnused:
modelXbrl.warning("EBA.3.4",
_("There SHOULD be no unused prefixes but these were declared: %(unusedPrefixes)s.'"),
modelObject=modelDocument, unusedPrefixes=', '.join(sorted(prefixesUnused)))
for ns, prefixes in namespacePrefixesUsed.items():
nsDocs = modelXbrl.namespaceDocs.get(ns)
if nsDocs:
for nsDoc in nsDocs:
nsDocPrefix = XmlUtil.xmlnsprefix(nsDoc.xmlRootElement, ns)
if any(prefix != nsDocPrefix for prefix in prefixes if prefix is not None):
modelXbrl.warning("EBA.3.5",
_("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s"),
modelObject=modelDocument, namespace=ns, declaredPrefix=nsDocPrefix, foundPrefixes=', '.join(sorted(prefixes - {None})))
modelXbrl.profileActivity(_statusMsg, minTimeToShow=0.0)
val.showStatus(None)
del val.prefixNamespace, val.namespacePrefix, val.idObjects, val.typedDomainElements
def runFromExcel(self, options):
#testGenFileName = options.excelfilename
testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation-index.xls"
testGenDir = os.path.dirname(testGenFileName)
schemaDir = os.path.dirname(testGenDir) + os.sep + "schema"
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
if options.testfiledate:
today = options.testfiledate
else:
today = XmlUtil.dateunionValue(datetime.date.today())
startedAt = time.time()
LogHandler(self) # start logger
self.logMessages = []
logMessagesFile = testGenDir + os.sep + 'log-generation-messages.txt'
modelTestcases = ModelXbrl.create(self.modelManager,
url=testGenFileName,
isEntry=True)
testcaseIndexBook = xlrd.open_workbook(testGenFileName)
testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0)
self.addToLog(
_("[info] xls loaded in {0:.2} secs at {1}").format(
time.time() - startedAt, timeNow))
# start index file
indexFiles = [
testGenDir + os.sep + 'creation-testcases-index.xml',
testGenDir + os.sep + 'consumption-testcases-index.xml'
]
indexDocs = []
testcasesElements = []
for purpose in ("Creation", "Consumption"):
file = io.StringIO(
#'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- XBRL Versioning 1.0 {0} Tests -->'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>'
'<testcases name="XBRL Versioning 1.0 {0} Tests" '
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">'
'</testcases>'.format(purpose, today))
doc = etree.parse(file)
file.close()
indexDocs.append(doc)
testcasesElements.append(doc.getroot())
priorTestcasesDir = None
testcaseFiles = None
testcaseDocs = None
for iRow in range(1, testcaseIndexSheet.nrows):
try:
row = testcaseIndexSheet.row(iRow)
if (row[0].ctype == xlrd.XL_CELL_EMPTY
or # must have directory
row[1].ctype == xlrd.XL_CELL_EMPTY or # from
row[2].ctype == xlrd.XL_CELL_EMPTY): # to
continue
testDir = row[0].value
uriFrom = row[1].value
uriTo = row[2].value
overrideReport = row[3].value
description = row[4].value
if description is None or len(description) == 0:
continue # test not ready to run
assignment = row[5].value
expectedEvents = row[
6].value # comma space separated if multiple
note = row[7].value
useCase = row[8].value
base = os.path.join(os.path.dirname(testGenFileName),
testDir) + os.sep
self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom))
if uriFrom and uriTo and assignment.lower() not in (
"n.a.", "error") and expectedEvents != "N.A.":
modelDTSfrom = modelDTSto = None
for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"),
True), (uriTo,
_("loading to DTS"),
False)):
if ',' not in URIs:
modelDTS = ModelXbrl.load(self.modelManager,
URIs,
msg,
base=base)
else:
modelDTS = ModelXbrl.create(
self.modelManager,
ModelDocument.Type.DTSENTRIES,
self.webCache.normalizeUrl(
URIs.replace(", ", "_") + ".dts", base),
isEntry=True)
DTSdoc = modelDTS.modelDocument
DTSdoc.inDTS = True
for uri in URIs.split(','):
doc = ModelDocument.load(modelDTS,
uri.strip(),
base=base)
if doc is not None:
DTSdoc.referencesDocument[
doc] = "import" #fake import
doc.inDTS = True
if isFrom: modelDTSfrom = modelDTS
else: modelDTSto = modelDTS
if modelDTSfrom is not None and modelDTSto is not None:
# generate differences report
reportUri = uriFrom.partition(',')[0] # first file
reportDir = os.path.dirname(reportUri)
if reportDir: reportDir += os.sep
reportName = os.path.basename(reportUri).replace(
"from.xsd", "report.xml")
reportFile = reportDir + "out" + os.sep + reportName
#reportFile = reportDir + "report" + os.sep + reportName
reportFullPath = self.webCache.normalizeUrl(
reportFile, base)
testcasesDir = os.path.dirname(
os.path.dirname(reportFullPath))
if testcasesDir != priorTestcasesDir:
# close prior report
if priorTestcasesDir:
for i, testcaseFile in enumerate(
testcaseFiles):
with open(testcaseFile,
"w",
encoding="utf-8") as fh:
XmlUtil.writexml(fh,
testcaseDocs[i],
encoding="utf-8")
testcaseName = os.path.basename(testcasesDir)
testcaseFiles = [
testcasesDir + os.sep + testcaseName +
"-creation-testcase.xml",
testcasesDir + os.sep + testcaseName +
"-consumption-testcase.xml"
]
for i, testcaseFile in enumerate(testcaseFiles):
etree.SubElement(
testcasesElements[i],
"testcase",
attrib={
"uri":
testcaseFile[len(testGenDir) +
1:].replace("\\", "/")
})
# start testcase file
testcaseDocs = []
testcaseElements = []
testcaseNumber = testcaseName[0:4]
if testcaseNumber.isnumeric():
testcaseNumberElement = "<number>{0}</number>".format(
testcaseNumber)
testcaseName = testcaseName[5:]
else:
testcaseNumberElement = ""
testDirSegments = testDir.split('/')
if len(testDirSegments
) >= 2 and '-' in testDirSegments[1]:
testedModule = testDirSegments[1][
testDirSegments[1].index('-') + 1:]
else:
testedModule = ''
for purpose in ("Creation", "Consumption"):
file = io.StringIO(
#'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>'
'<testcase '
' xmlns="http://xbrl.org/2008/conformance"'
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">'
'<creator>'
'<name>Roland Hommes</name>'
'<email>[email protected]</email>'
'</creator>'
'{0}'
'<name>{1}</name>'
# '<description>{0}</description>'
'<reference>'
'{2}'
'{3}'
'</reference>'
'</testcase>'.format(
testcaseNumberElement, testcaseName,
'<name>{0}</name>'.format(testedModule)
if testedModule else '',
'<id>{0}</id>'.format(useCase)
if useCase else ''))
doc = etree.parse(file)
file.close()
testcaseDocs.append(doc)
testcaseElements.append(doc.getroot())
priorTestcasesDir = testcasesDir
variationSeq = 1
try:
os.makedirs(os.path.dirname(reportFullPath))
except WindowsError:
pass # dir already exists
modelVersReport = ModelVersReport.ModelVersReport(
modelTestcases)
modelVersReport.diffDTSes(reportFullPath,
modelDTSfrom,
modelDTSto,
assignment=assignment,
schemaDir=schemaDir)
# check for expected elements
if expectedEvents:
for expectedEvent in expectedEvents.split(","):
if expectedEvent not in ("No change", "N.A."):
prefix, sep, localName = expectedEvent.partition(
':')
if sep and len(
modelVersReport.xmlDocument.
findall('//{{{0}}}{1}'.format(
XbrlConst.verPrefixNS.get(
prefix), localName))) == 0:
modelTestcases.warning(
"warning",
"Generated test case %(reportName)s missing expected event %(event)s",
reportName=reportName,
event=expectedEvent)
modelVersReport.close()
uriFromParts = uriFrom.split('_')
if len(uriFromParts) >= 2:
variationId = uriFromParts[1]
else:
variationId = "_{0:02n}".format(variationSeq)
for i, testcaseElt in enumerate(testcaseElements):
variationElement = etree.SubElement(
testcaseElt,
"{http://xbrl.org/2008/conformance}variation",
attrib={"id": variationId})
nameElement = etree.SubElement(
variationElement,
"{http://xbrl.org/2008/conformance}description"
)
nameElement.text = description
''' (removed per RH 2011/10/04
if note:
paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}description")
paramElement.text = "Note: " + note
if useCase:
paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}reference")
paramElement.set("specification", "versioning-requirements")
paramElement.set("useCase", useCase)
'''
dataElement = etree.SubElement(
variationElement,
"{http://xbrl.org/2008/conformance}data")
if i == 0: # result is report
if expectedEvents:
paramElement = etree.SubElement(
dataElement,
"{http://xbrl.org/2008/conformance}parameter",
attrib={
"name":
"expectedEvent",
"value":
expectedEvents.replace(',', ' ')
},
nsmap={
"conf":
"http://xbrl.org/2008/conformance",
None: ""
})
if assignment:
paramElement = etree.SubElement(
dataElement,
"{http://xbrl.org/2008/conformance}parameter",
attrib={
"name": "assignment",
"value": assignment
},
nsmap={
"conf":
"http://xbrl.org/2008/conformance",
None: ""
})
for schemaURIs, dtsAttr in ((uriFrom, "from"),
(uriTo, "to")):
for schemaURI in schemaURIs.split(","):
schemaElement = etree.SubElement(
dataElement,
"{http://xbrl.org/2008/conformance}schema"
)
schemaElement.set("dts", dtsAttr)
if i == 0:
schemaElement.set(
"readMeFirst", "true")
schemaElement.text = os.path.basename(
schemaURI.strip())
resultElement = etree.SubElement(
variationElement,
"{http://xbrl.org/2008/conformance}result")
reportElement = etree.SubElement(
resultElement if i == 0 else dataElement,
"{http://xbrl.org/2008/conformance}versioningReport"
)
if i == 1:
reportElement.set("readMeFirst", "true")
reportElement.text = "report/" + reportName
variationSeq += 1
except Exception as err:
modelTestcases.error(
"exception",
_("Exception: %(error)s, Excel row: %(excelRow)s"),
error=err,
excelRow=iRow,
exc_info=True)
# add tests-error-code index files to consumption
for testcaseFile in self.testcaseFiles(testGenDir + os.sep +
"tests-error-code"):
etree.SubElement(testcasesElements[1],
"testcase",
attrib={
"uri":
testcaseFile[len(testGenDir) + 1:].replace(
"\\", "/")
})
with open(logMessagesFile, "w") as fh:
fh.writelines(self.logMessages)
if priorTestcasesDir:
for i, testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
for i, indexFile in enumerate(indexFiles):
with open(indexFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def runFromExcel(self, options):
testGenFileName = options.excelfilename
#testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation\1000-2000-index.xls"
testGenDir = os.path.dirname(testGenFileName)
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
if options.testfiledate:
today = options.testfiledate
else:
today = XmlUtil.dateunionValue(datetime.date.today())
startedAt = time.time()
self.logMessages = []
logMessagesFile = testGenDir + os.sep + 'logGenerationMessages.txt'
modelTestcases = ModelXbrl.create(self.modelManager)
testcaseIndexBook = xlrd.open_workbook(testGenFileName)
testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0)
self.addToLog(
_("[info] xls loaded in {0:.2} secs at {1}").format(
time.time() - startedAt, timeNow))
# start index file
indexFiles = [
testGenDir + os.sep + 'creationTestcasesIndex.xml',
testGenDir + os.sep + 'consumptionTestcasesIndex.xml'
]
indexDocs = [
xml.dom.minidom.parseString(
'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- XBRL Versioning 1.0 {0} Tests -->'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>'
'<testcases name="XBRL Versioning 1.0 Consumption Tests" date="{1}" '
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">'
'</testcases>'.format(purpose, today))
for purpose in ("Creation", "Consumption")
]
testcasesElements = [
XmlUtil.child(indexDoc, None, "testcases")
for indexDoc in indexDocs
]
priorTestcasesDir = None
testcaseFiles = None
testcaseDocs = None
for iRow in range(1, testcaseIndexSheet.nrows):
row = testcaseIndexSheet.row(iRow)
if row[0].ctype == xlrd.XL_CELL_EMPTY or row[
1].ctype == xlrd.XL_CELL_EMPTY or row[
2].ctype == xlrd.XL_CELL_EMPTY:
continue
testDir = row[0].value
uriFrom = row[1].value
uriTo = row[2].value
intention = row[3].value
if intention is None or len(intention) == 0:
continue # test not ready to run
reason = row[4].value
expectedEvent = row[5].value
base = os.path.join(os.path.dirname(testGenFileName),
testDir) + os.sep
self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom))
if uriFrom and uriTo and reason.lower() not in (
"n.a.", "error") and expectedEvent != "N.A.":
for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"),
True), (uriTo, _("loading to DTS"),
False)):
if ',' not in URIs:
modelDTS = ModelXbrl.load(self.modelManager,
URIs,
msg,
base=base)
else:
modelDTS = ModelXbrl.create(
self.modelManager,
ModelDocument.Type.DTSENTRIES,
self.webCache.normalizeUrl(
URIs.replace(", ", "_") + ".dts", base),
isEntry=True)
DTSdoc = modelDTS.modelDocument
DTSdoc.inDTS = True
for uri in URIs.split(','):
doc = ModelDocument.load(modelDTS,
uri.strip(),
base=base)
DTSdoc.referencesDocument[
doc] = "import" #fake import
doc.inDTS = True
if isFrom: modelDTSfrom = modelDTS
else: modelDTSto = modelDTS
if modelDTSfrom and modelDTSto:
# generate differences report
reportUri = uriFrom.partition(',')[0] # first file
reportDir = os.path.dirname(reportUri)
if reportDir: reportDir += os.sep
reportName = os.path.basename(reportUri).replace(
"from.xsd", "report.xml")
reportFile = reportDir + "report" + os.sep + reportName
reportFullPath = self.webCache.normalizeUrl(
reportFile, base)
testcasesDir = os.path.dirname(
os.path.dirname(reportFullPath))
if testcasesDir != priorTestcasesDir:
# close prior report
if priorTestcasesDir:
for i, testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w",
encoding="utf-8") as fh:
XmlUtil.writexml(fh,
testcaseDocs[i],
encoding="utf-8")
testcaseName = os.path.basename(testcasesDir)
testcaseFiles = [
testcasesDir + os.sep + testcaseName +
"-creation-testcase.xml", testcasesDir + os.sep +
testcaseName + "-consumption-testcase.xml"
]
for i, testcaseFile in enumerate(testcaseFiles):
XmlUtil.addChild(
testcasesElements[i], None, "testcase",
("uri", testcaseFile[len(testGenDir) +
1:].replace("\\", "/")))
# start testcase file
testcaseDocs = [
xml.dom.minidom.parseString(
'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>'
'<testcase name="XBRL Versioning 1.0 {1} Tests" date="{2}" '
' xmlns="http://xbrl.org/2008/conformance"'
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">'
'<creator>'
'<name>Roland Hommes</name>'
'<email>[email protected]</email>'
'</creator>'
'<name>{0}</name>'
'<description>{0}</description>'
'</testcase>'.format(testcaseName, purpose,
today))
for purpose in ("Creation", "Consumption")
]
testcaseElements = [
XmlUtil.child(testcaseDoc, conformanceNS,
"testcase")
for testcaseDoc in testcaseDocs
]
priorTestcasesDir = testcasesDir
variationID = 1
try:
os.makedirs(os.path.dirname(reportFullPath))
except WindowsError:
pass # dir already exists
modelVersReport = ModelVersReport.ModelVersReport(
modelTestcases)
modelVersReport.diffDTSes(reportFullPath, modelDTSfrom,
modelDTSto)
# check for expected elements
if expectedEvent and expectedEvent not in ("No change",
"N.A."):
if len(
modelVersReport.xmlDocument.
getElementsByTagNameNS('*',
expectedEvent)) == 0:
modelTestcases.error(
"Generated test case {0} missing expected event {1}"
.format(reportName,
expectedEvent), "wrn", "missingEvent")
modelVersReport.close([])
for i, testcaseElt in enumerate(testcaseElements):
variationElement = XmlUtil.addChild(
testcaseElt,
conformanceNS,
"variation",
attributes=("id", "_{0:02n}".format(variationID)))
XmlUtil.addChild(variationElement,
conformanceNS,
"name",
text=intention)
dataElement = XmlUtil.addChild(variationElement,
conformanceNS, "data")
for schemaURIs, dtsAttr in ((uriFrom, "from"), (uriTo,
"to")):
for schemaURI in schemaURIs.split(","):
XmlUtil.addChild(
dataElement,
conformanceNS,
"schema",
attributes=((("dts", dtsAttr), ) +
((("readMeFirst",
"true"), ) if i == 0 else
())),
text=os.path.basename(schemaURI.strip()))
resultElement = XmlUtil.addChild(
variationElement, conformanceNS, "result")
XmlUtil.addChild(
resultElement if i == 0 else dataElement,
conformanceNS,
"versioningReport",
attributes=(("readMeFirst", "true") if i == 1 else
()),
text="report/" + reportName)
variationID += 1
with open(logMessagesFile, "w") as fh:
fh.writelines(self.logMessages)
if priorTestcasesDir:
for i, testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
for i, indexFile in enumerate(indexFiles):
with open(indexFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def runFromExcel(self, options):
testGenFileName = options.excelfilename
#testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation\1000-2000-index.xls"
testGenDir = os.path.dirname(testGenFileName)
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
if options.testfiledate:
today = options.testfiledate
else:
today = XmlUtil.dateunionValue(datetime.date.today())
startedAt = time.time()
self.logMessages = []
logMessagesFile = testGenDir + os.sep + 'logGenerationMessages.txt'
modelTestcases = ModelXbrl.create(self.modelManager)
testcaseIndexBook = xlrd.open_workbook(testGenFileName)
testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0)
self.addToLog(_("[info] xls loaded in {0:.2} secs at {1}").format(time.time() - startedAt, timeNow))
# start index file
indexFiles = [testGenDir + os.sep + 'creationTestcasesIndex.xml',
testGenDir + os.sep + 'consumptionTestcasesIndex.xml']
indexDocs = [xml.dom.minidom.parseString(
'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- XBRL Versioning 1.0 {0} Tests -->'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>'
'<testcases name="XBRL Versioning 1.0 Consumption Tests" date="{1}" '
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">'
'</testcases>'.format(purpose, today)
) for purpose in ("Creation","Consumption")]
testcasesElements = [XmlUtil.child(indexDoc, None, "testcases") for indexDoc in indexDocs]
priorTestcasesDir = None
testcaseFiles = None
testcaseDocs = None
for iRow in range(1, testcaseIndexSheet.nrows):
row = testcaseIndexSheet.row(iRow)
if row[0].ctype == xlrd.XL_CELL_EMPTY or row[1].ctype == xlrd.XL_CELL_EMPTY or row[2].ctype == xlrd.XL_CELL_EMPTY:
continue
testDir = row[0].value
uriFrom = row[1].value
uriTo = row[2].value
intention = row[3].value
if intention is None or len(intention) == 0:
continue # test not ready to run
reason = row[4].value
expectedEvent = row[5].value
base = os.path.join(os.path.dirname(testGenFileName),testDir) + os.sep
self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom))
if uriFrom and uriTo and reason.lower() not in ("n.a.", "error") and expectedEvent != "N.A.":
for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"), True), (uriTo, _("loading to DTS"), False)):
if ',' not in URIs:
modelDTS = ModelXbrl.load(self.modelManager, URIs, msg, base=base)
else:
modelDTS = ModelXbrl.create(self.modelManager,
ModelDocument.Type.DTSENTRIES,
self.webCache.normalizeUrl(URIs.replace(", ","_") + ".dts",
base),
isEntry=True)
DTSdoc = modelDTS.modelDocument
DTSdoc.inDTS = True
for uri in URIs.split(','):
doc = ModelDocument.load(modelDTS, uri.strip(), base=base)
DTSdoc.referencesDocument[doc] = "import" #fake import
doc.inDTS = True
if isFrom: modelDTSfrom = modelDTS
else: modelDTSto = modelDTS
if modelDTSfrom and modelDTSto:
# generate differences report
reportUri = uriFrom.partition(',')[0] # first file
reportDir = os.path.dirname(reportUri)
if reportDir: reportDir += os.sep
reportName = os.path.basename(reportUri).replace("from.xsd","report.xml")
reportFile = reportDir + "report" + os.sep + reportName
reportFullPath = self.webCache.normalizeUrl(
reportFile,
base)
testcasesDir = os.path.dirname(os.path.dirname(reportFullPath))
if testcasesDir != priorTestcasesDir:
# close prior report
if priorTestcasesDir:
for i,testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
testcaseName = os.path.basename(testcasesDir)
testcaseFiles = [testcasesDir + os.sep + testcaseName + "-creation-testcase.xml",
testcasesDir + os.sep + testcaseName + "-consumption-testcase.xml"]
for i,testcaseFile in enumerate(testcaseFiles):
XmlUtil.addChild(testcasesElements[i], None, "testcase",
("uri",
testcaseFile[len(testGenDir)+1:].replace("\\","/")) )
# start testcase file
testcaseDocs = [xml.dom.minidom.parseString(
'<?xml version="1.0" encoding="UTF-8"?>'
'<!-- Copyright 2011 XBRL International. All Rights Reserved. -->'
'<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>'
'<testcase name="XBRL Versioning 1.0 {1} Tests" date="{2}" '
' xmlns="http://xbrl.org/2008/conformance"'
' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">'
'<creator>'
'<name>Roland Hommes</name>'
'<email>[email protected]</email>'
'</creator>'
'<name>{0}</name>'
'<description>{0}</description>'
'</testcase>'.format(testcaseName,purpose,today)
) for purpose in ("Creation","Consumption")]
testcaseElements = [XmlUtil.child(testcaseDoc, conformanceNS, "testcase") for testcaseDoc in testcaseDocs]
priorTestcasesDir = testcasesDir
variationID = 1
try:
os.makedirs(os.path.dirname(reportFullPath))
except WindowsError:
pass # dir already exists
modelVersReport = ModelVersReport.ModelVersReport(modelTestcases)
modelVersReport.diffDTSes(reportFullPath,modelDTSfrom, modelDTSto)
# check for expected elements
if expectedEvent and expectedEvent not in (
"No change", "N.A."):
if len(modelVersReport.xmlDocument.getElementsByTagNameNS('*',expectedEvent)) == 0:
modelTestcases.error(
"Generated test case {0} missing expected event {1}".format(
reportName,
expectedEvent),
"wrn", "missingEvent")
modelVersReport.close([])
for i,testcaseElt in enumerate(testcaseElements):
variationElement = XmlUtil.addChild(testcaseElt, conformanceNS, "variation",
attributes=("id", "_{0:02n}".format(variationID)))
XmlUtil.addChild(variationElement, conformanceNS, "name", text=intention)
dataElement = XmlUtil.addChild(variationElement, conformanceNS, "data")
for schemaURIs, dtsAttr in ((uriFrom,"from"), (uriTo,"to")):
for schemaURI in schemaURIs.split(","):
XmlUtil.addChild(dataElement, conformanceNS, "schema",
attributes=((("dts",dtsAttr),) +
((("readMeFirst","true"),) if i == 0 else ())),
text=os.path.basename(schemaURI.strip()))
resultElement = XmlUtil.addChild(variationElement, conformanceNS, "result")
XmlUtil.addChild(resultElement if i == 0 else dataElement,
conformanceNS, "versioningReport",
attributes=(("readMeFirst","true") if i == 1 else ()),
text="report/" + reportName)
variationID += 1
with open(logMessagesFile, "w") as fh:
fh.writelines(self.logMessages)
if priorTestcasesDir:
for i,testcaseFile in enumerate(testcaseFiles):
with open(testcaseFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8")
for i,indexFile in enumerate(indexFiles):
with open(indexFile, "w", encoding="utf-8") as fh:
XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def final(val):
if not (val.validateEBA or val.validateEIOPA):
return
modelXbrl = val.modelXbrl
modelDocument = modelXbrl.modelDocument
_statusMsg = _("validating {0} filing rules").format(val.disclosureSystem.name)
modelXbrl.profileActivity()
modelXbrl.modelManager.showStatus(_statusMsg)
if modelDocument.type == ModelDocument.Type.INSTANCE and (val.validateEBA or val.validateEIOPA):
if not modelDocument.uri.endswith(".xbrl"):
modelXbrl.warning("EBA.1.1",
_('XBRL instance documents SHOULD use the extension ".xbrl" but it is "%(extension)s"'),
modelObject=modelDocument, extension=os.path.splitext(modelDocument.basename)[1])
modelXbrl.error("EIOPA.S.1.1.a",
_('XBRL instance documents MUST use the extension ".xbrl" but it is "%(extension)s"'),
modelObject=modelDocument, extension=os.path.splitext(modelDocument.basename)[1])
if val.isEIOPA_2_0_1: _encodings = ("UTF-8", "utf-8-sig")
else: _encodings = ("utf-8", "UTF-8", "utf-8-sig")
if modelDocument.documentEncoding not in _encodings:
modelXbrl.error(("EBA.1.4", "EIOPA.1.4"),
_('XBRL instance documents MUST use "UTF-8" encoding but is "%(xmlEncoding)s"'),
modelObject=modelDocument, xmlEncoding=modelDocument.documentEncoding)
schemaRefElts = []
schemaRefFileNames = []
for doc, docRef in modelDocument.referencesDocument.items():
if docRef.referenceType == "href":
if docRef.referringModelObject.localName == "schemaRef":
schemaRefElts.append(docRef.referringModelObject)
schemaRefFileNames.append(doc.basename)
if not UrlUtil.isAbsolute(doc.uri):
modelXbrl.error(("EBA.2.2", "EIOPA.S.1.5.a" if val.isEIOPAfullVersion else "EIOPA.S.1.5.b"),
_('The link:schemaRef element in submitted instances MUST resolve to the full published entry point URL: %(url)s.'),
modelObject=docRef.referringModelObject, url=doc.uri,
messageCodes=("EBA.2.2", "EIOPA.S.1.5.a","EIOPA.S.1.5.b"))
elif docRef.referringModelObject.localName == "linkbaseRef":
modelXbrl.error(("EBA.2.3","EIOPA.S.1.5.a"),
_('The link:linkbaseRef element is not allowed: %(fileName)s.'),
modelObject=docRef.referringModelObject, fileName=doc.basename)
_numSchemaRefs = len(XmlUtil.children(modelDocument.xmlRootElement, XbrlConst.link, "schemaRef"))
if _numSchemaRefs > 1:
modelXbrl.error(("EIOPA.S.1.5.a", "EBA.1.5"),
_('XBRL instance documents MUST reference only one entry point schema but %(numEntryPoints)s were found: %(entryPointNames)s'),
modelObject=modelDocument, numEntryPoints=_numSchemaRefs, entryPointNames=', '.join(sorted(schemaRefFileNames)))
### check entry point names appropriate for filing indicator (DPM DB?)
if len(schemaRefElts) != 1:
modelXbrl.error("EBA.2.3",
_('Any reported XBRL instance document MUST contain only one xbrli:xbrl/link:schemaRef node, but %(entryPointCount)s.'),
modelObject=schemaRefElts, entryPointCount=len(schemaRefElts))
# non-streaming EBA checks
if not getattr(modelXbrl, "isStreamingMode", False):
val.qnReportedCurrency = None
if val.isEIOPA_2_0_1 and qnMetReportingCurrency in modelXbrl.factsByQname:
for _multiCurrencyFact in modelXbrl.factsByQname[qnMetReportingCurrency]:
# multi-currency fact
val.qnReportedCurrency = _multiCurrencyFact.xValue
break
validateFacts(val, modelXbrl.facts)
# check sum of fact md5s (otherwise checked in streaming process)
xbrlFactsCheckVersion = None
expectedSumOfFactMd5s = None
for pi in modelDocument.xmlRootElement.getchildren():
if isinstance(pi, etree._ProcessingInstruction) and pi.target == "xbrl-facts-check":
_match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']", pi.text)
if _match:
_matchGroups = _match.groups()
if len(_matchGroups) == 2:
if _matchGroups[0] == "version":
xbrlFactsCheckVersion = _matchGroups[1]
elif _matchGroups[0] == "sum-of-fact-md5s":
try:
expectedSumOfFactMd5s = Md5Sum(_matchGroups[1])
except ValueError:
modelXbrl.error("EIOPA:xbrlFactsCheckError",
_("Invalid sum-of-md5s %(sumOfMd5)s"),
modelObject=modelXbrl, sumOfMd5=_matchGroups[1])
if xbrlFactsCheckVersion and expectedSumOfFactMd5s:
sumOfFactMd5s = Md5Sum()
for f in modelXbrl.factsInInstance:
sumOfFactMd5s += f.md5sum
if sumOfFactMd5s != expectedSumOfFactMd5s:
modelXbrl.warning("EIOPA:xbrlFactsCheckWarning",
_("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"),
modelObject=modelXbrl, expectedMd5=expectedSumOfFactMd5s, actualMd5Sum=sumOfFactMd5s)
else:
modelXbrl.info("info",
_("Successful XBRL facts sum of md5s."),
modelObject=modelXbrl)
if any(badError in modelXbrl.errors
for badError in ("EBA.2.1", "EIOPA.2.1", "EIOPA.S.1.5.a/EIOPA.S.1.5.b")):
pass # skip checking filingIndicators if bad errors
elif not val.filingIndicators:
modelXbrl.error(("EBA.1.6", "EIOPA.1.6.a"),
_('Missing filing indicators. Reported XBRL instances MUST include appropriate (positive) filing indicator elements'),
modelObject=modelDocument)
elif all(filed == False for filed in val.filingIndicators.values()):
modelXbrl.error(("EBA.1.6", "EIOPA.1.6.a"),
_('All filing indicators are filed="false". Reported XBRL instances MUST include appropriate (positive) filing indicator elements'),
modelObject=modelDocument)
if val.numFilingIndicatorTuples > 1:
modelXbrl.warning(("EBA.1.6.2", "EIOPA.1.6.2"),
_('Multiple filing indicators tuples when not in streaming mode (info).'),
modelObject=modelXbrl.factsByQname[qnFIndicators])
if len(val.cntxDates) > 1:
modelXbrl.error(("EBA.2.13","EIOPA.2.13"),
_('Contexts must have the same date: %(dates)s.'),
# when streaming values are no longer available, but without streaming they can be logged
modelObject=set(_cntx for _cntxs in val.cntxDates.values() for _cntx in _cntxs),
dates=', '.join(XmlUtil.dateunionValue(_dt, subtractOneDay=True)
for _dt in val.cntxDates.keys()))
if val.unusedCntxIDs:
if val.isEIOPA_2_0_1:
modelXbrl.error("EIOPA.2.7",
_('Unused xbrli:context nodes MUST NOT be present in the instance: %(unusedContextIDs)s.'),
modelObject=[modelXbrl.contexts[unusedCntxID] for unusedCntxID in val.unusedCntxIDs if unusedCntxID in modelXbrl.contexts],
unusedContextIDs=", ".join(sorted(val.unusedCntxIDs)))
else:
modelXbrl.warning(("EBA.2.7", "EIOPA.2.7"),
_('Unused xbrli:context nodes SHOULD NOT be present in the instance: %(unusedContextIDs)s.'),
modelObject=[modelXbrl.contexts[unusedCntxID] for unusedCntxID in val.unusedCntxIDs if unusedCntxID in modelXbrl.contexts],
unusedContextIDs=", ".join(sorted(val.unusedCntxIDs)))
if len(val.cntxEntities) > 1:
modelXbrl.error(("EBA.2.9", "EIOPA.2.9"),
_('All entity identifiers and schemes MUST be the same, %(count)s found: %(entities)s.'),
modelObject=modelDocument, count=len(val.cntxEntities),
entities=", ".join(sorted(str(cntxEntity) for cntxEntity in val.cntxEntities)))
for _scheme, _LEI in val.cntxEntities:
if (_scheme in ("http://standards.iso.org/iso/17442", "http://standard.iso.org/iso/17442", "LEI") or
(not val.isEIOPAfullVersion and _scheme == "PRE-LEI")):
if _scheme == "http://standard.iso.org/iso/17442":
modelXbrl.warning(("EBA.3.6", "EIOPA.S.2.8.c"),
_("Warning, context has entity scheme %(scheme)s should be plural: http://standards.iso.org/iso/17442."),
modelObject=modelDocument, scheme=_scheme)
result = LeiUtil.checkLei(_LEI)
if result == LeiUtil.LEI_INVALID_LEXICAL:
modelXbrl.error("EIOPA.S.2.8.c",
_("Context has lexically invalid LEI %(lei)s."),
modelObject=modelDocument, lei=_LEI)
elif result == LeiUtil.LEI_INVALID_CHECKSUM:
modelXbrl.error("EIOPA.S.2.8.c",
_("Context has LEI checksum error in %(lei)s."),
modelObject=modelDocument, lei=_LEI)
elif _scheme == "SC":
pass # anything is ok for Specific Code
else:
modelXbrl.error("EIOPA.S.2.8.c",
_("Context has unrecognized entity scheme %(scheme)s."),
modelObject=modelDocument, scheme=_scheme)
if val.unusedUnitIDs:
if val.isEIOPA_2_0_1:
modelXbrl.error("EIOPA.2.22",
_('Unused xbrli:unit nodes MUST NOT be present in the instance: %(unusedUnitIDs)s.'),
modelObject=[modelXbrl.units[unusedUnitID] for unusedUnitID in val.unusedUnitIDs if unusedUnitID in modelXbrl.units],
unusedUnitIDs=", ".join(sorted(val.unusedUnitIDs)))
else:
modelXbrl.warning(("EBA.2.22", "EIOPA.2.22"),
_('Unused xbrli:unit nodes SHOULD NOT be present in the instance: %(unusedUnitIDs)s.'),
modelObject=[modelXbrl.units[unusedUnitID] for unusedUnitID in val.unusedUnitIDs if unusedUnitID in modelXbrl.units],
unusedUnitIDs=", ".join(sorted(val.unusedUnitIDs)))
if len(val.currenciesUsed) > 1:
modelXbrl.error(("EBA.3.1","EIOPA.3.1"),
_("There MUST be only one currency but %(numCurrencies)s were found: %(currencies)s.'"),
modelObject=val.currenciesUsed.values(), numCurrencies=len(val.currenciesUsed), currencies=", ".join(str(c) for c in val.currenciesUsed.keys()))
elif val.isEIOPA_2_0_1 and any(_measure.localName != val.reportingCurrency for _measure in val.currenciesUsed.keys()):
modelXbrl.error("EIOPA.3.1",
_("There MUST be only one currency but reporting currency %(reportingCurrency)s differs from unit currencies: %(currencies)s.'"),
modelObject=val.currenciesUsed.values(), reportingCurrency=val.reportingCurrency, currencies=", ".join(str(c) for c in val.currenciesUsed.keys()))
if val.prefixesUnused:
modelXbrl.warning(("EBA.3.4", "EIOPA.3.4"),
_("There SHOULD be no unused prefixes but these were declared: %(unusedPrefixes)s.'"),
modelObject=modelDocument, unusedPrefixes=', '.join(sorted(val.prefixesUnused)))
for ns, prefixes in val.namespacePrefixesUsed.items():
nsDocs = modelXbrl.namespaceDocs.get(ns)
if nsDocs:
for nsDoc in nsDocs:
nsDocPrefix = XmlUtil.xmlnsprefix(nsDoc.xmlRootElement, ns)
if any(prefix != nsDocPrefix for prefix in prefixes if prefix is not None):
modelXbrl.warning(("EBA.3.5", "EIOPA.3.5"),
_("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s"),
modelObject=modelDocument, namespace=ns, declaredPrefix=nsDocPrefix, foundPrefixes=', '.join(sorted(prefixes - {None})))
elif ns in CANONICAL_PREFIXES and any(prefix != CANONICAL_PREFIXES[ns] for prefix in prefixes if prefix is not None):
modelXbrl.warning(("EBA.3.5", "EIOPA.3.5"),
_("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s"),
modelObject=modelDocument, namespace=ns, declaredPrefix=CANONICAL_PREFIXES[ns], foundPrefixes=', '.join(sorted(prefixes - {None})))
modelXbrl.profileActivity(_statusMsg, minTimeToShow=0.0)
modelXbrl.modelManager.showStatus(None)
del val.prefixNamespace, val.namespacePrefix, val.idObjects, val.typedDomainElements
del val.utrValidator, val.firstFact, val.footnotesRelationshipSet
def run(self, options, sourceZipStream=None):
self.entrypointFile = options.entrypointFile
filesource = FileSource.openFileSource(self.entrypointFile, self, sourceZipStream)
if options.validateEFM:
if options.gfmName:
self.addToLog(_("both --efm and --gfm validation are requested, proceeding with --efm only"),
messageCode="info", file=self.entrypointFile)
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select("efm")
elif options.gfmName:
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select(options.gfmName)
elif options.validateHMRC:
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select("hmrc")
else:
self.modelManager.disclosureSystem.select(None) # just load ordinary mappings
if options.calcDecimals:
if options.calcPrecision:
self.addToLog(_("both --calcDecimals and --calcPrecision validation are requested, proceeding with --calcDecimals only"),
messageCode="info", file=self.entrypointFile)
self.modelManager.validateInferDecimals = True
self.modelManager.validateCalcLB = True
elif options.calcPrecision:
self.modelManager.validateInferDecimals = False
self.modelManager.validateCalcLB = True
if options.utrValidate:
self.modelManager.validateUtr = True
fo = FormulaOptions()
if options.parameters:
fo.parameterValues = dict(((qname(key, noPrefixIsNoNamespace=True),(None,value))
for param in options.parameters.split(',')
for key,sep,value in (param.partition('='),) ) )
if options.formulaParamExprResult:
fo.traceParameterExpressionResult = True
if options.formulaParamInputValue:
fo.traceParameterInputValue = True
if options.formulaCallExprSource:
fo.traceCallExpressionSource = True
if options.formulaCallExprCode:
fo.traceCallExpressionCode = True
if options.formulaCallExprEval:
fo.traceCallExpressionEvaluation = True
if options.formulaCallExprResult:
fo.traceCallExpressionResult = True
if options.formulaVarSetExprEval:
fo.traceVariableSetExpressionEvaluation = True
if options.formulaVarSetExprResult:
fo.traceVariableSetExpressionResult = True
if options.formulaAsserResultCounts:
fo.traceAssertionResultCounts = True
if options.formulaFormulaRules:
fo.traceFormulaRules = True
if options.formulaVarsOrder:
fo.traceVariablesOrder = True
if options.formulaVarExpressionSource:
fo.traceVariableExpressionSource = True
if options.formulaVarExpressionCode:
fo.traceVariableExpressionCode = True
if options.formulaVarExpressionEvaluation:
fo.traceVariableExpressionEvaluation = True
if options.formulaVarExpressionResult:
fo.traceVariableExpressionResult = True
if options.formulaVarFilterWinnowing:
fo.traceVariableFilterWinnowing = True
if options.formulaVarFiltersResult:
fo.traceVariableFiltersResult = True
self.modelManager.formulaOptions = fo
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
startedAt = time.time()
modelDiffReport = None
success = True
modelXbrl = None
try:
modelXbrl = self.modelManager.load(filesource, _("views loading"))
except Exception as err:
self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format(
err,
traceback.format_tb(sys.exc_info()[2])))
success = False # loading errors, don't attempt to utilize loaded DTS
if modelXbrl and modelXbrl.modelDocument:
self.addToLog(format_string(self.modelManager.locale,
_("loaded in %.2f secs at %s"),
(time.time() - startedAt, timeNow)),
messageCode="info", file=self.entrypointFile)
if options.importFiles:
for importFile in options.importFiles.split("|"):
ModelDocument.load(modelXbrl, importFile.strip())
self.addToLog(format_string(self.modelManager.locale,
_("imported in %.2f secs at %s"),
(time.time() - startedAt, timeNow)),
messageCode="info", file=importFile)
if modelXbrl.errors:
success = False # loading errors, don't attempt to utilize loaded DTS
else:
success = False
if success and options.diffFile and options.versReportFile:
diffFilesource = FileSource.FileSource(options.diffFile,self)
startedAt = time.time()
modelXbrl2 = self.modelManager.load(diffFilesource, _("views loading"))
if modelXbrl2.errors:
if not options.keepOpen:
modelXbrl2.close()
success = False
else:
self.addToLog(format_string(self.modelManager.locale,
_("diff comparison DTS loaded in %.2f secs"),
time.time() - startedAt),
messageCode="info", file=self.entrypointFile)
startedAt = time.time()
modelDiffReport = self.modelManager.compareDTSes(options.versReportFile)
self.addToLog(format_string(self.modelManager.locale,
_("compared in %.2f secs"),
time.time() - startedAt),
messageCode="info", file=self.entrypointFile)
if success:
try:
if options.validate:
startedAt = time.time()
self.modelManager.validate()
self.addToLog(format_string(self.modelManager.locale,
_("validated in %.2f secs"),
time.time() - startedAt),
messageCode="info", file=self.entrypointFile)
if (options.testReport and
self.modelManager.modelXbrl.modelDocument.type in
(ModelDocument.Type.TESTCASESINDEX,
ModelDocument.Type.TESTCASE,
ModelDocument.Type.REGISTRY)):
ViewFileTests.viewTests(self.modelManager.modelXbrl, options.testReport)
if options.DTSFile:
ViewFileDTS.viewDTS(modelXbrl, options.DTSFile)
if options.factsFile:
ViewFileFactList.viewFacts(modelXbrl, options.factsFile, labelrole=options.labelRole, lang=options.labelLang, cols=options.factListCols)
if options.factTableFile:
ViewFileFactTable.viewFacts(modelXbrl, options.factTableFile, labelrole=options.labelRole, lang=options.labelLang)
if options.conceptsFile:
ViewFileConcepts.viewConcepts(modelXbrl, options.conceptsFile, labelrole=options.labelRole, lang=options.labelLang)
if options.preFile:
ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.preFile, "Presentation Linkbase", "http://www.xbrl.org/2003/arcrole/parent-child", labelrole=options.labelRole, lang=options.labelLang)
if options.calFile:
ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.calFile, "Calculation Linkbase", "http://www.xbrl.org/2003/arcrole/summation-item", labelrole=options.labelRole, lang=options.labelLang)
if options.dimFile:
ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.dimFile, "Dimensions", "XBRL-dimensions", labelrole=options.labelRole, lang=options.labelLang)
if options.formulaeFile:
ViewFileFormulae.viewFormulae(modelXbrl, options.formulaeFile, "Formulae", lang=options.labelLang)
for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Run"):
pluginXbrlMethod(self, options, modelXbrl)
except (IOError, EnvironmentError) as err:
self.addToLog(_("[IOError] Failed to save output:\n {0}").format(err))
success = False
except Exception as err:
self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format(
err,
traceback.format_tb(sys.exc_info()[2])))
success = False
if not options.keepOpen:
if modelDiffReport:
modelDiffReport.close()
elif modelXbrl:
modelXbrl.close()
return success
def produceOutputFact(xpCtx, formula, result):
priorErrorCount = len(xpCtx.modelXbrl.errors)
# assemble context
conceptQname = aspectValue(xpCtx, formula, Aspect.CONCEPT,
"xbrlfe:missingConceptRule")
if isinstance(conceptQname, VariableBindingError):
xpCtx.modelXbrl.error(
_("Formula {0} concept: {1}").format(formula, conceptQname.msg),
"err", conceptQname.err)
modelConcept = None
else:
modelConcept = xpCtx.modelXbrl.qnameConcepts[conceptQname]
if modelConcept is None or not modelConcept.isItem:
xpCtx.modelXbrl.error(
_("Formula {0} concept {1} is not an item").format(
formula, conceptQname), "err", "xbrlfe:missingConceptRule")
# entity
entityIdentScheme = aspectValue(xpCtx, formula, Aspect.SCHEME,
"xbrlfe:missingEntityIdentifierRule")
if isinstance(entityIdentScheme, VariableBindingError):
xpCtx.modelXbrl.error(
_("Formula {0} entity identifier scheme: {1}").format(
formula, entityIdentScheme.msg), "err", str(entityIdentScheme))
entityIdentValue = None
else:
entityIdentValue = aspectValue(xpCtx, formula, Aspect.VALUE,
"xbrlfe:missingEntityIdentifierRule")
if isinstance(entityIdentValue, VariableBindingError):
xpCtx.modelXbrl.error(
_("Formula {0} entity identifier value: {1}").format(
formula, entityIdentValue.msg), "err",
str(entityIdentScheme))
# period
periodType = aspectValue(xpCtx, formula, Aspect.PERIOD_TYPE,
"xbrlfe:missingPeriodRule")
periodStart = None
periodEndInstant = None
if isinstance(periodType, VariableBindingError):
xpCtx.modelXbrl.error(
_("Formula {0} period type: {1}").format(formula, periodType.msg),
"err", str(periodType))
elif periodType == "instant":
periodEndInstant = aspectValue(xpCtx, formula, Aspect.INSTANT,
"xbrlfe:missingPeriodRule")
if isinstance(periodEndInstant, VariableBindingError):
xpCtx.modelXbrl.error(
_("Formula {0} period start: {1}").format(
formula, periodEndInstant.msg), "err",
str(periodEndInstant))
elif periodType == "duration":
periodStart = aspectValue(xpCtx, formula, Aspect.START,
"xbrlfe:missingPeriodRule")
if isinstance(periodStart, VariableBindingError):
xpCtx.modelXbrl.error(
_("Formula {0} period start: {1}").format(
formula, periodStart.msg), "err", str(periodStart))
periodEndInstant = aspectValue(xpCtx, formula, Aspect.END,
"xbrlfe:missingPeriodRule")
if isinstance(periodEndInstant, VariableBindingError):
xpCtx.modelXbrl.error(
_("Formula {0} period end: {1}").format(
formula, periodEndInstant.msg), "err",
str(periodEndInstant))
# unit
if modelConcept and modelConcept.isNumeric:
unitSource = aspectValue(xpCtx, formula, Aspect.UNIT_MEASURES, None)
multDivBy = aspectValue(xpCtx, formula, Aspect.MULTIPLY_BY,
"xbrlfe:missingUnitRule")
if isinstance(multDivBy, VariableBindingError):
xpCtx.modelXbrl.error(
_("Formula {0} unit: {1}").format(formula,
multDivBy.msg), "err",
str(multDivBy) if isinstance(multDivBy, VariableBindingError)
else "xbrlfe:missingUnitRule")
multiplyBy = ()
divideBy = () # prevent errors later if bad
else:
divMultBy = aspectValue(xpCtx, formula, Aspect.DIVIDE_BY,
"xbrlfe:missingUnitRule")
if isinstance(divMultBy, VariableBindingError):
xpCtx.modelXbrl.error(
_("Formula {0} unit: {1}").format(formula,
divMultBy.msg), "err",
str(multDivBy)
if isinstance(divMultBy, VariableBindingError) else
"xbrlfe:missingUnitRule")
multiplyBy = ()
divideBy = () # prevent errors later if bad
else:
multiplyBy = unitSource[0] + multDivBy[0] + divMultBy[1]
divideBy = unitSource[1] + multDivBy[1] + divMultBy[0]
# remove cancelling mult/div units
lookForCommonUnits = True
while lookForCommonUnits:
lookForCommonUnits = False
for commonUnit in multiplyBy:
if commonUnit in divideBy:
multiplyBy.remove(commonUnit)
divideBy.remove(commonUnit)
lookForCommonUnits = True
break
if len(multiplyBy) == 0: # if no units add pure
multiplyBy.append(XbrlConst.qnXbrliPure)
# dimensions
segOCCs = []
scenOCCs = []
if formula.aspectModel == "dimensional":
dimAspects = {}
dimQnames = aspectValue(xpCtx, formula, Aspect.DIMENSIONS, None)
if dimQnames:
for dimQname in dimQnames:
dimConcept = xpCtx.modelXbrl.qnameConcepts[dimQname]
dimErr = "xbrlfe:missing{0}DimensionRule".format(
"typed" if dimConcept and dimConcept.isTypedDimension else
"explicit")
dimValue = aspectValue(xpCtx, formula, dimQname, dimErr)
if isinstance(dimValue, VariableBindingError):
xpCtx.modelXbrl.error(
_("Formula {0} dimension {1}: {2}").format(
formula, dimQname, dimValue.msg), "err", dimErr)
elif dimValue and xpCtx.modelXbrl.qnameDimensionDefaults.get(
dimQname) != dimValue:
dimAspects[dimQname] = dimValue
segOCCs = aspectValue(xpCtx, formula, Aspect.NON_XDT_SEGMENT, None)
scenOCCs = aspectValue(xpCtx, formula, Aspect.NON_XDT_SCENARIO, None)
else:
dimAspects = None # non-dimensional
segOCCs = aspectValue(xpCtx, formula, Aspect.COMPLETE_SEGMENT, None)
scenOCCs = aspectValue(xpCtx, formula, Aspect.COMPLETE_SCENARIO, None)
if priorErrorCount < len(xpCtx.modelXbrl.errors):
return None # had errors, don't produce output fact
# does context exist in out instance document
outputInstanceQname = formula.outputInstanceQname
outputXbrlInstance = xpCtx.inScopeVars[outputInstanceQname]
xbrlElt = outputXbrlInstance.modelDocument.xmlRootElement
# in source instance document
# add context
prevCntx = outputXbrlInstance.matchContext(entityIdentScheme,
entityIdentValue, periodType,
periodStart, periodEndInstant,
dimAspects, segOCCs, scenOCCs)
if prevCntx:
cntxId = prevCntx.id
newCntxElt = prevCntx.element
else:
cntxId = 'c-{0:02n}'.format(len(outputXbrlInstance.contexts) + 1)
newCntxElt = XmlUtil.addChild(
xbrlElt,
XbrlConst.xbrli,
"context",
attributes=("id", cntxId),
afterSibling=xpCtx.outputLastContext.get(outputInstanceQname))
xpCtx.outputLastContext[outputInstanceQname] = newCntxElt
entityElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "entity")
XmlUtil.addChild(entityElt,
XbrlConst.xbrli,
"identifier",
attributes=("scheme", entityIdentScheme),
text=entityIdentValue)
periodElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "period")
if periodType == "forever":
XmlUtil.addChild(periodElt, XbrlConst.xbrli, "forever")
elif periodType == "instant":
XmlUtil.addChild(periodElt,
XbrlConst.xbrli,
"instant",
text=XmlUtil.dateunionValue(periodEndInstant,
subtractOneDay=True))
elif periodType == "duration":
XmlUtil.addChild(periodElt,
XbrlConst.xbrli,
"startDate",
text=XmlUtil.dateunionValue(periodStart))
XmlUtil.addChild(periodElt,
XbrlConst.xbrli,
"endDate",
text=XmlUtil.dateunionValue(periodEndInstant,
subtractOneDay=True))
segmentElt = None
scenarioElt = None
from arelle.ModelObject import ModelDimensionValue
if dimAspects:
for dimQname in sorted(dimAspects.keys()):
dimValue = dimAspects[dimQname]
if isinstance(dimValue, ModelDimensionValue):
if dimValue.isExplicit:
dimMemberQname = dimValue.memberQname
contextEltName = dimValue.contextElement
else: # qname for explicit or node for typed
dimMemberQname = dimValue
contextEltName = xpCtx.modelXbrl.qnameDimensionContextElement.get(
dimQname)
if contextEltName == "segment":
if not segmentElt:
segmentElt = XmlUtil.addChild(entityElt,
XbrlConst.xbrli,
"segment")
contextElt = segmentElt
elif contextEltName == "scenario":
if not scenarioElt:
scenarioElt = XmlUtil.addChild(newCntxElt,
XbrlConst.xbrli,
"scenario")
contextElt = scenarioElt
else:
continue
dimConcept = xpCtx.modelXbrl.qnameConcepts[dimQname]
dimAttr = ("dimension",
XmlUtil.addQnameValue(xbrlElt, dimConcept.qname))
if dimConcept.isTypedDimension:
dimElt = XmlUtil.addChild(contextElt,
XbrlConst.xbrldi,
"typedMember",
attributes=dimAttr)
if isinstance(dimValue,
ModelDimensionValue) and dimValue.isTyped:
XmlUtil.copyChildren(dimElt, dimValue.typedMember)
elif dimMemberQname:
dimElt = XmlUtil.addChild(contextElt,
XbrlConst.xbrldi,
"explicitMember",
attributes=dimAttr,
text=XmlUtil.addQnameValue(
xbrlElt, dimMemberQname))
if segOCCs:
if not segmentElt:
segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli,
"segment")
XmlUtil.copyNodes(segmentElt, segOCCs)
if scenOCCs:
if not scenarioElt:
scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli,
"scenario")
XmlUtil.copyNodes(scenarioElt, scenOCCs)
outputXbrlInstance.modelDocument.contextDiscover(newCntxElt)
# does unit exist
# add unit
if modelConcept.isNumeric:
prevUnit = outputXbrlInstance.matchUnit(multiplyBy, divideBy)
if prevUnit:
unitId = prevUnit.id
newUnitElt = prevUnit.element
else:
unitId = 'u-{0:02n}'.format(len(outputXbrlInstance.units) + 1)
newUnitElt = XmlUtil.addChild(
xbrlElt,
XbrlConst.xbrli,
"unit",
attributes=("id", unitId),
afterSibling=xpCtx.outputLastUnit.get(outputInstanceQname))
xpCtx.outputLastUnit[outputInstanceQname] = newUnitElt
if len(divideBy) == 0:
for multiply in multiplyBy:
XmlUtil.addChild(newUnitElt,
XbrlConst.xbrli,
"measure",
text=XmlUtil.addQnameValue(
xbrlElt, multiply))
else:
divElt = XmlUtil.addChild(newUnitElt, XbrlConst.xbrli,
"divide")
numElt = XmlUtil.addChild(divElt, XbrlConst.xbrli,
"unitNumerator")
denElt = XmlUtil.addChild(divElt, XbrlConst.xbrli,
"unitDenominator")
for multiply in multiplyBy:
XmlUtil.addChild(numElt,
XbrlConst.xbrli,
"measure",
text=XmlUtil.addQnameValue(
xbrlElt, multiply))
for divide in divideBy:
XmlUtil.addChild(denElt,
XbrlConst.xbrli,
"measure",
text=XmlUtil.addQnameValue(
xbrlElt, divide))
outputXbrlInstance.modelDocument.unitDiscover(newUnitElt)
# add fact
attrs = [("contextRef", cntxId)]
precision = None
decimals = None
if modelConcept.isNumeric:
attrs.append(("unitRef", unitId))
value = formula.evaluate(xpCtx)
valueSeqLen = len(value)
if valueSeqLen > 1:
xpCtx.modelXbrl.error(
_("Formula {0} value is a sequence of length {1}").format(
formula, valueSeqLen), "err", "xbrlfe:nonSingletonOutputValue")
else:
if valueSeqLen == 0: #xsi:nil if no value
attrs.append((XbrlConst.qnXsiNil, "true"))
v = None
else:
# add precision/decimals for non-fraction numerics
if modelConcept.isNumeric and not modelConcept.isFraction:
if formula.hasDecimals:
decimals = formula.evaluateRule(xpCtx, Aspect.DECIMALS)
attrs.append(("decimals", decimals))
else:
if formula.hasPrecision:
precision = formula.evaluateRule(
xpCtx, Aspect.PRECISION)
else:
precision = 0
attrs.append(("precision", precision))
x = value[0]
if isinstance(x, float):
from math import (log10, isnan, isinf, fabs)
if (isnan(x)
or (precision and (isinf(precision) or precision == 0))
or (decimals and isinf(decimals))):
v = string(xpCtx, x)
elif decimals is not None:
v = "%.*f" % (int(decimals), x)
elif precision is not None:
a = fabs(x)
log = log10(a) if a != 0 else 0
v = "%.*f" % (int(precision) - int(log) -
(1 if a >= 1 else 0), x)
else: # no implicit precision yet
v = string(xpCtx, x)
elif isinstance(x, QName):
v = XmlUtil.addQnameValue(xbrlElt, x)
elif isinstance(x, datetime.datetime):
v = XmlUtil.dateunionValue(x)
else:
v = string(xpCtx, x)
itemElt = XmlUtil.addChild(
xbrlElt,
conceptQname,
attributes=attrs,
text=v,
afterSibling=xpCtx.outputLastFact.get(outputInstanceQname))
xpCtx.outputLastFact[outputInstanceQname] = itemElt
newFact = outputXbrlInstance.modelDocument.factDiscover(
itemElt, outputXbrlInstance.facts)
return newFact
def setArgForFactProperty(param, modelFact, propertyNameParts):
propVal = None
property = propertyNameParts[0]
if property == "value":
if isinstance(modelFact.xValue, Decimal):
propVal = "{:,}".format(modelFact.xValue)
else:
propVal = modelFact.value
elif property == "decimals":
propVal = modelFact.decimals
elif property == "label" and modelFact.concept is not None:
propVal = modelFact.concept.label(labelrole,
lang=lang,
linkroleHint=XbrlConst.defaultLinkRole)
elif property == "name":
propVal = str(modelFact.qname)
elif property == "localName":
propVal = modelFact.qname.localName
else:
cntx = modelFact.context
unit = modelFact.unit
if cntx is not None:
if property == "period":
if len(propertyNameParts) == 1:
if cntx.isForeverPeriod:
propVal = "forever"
elif cntx.isInstantPeriod:
propVal = XmlUtil.dateunionValue(cntx.instantDatetime, subtractOneDay=True)
else:
propVal = "{} to {}".format(XmlUtil.dateunionValue(cntx.startDatetime),
XmlUtil.dateunionValue(cntx.endDatetime, subtractOneDay=True))
else:
dateSelection = propertyNameParts[1]
if dateSelection == "startDate":
propVal = XmlUtil.dateunionValue(cntx.startDatetime)
elif dateSelection == "endDate":
propVal = XmlUtil.dateunionValue(cntx.endDatetime, subtractOneDay=True)
elif dateSelection == "instant":
propVal = XmlUtil.dateunionValue(cntx.instant, subtractOneDay=True)
elif dateSelection == "durationDays":
propVal = str((cntx.endDatetime - cntx.startDatetime).days)
elif property == "dimensions":
if cntx.qnameDims:
propVal = "\n".join("{} = {}".format(d.dimensionQname,
d.memberQname if d.isExplicit else
XmlUtil.xmlstring( XmlUtil.child(d), stripXmlns=True, prettyPrint=True ))
for d in cntx.qnameDims.values())
else:
propVal = "none"
if property == "unit":
if unit is None:
propVal = "none"
else:
measures = unit.measures
if measures[1]:
propVal = 'mul {}\ndiv {} '.format(
', '.join(measureFormat(m) for m in measures[0]),
', '.join(measureFormat(m) for m in measures[1]))
else:
propVal = ', '.join(measureFormat(m) for m in measures[0])
if propVal is not None:
fmtArgs[param] = propVal
def jsonDefaultEncoder(obj):
if isinstance(obj, Decimal):
return float(obj)
elif isinstance(obj, (datetime.date, datetime.datetime)):
return XmlUtil.dateunionValue(obj)
raise TypeError("Type {} is not supported for json output".format(type(obj).__name__))
def __init__(self, modelXbrl, xfFile):
self.modelXbrl = modelXbrl
self.xfFile = xfFile
self.xfLines = []
self.xmlns = {}
self.eltTypeCount = {}
for cfQnameArity in sorted(qnameArity
for qnameArity in self.modelXbrl.modelCustomFunctionSignatures.keys()
if isinstance(qnameArity, (tuple,list))):
cfObject = self.modelXbrl.modelCustomFunctionSignatures[cfQnameArity]
self.doObject(cfObject, None, "", set())
rootObjects = rootFormulaObjects(self) # sets var sets up
# put parameters at root regardless of whether linked to
for qn, param in sorted(self.modelXbrl.qnameParameters.items(), key=lambda i:i[0]):
self.doObject(param, None, "", set())
for rootObject in sorted(rootObjects, key=formulaObjSortKey):
self.doObject(rootObject, None, "", set())
if self.xmlns:
self.xfLines.insert(0, "")
for prefix, ns in sorted(self.xmlns.items(), reverse=True):
self.xfLines.insert(0, "namespace \"{}\" = \"{}\";".format(prefix, ns))
self.xfLines.insert(0, "")
self.xfLines.insert(0, "(: Generated from {} by Arelle on {} :)".format(modelXbrl.modelDocument.basename, XmlUtil.dateunionValue(datetime.datetime.now())))
with open(xfFile, "w", encoding="utf-8") as fh:
fh.write("\n".join(self.xfLines))
modelXbrl.info("info", "saved formula file %(file)s", file=xfFile)
def insertDataPoints(self):
# separate graph
# document-> dataTypeSet -> dataType
self.showStatus("insert DataPoints")
# note these initial aspects Qnames used also must be in conceptsUsed above
dimensions = [] # index by hash of dimension
dimensionIds = {} # index for dimension
if self.modelXbrl.modelDocument.type in (Type.INSTANCE, Type.INLINEXBRL):
contextAspectValueSelections = {} # contexts processed already
unitIDs = set() # units processed already
periodProxies = {}
entityIdentifierAspectProxies = {}
dataPoints = self.report['dataPoints']
for fact in self.modelXbrl.factsInInstance:
self.insertAspectProxies( (fact.qname,) )
factId = XmlUtil.elementFragmentIdentifier(fact)
dataPoints[factId] = dataPoint = {
'document': modelObjectDocumentUri(fact),
'id': factId,
'sourceLine': fact.sourceline,
'dataPointUrl': modelObjectUri(fact),
'baseItem': self.aspectQnameProxyId(fact.qname)
}
context = fact.context
concept = fact.concept
if context is not None:
if context.entityIdentifier not in entityIdentifierAspectProxies:
entityScheme, entityIdentifier = context.entityIdentifier
entityIdentifierAspectProxy = "{}/{}".format(
qnamePrefix_Name(XbrlConst.qnXbrliIdentifier),
entityIdentifier)
e = self.insertAspectProxy(XbrlConst.qnXbrliIdentifier, entityIdentifierAspectProxy)
e['scheme'] = entityScheme
e['identifier'] = entityIdentifier
entityIdentifierAspectProxies[context.entityIdentifier] = entityIdentifierAspectProxy
else:
entityIdentifierAspectProxy = entityIdentifierAspectProxies[context.entityIdentifier]
dataPoint['entityIdentifier'] = entityIdentifierAspectProxy
if context.isForeverPeriod:
period = "forever"
if context.isInstantPeriod:
endDate = XmlUtil.dateunionValue(context.instantDatetime, subtractOneDay=True).replace(':','_')
period = "instant/{}".format(endDate)
else:
startDate = XmlUtil.dateunionValue(context.startDatetime).replace(':','_')
endDate = XmlUtil.dateunionValue(context.endDatetime, subtractOneDay=True).replace(':','_')
period = "duration/{}/{}".format(startDate, endDate)
if period not in periodProxies:
periodProxy = "{}/{}".format(
qnamePrefix_Name(XbrlConst.qnXbrliPeriod),
period)
p = self.insertAspectProxy(XbrlConst.qnXbrliPeriod, periodProxy)
p['isForever'] = context.isForeverPeriod
p['isInstant'] = context.isInstantPeriod
if context.isStartEndPeriod:
d = context.startDatetime
if d.hour == 0 and d.minute == 0 and d.second == 0:
d = d.date()
p['startDate'] = d
if context.isStartEndPeriod or context.isInstantPeriod:
d = context.endDatetime
if d.hour == 0 and d.minute == 0 and d.second == 0:
d = (d - datetime.timedelta(1)).date()
p['endDate'] = d
periodProxies[period] = periodProxy
else:
periodProxy = periodProxies[period]
dataPoint['period'] = periodProxy
dataPoint['contextUrl'] = modelObjectUri(context)
dataPoint['contextId'] = context.id
if context.id not in contextAspectValueSelections:
contextAspectValueSelections[context.id] = contextAspectValueSelection = []
for dimVal in context.qnameDims.values():
dim = qnamePrefix_Name(dimVal.dimensionQname)
if dimVal.isExplicit:
self.insertAspectProxies( (dimVal.memberQname,) ) # need imediate use of proxy
v = self.aspectQnameProxyId(dimVal.memberQname)
else:
v = dimVal.typedMember.stringValue
dimProxy = "{}/{}".format(dim, v)
d = self.insertAspectProxy(dimVal.dimensionQname, dimProxy)
contextAspectValueSelection.append(dimProxy)
d['aspect'] = dim
if dimVal.isExplicit:
d['aspectValue'] = v
else:
d['typedValue'] = v
else:
contextAspectValueSelection = contextAspectValueSelections[context.id]
dataPoint['aspectValueSelections'] = contextAspectValueSelection
if fact.isNumeric:
if fact.precision == "INF":
dataPoint['precision'] = "INF"
elif fact.precision is not None:
dataPoint['precision'] = fact.precision
if fact.decimals == "INF":
dataPoint['decimals'] = "INF"
elif fact.decimals is not None:
dataPoint['decimals'] = fact.decimals
if fact.unit is not None:
unit = fact.unit
unitProxy = "{}/{}".format(
qnamePrefix_Name(XbrlConst.qnXbrliUnit),
unit.id)
dataPoint['unit'] = unitProxy
if unit.id not in unitIDs:
unitIDs.add(unit.id)
u = self.insertAspectProxy(XbrlConst.qnXbrliUnit, unitProxy)
u['unitId'] = unit.id
mults, divs = unit.measures
u['multiplyMeasures'] = [qnameUri(qn) for qn in mults]
if divs:
u['divideMeasures'] = [qnameUri(qn) for qn in divs]
if fact.xmlLang is None and fact.concept is not None and fact.concept.baseXsdType is not None:
dataPoint['value'] = fact.xValue
# The insert with base XSD type but no language
elif fact.xmlLang is not None:
# assuming string type with language
dataPoint['language'] = fact.xmlLang
dataPoint['value'] = fact.value
else:
# Otherwise insert as plain liternal with no language or datatype
dataPoint['value'] = fact.value
if fact.modelTupleFacts:
dataPoint['tuple'] = [XmlUtil.elementFragmentIdentifier(tupleFact)
for tupleFact in fact.modelTupleFacts]
def createContext(self, entityIdentScheme, entityIdentValue, periodType, periodStart, periodEndInstant, priItem, dims, segOCCs, scenOCCs,
afterSibling=None, beforeSibling=None):
xbrlElt = self.modelDocument.xmlRootElement
if afterSibling == AUTO_LOCATE_ELEMENT:
afterSibling = XmlUtil.lastChild(xbrlElt, XbrlConst.xbrli, ("schemaLocation", "roleType", "arcroleType", "context"))
cntxId = 'c-{0:02n}'.format( len(self.contexts) + 1)
newCntxElt = XmlUtil.addChild(xbrlElt, XbrlConst.xbrli, "context", attributes=("id", cntxId),
afterSibling=afterSibling, beforeSibling=beforeSibling)
entityElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "entity")
XmlUtil.addChild(entityElt, XbrlConst.xbrli, "identifier",
attributes=("scheme", entityIdentScheme),
text=entityIdentValue)
periodElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "period")
if periodType == "forever":
XmlUtil.addChild(periodElt, XbrlConst.xbrli, "forever")
elif periodType == "instant":
XmlUtil.addChild(periodElt, XbrlConst.xbrli, "instant",
text=XmlUtil.dateunionValue(periodEndInstant, subtractOneDay=True))
elif periodType == "duration":
XmlUtil.addChild(periodElt, XbrlConst.xbrli, "startDate",
text=XmlUtil.dateunionValue(periodStart))
XmlUtil.addChild(periodElt, XbrlConst.xbrli, "endDate",
text=XmlUtil.dateunionValue(periodEndInstant, subtractOneDay=True))
segmentElt = None
scenarioElt = None
from arelle.ModelInstanceObject import ModelDimensionValue
if dims: # requires primary item to determin ambiguous concepts
''' in theory we have to check full set of dimensions for validity in source or any other
context element, but for shortcut will see if each dimension is already reported in an
unambiguous valid contextElement
'''
from arelle.PrototypeInstanceObject import FactPrototype, ContextPrototype, DimValuePrototype
fp = FactPrototype(self, priItem, dims.items())
# force trying a valid prototype's context Elements
if not isFactDimensionallyValid(self, fp, setPrototypeContextElements=True):
self.info("arelleLinfo",
_("Create context for %(priItem)s, cannot determine valid context elements, no suitable hypercubes"),
modelObject=self, priItem=priItem)
fpDims = fp.context.qnameDims
for dimQname in sorted(fpDims.keys()):
dimValue = fpDims[dimQname]
if isinstance(dimValue, DimValuePrototype):
dimMemberQname = dimValue.memberQname # None if typed dimension
contextEltName = dimValue.contextElement
else: # qname for explicit or node for typed
dimMemberQname = None
contextEltName = None
if contextEltName == "segment":
if segmentElt is None:
segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli, "segment")
contextElt = segmentElt
elif contextEltName == "scenario":
if scenarioElt is None:
scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "scenario")
contextElt = scenarioElt
else:
self.info("arelleLinfo",
_("Create context, %(dimension)s, cannot determine context element, either no all relationship or validation issue"),
modelObject=self, dimension=dimQname),
continue
dimConcept = self.qnameConcepts[dimQname]
dimAttr = ("dimension", XmlUtil.addQnameValue(xbrlElt, dimConcept.qname))
if dimConcept.isTypedDimension:
dimElt = XmlUtil.addChild(contextElt, XbrlConst.xbrldi, "xbrldi:typedMember",
attributes=dimAttr)
if isinstance(dimValue, (ModelDimensionValue, DimValuePrototype)) and dimValue.isTyped:
XmlUtil.copyNodes(dimElt, dimValue.typedMember)
elif dimMemberQname:
dimElt = XmlUtil.addChild(contextElt, XbrlConst.xbrldi, "xbrldi:explicitMember",
attributes=dimAttr,
text=XmlUtil.addQnameValue(xbrlElt, dimMemberQname))
if segOCCs:
if segmentElt is None:
segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli, "segment")
XmlUtil.copyNodes(segmentElt, segOCCs)
if scenOCCs:
if scenarioElt is None:
scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "scenario")
XmlUtil.copyNodes(scenarioElt, scenOCCs)
self.modelDocument.contextDiscover(newCntxElt)
XmlValidate.validate(self, newCntxElt)
return newCntxElt
def produceOutputFact(xpCtx, formula, result):
priorErrorCount = len(xpCtx.modelXbrl.errors)
isTuple = isinstance(formula,ModelTuple)
# assemble context
conceptQname = aspectValue(xpCtx, formula, Aspect.CONCEPT, "xbrlfe:missingConceptRule")
if isinstance(conceptQname, VariableBindingError):
xpCtx.modelXbrl.error(conceptQname.err,
_("Formula %(xlinkLabel)s concept: %(concept)s"),
modelObject=formula, xlinkLabel=formula.xlinkLabel, concept=conceptQname.msg)
modelConcept = None
else:
modelConcept = xpCtx.modelXbrl.qnameConcepts[conceptQname]
if modelConcept is None or (not modelConcept.isTuple if isTuple else not modelConcept.isItem):
xpCtx.modelXbrl.error("xbrlfe:missingConceptRule",
_("Formula %(xlinkLabel)s concept %(concept)s is not a %(element)s"),
modelObject=formula, xlinkLabel=formula.xlinkLabel, concept=conceptQname, element=formula.localName)
outputLocation = aspectValue(xpCtx, formula, Aspect.LOCATION_RULE, None)
if not isTuple:
# entity
entityIdentScheme = aspectValue(xpCtx, formula, Aspect.SCHEME, "xbrlfe:missingEntityIdentifierRule")
if isinstance(entityIdentScheme, VariableBindingError):
xpCtx.modelXbrl.error(str(entityIdentScheme),
_("Formula %(xlinkLabel)s entity identifier scheme: %(scheme)s"),
modelObject=formula, xlinkLabel=formula.xlinkLabel, scheme=entityIdentScheme.msg)
entityIdentValue = None
else:
entityIdentValue = aspectValue(xpCtx, formula, Aspect.VALUE, "xbrlfe:missingEntityIdentifierRule")
if isinstance(entityIdentValue, VariableBindingError):
xpCtx.modelXbrl.error(str(entityIdentScheme),
_("Formula %(xlinkLabel)s entity identifier value: %(entityIdentifier)s"),
modelObject=formula, xlinkLabel=formula.xlinkLabel, entityIdentifier=entityIdentValue.msg)
# period
periodType = aspectValue(xpCtx, formula, Aspect.PERIOD_TYPE, "xbrlfe:missingPeriodRule")
periodStart = None
periodEndInstant = None
if isinstance(periodType, VariableBindingError):
xpCtx.modelXbrl.error(str(periodType),
_("Formula %(xlinkLabel)s period type: %(periodType)s"),
modelObject=formula, xlinkLabel=formula.xlinkLabel, periodType=periodType.msg)
elif periodType == "instant":
periodEndInstant = aspectValue(xpCtx, formula, Aspect.INSTANT, "xbrlfe:missingPeriodRule")
if isinstance(periodEndInstant, VariableBindingError):
xpCtx.modelXbrl.error(str(periodEndInstant),
_("Formula %(xlinkLabel)s period end: %(period)s"),
modelObject=formula, xlinkLabel=formula.xlinkLabel, period=periodEndInstant.msg)
elif periodType == "duration":
periodStart = aspectValue(xpCtx, formula, Aspect.START, "xbrlfe:missingPeriodRule")
if isinstance(periodStart, VariableBindingError):
xpCtx.modelXbrl.error(str(periodStart),
_("Formula %(xlinkLabel)s period start: %(period)s"),
modelObject=formula, xlinkLabel=formula.xlinkLabel, period=periodStart.msg)
periodEndInstant = aspectValue(xpCtx, formula, Aspect.END, "xbrlfe:missingPeriodRule")
if isinstance(periodEndInstant, VariableBindingError):
xpCtx.modelXbrl.error(str(periodEndInstant),
_("Formula %(xlinkLabel)s period end: %(period)s"),
modelObject=formula, xlinkLabel=formula.xlinkLabel, period=periodEndInstant.msg)
# unit
if modelConcept is not None and modelConcept.isNumeric:
unitSource = aspectValue(xpCtx, formula, Aspect.UNIT_MEASURES, None)
multDivBy = aspectValue(xpCtx, formula, Aspect.MULTIPLY_BY, "xbrlfe:missingUnitRule")
if isinstance(multDivBy, VariableBindingError):
xpCtx.modelXbrl.error(str(multDivBy) if isinstance(multDivBy, VariableBindingError) else "xbrlfe:missingUnitRule",
_("Formula %(xlinkLabel)s unit: %(unit)s"),
modelObject=formula, xlinkLabel=formula.xlinkLabel, unit=multDivBy.msg)
multiplyBy = (); divideBy = () # prevent errors later if bad
else:
divMultBy = aspectValue(xpCtx, formula, Aspect.DIVIDE_BY, "xbrlfe:missingUnitRule")
if isinstance(divMultBy, VariableBindingError):
xpCtx.modelXbrl.error(str(multDivBy) if isinstance(divMultBy, VariableBindingError) else "xbrlfe:missingUnitRule",
_("Formula %(xlinkLabel)s unit: %(unit)s"),
modelObject=formula, xlinkLabel=formula.xlinkLabel, unit=divMultBy.msg)
multiplyBy = (); divideBy = () # prevent errors later if bad
else:
multiplyBy = unitSource[0] + multDivBy[0] + divMultBy[1]
divideBy = unitSource[1] + multDivBy[1] + divMultBy[0]
# remove cancelling mult/div units
lookForCommonUnits = True
while lookForCommonUnits:
lookForCommonUnits = False
for commonUnit in multiplyBy:
if commonUnit in divideBy:
multiplyBy.remove(commonUnit)
divideBy.remove(commonUnit)
lookForCommonUnits = True
break
if len(multiplyBy) == 0: # if no units add pure
multiplyBy.append(XbrlConst.qnXbrliPure)
# dimensions
segOCCs = []
scenOCCs = []
if formula.aspectModel == "dimensional":
dimAspects = {}
dimQnames = aspectValue(xpCtx, formula, Aspect.DIMENSIONS, None)
if dimQnames:
for dimQname in dimQnames:
dimConcept = xpCtx.modelXbrl.qnameConcepts[dimQname]
dimErr = "xbrlfe:missing{0}DimensionRule".format("typed" if dimConcept is not None and dimConcept.isTypedDimension else "explicit")
dimValue = aspectValue(xpCtx, formula, dimQname, dimErr)
if isinstance(dimValue, VariableBindingError):
xpCtx.modelXbrl.error(dimErr,
_("Formula %(xlinkLabel)s dimension %(dimension)s: %(value)s"),
modelObject=formula, xlinkLabel=formula.xlinkLabel,
dimension=dimQname, value=dimValue.msg)
elif dimConcept.isTypedDimension:
if isinstance(dimValue, list): # result of flatten, always a list
if len(dimValue) != 1 or not isinstance(dimValue[0], ModelObject):
xpCtx.modelXbrl.error("xbrlfe:wrongXpathResultForTypedDimensionRule",
_("Formula %(xlinkLabel)s dimension %(dimension)s value is not a node: %(value)s"),
modelObject=formula, xlinkLabel=formula.xlinkLabel,
dimension=dimQname, value=dimValue)
continue
dimValue = dimValue[0]
dimAspects[dimQname] = dimValue
elif dimValue is not None and xpCtx.modelXbrl.qnameDimensionDefaults.get(dimQname) != dimValue:
dimAspects[dimQname] = dimValue
segOCCs = aspectValue(xpCtx, formula, Aspect.NON_XDT_SEGMENT, None)
scenOCCs = aspectValue(xpCtx, formula, Aspect.NON_XDT_SCENARIO, None)
for occElt in xpCtx.flattenSequence((segOCCs, scenOCCs)):
if isinstance(occElt, ModelObject) and occElt.namespaceURI == XbrlConst.xbrldi:
xpCtx.modelXbrl.error("xbrlfe:badSubsequentOCCValue",
_("Formula %(xlinkLabel)s OCC element %(occ)s covers a dimensional aspect"),
modelObject=(formula,occElt), xlinkLabel=formula.xlinkLabel,
occ=occElt.elementQname)
else:
dimAspects = None # non-dimensional
segOCCs = aspectValue(xpCtx, formula, Aspect.COMPLETE_SEGMENT, None)
scenOCCs = aspectValue(xpCtx, formula, Aspect.COMPLETE_SCENARIO, None)
if priorErrorCount < len(xpCtx.modelXbrl.errors):
return None # had errors, don't produce output fact
# does context exist in out instance document
outputInstanceQname = formula.outputInstanceQname
outputXbrlInstance = xpCtx.inScopeVars[outputInstanceQname]
xbrlElt = outputXbrlInstance.modelDocument.xmlRootElement
# in source instance document
newFact = None
if isTuple:
newFact = outputXbrlInstance.createFact(conceptQname, parent=outputLocation,
afterSibling=xpCtx.outputLastFact.get(outputInstanceQname))
else:
# add context
prevCntx = outputXbrlInstance.matchContext(
entityIdentScheme, entityIdentValue, periodType, periodStart, periodEndInstant,
dimAspects, segOCCs, scenOCCs)
if prevCntx is not None:
cntxId = prevCntx.id
newCntxElt = prevCntx
else:
newCntxElt = outputXbrlInstance.createContext(entityIdentScheme, entityIdentValue,
periodType, periodStart, periodEndInstant, conceptQname, dimAspects, segOCCs, scenOCCs,
afterSibling=xpCtx.outputLastContext.get(outputInstanceQname),
beforeSibling=xpCtx.outputFirstFact.get(outputInstanceQname))
cntxId = newCntxElt.id
xpCtx.outputLastContext[outputInstanceQname] = newCntxElt
# does unit exist
# add unit
if modelConcept.isNumeric:
prevUnit = outputXbrlInstance.matchUnit(multiplyBy, divideBy)
if prevUnit is not None:
unitId = prevUnit.id
newUnitElt = prevUnit
else:
newUnitElt = outputXbrlInstance.createUnit(multiplyBy, divideBy,
afterSibling=xpCtx.outputLastUnit.get(outputInstanceQname),
beforeSibling=xpCtx.outputFirstFact.get(outputInstanceQname))
unitId = newUnitElt.id
xpCtx.outputLastUnit[outputInstanceQname] = newUnitElt
# add fact
attrs = [("contextRef", cntxId)]
precision = None
decimals = None
if modelConcept.isNumeric:
attrs.append(("unitRef", unitId))
value = formula.evaluate(xpCtx)
valueSeqLen = len(value)
if valueSeqLen > 1:
xpCtx.modelXbrl.error("xbrlfe:nonSingletonOutputValue",
_("Formula %(xlinkLabel)s value is a sequence of length %(valueSequenceLength)s"),
modelObject=formula, xlinkLabel=formula.xlinkLabel, valueSequenceLength=valueSeqLen)
else:
if valueSeqLen == 0: #xsi:nil if no value
attrs.append((XbrlConst.qnXsiNil, "true"))
v = None
else:
# add precision/decimals for non-fraction numerics
if modelConcept.isNumeric and not modelConcept.isFraction:
if formula.hasDecimals:
decimals = formula.evaluateRule(xpCtx, Aspect.DECIMALS)
attrs.append(("decimals", decimals))
else:
if formula.hasPrecision:
precision = formula.evaluateRule(xpCtx, Aspect.PRECISION)
else:
precision = 0
attrs.append(("precision", precision))
x = value[0]
if isinstance(x,float):
from math import (log10, isnan, isinf, fabs)
if (isnan(x) or
(precision and (isinf(precision) or precision == 0)) or
(decimals and isinf(decimals))):
v = xsString(xpCtx, None, x)
elif decimals is not None:
v = "%.*f" % ( int(decimals), x)
elif precision is not None and precision != 0:
a = fabs(x)
log = log10(a) if a != 0 else 0
v = "%.*f" % ( int(precision) - int(log) - (1 if a >= 1 else 0), x)
else: # no implicit precision yet
v = xsString(xpCtx, None, x)
elif isinstance(x,QName):
v = XmlUtil.addQnameValue(xbrlElt, x)
elif isinstance(x,datetime.datetime):
v = XmlUtil.dateunionValue(x)
else:
v = xsString(xpCtx, None, x)
newFact = outputXbrlInstance.createFact(conceptQname, attributes=attrs, text=v,
parent=outputLocation,
afterSibling=xpCtx.outputLastFact.get(outputInstanceQname))
if newFact is not None:
xpCtx.outputLastFact[outputInstanceQname] = newFact
if outputInstanceQname not in xpCtx.outputFirstFact:
xpCtx.outputFirstFact[outputInstanceQname] = newFact
return newFact
def final(val):
if not (val.validateEBA or val.validateEIOPA):
return
modelXbrl = val.modelXbrl
modelDocument = modelXbrl.modelDocument
_statusMsg = _("validating {0} filing rules").format(val.disclosureSystem.name)
modelXbrl.profileActivity()
modelXbrl.modelManager.showStatus(_statusMsg)
if modelDocument.type == ModelDocument.Type.INSTANCE and (val.validateEBA or val.validateEIOPA):
if not modelDocument.uri.endswith(".xbrl"):
modelXbrl.warning("EBA.1.1",
_('XBRL instance documents SHOULD use the extension ".xbrl" but it is "%(extension)s"'),
modelObject=modelDocument, extension=os.path.splitext(modelDocument.basename)[1])
if modelDocument.documentEncoding.lower() not in ("utf-8", "utf-8-sig"):
modelXbrl.error("EBA.1.4",
_('XBRL instance documents MUST use "UTF-8" encoding but is "%(xmlEncoding)s"'),
modelObject=modelDocument, xmlEncoding=modelDocument.documentEncoding)
schemaRefElts = []
schemaRefFileNames = []
for doc, docRef in modelDocument.referencesDocument.items():
if docRef.referenceType == "href":
if docRef.referringModelObject.localName == "schemaRef":
schemaRefElts.append(docRef.referringModelObject)
schemaRefFileNames.append(doc.basename)
if not UrlUtil.isAbsolute(doc.uri):
modelXbrl.error("EBA.2.2",
_('The link:schemaRef element in submitted instances MUST resolve to the full published entry point URL: %(url)s.'),
modelObject=docRef.referringModelObject, url=doc.uri)
elif docRef.referringModelObject.localName == "linkbaseRef":
modelXbrl.error("EBA.2.3",
_('The link:linkbaseRef element is not allowed: %(fileName)s.'),
modelObject=docRef.referringModelObject, fileName=doc.basename)
if len(schemaRefFileNames) > 1:
modelXbrl.error("EBA.1.5",
_('XBRL instance documents MUST reference only one entry point schema but %(numEntryPoints)s were found: %(entryPointNames)s'),
modelObject=modelDocument, numEntryPoints=len(schemaRefFileNames), entryPointNames=', '.join(sorted(schemaRefFileNames)))
### check entry point names appropriate for filing indicator (DPM DB?)
if len(schemaRefElts) != 1:
modelXbrl.error("EBA.2.3",
_('Any reported XBRL instance document MUST contain only one xbrli:xbrl/link:schemaRef node, but %(entryPointCount)s.'),
modelObject=schemaRefElts, entryPointCount=len(schemaRefElts))
# non-streaming EBA checks
if not getattr(modelXbrl, "isStreamingMode", False):
validateFacts(val, modelXbrl.facts)
# check sum of fact md5s (otherwise checked in streaming process)
xbrlFactsCheckVersion = None
expectedSumOfFactMd5s = None
for pi in modelDocument.xmlRootElement.getchildren():
if isinstance(pi, etree._ProcessingInstruction) and pi.target == "xbrl-facts-check":
_match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']", pi.text)
if _match:
_matchGroups = _match.groups()
if len(_matchGroups) == 2:
if _matchGroups[0] == "version":
xbrlFactsCheckVersion = _matchGroups[1]
elif _matchGroups[0] == "sum-of-fact-md5s":
try:
expectedSumOfFactMd5s = Md5Sum(_matchGroups[1])
except ValueError:
modelXbrl.error("EIOPA:xbrlFactsCheckError",
_("Invalid sum-of-md5s %(sumOfMd5)s"),
modelObject=modelXbrl, sumOfMd5=_matchGroups[1])
if xbrlFactsCheckVersion and expectedSumOfFactMd5s:
sumOfFactMd5s = Md5Sum()
for f in modelXbrl.factsInInstance:
sumOfFactMd5s += f.md5sum
if sumOfFactMd5s != expectedSumOfFactMd5s:
modelXbrl.warning("EIOPA:xbrlFactsCheckWarning",
_("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"),
modelObject=modelXbrl, expectedMd5=expectedSumOfFactMd5s, actualMd5Sum=sumOfFactMd5s)
else:
modelXbrl.info("info",
_("Successful XBRL facts sum of md5s."),
modelObject=modelXbrl)
if not val.filingIndicators:
modelXbrl.error("EBA.1.6",
_('Missing filing indicators. Reported XBRL instances MUST include appropriate filing indicator elements'),
modelObject=modelDocument)
if val.numFilingIndicatorTuples > 1:
modelXbrl.info("EBA.1.6.2",
_('Multiple filing indicators tuples when not in streaming mode (info).'),
modelObject=modelXbrl.factsByQname[qnFIndicators])
if len(val.cntxDates) > 1:
modelXbrl.error("EBA.2.13",
_('Contexts must have the same date: %(dates)s.'),
# when streaming values are no longer available, but without streaming they can be logged
modelObject=set(_cntx for _cntxs in val.cntxDates.values() for _cntx in _cntxs),
dates=', '.join(XmlUtil.dateunionValue(_dt, subtractOneDay=True)
for _dt in val.cntxDates.keys()))
if val.unusedCntxIDs:
modelXbrl.warning("EBA.2.7",
_('Unused xbrli:context nodes SHOULD NOT be present in the instance: %(unusedContextIDs)s.'),
modelObject=[modelXbrl.contexts[unusedCntxID] for unusedCntxID in val.unusedCntxIDs if unusedCntxID in modelXbrl.contexts],
unusedContextIDs=", ".join(sorted(val.unusedCntxIDs)))
if len(val.cntxEntities) > 1:
modelXbrl.warning("EBA.2.9",
_('All entity identifiers and schemes must be the same, %(count)s found: %(entities)s.'),
modelObject=modelDocument, count=len(val.cntxEntities),
entities=", ".join(sorted(str(cntxEntity) for cntxEntity in val.cntxEntities)))
if val.unusedUnitIDs:
modelXbrl.warning("EBA.2.21",
_('Unused xbrli:unit nodes SHOULD NOT be present in the instance: %(unusedUnitIDs)s.'),
modelObject=[modelXbrl.units[unusedUnitID] for unusedUnitID in val.unusedUnitIDs if unusedUnitID in modelXbrl.units],
unusedUnitIDs=", ".join(sorted(val.unusedUnitIDs)))
if len(val.currenciesUsed) > 1:
modelXbrl.error("EBA.3.1",
_("There MUST be only one currency but %(numCurrencies)s were found: %(currencies)s.'"),
modelObject=val.currenciesUsed.values(), numCurrencies=len(val.currenciesUsed), currencies=", ".join(str(c) for c in val.currenciesUsed.keys()))
if val.prefixesUnused:
modelXbrl.warning("EBA.3.4",
_("There SHOULD be no unused prefixes but these were declared: %(unusedPrefixes)s.'"),
modelObject=modelDocument, unusedPrefixes=', '.join(sorted(val.prefixesUnused)))
for ns, prefixes in val.namespacePrefixesUsed.items():
nsDocs = modelXbrl.namespaceDocs.get(ns)
if nsDocs:
for nsDoc in nsDocs:
nsDocPrefix = XmlUtil.xmlnsprefix(nsDoc.xmlRootElement, ns)
if any(prefix != nsDocPrefix for prefix in prefixes if prefix is not None):
modelXbrl.warning("EBA.3.5",
_("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s"),
modelObject=modelDocument, namespace=ns, declaredPrefix=nsDocPrefix, foundPrefixes=', '.join(sorted(prefixes - {None})))
modelXbrl.profileActivity(_statusMsg, minTimeToShow=0.0)
modelXbrl.modelManager.showStatus(None)
del val.prefixNamespace, val.namespacePrefix, val.idObjects, val.typedDomainElements
del val.utrValidator
def run(self, options):
if options.logFile:
self.messages = []
else:
self.messages = None
self.filename = options.filename
filesource = FileSource.FileSource(self.filename, self)
if options.validateEFM:
if options.gfmName:
self.addToLog(
_("[info] both --efm and --gfm validation are requested, proceeding with --efm only"
))
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select("efm")
elif options.gfmName:
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select(options.gfmName)
else:
self.modelManager.disclosureSystem.select(
None) # just load ordinary mappings
if options.calcDecimals:
if options.calcPrecision:
self.addToLog(
_("[info] both --calcDecimals and --calcPrecision validation are requested, proceeding with --calcDecimals only"
))
self.modelManager.validateInferDecimals = True
self.modelManager.validateCalcLB = True
elif options.calcPrecision:
self.modelManager.validateInferDecimals = False
self.modelManager.validateCalcLB = True
if options.utrValidate:
self.modelManager.validateUtr = True
fo = FormulaOptions()
if options.formulaParamExprResult:
fo.traceParameterExpressionResult = True
if options.formulaParamInputValue:
fo.traceParameterInputValue = True
if options.formulaCallExprSource:
fo.traceCallExpressionSource = True
if options.formulaCallExprCode:
fo.traceCallExpressionCode = True
if options.formulaCallExprEval:
fo.traceCallExpressionEvaluation = True
if options.formulaCallExprResult:
fo.traceCallExpressionResult = True
if options.formulaVarSetExprEval:
fo.traceVariableSetExpressionEvaluation = True
if options.formulaVarSetExprResult:
fo.traceVariableSetExpressionResult = True
if options.formulaAsserResultCounts:
fo.traceAssertionResultCounts = True
if options.formulaFormulaRules:
fo.traceFormulaRules = True
if options.formulaVarsOrder:
fo.traceVariablesOrder = True
if options.formulaVarExpressionSource:
fo.traceVariableExpressionSource = True
if options.formulaVarExpressionCode:
fo.traceVariableExpressionCode = True
if options.formulaVarExpressionEvaluation:
fo.traceVariableExpressionEvaluation = True
if options.formulaVarExpressionResult:
fo.traceVariableExpressionResult = True
if options.formulaVarFiltersResult:
fo.traceVariableFiltersResult = True
self.modelManager.formulaOptions = fo
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
startedAt = time.time()
modelXbrl = self.modelManager.load(filesource, _("views loading"))
self.addToLog(
format_string(self.modelManager.locale,
_("[info] loaded in %.2f secs at %s"),
(time.time() - startedAt, timeNow)))
if options.diffFilename and options.versReportFilename:
diffFilesource = FileSource.FileSource(self.diffFilename, self)
startedAt = time.time()
modelXbrl = self.modelManager.load(diffFilesource,
_("views loading"))
self.addToLog(
format_string(
self.modelManager.locale,
_("[info] diff comparison DTS loaded in %.2f secs"),
time.time() - startedAt))
startedAt = time.time()
self.modelManager.compareDTSes(options.versReportFilename)
self.addToLog(
format_string(self.modelManager.locale,
_("[info] compared in %.2f secs"),
time.time() - startedAt))
try:
if options.validate:
startedAt = time.time()
self.modelManager.validate()
self.addToLog(
format_string(self.modelManager.locale,
_("[info] validated in %.2f secs"),
time.time() - startedAt))
if (options.csvTestReport
and self.modelManager.modelXbrl.modelDocument.type
in (ModelDocument.Type.TESTCASESINDEX,
ModelDocument.Type.REGISTRY)):
ViewCsvTests.viewTests(self.modelManager.modelXbrl,
options.csvTestReport)
if options.csvDTS:
ViewCsvDTS.viewDTS(modelXbrl, options.csvDTS)
if options.csvFactList:
ViewCsvFactList.viewFacts(modelXbrl, options.csvFactList)
if options.csvConcepts:
ViewCsvConcepts.viewConcepts(modelXbrl, options.csvConcepts)
if options.csvPre:
ViewCsvRelationshipSet.viewRelationshipSet(
modelXbrl, options.csvPre, "Presentation",
"http://www.xbrl.org/2003/arcrole/parent-child")
if options.csvCal:
ViewCsvRelationshipSet.viewRelationshipSet(
modelXbrl, options.csvCal, "Calculation",
"http://www.xbrl.org/2003/arcrole/summation-item")
if options.csvDim:
ViewCsvRelationshipSet.viewRelationshipSet(
modelXbrl, options.csvDim, "Dimension", "XBRL-dimensions")
except (IOError, EnvironmentError) as err:
self.addToLog(
_("[IOError] Failed to save output:\n {0}").format(err))
if self.messages:
try:
with open(options.logFile, "w", encoding="utf-8") as fh:
fh.writelines(self.messages)
except (IOError, EnvironmentError) as err:
print("Unable to save log to file: " + err)
def final(val):
if not (val.validateEBA or val.validateEIOPA):
return
modelXbrl = val.modelXbrl
modelDocument = modelXbrl.modelDocument
_statusMsg = _("validating {0} filing rules").format(val.disclosureSystem.name)
modelXbrl.profileActivity()
modelXbrl.modelManager.showStatus(_statusMsg)
if modelDocument.type == ModelDocument.Type.INSTANCE and (val.validateEBA or val.validateEIOPA):
if not modelDocument.uri.endswith(".xbrl"):
modelXbrl.warning("EBA.1.1",
_('XBRL instance documents SHOULD use the extension ".xbrl" but it is "%(extension)s"'),
modelObject=modelDocument, extension=os.path.splitext(modelDocument.basename)[1])
modelXbrl.error("EIOPA.S.1.1.a",
_('XBRL instance documents MUST use the extension ".xbrl" but it is "%(extension)s"'),
modelObject=modelDocument, extension=os.path.splitext(modelDocument.basename)[1])
if val.isEIOPA_2_0_1: _encodings = ("UTF-8", "utf-8-sig")
else: _encodings = ("utf-8", "UTF-8", "utf-8-sig")
if modelDocument.documentEncoding not in _encodings:
modelXbrl.error(("EBA.1.4", "EIOPA.1.4"),
_('XBRL instance documents MUST use "UTF-8" encoding but is "%(xmlEncoding)s"'),
modelObject=modelDocument, xmlEncoding=modelDocument.documentEncoding)
schemaRefElts = []
schemaRefFileNames = []
for doc, docRef in modelDocument.referencesDocument.items():
if docRef.referenceType == "href":
if docRef.referringModelObject.localName == "schemaRef":
schemaRefElts.append(docRef.referringModelObject)
schemaRefFileNames.append(doc.basename)
if not UrlUtil.isAbsolute(doc.uri):
modelXbrl.error(("EBA.2.2", "EIOPA.S.1.5.a" if val.isEIOPAfullVersion else "EIOPA.S.1.5.b"),
_('The link:schemaRef element in submitted instances MUST resolve to the full published entry point URL: %(url)s.'),
modelObject=docRef.referringModelObject, url=doc.uri,
messageCodes=("EBA.2.2", "EIOPA.S.1.5.a","EIOPA.S.1.5.b"))
elif docRef.referringModelObject.localName == "linkbaseRef":
modelXbrl.error(("EBA.2.3","EIOPA.S.1.5.a"),
_('The link:linkbaseRef element is not allowed: %(fileName)s.'),
modelObject=docRef.referringModelObject, fileName=doc.basename)
_numSchemaRefs = len(XmlUtil.children(modelDocument.xmlRootElement, XbrlConst.link, "schemaRef"))
if _numSchemaRefs > 1:
modelXbrl.error(("EIOPA.S.1.5.a", "EBA.1.5"),
_('XBRL instance documents MUST reference only one entry point schema but %(numEntryPoints)s were found: %(entryPointNames)s'),
modelObject=modelDocument, numEntryPoints=_numSchemaRefs, entryPointNames=', '.join(sorted(schemaRefFileNames)))
### check entry point names appropriate for filing indicator (DPM DB?)
if len(schemaRefElts) != 1:
modelXbrl.error("EBA.2.3",
_('Any reported XBRL instance document MUST contain only one xbrli:xbrl/link:schemaRef node, but %(entryPointCount)s.'),
modelObject=schemaRefElts, entryPointCount=len(schemaRefElts))
# non-streaming EBA checks
if not getattr(modelXbrl, "isStreamingMode", False):
val.qnReportedCurrency = None
if val.isEIOPA_2_0_1 and qnMetReportingCurrency in modelXbrl.factsByQname:
for _multiCurrencyFact in modelXbrl.factsByQname[qnMetReportingCurrency]:
# multi-currency fact
val.qnReportedCurrency = _multiCurrencyFact.xValue
break
validateFacts(val, modelXbrl.facts)
# check sum of fact md5s (otherwise checked in streaming process)
xbrlFactsCheckVersion = None
expectedSumOfFactMd5s = None
for pi in modelDocument.xmlRootElement.getchildren():
if isinstance(pi, etree._ProcessingInstruction) and pi.target == "xbrl-facts-check":
_match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']", pi.text)
if _match:
_matchGroups = _match.groups()
if len(_matchGroups) == 2:
if _matchGroups[0] == "version":
xbrlFactsCheckVersion = _matchGroups[1]
elif _matchGroups[0] == "sum-of-fact-md5s":
try:
expectedSumOfFactMd5s = Md5Sum(_matchGroups[1])
except ValueError:
modelXbrl.error("EIOPA:xbrlFactsCheckError",
_("Invalid sum-of-md5s %(sumOfMd5)s"),
modelObject=modelXbrl, sumOfMd5=_matchGroups[1])
if xbrlFactsCheckVersion and expectedSumOfFactMd5s:
sumOfFactMd5s = Md5Sum()
for f in modelXbrl.factsInInstance:
sumOfFactMd5s += f.md5sum
if sumOfFactMd5s != expectedSumOfFactMd5s:
modelXbrl.warning("EIOPA:xbrlFactsCheckWarning",
_("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"),
modelObject=modelXbrl, expectedMd5=expectedSumOfFactMd5s, actualMd5Sum=sumOfFactMd5s)
else:
modelXbrl.info("info",
_("Successful XBRL facts sum of md5s."),
modelObject=modelXbrl)
if any(badError in modelXbrl.errors
for badError in ("EBA.2.1", "EIOPA.2.1", "EIOPA.S.1.5.a/EIOPA.S.1.5.b")):
pass # skip checking filingIndicators if bad errors
elif not val.filingIndicators:
modelXbrl.error(("EBA.1.6", "EIOPA.1.6.a"),
_('Missing filing indicators. Reported XBRL instances MUST include appropriate (positive) filing indicator elements'),
modelObject=modelDocument)
elif all(filed == False for filed in val.filingIndicators.values()):
modelXbrl.error(("EBA.1.6", "EIOPA.1.6.a"),
_('All filing indicators are filed="false". Reported XBRL instances MUST include appropriate (positive) filing indicator elements'),
modelObject=modelDocument)
if val.numFilingIndicatorTuples > 1:
modelXbrl.warning(("EBA.1.6.2", "EIOPA.1.6.2"),
_('Multiple filing indicators tuples when not in streaming mode (info).'),
modelObject=modelXbrl.factsByQname[qnFIndicators])
if len(val.cntxDates) > 1:
modelXbrl.error(("EBA.2.13","EIOPA.2.13"),
_('Contexts must have the same date: %(dates)s.'),
# when streaming values are no longer available, but without streaming they can be logged
modelObject=set(_cntx for _cntxs in val.cntxDates.values() for _cntx in _cntxs),
dates=', '.join(XmlUtil.dateunionValue(_dt, subtractOneDay=True)
for _dt in val.cntxDates.keys()))
if val.unusedCntxIDs:
if val.isEIOPA_2_0_1:
modelXbrl.error("EIOPA.2.7",
_('Unused xbrli:context nodes MUST NOT be present in the instance: %(unusedContextIDs)s.'),
modelObject=[modelXbrl.contexts[unusedCntxID] for unusedCntxID in val.unusedCntxIDs if unusedCntxID in modelXbrl.contexts],
unusedContextIDs=", ".join(sorted(val.unusedCntxIDs)))
else:
modelXbrl.warning(("EBA.2.7", "EIOPA.2.7"),
_('Unused xbrli:context nodes SHOULD NOT be present in the instance: %(unusedContextIDs)s.'),
modelObject=[modelXbrl.contexts[unusedCntxID] for unusedCntxID in val.unusedCntxIDs if unusedCntxID in modelXbrl.contexts],
unusedContextIDs=", ".join(sorted(val.unusedCntxIDs)))
if len(val.cntxEntities) > 1:
modelXbrl.error(("EBA.2.9", "EIOPA.2.9"),
_('All entity identifiers and schemes MUST be the same, %(count)s found: %(entities)s.'),
modelObject=modelDocument, count=len(val.cntxEntities),
entities=", ".join(sorted(str(cntxEntity) for cntxEntity in val.cntxEntities)))
for _scheme, _LEI in val.cntxEntities:
if (_scheme in ("http://standards.iso.org/iso/17442", "http://standard.iso.org/iso/17442", "LEI") or
(not val.isEIOPAfullVersion and _scheme == "PRE-LEI")):
result = LeiUtil.checkLei(_LEI)
if result == LeiUtil.LEI_INVALID_LEXICAL:
modelXbrl.error("EIOPA.S.2.8.c",
_("Context has lexically invalid LEI %(lei)s."),
modelObject=modelDocument, lei=_LEI)
elif result == LeiUtil.LEI_INVALID_CHECKSUM:
modelXbrl.error("EIOPA.S.2.8.c",
_("Context has LEI checksum error in %(lei)s."),
modelObject=modelDocument, lei=_LEI)
if _scheme == "http://standard.iso.org/iso/17442":
modelXbrl.warning("EIOPA.S.2.8.c",
_("Warning, context has entity scheme %(scheme)s should be plural: http://standards.iso.org/iso/17442."),
modelObject=modelDocument, scheme=_scheme)
elif _scheme == "SC":
pass # anything is ok for Specific Code
else:
modelXbrl.error("EIOPA.S.2.8.c",
_("Context has unrecognized entity scheme %(scheme)s."),
modelObject=modelDocument, scheme=_scheme)
if val.unusedUnitIDs:
if val.isEIOPA_2_0_1:
modelXbrl.error("EIOPA.2.22",
_('Unused xbrli:unit nodes MUST NOT be present in the instance: %(unusedUnitIDs)s.'),
modelObject=[modelXbrl.units[unusedUnitID] for unusedUnitID in val.unusedUnitIDs if unusedUnitID in modelXbrl.units],
unusedUnitIDs=", ".join(sorted(val.unusedUnitIDs)))
else:
modelXbrl.warning(("EBA.2.22", "EIOPA.2.22"),
_('Unused xbrli:unit nodes SHOULD NOT be present in the instance: %(unusedUnitIDs)s.'),
modelObject=[modelXbrl.units[unusedUnitID] for unusedUnitID in val.unusedUnitIDs if unusedUnitID in modelXbrl.units],
unusedUnitIDs=", ".join(sorted(val.unusedUnitIDs)))
if len(val.currenciesUsed) > 1:
modelXbrl.error(("EBA.3.1","EIOPA.3.1"),
_("There MUST be only one currency but %(numCurrencies)s were found: %(currencies)s.'"),
modelObject=val.currenciesUsed.values(), numCurrencies=len(val.currenciesUsed), currencies=", ".join(str(c) for c in val.currenciesUsed.keys()))
elif val.isEIOPA_2_0_1 and any(_measure.localName != val.reportingCurrency for _measure in val.currenciesUsed.keys()):
modelXbrl.error("EIOPA.3.1",
_("There MUST be only one currency but reporting currency %(reportingCurrency)s differs from unit currencies: %(currencies)s.'"),
modelObject=val.currenciesUsed.values(), reportingCurrency=val.reportingCurrency, currencies=", ".join(str(c) for c in val.currenciesUsed.keys()))
if val.prefixesUnused:
modelXbrl.warning(("EBA.3.4", "EIOPA.3.4"),
_("There SHOULD be no unused prefixes but these were declared: %(unusedPrefixes)s.'"),
modelObject=modelDocument, unusedPrefixes=', '.join(sorted(val.prefixesUnused)))
for ns, prefixes in val.namespacePrefixesUsed.items():
nsDocs = modelXbrl.namespaceDocs.get(ns)
if nsDocs:
for nsDoc in nsDocs:
nsDocPrefix = XmlUtil.xmlnsprefix(nsDoc.xmlRootElement, ns)
if any(prefix != nsDocPrefix for prefix in prefixes if prefix is not None):
modelXbrl.warning(("EBA.3.5", "EIOPA.3.5"),
_("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s"),
modelObject=modelDocument, namespace=ns, declaredPrefix=nsDocPrefix, foundPrefixes=', '.join(sorted(prefixes - {None})))
elif ns in CANONICAL_PREFIXES and any(prefix != CANONICAL_PREFIXES[ns] for prefix in prefixes if prefix is not None):
modelXbrl.warning(("EBA.3.5", "EIOPA.3.5"),
_("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s"),
modelObject=modelDocument, namespace=ns, declaredPrefix=CANONICAL_PREFIXES[ns], foundPrefixes=', '.join(sorted(prefixes - {None})))
modelXbrl.profileActivity(_statusMsg, minTimeToShow=0.0)
modelXbrl.modelManager.showStatus(None)
del val.prefixNamespace, val.namespacePrefix, val.idObjects, val.typedDomainElements
del val.utrValidator, val.firstFact, val.footnotesRelationshipSet
def final(val):
if not (val.validateEBA or val.validateEIOPA):
return
modelXbrl = val.modelXbrl
modelDocument = modelXbrl.modelDocument
_statusMsg = _("validating {0} filing rules").format(
val.disclosureSystem.name)
modelXbrl.profileActivity()
modelXbrl.modelManager.showStatus(_statusMsg)
if modelDocument.type == ModelDocument.Type.INSTANCE and (
val.validateEBA or val.validateEIOPA):
if not modelDocument.uri.endswith(".xbrl"):
modelXbrl.warning(
"EBA.1.1",
_('XBRL instance documents SHOULD use the extension ".xbrl" but it is "%(extension)s"'
),
modelObject=modelDocument,
extension=os.path.splitext(modelDocument.basename)[1])
if modelDocument.documentEncoding.lower() not in ("utf-8",
"utf-8-sig"):
modelXbrl.error(
"EBA.1.4",
_('XBRL instance documents MUST use "UTF-8" encoding but is "%(xmlEncoding)s"'
),
modelObject=modelDocument,
xmlEncoding=modelDocument.documentEncoding)
schemaRefElts = []
schemaRefFileNames = []
for doc, docRef in modelDocument.referencesDocument.items():
if docRef.referenceType == "href":
if docRef.referringModelObject.localName == "schemaRef":
schemaRefElts.append(docRef.referringModelObject)
schemaRefFileNames.append(doc.basename)
if not UrlUtil.isAbsolute(doc.uri):
modelXbrl.error(
"EBA.2.2",
_('The link:schemaRef element in submitted instances MUST resolve to the full published entry point URL: %(url)s.'
),
modelObject=docRef.referringModelObject,
url=doc.uri)
elif docRef.referringModelObject.localName == "linkbaseRef":
modelXbrl.error(
"EBA.2.3",
_('The link:linkbaseRef element is not allowed: %(fileName)s.'
),
modelObject=docRef.referringModelObject,
fileName=doc.basename)
if len(schemaRefFileNames) > 1:
modelXbrl.error(
"EBA.1.5",
_('XBRL instance documents MUST reference only one entry point schema but %(numEntryPoints)s were found: %(entryPointNames)s'
),
modelObject=modelDocument,
numEntryPoints=len(schemaRefFileNames),
entryPointNames=', '.join(sorted(schemaRefFileNames)))
### check entry point names appropriate for filing indicator (DPM DB?)
if len(schemaRefElts) != 1:
modelXbrl.error(
"EBA.2.3",
_('Any reported XBRL instance document MUST contain only one xbrli:xbrl/link:schemaRef node, but %(entryPointCount)s.'
),
modelObject=schemaRefElts,
entryPointCount=len(schemaRefElts))
# non-streaming EBA checks
if not getattr(modelXbrl, "isStreamingMode", False):
validateFacts(val, modelXbrl.facts)
# check sum of fact md5s (otherwise checked in streaming process)
xbrlFactsCheckVersion = None
expectedSumOfFactMd5s = None
for pi in modelDocument.xmlRootElement.getchildren():
if isinstance(pi, etree._ProcessingInstruction
) and pi.target == "xbrl-facts-check":
_match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']",
pi.text)
if _match:
_matchGroups = _match.groups()
if len(_matchGroups) == 2:
if _matchGroups[0] == "version":
xbrlFactsCheckVersion = _matchGroups[1]
elif _matchGroups[0] == "sum-of-fact-md5s":
try:
expectedSumOfFactMd5s = Md5Sum(
_matchGroups[1])
except ValueError:
modelXbrl.error(
"EIOPA:xbrlFactsCheckError",
_("Invalid sum-of-md5s %(sumOfMd5)s"),
modelObject=modelXbrl,
sumOfMd5=_matchGroups[1])
if xbrlFactsCheckVersion and expectedSumOfFactMd5s:
sumOfFactMd5s = Md5Sum()
for f in modelXbrl.factsInInstance:
sumOfFactMd5s += f.md5sum
if sumOfFactMd5s != expectedSumOfFactMd5s:
modelXbrl.warning(
"EIOPA:xbrlFactsCheckWarning",
_("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"
),
modelObject=modelXbrl,
expectedMd5=expectedSumOfFactMd5s,
actualMd5Sum=sumOfFactMd5s)
else:
modelXbrl.info("info",
_("Successful XBRL facts sum of md5s."),
modelObject=modelXbrl)
if not val.filingIndicators:
modelXbrl.error(
"EBA.1.6",
_('Missing filing indicators. Reported XBRL instances MUST include appropriate filing indicator elements'
),
modelObject=modelDocument)
if val.numFilingIndicatorTuples > 1:
modelXbrl.info(
"EBA.1.6.2",
_('Multiple filing indicators tuples when not in streaming mode (info).'
),
modelObject=modelXbrl.factsByQname[qnFIndicators])
if len(val.cntxDates) > 1:
modelXbrl.error(
"EBA.2.13",
_('Contexts must have the same date: %(dates)s.'),
# when streaming values are no longer available, but without streaming they can be logged
modelObject=set(_cntx for _cntxs in val.cntxDates.values()
for _cntx in _cntxs),
dates=', '.join(
XmlUtil.dateunionValue(_dt, subtractOneDay=True)
for _dt in val.cntxDates.keys()))
if val.unusedCntxIDs:
modelXbrl.warning(
"EBA.2.7",
_('Unused xbrli:context nodes SHOULD NOT be present in the instance: %(unusedContextIDs)s.'
),
modelObject=[
modelXbrl.contexts[unusedCntxID]
for unusedCntxID in val.unusedCntxIDs
if unusedCntxID in modelXbrl.contexts
],
unusedContextIDs=", ".join(sorted(val.unusedCntxIDs)))
if len(val.cntxEntities) > 1:
modelXbrl.warning(
"EBA.2.9",
_('All entity identifiers and schemes must be the same, %(count)s found: %(entities)s.'
),
modelObject=modelDocument,
count=len(val.cntxEntities),
entities=", ".join(
sorted(str(cntxEntity)
for cntxEntity in val.cntxEntities)))
if val.unusedUnitIDs:
modelXbrl.warning(
"EBA.2.21",
_('Unused xbrli:unit nodes SHOULD NOT be present in the instance: %(unusedUnitIDs)s.'
),
modelObject=[
modelXbrl.units[unusedUnitID]
for unusedUnitID in val.unusedUnitIDs
if unusedUnitID in modelXbrl.units
],
unusedUnitIDs=", ".join(sorted(val.unusedUnitIDs)))
if len(val.currenciesUsed) > 1:
modelXbrl.error(
"EBA.3.1",
_("There MUST be only one currency but %(numCurrencies)s were found: %(currencies)s.'"
),
modelObject=val.currenciesUsed.values(),
numCurrencies=len(val.currenciesUsed),
currencies=", ".join(
str(c) for c in val.currenciesUsed.keys()))
if val.prefixesUnused:
modelXbrl.warning(
"EBA.3.4",
_("There SHOULD be no unused prefixes but these were declared: %(unusedPrefixes)s.'"
),
modelObject=modelDocument,
unusedPrefixes=', '.join(sorted(val.prefixesUnused)))
for ns, prefixes in val.namespacePrefixesUsed.items():
nsDocs = modelXbrl.namespaceDocs.get(ns)
if nsDocs:
for nsDoc in nsDocs:
nsDocPrefix = XmlUtil.xmlnsprefix(nsDoc.xmlRootElement, ns)
if any(prefix != nsDocPrefix for prefix in prefixes
if prefix is not None):
modelXbrl.warning(
"EBA.3.5",
_("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s"
),
modelObject=modelDocument,
namespace=ns,
declaredPrefix=nsDocPrefix,
foundPrefixes=', '.join(sorted(prefixes - {None})))
modelXbrl.profileActivity(_statusMsg, minTimeToShow=0.0)
modelXbrl.modelManager.showStatus(None)
del val.prefixNamespace, val.namespacePrefix, val.idObjects, val.typedDomainElements
del val.utrValidator
def run(self, options):
self.filename = options.filename
filesource = FileSource.openFileSource(self.filename,self)
if options.validateEFM:
if options.gfmName:
self.addToLog(_("both --efm and --gfm validation are requested, proceeding with --efm only"),
messageCode="info", file=self.filename)
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select("efm")
elif options.gfmName:
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select(options.gfmName)
else:
self.modelManager.disclosureSystem.select(None) # just load ordinary mappings
if options.calcDecimals:
if options.calcPrecision:
self.addToLog(_("both --calcDecimals and --calcPrecision validation are requested, proceeding with --calcDecimals only"),
messageCode="info", file=self.filename)
self.modelManager.validateInferDecimals = True
self.modelManager.validateCalcLB = True
elif options.calcPrecision:
self.modelManager.validateInferDecimals = False
self.modelManager.validateCalcLB = True
if options.utrValidate:
self.modelManager.validateUtr = True
fo = FormulaOptions()
if options.formulaParamExprResult:
fo.traceParameterExpressionResult = True
if options.formulaParamInputValue:
fo.traceParameterInputValue = True
if options.formulaCallExprSource:
fo.traceCallExpressionSource = True
if options.formulaCallExprCode:
fo.traceCallExpressionCode = True
if options.formulaCallExprEval:
fo.traceCallExpressionEvaluation = True
if options.formulaCallExprResult:
fo.traceCallExpressionResult = True
if options.formulaVarSetExprEval:
fo.traceVariableSetExpressionEvaluation = True
if options.formulaVarSetExprResult:
fo.traceVariableSetExpressionResult = True
if options.formulaAsserResultCounts:
fo.traceAssertionResultCounts = True
if options.formulaFormulaRules:
fo.traceFormulaRules = True
if options.formulaVarsOrder:
fo.traceVariablesOrder = True
if options.formulaVarExpressionSource:
fo.traceVariableExpressionSource = True
if options.formulaVarExpressionCode:
fo.traceVariableExpressionCode = True
if options.formulaVarExpressionEvaluation:
fo.traceVariableExpressionEvaluation = True
if options.formulaVarExpressionResult:
fo.traceVariableExpressionResult = True
if options.formulaVarFiltersResult:
fo.traceVariableFiltersResult = True
self.modelManager.formulaOptions = fo
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
startedAt = time.time()
modelXbrl = self.modelManager.load(filesource, _("views loading"))
self.addToLog(format_string(self.modelManager.locale,
_("loaded in %.2f secs at %s"),
(time.time() - startedAt, timeNow)),
messageCode="info", file=self.filename)
if options.diffFilename and options.versReportFilename:
diffFilesource = FileSource.FileSource(self.diffFilename,self)
startedAt = time.time()
modelXbrl = self.modelManager.load(diffFilesource, _("views loading"))
self.addToLog(format_string(self.modelManager.locale,
_("diff comparison DTS loaded in %.2f secs"),
time.time() - startedAt),
messageCode="info", file=self.filename)
startedAt = time.time()
self.modelManager.compareDTSes(options.versReportFilename)
self.addToLog(format_string(self.modelManager.locale,
_("compared in %.2f secs"),
time.time() - startedAt),
messageCode="info", file=self.filename)
try:
if options.validate:
startedAt = time.time()
self.modelManager.validate()
self.addToLog(format_string(self.modelManager.locale,
_("validated in %.2f secs"),
time.time() - startedAt),
messageCode="info", file=self.filename)
if (options.csvTestReport and
self.modelManager.modelXbrl.modelDocument.type in
(ModelDocument.Type.TESTCASESINDEX,
ModelDocument.Type.TESTCASE,
ModelDocument.Type.REGISTRY)):
ViewCsvTests.viewTests(self.modelManager.modelXbrl, options.csvTestReport)
if options.csvDTS:
ViewCsvDTS.viewDTS(modelXbrl, options.csvDTS)
if options.csvFactList:
ViewCsvFactList.viewFacts(modelXbrl, options.csvFactList, cols=options.csvFactListCols)
if options.csvConcepts:
ViewCsvConcepts.viewConcepts(modelXbrl, options.csvConcepts)
if options.csvPre:
ViewCsvRelationshipSet.viewRelationshipSet(modelXbrl, options.csvPre, "Presentation", "http://www.xbrl.org/2003/arcrole/parent-child")
if options.csvCal:
ViewCsvRelationshipSet.viewRelationshipSet(modelXbrl, options.csvCal, "Calculation", "http://www.xbrl.org/2003/arcrole/summation-item")
if options.csvDim:
ViewCsvRelationshipSet.viewRelationshipSet(modelXbrl, options.csvDim, "Dimension", "XBRL-dimensions")
except (IOError, EnvironmentError) as err:
self.addToLog(_("[IOError] Failed to save output:\n {0}").format(err))
except Exception as err:
self.addToLog(_("[Exception] Failed to complete validation: \n{0} \n{1}").format(
err,
traceback.format_tb(sys.exc_info()[2])))
def produceOutputFact(xpCtx, formula, result):
priorErrorCount = len(xpCtx.modelXbrl.errors)
# assemble context
conceptQname = aspectValue(xpCtx, formula, Aspect.CONCEPT, "xbrlfe:missingConceptRule")
if isinstance(conceptQname, VariableBindingError):
xpCtx.modelXbrl.error( _("Formula {0} concept: {1}").format( formula, conceptQname.msg),
"err", conceptQname.err)
modelConcept = None
else:
modelConcept = xpCtx.modelXbrl.qnameConcepts[conceptQname]
if modelConcept is None or not modelConcept.isItem:
xpCtx.modelXbrl.error( _("Formula {0} concept {1} is not an item").format( formula, conceptQname),
"err", "xbrlfe:missingConceptRule")
# entity
entityIdentScheme = aspectValue(xpCtx, formula, Aspect.SCHEME, "xbrlfe:missingEntityIdentifierRule")
if isinstance(entityIdentScheme, VariableBindingError):
xpCtx.modelXbrl.error( _("Formula {0} entity identifier scheme: {1}").format( formula, entityIdentScheme.msg ),
"err", str(entityIdentScheme))
entityIdentValue = None
else:
entityIdentValue = aspectValue(xpCtx, formula, Aspect.VALUE, "xbrlfe:missingEntityIdentifierRule")
if isinstance(entityIdentValue, VariableBindingError):
xpCtx.modelXbrl.error( _("Formula {0} entity identifier value: {1}").format( formula, entityIdentValue.msg ),
"err", str(entityIdentScheme))
# period
periodType = aspectValue(xpCtx, formula, Aspect.PERIOD_TYPE, "xbrlfe:missingPeriodRule")
periodStart = None
periodEndInstant = None
if isinstance(periodType, VariableBindingError):
xpCtx.modelXbrl.error( _("Formula {0} period type: {1}").format( formula, periodType.msg ),
"err", str(periodType))
elif periodType == "instant":
periodEndInstant = aspectValue(xpCtx, formula, Aspect.INSTANT, "xbrlfe:missingPeriodRule")
if isinstance(periodEndInstant, VariableBindingError):
xpCtx.modelXbrl.error( _("Formula {0} period start: {1}").format( formula, periodEndInstant.msg ),
"err", str(periodEndInstant))
elif periodType == "duration":
periodStart = aspectValue(xpCtx, formula, Aspect.START, "xbrlfe:missingPeriodRule")
if isinstance(periodStart, VariableBindingError):
xpCtx.modelXbrl.error( _("Formula {0} period start: {1}").format( formula, periodStart.msg ),
"err", str(periodStart))
periodEndInstant = aspectValue(xpCtx, formula, Aspect.END, "xbrlfe:missingPeriodRule")
if isinstance(periodEndInstant, VariableBindingError):
xpCtx.modelXbrl.error( _("Formula {0} period end: {1}").format( formula, periodEndInstant.msg ),
"err", str(periodEndInstant))
# unit
if modelConcept and modelConcept.isNumeric:
unitSource = aspectValue(xpCtx, formula, Aspect.UNIT_MEASURES, None)
multDivBy = aspectValue(xpCtx, formula, Aspect.MULTIPLY_BY, "xbrlfe:missingUnitRule")
if isinstance(multDivBy, VariableBindingError):
xpCtx.modelXbrl.error( _("Formula {0} unit: {1}").format( formula, multDivBy.msg ),
"err", str(multDivBy) if isinstance(multDivBy, VariableBindingError) else "xbrlfe:missingUnitRule")
multiplyBy = (); divideBy = () # prevent errors later if bad
else:
divMultBy = aspectValue(xpCtx, formula, Aspect.DIVIDE_BY, "xbrlfe:missingUnitRule")
if isinstance(divMultBy, VariableBindingError):
xpCtx.modelXbrl.error( _("Formula {0} unit: {1}").format( formula, divMultBy.msg ),
"err", str(multDivBy) if isinstance(divMultBy, VariableBindingError) else "xbrlfe:missingUnitRule")
multiplyBy = (); divideBy = () # prevent errors later if bad
else:
multiplyBy = unitSource[0] + multDivBy[0] + divMultBy[1]
divideBy = unitSource[1] + multDivBy[1] + divMultBy[0]
# remove cancelling mult/div units
lookForCommonUnits = True
while lookForCommonUnits:
lookForCommonUnits = False
for commonUnit in multiplyBy:
if commonUnit in divideBy:
multiplyBy.remove(commonUnit)
divideBy.remove(commonUnit)
lookForCommonUnits = True
break
if len(multiplyBy) == 0: # if no units add pure
multiplyBy.append(XbrlConst.qnXbrliPure)
# dimensions
segOCCs = []
scenOCCs = []
if formula.aspectModel == "dimensional":
dimAspects = {}
dimQnames = aspectValue(xpCtx, formula, Aspect.DIMENSIONS, None)
if dimQnames:
for dimQname in dimQnames:
dimConcept = xpCtx.modelXbrl.qnameConcepts[dimQname]
dimErr = "xbrlfe:missing{0}DimensionRule".format("typed" if dimConcept and dimConcept.isTypedDimension else "explicit")
dimValue = aspectValue(xpCtx, formula, dimQname, dimErr)
if isinstance(dimValue, VariableBindingError):
xpCtx.modelXbrl.error( _("Formula {0} dimension {1}: {2}").format( formula, dimQname, dimValue.msg ),
"err", dimErr)
elif dimValue and xpCtx.modelXbrl.qnameDimensionDefaults.get(dimQname) != dimValue:
dimAspects[dimQname] = dimValue
segOCCs = aspectValue(xpCtx, formula, Aspect.NON_XDT_SEGMENT, None)
scenOCCs = aspectValue(xpCtx, formula, Aspect.NON_XDT_SCENARIO, None)
else:
dimAspects = None # non-dimensional
segOCCs = aspectValue(xpCtx, formula, Aspect.COMPLETE_SEGMENT, None)
scenOCCs = aspectValue(xpCtx, formula, Aspect.COMPLETE_SCENARIO, None)
if priorErrorCount < len(xpCtx.modelXbrl.errors):
return None # had errors, don't produce output fact
# does context exist in out instance document
outputInstanceQname = formula.outputInstanceQname
outputXbrlInstance = xpCtx.inScopeVars[outputInstanceQname]
xbrlElt = outputXbrlInstance.modelDocument.xmlRootElement
# in source instance document
# add context
prevCntx = outputXbrlInstance.matchContext(
entityIdentScheme, entityIdentValue, periodType, periodStart, periodEndInstant,
dimAspects, segOCCs, scenOCCs)
if prevCntx:
cntxId = prevCntx.id
newCntxElt = prevCntx.element
else:
cntxId = 'c-{0:02n}'.format( len(outputXbrlInstance.contexts) + 1)
newCntxElt = XmlUtil.addChild(xbrlElt, XbrlConst.xbrli, "context", attributes=("id", cntxId),
afterSibling=xpCtx.outputLastContext.get(outputInstanceQname))
xpCtx.outputLastContext[outputInstanceQname] = newCntxElt
entityElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "entity")
XmlUtil.addChild(entityElt, XbrlConst.xbrli, "identifier",
attributes=("scheme", entityIdentScheme),
text=entityIdentValue)
periodElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "period")
if periodType == "forever":
XmlUtil.addChild(periodElt, XbrlConst.xbrli, "forever")
elif periodType == "instant":
XmlUtil.addChild(periodElt, XbrlConst.xbrli, "instant",
text=XmlUtil.dateunionValue(periodEndInstant, subtractOneDay=True))
elif periodType == "duration":
XmlUtil.addChild(periodElt, XbrlConst.xbrli, "startDate",
text=XmlUtil.dateunionValue(periodStart))
XmlUtil.addChild(periodElt, XbrlConst.xbrli, "endDate",
text=XmlUtil.dateunionValue(periodEndInstant, subtractOneDay=True))
segmentElt = None
scenarioElt = None
from arelle.ModelObject import ModelDimensionValue
if dimAspects:
for dimQname in sorted(dimAspects.keys()):
dimValue = dimAspects[dimQname]
if isinstance(dimValue, ModelDimensionValue):
if dimValue.isExplicit:
dimMemberQname = dimValue.memberQname
contextEltName = dimValue.contextElement
else: # qname for explicit or node for typed
dimMemberQname = dimValue
contextEltName = xpCtx.modelXbrl.qnameDimensionContextElement.get(dimQname)
if contextEltName == "segment":
if not segmentElt:
segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli, "segment")
contextElt = segmentElt
elif contextEltName == "scenario":
if not scenarioElt:
scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "scenario")
contextElt = scenarioElt
else:
continue
dimConcept = xpCtx.modelXbrl.qnameConcepts[dimQname]
dimAttr = ("dimension", XmlUtil.addQnameValue(xbrlElt, dimConcept.qname))
if dimConcept.isTypedDimension:
dimElt = XmlUtil.addChild(contextElt, XbrlConst.xbrldi, "typedMember",
attributes=dimAttr)
if isinstance(dimValue, ModelDimensionValue) and dimValue.isTyped:
XmlUtil.copyChildren(dimElt, dimValue.typedMember)
elif dimMemberQname:
dimElt = XmlUtil.addChild(contextElt, XbrlConst.xbrldi, "explicitMember",
attributes=dimAttr,
text=XmlUtil.addQnameValue(xbrlElt, dimMemberQname))
if segOCCs:
if not segmentElt:
segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli, "segment")
XmlUtil.copyNodes(segmentElt, segOCCs)
if scenOCCs:
if not scenarioElt:
scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "scenario")
XmlUtil.copyNodes(scenarioElt, scenOCCs)
outputXbrlInstance.modelDocument.contextDiscover(newCntxElt)
# does unit exist
# add unit
if modelConcept.isNumeric:
prevUnit = outputXbrlInstance.matchUnit(multiplyBy, divideBy)
if prevUnit:
unitId = prevUnit.id
newUnitElt = prevUnit.element
else:
unitId = 'u-{0:02n}'.format( len(outputXbrlInstance.units) + 1)
newUnitElt = XmlUtil.addChild(xbrlElt, XbrlConst.xbrli, "unit", attributes=("id", unitId),
afterSibling=xpCtx.outputLastUnit.get(outputInstanceQname))
xpCtx.outputLastUnit[outputInstanceQname] = newUnitElt
if len(divideBy) == 0:
for multiply in multiplyBy:
XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, multiply))
else:
divElt = XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "divide")
numElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitNumerator")
denElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitDenominator")
for multiply in multiplyBy:
XmlUtil.addChild(numElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, multiply))
for divide in divideBy:
XmlUtil.addChild(denElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, divide))
outputXbrlInstance.modelDocument.unitDiscover(newUnitElt)
# add fact
attrs = [("contextRef", cntxId)]
precision = None
decimals = None
if modelConcept.isNumeric:
attrs.append(("unitRef", unitId))
value = formula.evaluate(xpCtx)
valueSeqLen = len(value)
if valueSeqLen > 1:
xpCtx.modelXbrl.error( _("Formula {0} value is a sequence of length {1}").format( formula, valueSeqLen ),
"err", "xbrlfe:nonSingletonOutputValue")
else:
if valueSeqLen == 0: #xsi:nil if no value
attrs.append((XbrlConst.qnXsiNil, "true"))
v = None
else:
# add precision/decimals for non-fraction numerics
if modelConcept.isNumeric and not modelConcept.isFraction:
if formula.hasDecimals:
decimals = formula.evaluateRule(xpCtx, Aspect.DECIMALS)
attrs.append(("decimals", decimals))
else:
if formula.hasPrecision:
precision = formula.evaluateRule(xpCtx, Aspect.PRECISION)
else:
precision = 0
attrs.append(("precision", precision))
x = value[0]
if isinstance(x,float):
from math import (log10, isnan, isinf, fabs)
if (isnan(x) or
(precision and (isinf(precision) or precision == 0)) or
(decimals and isinf(decimals))):
v = string(xpCtx, x)
elif decimals is not None:
v = "%.*f" % ( int(decimals), x)
elif precision is not None:
a = fabs(x)
log = log10(a) if a != 0 else 0
v = "%.*f" % ( int(precision) - int(log) - (1 if a >= 1 else 0), x)
else: # no implicit precision yet
v = string(xpCtx, x)
elif isinstance(x,QName):
v = XmlUtil.addQnameValue(xbrlElt, x)
elif isinstance(x,datetime.datetime):
v = XmlUtil.dateunionValue(x)
else:
v = string(xpCtx, x)
itemElt = XmlUtil.addChild(xbrlElt, conceptQname,
attributes=attrs, text=v,
afterSibling=xpCtx.outputLastFact.get(outputInstanceQname))
xpCtx.outputLastFact[outputInstanceQname] = itemElt
newFact = outputXbrlInstance.modelDocument.factDiscover(itemElt, outputXbrlInstance.facts)
return newFact
