def __init__(self, savedOptions=None, xbrlInstance=None): self.entityIdentScheme = "" self.entityIdentValue = "" self.startDate = "" # use string values so structure can be json-saved self.endDate = "" self.monetaryUnit = "" self.monetaryDecimals = "" self.nonMonetaryDecimals = "" if savedOptions is not None: self.__dict__.update(savedOptions) elif xbrlInstance is not None: for fact in xbrlInstance.facts: cntx = fact.context unit = fact.unit if fact.isItem and cntx is not None: if not self.entityIdentScheme: self.entityIdentScheme, self.entityIdentValue = cntx.entityIdentifier if not self.startDate and cntx.isStartEndPeriod: self.startDate = XmlUtil.dateunionValue(cntx.startDatetime) if not self.startDate and (cntx.isStartEndPeriod or cntx.isInstantPeriod): self.endDate = XmlUtil.dateunionValue(cntx.endDatetime, subtractOneDay=True) if fact.isNumeric and unit is not None: if fact.concept.isMonetary: if not self.monetaryUnit and unit.measures[0] and unit.measures[0][0].namespaceURI == XbrlConst.iso4217: self.monetaryUnit = unit.measures[0][0].localName if not self.monetaryDecimals: self.monetaryDecimals = fact.decimals elif not self.nonMonetaryDecimals: self.nonMonetaryDecimals = fact.decimals if self.entityIdentScheme and self.startDate and self.monetaryUnit and self.monetaryDecimals and self.nonMonetaryDecimals: break
def factAspectValue(fact, aspect, view=False): if aspect == Aspect.LOCATION: parentQname = fact.getparent().qname if parentQname == XbrlConst.qnXbrliXbrl: # not tuple return NONE return parentQname # tuple elif aspect == Aspect.CONCEPT: return fact.qname elif fact.isTuple or fact.context is None: return NONE #subsequent aspects don't exist for tuples elif aspect == Aspect.UNIT: if fact.unit is None: return NONE measures = fact.unit.measures if measures[1]: return "{0} / {1}".format(' '.join(str(m) for m in measures[0]), ' '.join(str(m) for m in measures[1])) else: return ' '.join(str(m) for m in measures[0]) else: context = fact.context if aspect == Aspect.PERIOD: return ("forever" if context.isForeverPeriod else XmlUtil.dateunionValue(context.instantDatetime, subtractOneDay=True) if context.isInstantPeriod else XmlUtil.dateunionValue(context.startDatetime) + "-" + XmlUtil.dateunionValue(context.endDatetime, subtractOneDay=True)) elif aspect == Aspect.ENTITY_IDENTIFIER: if view: return context.entityIdentifier[1] else: return context.entityIdentifier # (scheme, identifier) elif aspect in (Aspect.COMPLETE_SEGMENT, Aspect.COMPLETE_SCENARIO, Aspect.NON_XDT_SEGMENT, Aspect.NON_XDT_SCENARIO): return ''.join(XmlUtil.xmlstring(elt, stripXmlns=True, prettyPrint=True) for elt in context.nonDimValues(aspect))
def __init__(self, mainWin, options): self.mainWin = mainWin parent = mainWin.parent super(DialogNewFactItemOptions, self).__init__(parent) self.parent = parent self.options = options parentGeometry = re.match("(\d+)x(\d+)[+]?([-]?\d+)[+]?([-]?\d+)", parent.geometry()) dialogX = int(parentGeometry.group(3)) dialogY = int(parentGeometry.group(4)) self.accepted = False self.transient(self.parent) self.title(_("New Fact Item Options")) frame = Frame(self) label(frame, 1, 1, "Entity scheme:") self.cellEntityIdentScheme = gridCell(frame, 2, 1, getattr(options,"entityIdentScheme",""), width=50) ToolTip(self.cellEntityIdentScheme, text=_("Enter the scheme for the context entity identifier"), wraplength=240) label(frame, 1, 2, "Entity identifier:") self.cellEntityIdentValue = gridCell(frame, 2, 2, getattr(options,"entityIdentValue","")) ToolTip(self.cellEntityIdentValue, text=_("Enter the entity identifier value (e.g., stock ticker)"), wraplength=240) label(frame, 1, 3, "Start date:") startDate = getattr(options,"startDate",None) self.cellStartDate = gridCell(frame, 2, 3, XmlUtil.dateunionValue(startDate) if startDate else "") ToolTip(self.cellStartDate, text=_("Enter the start date for the report period (e.g., 2010-01-01)"), wraplength=240) label(frame, 1, 4, "End date:") endDate = getattr(options,"endDate",None) self.cellEndDate = gridCell(frame, 2, 4, XmlUtil.dateunionValue(endDate, subtractOneDay=True) if endDate else "") ToolTip(self.cellEndDate, text=_("Enter the end date for the report period (e.g., 2010-12-31)"), wraplength=240) label(frame, 1, 5, "Monetary unit:") self.cellMonetaryUnit = gridCombobox(frame, 2, 5, getattr(options,"monetaryUnit",""), values=monetaryUnits) ToolTip(self.cellMonetaryUnit, text=_("Select a monetary unit (e.g., EUR)"), wraplength=240) label(frame, 1, 6, "Monetary decimals:") self.cellMonetaryDecimals = gridCell(frame, 2, 6, getattr(options,"monetaryDecimals","2")) ToolTip(self.cellMonetaryDecimals, text=_("Enter decimals for monetary items"), wraplength=240) label(frame, 1, 7, "Non-monetary decimals:") self.cellNonMonetaryDecimals = gridCell(frame, 2, 7, getattr(options,"nonMonetaryDecimals","0")) ToolTip(self.cellNonMonetaryDecimals, text=_("Enter decimals for non-monetary items (e.g., stock shares)"), wraplength=240) cancelButton = Button(frame, text=_("Cancel"), width=8, command=self.close) ToolTip(cancelButton, text=_("Cancel operation, discarding changes and entries")) okButton = Button(frame, text=_("OK"), width=8, command=self.ok) ToolTip(okButton, text=_("Accept the options as entered above")) cancelButton.grid(row=8, column=1, columnspan=3, sticky=E, pady=3, padx=3) okButton.grid(row=8, column=1, columnspan=3, sticky=E, pady=3, padx=86) frame.grid(row=0, column=0, sticky=(N,S,E,W)) frame.columnconfigure(2, weight=1) window = self.winfo_toplevel() window.columnconfigure(0, weight=1) self.geometry("+{0}+{1}".format(dialogX+50,dialogY+100)) #self.bind("<Return>", self.ok) #self.bind("<Escape>", self.close) self.protocol("WM_DELETE_WINDOW", self.close) self.grab_set() self.wait_window(self)
def __repr__(self): return ("modelContext[{0}, period: {1}, {2}{3} line {4}]" .format(self.id, "forever" if self.isForeverPeriod else "instant " + XmlUtil.dateunionValue(self.instantDatetime, subtractOneDay=True) if self.isInstantPeriod else "duration " + XmlUtil.dateunionValue(self.startDatetime) + " - " + XmlUtil.dateunionValue(self.endDatetime, subtractOneDay=True), "dimensions: ({0}) {1},".format(len(self.qnameDims), tuple(mem.propertyView for dim,mem in sorted(self.qnameDims.items()))) if self.qnameDims else "", self.modelDocument.basename, self.sourceline))
def propertyView(self): scheme, entityId = self.entityIdentifier return ((("entity", entityId, (("scheme", scheme),)),) + ((("forever", ""),) if self.isForeverPeriod else (("instant", XmlUtil.dateunionValue(self.instantDatetime, subtractOneDay=True)),) if self.isInstantPeriod else (("startDate", XmlUtil.dateunionValue(self.startDatetime)),("endDate", XmlUtil.dateunionValue(self.endDatetime, subtractOneDay=True)))) + (("dimensions", "({0})".format(len(self.qnameDims)), tuple(mem.propertyView for dim,mem in sorted(self.qnameDims.items()))) if self.qnameDims else (), ))
def insertAccession(self, rssItem): self.showStatus("insert accession") # accession graph -> document vertices new_accession = {'class':'accession', 'is_most_current': True} if self.modelXbrl.modelDocument.creationSoftwareComment: new_accession['creation_software'] = self.modelXbrl.modelDocument.creationSoftwareComment datetimeNow = datetime.datetime.now() datetimeNowStr = XmlUtil.dateunionValue(datetimeNow) if rssItem is not None: # sec accession # set self. accessionType = "SEC_accession" # for an RSS Feed entry from SEC, use rss item's accession information new_accession['accepted_timestamp'] = XmlUtil.dateunionValue(rssItem.acceptanceDatetime) new_accession['filing_date'] = XmlUtil.dateunionValue(rssItem.filingDate) new_accession['entity_id'] = rssItem.cikNumber new_accession['entity_name'] = rssItem.companyName new_accession['standard_industrial_classification'] = rssItem.assignedSic new_accession['sec_html_url'] = rssItem.htmlUrl new_accession['entry_url'] = rssItem.url new_accession['filing_accession_number'] = filing_accession_number = rssItem.accessionNumber else: # not an RSS Feed item, make up our own accession ID (the time in seconds of epoch) self.accessionId = int(time.time()) # only available if entered from an SEC filing intNow = int(time.time()) accessionType = "independent_filing" new_accession['accepted_timestamp'] = datetimeNowStr new_accession['filing_date'] = datetimeNowStr new_accession['entry_url'] = self.modelXbrl.fileSource.url new_accession['filing_accession_number'] = filing_accession_number = str(intNow) for id in self.execute("Insert accession " + accessionType, """ r = g.v(root_accessions_id) // check if accession already has a vertex vIt = g.V('filing_accession_number', new_accession.filing_accession_number) // use prior vertex, or if none, create new vertex for it accession = (vIt.hasNext() ? vIt.next() : g.addVertex(new_accession) ) // TBD: modify accession timestamp (last-updated-at, if it already existed) // check if vertex has edge to root_accessions vertex vIn = accession.in // if no edge, add one vIn.hasNext() && vIn.next() == r ?: g.addEdge(r, accession, accession_type) accession.id """, params={'root_accessions_id': self.root_accessions_id, 'new_accession': new_accession, 'accession_type': accessionType, 'datetime_now': datetimeNowStr, })["results"]: self.accession_id = int(id) # relationshipSets are a dts property self.relationshipSets = [(arcrole, ELR, linkqname, arcqname) for arcrole, ELR, linkqname, arcqname in self.modelXbrl.baseSets.keys() if ELR and linkqname and arcqname and not arcrole.startswith("XBRL-")]
def factAspectValue(fact, aspect, view=False): if fact is DEFAULT: return 'none' elif fact is NONDEFAULT: return '*' elif fact is DEFAULTorNONDEFAULT: return '**' elif aspect == Aspect.LOCATION: parentQname = fact.getparent().qname if parentQname == XbrlConst.qnXbrliXbrl: # not tuple return NONE return parentQname # tuple elif aspect == Aspect.CONCEPT: return fact.qname elif fact.isTuple or fact.context is None: return NONE #subsequent aspects don't exist for tuples elif aspect == Aspect.UNIT: if fact.unit is None: return NONE measures = fact.unit.measures if measures[1]: return "{0} / {1}".format(' '.join(str(m) for m in measures[0]), ' '.join(str(m) for m in measures[1])) else: return ' '.join(str(m) for m in measures[0]) else: context = fact.context if aspect == Aspect.PERIOD: return ("forever" if context.isForeverPeriod else XmlUtil.dateunionValue(context.instantDatetime, subtractOneDay=True) if context.isInstantPeriod else XmlUtil.dateunionValue(context.startDatetime) + "-" + XmlUtil.dateunionValue(context.endDatetime, subtractOneDay=True)) elif aspect == Aspect.ENTITY_IDENTIFIER: if view: return context.entityIdentifier[1] else: return context.entityIdentifier # (scheme, identifier) elif aspect in (Aspect.COMPLETE_SEGMENT, Aspect.COMPLETE_SCENARIO, Aspect.NON_XDT_SEGMENT, Aspect.NON_XDT_SCENARIO): return ''.join(XmlUtil.xmlstring(elt, stripXmlns=True, prettyPrint=True) for elt in context.nonDimValues(aspect)) elif aspect == Aspect.DIMENSIONS: return context.dimAspects(fact.xpCtx.defaultDimensionAspects) elif isinstance(aspect, QName): dimValue = context.dimValue(aspect) if dimValue is None: return NONE else: if isinstance(dimValue, QName): #default dim return dimValue elif dimValue.isExplicit: return dimValue.memberQname else: # explicit return dimValue.typedMember.xValue # typed element value
def insertFiling(self, rssItem, g): self.showStatus("insert filing") # accession graph -> document vertices new_filing = {'documents': []} if self.modelXbrl.modelDocument.creationSoftwareComment: new_filing['creation_software'] = self.modelXbrl.modelDocument.creationSoftwareComment datetimeNow = datetime.datetime.now() datetimeNowStr = XmlUtil.dateunionValue(datetimeNow) entryUri = modelObjectDocumentUri(self.modelXbrl) if rssItem is not None: # sec accession # set self. new_filing['filingType'] = "SEC filing" # for an RSS Feed entry from SEC, use rss item's accession information new_filing['filingNumber'] = filingNumber = rssItem.accessionNumber new_filing['acceptedTimestamp'] = XmlUtil.dateunionValue(rssItem.acceptanceDatetime) new_filing['filingDate'] = XmlUtil.dateunionValue(rssItem.filingDate) new_filing['entityId'] = rssItem.cikNumber new_filing['entityName'] = rssItem.companyName new_filing['SICCode'] = rssItem.assignedSic new_filing['SECHtmlUrl'] = rssItem.htmlUrl new_filing['entryUrl'] = rssItem.url self.filingURI = rssItem.htmlUrl else: # not an RSS Feed item, make up our own accession ID (the time in seconds of epoch) intNow = int(time.time()) new_filing['filingNumber'] = filingNumber = str(intNow) self.filingId = int(time.time()) # only available if entered from an SEC filing new_filing['filingType'] = "independent filing" new_filing['acceptedTimestamp'] = datetimeNowStr new_filing['filingDate'] = datetimeNowStr new_filing['entryUrl'] = UrlUtil.ensureUrl(self.modelXbrl.fileSource.url) self.filingURI = filingNumber g[FILINGS][self.filingURI] = new_filing self.filing = new_filing # for now only one report per filing (but SEC may have multiple in future, such as form SD) self.reportURI = modelObjectDocumentUri(self.modelXbrl) self.report = {'filing': self.filingURI, 'aspectProxies': {}, 'relationshipSets': {}, 'dataPoints': {}, 'messages': {}} new_filing['reports'] = {self.reportURI: self.report} # relationshipSets are a dts property self.relationshipSets = [(arcrole, ELR, linkqname, arcqname) for arcrole, ELR, linkqname, arcqname in self.modelXbrl.baseSets.keys() if ELR and (arcrole.startswith("XBRL-") or (linkqname and arcqname))]
def __init__(self, cntlr, xfFile): self.modelXbrl = cntlr.modelManager.modelXbrl self.xfFile = xfFile self.xfLines = [] self.xmlns = {} self.eltTypeCount = {} self.nextIdDupNbr = {} cntlr.showStatus(_("Initializing Formula Grammar")) XPathParser.initializeParser(cntlr.modelManager) cntlr.showStatus(None) for cfQnameArity in sorted( qnameArity for qnameArity in self.modelXbrl.modelCustomFunctionSignatures.keys() if isinstance(qnameArity, (tuple, list))): cfObject = self.modelXbrl.modelCustomFunctionSignatures[ cfQnameArity] self.doObject(cfObject, None, "", set()) rootObjects = rootFormulaObjects(self) # sets var sets up # put parameters at root regardless of whether linked to for qn, param in sorted(self.modelXbrl.qnameParameters.items(), key=lambda i: i[0]): self.doObject(param, None, "", set()) for rootObject in sorted(rootObjects, key=formulaObjSortKey): self.doObject(rootObject, None, "", set()) if self.xmlns: self.xfLines.insert(0, "") for prefix, ns in sorted(self.xmlns.items(), reverse=True): self.xfLines.insert( 0, "namespace {} = \"{}\";".format(prefix, ns)) self.xfLines.insert(0, "") self.xfLines.insert( 0, "(: Generated from {} by Arelle on {} :)".format( self.modelXbrl.modelDocument.basename, XmlUtil.dateunionValue(datetime.datetime.now()))) with open(xfFile, "w", encoding="utf-8") as fh: fh.write("\n".join(self.xfLines)) self.modelXbrl.info("info", "saved formula file %(file)s", file=xfFile)
def run(self, options, sourceZipStream=None): """Process command line arguments or web service request, such as to load and validate an XBRL document, or start web server. When a web server has been requested, this method may be called multiple times, once for each web service (REST) request that requires processing. Otherwise (when called for a command line request) this method is called only once for the command line arguments request. :param options: OptionParser options from parse_args of main argv arguments (when called from command line) or corresponding arguments from web service (REST) request. :type options: optparse.Values """ if options.showOptions: # debug options for optName, optValue in sorted(options.__dict__.items(), key=lambda optItem: optItem[0]): self.addToLog("Option {0}={1}".format(optName, optValue), messageCode="info") self.addToLog("sys.argv {0}".format(sys.argv), messageCode="info") if options.uiLang: # set current UI Lang (but not config setting) self.setUiLanguage(options.uiLang) if options.proxy: if options.proxy != "show": proxySettings = proxyTuple(options.proxy) self.webCache.resetProxies(proxySettings) self.config["proxySettings"] = proxySettings self.saveConfig() self.addToLog(_("Proxy configuration has been set."), messageCode="info") useOsProxy, urlAddr, urlPort, user, password = self.config.get("proxySettings", proxyTuple("none")) if useOsProxy: self.addToLog(_("Proxy configured to use {0}.").format( _('Microsoft Windows Internet Settings') if sys.platform.startswith("win") else (_('Mac OS X System Configuration') if sys.platform in ("darwin", "macos") else _('environment variables'))), messageCode="info") elif urlAddr: self.addToLog(_("Proxy setting: http://{0}{1}{2}{3}{4}").format( user if user else "", ":****" if password else "", "@" if (user or password) else "", urlAddr, ":{0}".format(urlPort) if urlPort else ""), messageCode="info") else: self.addToLog(_("Proxy is disabled."), messageCode="info") if options.plugins: from arelle import PluginManager resetPlugins = False savePluginChanges = True showPluginModules = False for pluginCmd in options.plugins.split('|'): cmd = pluginCmd.strip() if cmd == "show": showPluginModules = True elif cmd == "temp": savePluginChanges = False elif cmd.startswith("+"): moduleInfo = PluginManager.addPluginModule(cmd[1:]) if moduleInfo: self.addToLog(_("Addition of plug-in {0} successful.").format(moduleInfo.get("name")), messageCode="info", file=moduleInfo.get("moduleURL")) resetPlugins = True else: self.addToLog(_("Unable to load plug-in."), messageCode="info", file=cmd[1:]) elif cmd.startswith("~"): if PluginManager.reloadPluginModule(cmd[1:]): self.addToLog(_("Reload of plug-in successful."), messageCode="info", file=cmd[1:]) resetPlugins = True else: self.addToLog(_("Unable to reload plug-in."), messageCode="info", file=cmd[1:]) elif cmd.startswith("-"): if PluginManager.removePluginModule(cmd[1:]): self.addToLog(_("Deletion of plug-in successful."), messageCode="info", file=cmd[1:]) resetPlugins = True else: self.addToLog(_("Unable to delete plug-in."), messageCode="info", file=cmd[1:]) else: # assume it is a module or package savePluginChanges = False moduleInfo = PluginManager.addPluginModule(cmd) if moduleInfo: self.addToLog(_("Activation of plug-in {0} successful.").format(moduleInfo.get("name")), messageCode="info", file=moduleInfo.get("moduleURL")) resetPlugins = True else: self.addToLog(_("Unable to load {0} as a plug-in or {0} is not recognized as a command. ").format(cmd), messageCode="info", file=cmd) if resetPlugins: PluginManager.reset() if savePluginChanges: PluginManager.save(self) if showPluginModules: self.addToLog(_("Plug-in modules:"), messageCode="info") for i, moduleItem in enumerate(sorted(PluginManager.pluginConfig.get("modules", {}).items())): moduleInfo = moduleItem[1] self.addToLog(_("Plug-in: {0}; author: {1}; version: {2}; status: {3}; date: {4}; description: {5}; license {6}.").format( moduleItem[0], moduleInfo.get("author"), moduleInfo.get("version"), moduleInfo.get("status"), moduleInfo.get("fileDate"), moduleInfo.get("description"), moduleInfo.get("license")), messageCode="info", file=moduleInfo.get("moduleURL")) # run utility command line options that don't depend on entrypoint Files hasUtilityPlugin = False for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Utility.Run"): hasUtilityPlugin = True pluginXbrlMethod(self, options) # if no entrypointFile is applicable, quit now if options.proxy or options.plugins or hasUtilityPlugin: if not options.entrypointFile: return True # success self.username = options.username self.password = options.password self.entrypointFile = options.entrypointFile if self.entrypointFile: filesource = FileSource.openFileSource(self.entrypointFile, self, sourceZipStream) else: filesource = None if options.validateEFM: if options.disclosureSystemName: self.addToLog(_("both --efm and --disclosureSystem validation are requested, proceeding with --efm only"), messageCode="info", file=self.entrypointFile) self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select("efm") elif options.disclosureSystemName: self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select(options.disclosureSystemName) elif options.validateHMRC: self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select("hmrc") else: self.modelManager.disclosureSystem.select(None) # just load ordinary mappings self.modelManager.validateDisclosureSystem = False if options.utrUrl: # override disclosureSystem utrUrl self.modelManager.disclosureSystem.utrUrl = options.utrUrl # can be set now because the utr is first loaded at validation time # disclosure system sets logging filters, override disclosure filters, if specified by command line if options.logLevelFilter: self.setLogLevelFilter(options.logLevelFilter) if options.logCodeFilter: self.setLogCodeFilter(options.logCodeFilter) if options.calcDecimals: if options.calcPrecision: self.addToLog(_("both --calcDecimals and --calcPrecision validation are requested, proceeding with --calcDecimals only"), messageCode="info", file=self.entrypointFile) self.modelManager.validateInferDecimals = True self.modelManager.validateCalcLB = True elif options.calcPrecision: self.modelManager.validateInferDecimals = False self.modelManager.validateCalcLB = True if options.utrValidate: self.modelManager.validateUtr = True if options.infosetValidate: self.modelManager.validateInfoset = True if options.abortOnMajorError: self.modelManager.abortOnMajorError = True if options.collectProfileStats: self.modelManager.collectProfileStats = True if options.internetConnectivity == "offline": self.webCache.workOffline = True elif options.internetConnectivity == "online": self.webCache.workOffline = False if options.internetTimeout is not None: self.webCache.timeout = (options.internetTimeout or None) # use None if zero specified to disable timeout fo = FormulaOptions() if options.parameters: parameterSeparator = (options.parameterSeparator or ',') fo.parameterValues = dict(((qname(key, noPrefixIsNoNamespace=True),(None,value)) for param in options.parameters.split(parameterSeparator) for key,sep,value in (param.partition('='),) ) ) if options.formulaParamExprResult: fo.traceParameterExpressionResult = True if options.formulaParamInputValue: fo.traceParameterInputValue = True if options.formulaCallExprSource: fo.traceCallExpressionSource = True if options.formulaCallExprCode: fo.traceCallExpressionCode = True if options.formulaCallExprEval: fo.traceCallExpressionEvaluation = True if options.formulaCallExprResult: fo.traceCallExpressionResult = True if options.formulaVarSetExprEval: fo.traceVariableSetExpressionEvaluation = True if options.formulaVarSetExprResult: fo.traceVariableSetExpressionResult = True if options.formulaAsserResultCounts: fo.traceAssertionResultCounts = True if options.formulaFormulaRules: fo.traceFormulaRules = True if options.formulaVarsOrder: fo.traceVariablesOrder = True if options.formulaVarExpressionSource: fo.traceVariableExpressionSource = True if options.formulaVarExpressionCode: fo.traceVariableExpressionCode = True if options.formulaVarExpressionEvaluation: fo.traceVariableExpressionEvaluation = True if options.formulaVarExpressionResult: fo.traceVariableExpressionResult = True if options.timeVariableSetEvaluation: fo.timeVariableSetEvaluation = True if options.formulaVarFilterWinnowing: fo.traceVariableFilterWinnowing = True if options.formulaVarFiltersResult: fo.traceVariableFiltersResult = True if options.formulaVarFiltersResult: fo.traceVariableFiltersResult = True self.modelManager.formulaOptions = fo timeNow = XmlUtil.dateunionValue(datetime.datetime.now()) firstStartedAt = startedAt = time.time() modelDiffReport = None success = True modelXbrl = None try: if filesource: modelXbrl = self.modelManager.load(filesource, _("views loading")) except ModelDocument.LoadingException: pass except Exception as err: self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format( err, traceback.format_tb(sys.exc_info()[2]))) success = False # loading errors, don't attempt to utilize loaded DTS if modelXbrl and modelXbrl.modelDocument: loadTime = time.time() - startedAt modelXbrl.profileStat(_("load"), loadTime) self.addToLog(format_string(self.modelManager.locale, _("loaded in %.2f secs at %s"), (loadTime, timeNow)), messageCode="info", file=self.entrypointFile) if options.importFiles: for importFile in options.importFiles.split("|"): fileName = importFile.strip() if sourceZipStream is not None and not (fileName.startswith('http://') or os.path.isabs(fileName)): fileName = os.path.dirname(modelXbrl.uri) + os.sep + fileName # make relative to sourceZipStream ModelDocument.load(modelXbrl, fileName) loadTime = time.time() - startedAt self.addToLog(format_string(self.modelManager.locale, _("import in %.2f secs at %s"), (loadTime, timeNow)), messageCode="info", file=importFile) modelXbrl.profileStat(_("import"), loadTime) if modelXbrl.errors: success = False # loading errors, don't attempt to utilize loaded DTS if modelXbrl.modelDocument.type in ModelDocument.Type.TESTCASETYPES: for pluginXbrlMethod in pluginClassMethods("Testcases.Start"): pluginXbrlMethod(self, options, modelXbrl) else: # not a test case, probably instance or DTS for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Loaded"): pluginXbrlMethod(self, options, modelXbrl) else: success = False if success and options.diffFile and options.versReportFile: try: diffFilesource = FileSource.FileSource(options.diffFile,self) startedAt = time.time() modelXbrl2 = self.modelManager.load(diffFilesource, _("views loading")) if modelXbrl2.errors: if not options.keepOpen: modelXbrl2.close() success = False else: loadTime = time.time() - startedAt modelXbrl.profileStat(_("load"), loadTime) self.addToLog(format_string(self.modelManager.locale, _("diff comparison DTS loaded in %.2f secs"), loadTime), messageCode="info", file=self.entrypointFile) startedAt = time.time() modelDiffReport = self.modelManager.compareDTSes(options.versReportFile) diffTime = time.time() - startedAt modelXbrl.profileStat(_("diff"), diffTime) self.addToLog(format_string(self.modelManager.locale, _("compared in %.2f secs"), diffTime), messageCode="info", file=self.entrypointFile) except ModelDocument.LoadingException: success = False except Exception as err: success = False self.addToLog(_("[Exception] Failed to doad diff file: \n{0} \n{1}").format( err, traceback.format_tb(sys.exc_info()[2]))) if success: try: modelXbrl = self.modelManager.modelXbrl hasFormulae = modelXbrl.hasFormulae if options.validate: startedAt = time.time() if options.formulaAction: # don't automatically run formulas modelXbrl.hasFormulae = False self.modelManager.validate() if options.formulaAction: # restore setting modelXbrl.hasFormulae = hasFormulae self.addToLog(format_string(self.modelManager.locale, _("validated in %.2f secs"), time.time() - startedAt), messageCode="info", file=self.entrypointFile) if options.formulaAction in ("validate", "run"): # do nothing here if "none" from arelle import ValidateXbrlDimensions, ValidateFormula startedAt = time.time() if not options.validate: ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # setup fresh parameters from formula optoins modelXbrl.parameters = fo.typedParameters() ValidateFormula.validate(modelXbrl, compileOnly=(options.formulaAction != "run")) self.addToLog(format_string(self.modelManager.locale, _("formula validation and execution in %.2f secs") if options.formulaAction == "run" else _("formula validation only in %.2f secs"), time.time() - startedAt), messageCode="info", file=self.entrypointFile) if options.testReport: ViewFileTests.viewTests(self.modelManager.modelXbrl, options.testReport, options.testReportCols) if options.rssReport: ViewFileRssFeed.viewRssFeed(self.modelManager.modelXbrl, options.rssReport, options.rssReportCols) if options.DTSFile: ViewFileDTS.viewDTS(modelXbrl, options.DTSFile) if options.factsFile: ViewFileFactList.viewFacts(modelXbrl, options.factsFile, labelrole=options.labelRole, lang=options.labelLang, cols=options.factListCols) if options.factTableFile: ViewFileFactTable.viewFacts(modelXbrl, options.factTableFile, labelrole=options.labelRole, lang=options.labelLang) if options.conceptsFile: ViewFileConcepts.viewConcepts(modelXbrl, options.conceptsFile, labelrole=options.labelRole, lang=options.labelLang) if options.preFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.preFile, "Presentation Linkbase", "http://www.xbrl.org/2003/arcrole/parent-child", labelrole=options.labelRole, lang=options.labelLang) if options.calFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.calFile, "Calculation Linkbase", "http://www.xbrl.org/2003/arcrole/summation-item", labelrole=options.labelRole, lang=options.labelLang) if options.dimFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.dimFile, "Dimensions", "XBRL-dimensions", labelrole=options.labelRole, lang=options.labelLang) if options.formulaeFile: ViewFileFormulae.viewFormulae(modelXbrl, options.formulaeFile, "Formulae", lang=options.labelLang) if options.viewArcrole and options.viewFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.viewFile, os.path.basename(options.viewArcrole), options.viewArcrole, labelrole=options.labelRole, lang=options.labelLang) for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Run"): pluginXbrlMethod(self, options, modelXbrl) except (IOError, EnvironmentError) as err: self.addToLog(_("[IOError] Failed to save output:\n {0}").format(err)) success = False except Exception as err: self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format( err, traceback.format_tb(sys.exc_info()[2]))) success = False if modelXbrl: modelXbrl.profileStat(_("total"), time.time() - firstStartedAt) if options.collectProfileStats and modelXbrl: modelXbrl.logProfileStats() if not options.keepOpen: if modelDiffReport: self.modelManager.close(modelDiffReport) elif modelXbrl: self.modelManager.close(modelXbrl) self.username = self.password = None #dereference password return success
def setArgForFactProperty(param, modelFact, propertyNameParts): propVal = None property = propertyNameParts[0] if property == "value": if isinstance(modelFact.xValue, Decimal): propVal = "{:,}".format(modelFact.xValue) else: propVal = modelFact.value elif property == "decimals": propVal = modelFact.decimals elif property == "label" and modelFact.concept is not None: propVal = modelFact.concept.label( labelrole, lang=lang, linkroleHint=XbrlConst.defaultLinkRole) elif property == "name": propVal = str(modelFact.qname) elif property == "localName": propVal = modelFact.qname.localName else: cntx = modelFact.context unit = modelFact.unit if cntx is not None: if property == "period": if len(propertyNameParts) == 1: if cntx.isForeverPeriod: propVal = "forever" elif cntx.isInstantPeriod: propVal = XmlUtil.dateunionValue( cntx.instantDatetime, subtractOneDay=True) else: propVal = "{} to {}".format( XmlUtil.dateunionValue(cntx.startDatetime), XmlUtil.dateunionValue( cntx.endDatetime, subtractOneDay=True)) else: dateSelection = propertyNameParts[1] if dateSelection == "startDate": propVal = XmlUtil.dateunionValue( cntx.startDatetime) elif dateSelection == "endDate": propVal = XmlUtil.dateunionValue( cntx.endDatetime, subtractOneDay=True) elif dateSelection == "instant": propVal = XmlUtil.dateunionValue( cntx.instant, subtractOneDay=True) elif dateSelection == "durationDays": propVal = str((cntx.endDatetime - cntx.startDatetime).days) elif property == "dimensions": if cntx.qnameDims: propVal = "\n".join( "{} = {}".format( d.dimensionQname, d.memberQname if d.isExplicit else XmlUtil. xmlstring(XmlUtil.child(d), stripXmlns=True, prettyPrint=True)) for d in cntx.qnameDims.values()) else: propVal = "none" if property == "unit": if unit is None: propVal = "none" else: measures = unit.measures if measures[1]: propVal = 'mul {}\ndiv {} '.format( ', '.join( measureFormat(m) for m in measures[0]), ', '.join( measureFormat(m) for m in measures[1])) else: propVal = ', '.join( measureFormat(m) for m in measures[0]) fmtArgs[param] = propVal
def runFromExcel(self, options): #testGenFileName = options.excelfilename testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation-index.xls" testGenDir = os.path.dirname(testGenFileName) schemaDir = os.path.dirname(testGenDir) + os.sep + "schema" timeNow = XmlUtil.dateunionValue(datetime.datetime.now()) if options.testfiledate: today = options.testfiledate else: today = XmlUtil.dateunionValue(datetime.date.today()) startedAt = time.time() LogHandler(self) # start logger self.logMessages = [] logMessagesFile = testGenDir + os.sep + 'log-generation-messages.txt' modelTestcases = ModelXbrl.create(self.modelManager, url=testGenFileName, isEntry=True) testcaseIndexBook = xlrd.open_workbook(testGenFileName) testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0) self.addToLog(_("[info] xls loaded in {0:.2} secs at {1}").format(time.time() - startedAt, timeNow)) # start index file indexFiles = [testGenDir + os.sep + 'creation-testcases-index.xml', testGenDir + os.sep + 'consumption-testcases-index.xml'] indexDocs = [] testcasesElements = [] for purpose in ("Creation","Consumption"): file = io.StringIO( #'<?xml version="1.0" encoding="UTF-8"?>' '<!-- XBRL Versioning 1.0 {0} Tests -->' '<!-- Copyright 2011 XBRL International. All Rights Reserved. -->' '<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>' '<testcases name="XBRL Versioning 1.0 {0} Tests" ' ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' ' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">' '</testcases>'.format(purpose, today) ) doc = etree.parse(file) file.close() indexDocs.append(doc) testcasesElements.append(doc.getroot()) priorTestcasesDir = None testcaseFiles = None testcaseDocs = None for iRow in range(1, testcaseIndexSheet.nrows): try: row = testcaseIndexSheet.row(iRow) if (row[0].ctype == xlrd.XL_CELL_EMPTY or # must have directory row[1].ctype == xlrd.XL_CELL_EMPTY or # from row[2].ctype == xlrd.XL_CELL_EMPTY): # to continue testDir = row[0].value uriFrom = row[1].value uriTo = row[2].value overrideReport = row[3].value description = row[4].value if description is None or len(description) == 0: continue # test not ready to run assignment = row[5].value expectedEvents = row[6].value # comma space separated if multiple note = row[7].value useCase = row[8].value base = os.path.join(os.path.dirname(testGenFileName),testDir) + os.sep self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom)) if uriFrom and uriTo and assignment.lower() not in ("n.a.", "error") and expectedEvents != "N.A.": modelDTSfrom = modelDTSto = None for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"), True), (uriTo, _("loading to DTS"), False)): if ',' not in URIs: modelDTS = ModelXbrl.load(self.modelManager, URIs, msg, base=base) else: modelDTS = ModelXbrl.create(self.modelManager, ModelDocument.Type.DTSENTRIES, self.webCache.normalizeUrl(URIs.replace(", ","_") + ".dts", base), isEntry=True) DTSdoc = modelDTS.modelDocument DTSdoc.inDTS = True for uri in URIs.split(','): doc = ModelDocument.load(modelDTS, uri.strip(), base=base) if doc is not None: DTSdoc.referencesDocument[doc] = "import" #fake import doc.inDTS = True if isFrom: modelDTSfrom = modelDTS else: modelDTSto = modelDTS if modelDTSfrom is not None and modelDTSto is not None: # generate differences report reportUri = uriFrom.partition(',')[0] # first file reportDir = os.path.dirname(reportUri) if reportDir: reportDir += os.sep reportName = os.path.basename(reportUri).replace("from.xsd","report.xml") reportFile = reportDir + "out" + os.sep + reportName #reportFile = reportDir + "report" + os.sep + reportName reportFullPath = self.webCache.normalizeUrl( reportFile, base) testcasesDir = os.path.dirname(os.path.dirname(reportFullPath)) if testcasesDir != priorTestcasesDir: # close prior report if priorTestcasesDir: for i,testcaseFile in enumerate(testcaseFiles): with open(testcaseFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8") testcaseName = os.path.basename(testcasesDir) testcaseFiles = [testcasesDir + os.sep + testcaseName + "-creation-testcase.xml", testcasesDir + os.sep + testcaseName + "-consumption-testcase.xml"] for i,testcaseFile in enumerate(testcaseFiles): etree.SubElement(testcasesElements[i], "testcase", attrib={"uri": testcaseFile[len(testGenDir)+1:].replace("\\","/")} ) # start testcase file testcaseDocs = [] testcaseElements = [] testcaseNumber = testcaseName[0:4] if testcaseNumber.isnumeric(): testcaseNumberElement = "<number>{0}</number>".format(testcaseNumber) testcaseName = testcaseName[5:] else: testcaseNumberElement = "" testDirSegments = testDir.split('/') if len(testDirSegments) >= 2 and '-' in testDirSegments[1]: testedModule = testDirSegments[1][testDirSegments[1].index('-') + 1:] else: testedModule = '' for purpose in ("Creation","Consumption"): file = io.StringIO( #'<?xml version="1.0" encoding="UTF-8"?>' '<!-- Copyright 2011 XBRL International. All Rights Reserved. -->' '<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>' '<testcase ' ' xmlns="http://xbrl.org/2008/conformance"' ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' ' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">' '<creator>' '<name>Roland Hommes</name>' '<email>[email protected]</email>' '</creator>' '{0}' '<name>{1}</name>' # '<description>{0}</description>' '<reference>' '{2}' '{3}' '</reference>' '</testcase>'.format(testcaseNumberElement, testcaseName, '<name>{0}</name>'.format(testedModule) if testedModule else '', '<id>{0}</id>'.format(useCase) if useCase else '') ) doc = etree.parse(file) file.close() testcaseDocs.append(doc) testcaseElements.append(doc.getroot()) priorTestcasesDir = testcasesDir variationSeq = 1 try: os.makedirs(os.path.dirname(reportFullPath)) except WindowsError: pass # dir already exists modelVersReport = ModelVersReport.ModelVersReport(modelTestcases) modelVersReport.diffDTSes(reportFullPath,modelDTSfrom, modelDTSto, assignment=assignment, schemaDir=schemaDir) # check for expected elements if expectedEvents: for expectedEvent in expectedEvents.split(","): if expectedEvent not in ("No change", "N.A."): prefix, sep, localName = expectedEvent.partition(':') if sep and len(modelVersReport.xmlDocument.findall( '//{{{0}}}{1}'.format( XbrlConst.verPrefixNS.get(prefix), localName))) == 0: modelTestcases.warning("warning", "Generated test case %(reportName)s missing expected event %(event)s", reportName=reportName, event=expectedEvent) modelVersReport.close() uriFromParts = uriFrom.split('_') if len(uriFromParts) >= 2: variationId = uriFromParts[1] else: variationId = "_{0:02n}".format(variationSeq) for i,testcaseElt in enumerate(testcaseElements): variationElement = etree.SubElement(testcaseElt, "{http://xbrl.org/2008/conformance}variation", attrib={"id": variationId}) nameElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}description") nameElement.text = description ''' (removed per RH 2011/10/04 if note: paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}description") paramElement.text = "Note: " + note if useCase: paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}reference") paramElement.set("specification", "versioning-requirements") paramElement.set("useCase", useCase) ''' dataElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}data") if i == 0: # result is report if expectedEvents: paramElement = etree.SubElement(dataElement, "{http://xbrl.org/2008/conformance}parameter", attrib={"name":"expectedEvent", "value":expectedEvents.replace(',',' ')}, nsmap={"conf":"http://xbrl.org/2008/conformance", None:""}) if assignment: paramElement = etree.SubElement(dataElement, "{http://xbrl.org/2008/conformance}parameter", attrib={"name":"assignment", "value":assignment}, nsmap={"conf":"http://xbrl.org/2008/conformance", None:""}) for schemaURIs, dtsAttr in ((uriFrom,"from"), (uriTo,"to")): for schemaURI in schemaURIs.split(","): schemaElement = etree.SubElement(dataElement, "{http://xbrl.org/2008/conformance}schema") schemaElement.set("dts",dtsAttr) if i == 0: schemaElement.set("readMeFirst","true") schemaElement.text=os.path.basename(schemaURI.strip()) resultElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}result") reportElement = etree.SubElement(resultElement if i == 0 else dataElement, "{http://xbrl.org/2008/conformance}versioningReport") if i == 1: reportElement.set("readMeFirst","true") reportElement.text = "report/" + reportName variationSeq += 1 except Exception as err: modelTestcases.error("exception", _("Exception: %(error)s, Excel row: %(excelRow)s"), error=err, excelRow=iRow, exc_info=True) # add tests-error-code index files to consumption for testcaseFile in self.testcaseFiles(testGenDir + os.sep + "tests-error-code"): etree.SubElement(testcasesElements[1], "testcase", attrib={"uri": testcaseFile[len(testGenDir)+1:].replace("\\","/")} ) with open(logMessagesFile, "w") as fh: fh.writelines(self.logMessages) if priorTestcasesDir: for i,testcaseFile in enumerate(testcaseFiles): with open(testcaseFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8") for i,indexFile in enumerate(indexFiles): with open(indexFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def createContext(self, entityIdentScheme, entityIdentValue, periodType, periodStart, periodEndInstant, priItem, dims, segOCCs, scenOCCs, afterSibling=None, beforeSibling=None): xbrlElt = self.modelDocument.xmlRootElement if afterSibling == AUTO_LOCATE_ELEMENT: afterSibling = XmlUtil.lastChild( xbrlElt, XbrlConst.xbrli, ("schemaLocation", "roleType", "arcroleType", "context")) cntxId = 'c-{0:02n}'.format(len(self.contexts) + 1) newCntxElt = XmlUtil.addChild(xbrlElt, XbrlConst.xbrli, "context", attributes=("id", cntxId), afterSibling=afterSibling, beforeSibling=beforeSibling) entityElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "entity") XmlUtil.addChild(entityElt, XbrlConst.xbrli, "identifier", attributes=("scheme", entityIdentScheme), text=entityIdentValue) periodElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "period") if periodType == "forever": XmlUtil.addChild(periodElt, XbrlConst.xbrli, "forever") elif periodType == "instant": XmlUtil.addChild(periodElt, XbrlConst.xbrli, "instant", text=XmlUtil.dateunionValue(periodEndInstant, subtractOneDay=True)) elif periodType == "duration": XmlUtil.addChild(periodElt, XbrlConst.xbrli, "startDate", text=XmlUtil.dateunionValue(periodStart)) XmlUtil.addChild(periodElt, XbrlConst.xbrli, "endDate", text=XmlUtil.dateunionValue(periodEndInstant, subtractOneDay=True)) segmentElt = None scenarioElt = None from arelle.ModelInstanceObject import ModelDimensionValue if dims: # requires primary item to determin ambiguous concepts ''' in theory we have to check full set of dimensions for validity in source or any other context element, but for shortcut will see if each dimension is already reported in an unambiguous valid contextElement ''' from arelle.PrototypeInstanceObject import FactPrototype, ContextPrototype, DimValuePrototype fp = FactPrototype(self, priItem, dims.items()) # force trying a valid prototype's context Elements if not isFactDimensionallyValid( self, fp, setPrototypeContextElements=True): self.info( "arelleLinfo", _("Create context for %(priItem)s, cannot determine valid context elements, no suitable hypercubes" ), modelObject=self, priItem=priItem) fpDims = fp.context.qnameDims for dimQname in sorted(fpDims.keys()): dimValue = fpDims[dimQname] if isinstance(dimValue, DimValuePrototype): dimMemberQname = dimValue.memberQname # None if typed dimension contextEltName = dimValue.contextElement else: # qname for explicit or node for typed dimMemberQname = None contextEltName = None if contextEltName == "segment": if segmentElt is None: segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli, "segment") contextElt = segmentElt elif contextEltName == "scenario": if scenarioElt is None: scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "scenario") contextElt = scenarioElt else: self.info( "arelleLinfo", _("Create context, %(dimension)s, cannot determine context element, either no all relationship or validation issue" ), modelObject=self, dimension=dimQname), continue dimConcept = self.qnameConcepts[dimQname] dimAttr = ("dimension", XmlUtil.addQnameValue(xbrlElt, dimConcept.qname)) if dimConcept.isTypedDimension: dimElt = XmlUtil.addChild(contextElt, XbrlConst.xbrldi, "xbrldi:typedMember", attributes=dimAttr) if isinstance(dimValue, (ModelDimensionValue, DimValuePrototype)) and dimValue.isTyped: XmlUtil.copyNodes(dimElt, dimValue.typedMember) elif dimMemberQname: dimElt = XmlUtil.addChild(contextElt, XbrlConst.xbrldi, "xbrldi:explicitMember", attributes=dimAttr, text=XmlUtil.addQnameValue( xbrlElt, dimMemberQname)) if segOCCs: if segmentElt is None: segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli, "segment") XmlUtil.copyNodes(segmentElt, segOCCs) if scenOCCs: if scenarioElt is None: scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "scenario") XmlUtil.copyNodes(scenarioElt, scenOCCs) self.modelDocument.contextDiscover(newCntxElt) XmlValidate.validate(self, newCntxElt) return newCntxElt
def final(val): if not (val.validateEBA or val.validateEIOPA): return modelXbrl = val.modelXbrl modelDocument = modelXbrl.modelDocument _statusMsg = _("validating {0} filing rules").format(val.disclosureSystem.name) modelXbrl.profileActivity() val.showStatus(_statusMsg) if modelDocument.type == ModelDocument.Type.INSTANCE and (val.validateEBA or val.validateEIOPA): if not modelDocument.uri.endswith(".xbrl"): modelXbrl.warning("EBA.1.1", _('XBRL instance documents SHOULD use the extension ".xbrl" but it is "%(extension)s"'), modelObject=modelDocument, xmlEncoding=os.path.splitext(modelDocument.basename)[1]) if modelDocument.documentEncoding.lower() not in ("utf-8", "utf-8-sig"): modelXbrl.error("EBA.1.4", _('XBRL instance documents MUST use "UTF-8" encoding but is "%(xmlEncoding)s"'), modelObject=modelDocument, xmlEncoding=modelDocument.documentEncoding) schemaRefElts = [] schemaRefFileNames = [] for doc, docRef in modelDocument.referencesDocument.items(): if docRef.referenceType == "href": if docRef.referringModelObject.localName == "schemaRef": schemaRefElts.append(docRef.referringModelObject) schemaRefFileNames.append(doc.basename) if not UrlUtil.isAbsolute(doc.uri): modelXbrl.error("EBA.2.2", _('The link:schemaRef element in submitted instances MUST resolve to the full published entry point URL: %(url)s.'), modelObject=docRef.referringModelObject, url=doc.uri) elif docRef.referringModelObject.localName == "linkbaseRef": modelXbrl.error("EBA.2.3", _('The link:linkbaseRef element is not allowed: %(fileName)s.'), modelObject=docRef.referringModelObject, fileName=doc.basename) if len(schemaRefFileNames) > 1: modelXbrl.error("EBA.1.5", _('XBRL instance documents MUST reference only one entry point schema but %(numEntryPoints)s were found: %(entryPointNames)s'), modelObject=modelDocument, numEntryPoints=len(schemaRefFileNames), entryPointNames=', '.join(sorted(schemaRefFileNames))) ### check entry point names appropriate for filing indicator (DPM DB?) if len(schemaRefElts) != 1: modelXbrl.error("EBA.2.3", _('Any reported XBRL instance document MUST contain only one xbrli:xbrl/link:schemaRef node, but %(entryPointCount)s.'), modelObject=schemaRefElts, entryPointCount=len(schemaRefElts)) filingIndicators = {} for fIndicator in modelXbrl.factsByQname[qnFilingIndicator]: _value = (fIndicator.xValue or fIndicator.value) # use validated xValue if DTS else value for skipDTS if _value in filingIndicators: modelXbrl.error("EBA.1.6.1", _('Multiple filing indicators facts for indicator %(filingIndicator)s.'), modelObject=(fIndicator, filingIndicators[_value]), filingIndicator=_value) filingIndicators[_value] = fIndicator if not filingIndicators: modelXbrl.error("EBA.1.6", _('Missing filing indicators. Reported XBRL instances MUST include appropriate filing indicator elements'), modelObject=modelDocument) numFilingIndicatorTuples = len(modelXbrl.factsByQname[qnFIndicators]) if numFilingIndicatorTuples > 1 and not getattr(modelXbrl, "isStreamingMode", False): modelXbrl.info("EBA.1.6.2", _('Multiple filing indicators tuples when not in streaming mode (info).'), modelObject=modelXbrl.factsByQname[qnFIndicators]) # note EBA 2.1 is in ModelDocument.py cntxIDs = set() cntxEntities = set() cntxDates = defaultdict(list) timelessDatePattern = re.compile(r"\s*([0-9]{4})-([0-9]{2})-([0-9]{2})\s*$") for cntx in modelXbrl.contexts.values(): cntxIDs.add(cntx.id) cntxEntities.add(cntx.entityIdentifier) dateElts = XmlUtil.descendants(cntx, XbrlConst.xbrli, ("startDate","endDate","instant")) if any(not timelessDatePattern.match(e.textValue) for e in dateElts): modelXbrl.error("EBA.2.10", _('Period dates must be whole dates without time or timezone: %(dates)s.'), modelObject=cntx, dates=", ".join(e.text for e in dateElts)) if cntx.isForeverPeriod: modelXbrl.error("EBA.2.11", _('Forever context period is not allowed.'), modelObject=cntx) elif cntx.isStartEndPeriod: modelXbrl.error("EBA.2.13", _('Start-End (flow) context period is not allowed.'), modelObject=cntx) elif cntx.isInstantPeriod: cntxDates[cntx.instantDatetime].append(cntx) if XmlUtil.hasChild(cntx, XbrlConst.xbrli, "segment"): modelXbrl.error("EBA.2.14", _("The segment element not allowed in context Id: %(context)s"), modelObject=cntx, context=cntx.contextID) for scenElt in XmlUtil.descendants(cntx, XbrlConst.xbrli, "scenario"): childTags = ", ".join([child.prefixedName for child in scenElt.iterchildren() if isinstance(child,ModelObject) and child.tag != "{http://xbrl.org/2006/xbrldi}explicitMember" and child.tag != "{http://xbrl.org/2006/xbrldi}typedMember"]) if len(childTags) > 0: modelXbrl.error("EBA.2.15", _("Scenario of context Id %(context)s has disallowed content: %(content)s"), modelObject=cntx, context=cntx.id, content=childTags) if len(cntxDates) > 1: modelXbrl.error("EBA.2.13", _('Contexts must have the same date: %(dates)s.'), modelObject=[_cntx for _cntxs in cntxDates.values() for _cntx in _cntxs], dates=', '.join(XmlUtil.dateunionValue(_dt, subtractOneDay=True) for _dt in cntxDates.keys())) unusedCntxIDs = cntxIDs - {fact.contextID for fact in modelXbrl.factsInInstance if fact.contextID} # skip tuples if unusedCntxIDs: modelXbrl.warning("EBA.2.7", _('Unused xbrli:context nodes SHOULD NOT be present in the instance: %(unusedContextIDs)s.'), modelObject=[modelXbrl.contexts[unusedCntxID] for unusedCntxID in unusedCntxIDs], unusedContextIDs=", ".join(sorted(unusedCntxIDs))) if len(cntxEntities) > 1: modelXbrl.warning("EBA.2.9", _('All entity identifiers and schemes must be the same, %(count)s found: %(entities)s.'), modelObject=modelDocument, count=len(cntxEntities), entities=", ".join(sorted(str(cntxEntity) for cntxEntity in cntxEntities))) otherFacts = {} # (contextHash, unitHash, xmlLangHash) : fact nilFacts = [] stringFactsWithoutXmlLang = [] nonMonetaryNonPureFacts = [] unitIDsUsed = set() currenciesUsed = {} for qname, facts in modelXbrl.factsByQname.items(): for f in facts: if modelXbrl.skipDTS: c = f.qname.localName[0] isNumeric = c in ('m', 'p', 'i') isMonetary = c == 'm' isInteger = c == 'i' isPercent = c == 'p' isString = c == 's' else: concept = f.concept if concept is not None: isNumeric = concept.isNumeric isMonetary = concept.isMonetary isInteger = concept.baseXbrliType in integerItemTypes isPercent = concept.typeQname == qnPercentItemType isString = concept.baseXbrliType in ("stringItemType", "normalizedStringItemType") else: isNumeric = isString = False # error situation k = (f.getparent().objectIndex, f.qname, f.context.contextDimAwareHash if f.context is not None else None, f.unit.hash if f.unit is not None else None, hash(f.xmlLang)) if f.qname == qnFIndicators and val.validateEIOPA: pass elif k not in otherFacts: otherFacts[k] = {f} else: matches = [o for o in otherFacts[k] if (f.getparent().objectIndex == o.getparent().objectIndex and f.qname == o.qname and f.context.isEqualTo(o.context) if f.context is not None and o.context is not None else True) and (f.unit.isEqualTo(o.unit) if f.unit is not None and o.unit is not None else True) and (f.xmlLang == o.xmlLang)] if matches: contexts = [f.contextID] + [o.contextID for o in matches] modelXbrl.error("EBA.2.16", _('Facts are duplicates %(fact)s contexts %(contexts)s.'), modelObject=[f] + matches, fact=f.qname, contexts=', '.join(contexts)) else: otherFacts[k].add(f) if isNumeric: if f.precision: modelXbrl.error("EBA.2.17", _("Numeric fact %(fact)s of context %(contextID)s has a precision attribute '%(precision)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, precision=f.precision) if f.decimals and f.decimals != "INF": try: dec = int(f.decimals) if isMonetary: if dec < -3: modelXbrl.error("EBA.2.17", _("Monetary fact %(fact)s of context %(contextID)s has a decimal attribute < -3: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) elif isInteger: if dec != 0: modelXbrl.error("EBA.2.17", _("Integer fact %(fact)s of context %(contextID)s has a decimal attribute \u2260 0: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) elif isPercent: if dec < 4: modelXbrl.error("EBA.2.17", _("Percent fact %(fact)s of context %(contextID)s has a decimal attribute < 4: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) except ValueError: pass # should have been reported as a schema error by loader '''' (not intended by EBA 2.18, paste here is from EFM) if not f.isNil and getattr(f,"xValid", 0) == 4: try: insignificance = insignificantDigits(f.xValue, decimals=f.decimals) if insignificance: # if not None, returns (truncatedDigits, insiginficantDigits) modelXbrl.error(("EFM.6.05.37", "GFM.1.02.26"), _("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s has nonzero digits in insignificant portion %(insignificantDigits)s."), modelObject=f1, fact=f1.qname, contextID=f1.contextID, decimals=f1.decimals, value=f1.xValue, truncatedDigits=insignificance[0], insignificantDigits=insignificance[1]) except (ValueError,TypeError): modelXbrl.error(("EBA.2.18"), _("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s causes Value Error exception."), modelObject=f1, fact=f1.qname, contextID=f1.contextID, decimals=f1.decimals, value=f1.value) ''' unit = f.unit if unit is not None: if isMonetary: if unit.measures[0]: currenciesUsed[unit.measures[0][0]] = unit elif not unit.isSingleMeasure or unit.measures[0][0] != XbrlConst.qnXbrliPure: nonMonetaryNonPureFacts.append(f) elif isString: if not f.xmlLang: stringFactsWithoutXmlLang.append(f) if f.unitID is not None: unitIDsUsed.add(f.unitID) if f.isNil: nilFacts.append(f) if nilFacts: modelXbrl.error("EBA.2.19", _('Nil facts MUST NOT be present in the instance: %(nilFacts)s.'), modelObject=nilFacts, nilFacts=", ".join(str(f.qname) for f in nilFacts)) if stringFactsWithoutXmlLang: modelXbrl.error("EBA.2.20", _("String facts need to report xml:lang: '%(langLessFacts)s'"), modelObject=stringFactsWithoutXmlLang, langLEssFacts=", ".join(str(f.qname) for f in stringFactsWithoutXmlLang)) if nonMonetaryNonPureFacts: modelXbrl.error("EBA.3.2", _("Non monetary (numeric) facts MUST use the pure unit: '%(langLessFacts)s'"), modelObject=nonMonetaryNonPureFacts, langLessFacts=", ".join(str(f.qname) for f in nonMonetaryNonPureFacts)) unusedUnitIDs = modelXbrl.units.keys() - unitIDsUsed if unusedUnitIDs: modelXbrl.warning("EBA.2.21", _('Unused xbrli:unit nodes SHOULD NOT be present in the instance: %(unusedUnitIDs)s.'), modelObject=[modelXbrl.units[unusedUnitID] for unusedUnitID in unusedUnitIDs], unusedUnitIDs=", ".join(sorted(unusedUnitIDs))) unitHashes = {} for unit in modelXbrl.units.values(): h = hash(unit) if h in unitHashes and unit.isEqualTo(unitHashes[h]): modelXbrl.warning("EBA.2.32", _("Duplicate units SHOULD NOT be reported, units %(unit1)s and %(unit2)s have same measures.'"), modelObject=(unit, unitHashes[h]), unit1=unit.id, unit2=unitHashes[h].id) else: unitHashes[h] = unit if len(currenciesUsed) > 1: modelXbrl.error("EBA.3.1", _("There MUST be only one currency but %(numCurrencies)s were found: %(currencies)s.'"), modelObject=currenciesUsed.values(), numCurrencies=len(currenciesUsed), currencies=", ".join(str(c) for c in currenciesUsed.keys())) namespacePrefixesUsed = defaultdict(set) prefixesUnused = set(modelDocument.xmlRootElement.keys()).copy() for elt in modelDocument.xmlRootElement.iter(): if isinstance(elt, ModelObject): # skip comments and processing instructions namespacePrefixesUsed[elt.qname.namespaceURI].add(elt.qname.prefix) prefixesUnused.discard(elt.qname.prefix) if prefixesUnused: modelXbrl.warning("EBA.3.4", _("There SHOULD be no unused prefixes but these were declared: %(unusedPrefixes)s.'"), modelObject=modelDocument, unusedPrefixes=', '.join(sorted(prefixesUnused))) for ns, prefixes in namespacePrefixesUsed.items(): nsDocs = modelXbrl.namespaceDocs.get(ns) if nsDocs: for nsDoc in nsDocs: nsDocPrefix = XmlUtil.xmlnsprefix(nsDoc.xmlRootElement, ns) if any(prefix != nsDocPrefix for prefix in prefixes if prefix is not None): modelXbrl.warning("EBA.3.5", _("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s"), modelObject=modelDocument, namespace=ns, declaredPrefix=nsDocPrefix, foundPrefixes=', '.join(sorted(prefixes - {None}))) modelXbrl.profileActivity(_statusMsg, minTimeToShow=0.0) val.showStatus(None) del val.prefixNamespace, val.namespacePrefix, val.idObjects, val.typedDomainElements
def runFromExcel(self, options): #testGenFileName = options.excelfilename testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation-index.xls" testGenDir = os.path.dirname(testGenFileName) schemaDir = os.path.dirname(testGenDir) + os.sep + "schema" timeNow = XmlUtil.dateunionValue(datetime.datetime.now()) if options.testfiledate: today = options.testfiledate else: today = XmlUtil.dateunionValue(datetime.date.today()) startedAt = time.time() LogHandler(self) # start logger self.logMessages = [] logMessagesFile = testGenDir + os.sep + 'log-generation-messages.txt' modelTestcases = ModelXbrl.create(self.modelManager, url=testGenFileName, isEntry=True) testcaseIndexBook = xlrd.open_workbook(testGenFileName) testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0) self.addToLog( _("[info] xls loaded in {0:.2} secs at {1}").format( time.time() - startedAt, timeNow)) # start index file indexFiles = [ testGenDir + os.sep + 'creation-testcases-index.xml', testGenDir + os.sep + 'consumption-testcases-index.xml' ] indexDocs = [] testcasesElements = [] for purpose in ("Creation", "Consumption"): file = io.StringIO( #'<?xml version="1.0" encoding="UTF-8"?>' '<!-- XBRL Versioning 1.0 {0} Tests -->' '<!-- Copyright 2011 XBRL International. All Rights Reserved. -->' '<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>' '<testcases name="XBRL Versioning 1.0 {0} Tests" ' ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' ' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">' '</testcases>'.format(purpose, today)) doc = etree.parse(file) file.close() indexDocs.append(doc) testcasesElements.append(doc.getroot()) priorTestcasesDir = None testcaseFiles = None testcaseDocs = None for iRow in range(1, testcaseIndexSheet.nrows): try: row = testcaseIndexSheet.row(iRow) if (row[0].ctype == xlrd.XL_CELL_EMPTY or # must have directory row[1].ctype == xlrd.XL_CELL_EMPTY or # from row[2].ctype == xlrd.XL_CELL_EMPTY): # to continue testDir = row[0].value uriFrom = row[1].value uriTo = row[2].value overrideReport = row[3].value description = row[4].value if description is None or len(description) == 0: continue # test not ready to run assignment = row[5].value expectedEvents = row[ 6].value # comma space separated if multiple note = row[7].value useCase = row[8].value base = os.path.join(os.path.dirname(testGenFileName), testDir) + os.sep self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom)) if uriFrom and uriTo and assignment.lower() not in ( "n.a.", "error") and expectedEvents != "N.A.": modelDTSfrom = modelDTSto = None for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"), True), (uriTo, _("loading to DTS"), False)): if ',' not in URIs: modelDTS = ModelXbrl.load(self.modelManager, URIs, msg, base=base) else: modelDTS = ModelXbrl.create( self.modelManager, ModelDocument.Type.DTSENTRIES, self.webCache.normalizeUrl( URIs.replace(", ", "_") + ".dts", base), isEntry=True) DTSdoc = modelDTS.modelDocument DTSdoc.inDTS = True for uri in URIs.split(','): doc = ModelDocument.load(modelDTS, uri.strip(), base=base) if doc is not None: DTSdoc.referencesDocument[ doc] = "import" #fake import doc.inDTS = True if isFrom: modelDTSfrom = modelDTS else: modelDTSto = modelDTS if modelDTSfrom is not None and modelDTSto is not None: # generate differences report reportUri = uriFrom.partition(',')[0] # first file reportDir = os.path.dirname(reportUri) if reportDir: reportDir += os.sep reportName = os.path.basename(reportUri).replace( "from.xsd", "report.xml") reportFile = reportDir + "out" + os.sep + reportName #reportFile = reportDir + "report" + os.sep + reportName reportFullPath = self.webCache.normalizeUrl( reportFile, base) testcasesDir = os.path.dirname( os.path.dirname(reportFullPath)) if testcasesDir != priorTestcasesDir: # close prior report if priorTestcasesDir: for i, testcaseFile in enumerate( testcaseFiles): with open(testcaseFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8") testcaseName = os.path.basename(testcasesDir) testcaseFiles = [ testcasesDir + os.sep + testcaseName + "-creation-testcase.xml", testcasesDir + os.sep + testcaseName + "-consumption-testcase.xml" ] for i, testcaseFile in enumerate(testcaseFiles): etree.SubElement( testcasesElements[i], "testcase", attrib={ "uri": testcaseFile[len(testGenDir) + 1:].replace("\\", "/") }) # start testcase file testcaseDocs = [] testcaseElements = [] testcaseNumber = testcaseName[0:4] if testcaseNumber.isnumeric(): testcaseNumberElement = "<number>{0}</number>".format( testcaseNumber) testcaseName = testcaseName[5:] else: testcaseNumberElement = "" testDirSegments = testDir.split('/') if len(testDirSegments ) >= 2 and '-' in testDirSegments[1]: testedModule = testDirSegments[1][ testDirSegments[1].index('-') + 1:] else: testedModule = '' for purpose in ("Creation", "Consumption"): file = io.StringIO( #'<?xml version="1.0" encoding="UTF-8"?>' '<!-- Copyright 2011 XBRL International. All Rights Reserved. -->' '<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>' '<testcase ' ' xmlns="http://xbrl.org/2008/conformance"' ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' ' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">' '<creator>' '<name>Roland Hommes</name>' '<email>[email protected]</email>' '</creator>' '{0}' '<name>{1}</name>' # '<description>{0}</description>' '<reference>' '{2}' '{3}' '</reference>' '</testcase>'.format( testcaseNumberElement, testcaseName, '<name>{0}</name>'.format(testedModule) if testedModule else '', '<id>{0}</id>'.format(useCase) if useCase else '')) doc = etree.parse(file) file.close() testcaseDocs.append(doc) testcaseElements.append(doc.getroot()) priorTestcasesDir = testcasesDir variationSeq = 1 try: os.makedirs(os.path.dirname(reportFullPath)) except WindowsError: pass # dir already exists modelVersReport = ModelVersReport.ModelVersReport( modelTestcases) modelVersReport.diffDTSes(reportFullPath, modelDTSfrom, modelDTSto, assignment=assignment, schemaDir=schemaDir) # check for expected elements if expectedEvents: for expectedEvent in expectedEvents.split(","): if expectedEvent not in ("No change", "N.A."): prefix, sep, localName = expectedEvent.partition( ':') if sep and len( modelVersReport.xmlDocument. findall('//{{{0}}}{1}'.format( XbrlConst.verPrefixNS.get( prefix), localName))) == 0: modelTestcases.warning( "warning", "Generated test case %(reportName)s missing expected event %(event)s", reportName=reportName, event=expectedEvent) modelVersReport.close() uriFromParts = uriFrom.split('_') if len(uriFromParts) >= 2: variationId = uriFromParts[1] else: variationId = "_{0:02n}".format(variationSeq) for i, testcaseElt in enumerate(testcaseElements): variationElement = etree.SubElement( testcaseElt, "{http://xbrl.org/2008/conformance}variation", attrib={"id": variationId}) nameElement = etree.SubElement( variationElement, "{http://xbrl.org/2008/conformance}description" ) nameElement.text = description ''' (removed per RH 2011/10/04 if note: paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}description") paramElement.text = "Note: " + note if useCase: paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}reference") paramElement.set("specification", "versioning-requirements") paramElement.set("useCase", useCase) ''' dataElement = etree.SubElement( variationElement, "{http://xbrl.org/2008/conformance}data") if i == 0: # result is report if expectedEvents: paramElement = etree.SubElement( dataElement, "{http://xbrl.org/2008/conformance}parameter", attrib={ "name": "expectedEvent", "value": expectedEvents.replace(',', ' ') }, nsmap={ "conf": "http://xbrl.org/2008/conformance", None: "" }) if assignment: paramElement = etree.SubElement( dataElement, "{http://xbrl.org/2008/conformance}parameter", attrib={ "name": "assignment", "value": assignment }, nsmap={ "conf": "http://xbrl.org/2008/conformance", None: "" }) for schemaURIs, dtsAttr in ((uriFrom, "from"), (uriTo, "to")): for schemaURI in schemaURIs.split(","): schemaElement = etree.SubElement( dataElement, "{http://xbrl.org/2008/conformance}schema" ) schemaElement.set("dts", dtsAttr) if i == 0: schemaElement.set( "readMeFirst", "true") schemaElement.text = os.path.basename( schemaURI.strip()) resultElement = etree.SubElement( variationElement, "{http://xbrl.org/2008/conformance}result") reportElement = etree.SubElement( resultElement if i == 0 else dataElement, "{http://xbrl.org/2008/conformance}versioningReport" ) if i == 1: reportElement.set("readMeFirst", "true") reportElement.text = "report/" + reportName variationSeq += 1 except Exception as err: modelTestcases.error( "exception", _("Exception: %(error)s, Excel row: %(excelRow)s"), error=err, excelRow=iRow, exc_info=True) # add tests-error-code index files to consumption for testcaseFile in self.testcaseFiles(testGenDir + os.sep + "tests-error-code"): etree.SubElement(testcasesElements[1], "testcase", attrib={ "uri": testcaseFile[len(testGenDir) + 1:].replace( "\\", "/") }) with open(logMessagesFile, "w") as fh: fh.writelines(self.logMessages) if priorTestcasesDir: for i, testcaseFile in enumerate(testcaseFiles): with open(testcaseFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8") for i, indexFile in enumerate(indexFiles): with open(indexFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def runFromExcel(self, options): testGenFileName = options.excelfilename #testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation\1000-2000-index.xls" testGenDir = os.path.dirname(testGenFileName) timeNow = XmlUtil.dateunionValue(datetime.datetime.now()) if options.testfiledate: today = options.testfiledate else: today = XmlUtil.dateunionValue(datetime.date.today()) startedAt = time.time() self.logMessages = [] logMessagesFile = testGenDir + os.sep + 'logGenerationMessages.txt' modelTestcases = ModelXbrl.create(self.modelManager) testcaseIndexBook = xlrd.open_workbook(testGenFileName) testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0) self.addToLog( _("[info] xls loaded in {0:.2} secs at {1}").format( time.time() - startedAt, timeNow)) # start index file indexFiles = [ testGenDir + os.sep + 'creationTestcasesIndex.xml', testGenDir + os.sep + 'consumptionTestcasesIndex.xml' ] indexDocs = [ xml.dom.minidom.parseString( '<?xml version="1.0" encoding="UTF-8"?>' '<!-- XBRL Versioning 1.0 {0} Tests -->' '<!-- Copyright 2011 XBRL International. All Rights Reserved. -->' '<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>' '<testcases name="XBRL Versioning 1.0 Consumption Tests" date="{1}" ' ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' ' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">' '</testcases>'.format(purpose, today)) for purpose in ("Creation", "Consumption") ] testcasesElements = [ XmlUtil.child(indexDoc, None, "testcases") for indexDoc in indexDocs ] priorTestcasesDir = None testcaseFiles = None testcaseDocs = None for iRow in range(1, testcaseIndexSheet.nrows): row = testcaseIndexSheet.row(iRow) if row[0].ctype == xlrd.XL_CELL_EMPTY or row[ 1].ctype == xlrd.XL_CELL_EMPTY or row[ 2].ctype == xlrd.XL_CELL_EMPTY: continue testDir = row[0].value uriFrom = row[1].value uriTo = row[2].value intention = row[3].value if intention is None or len(intention) == 0: continue # test not ready to run reason = row[4].value expectedEvent = row[5].value base = os.path.join(os.path.dirname(testGenFileName), testDir) + os.sep self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom)) if uriFrom and uriTo and reason.lower() not in ( "n.a.", "error") and expectedEvent != "N.A.": for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"), True), (uriTo, _("loading to DTS"), False)): if ',' not in URIs: modelDTS = ModelXbrl.load(self.modelManager, URIs, msg, base=base) else: modelDTS = ModelXbrl.create( self.modelManager, ModelDocument.Type.DTSENTRIES, self.webCache.normalizeUrl( URIs.replace(", ", "_") + ".dts", base), isEntry=True) DTSdoc = modelDTS.modelDocument DTSdoc.inDTS = True for uri in URIs.split(','): doc = ModelDocument.load(modelDTS, uri.strip(), base=base) DTSdoc.referencesDocument[ doc] = "import" #fake import doc.inDTS = True if isFrom: modelDTSfrom = modelDTS else: modelDTSto = modelDTS if modelDTSfrom and modelDTSto: # generate differences report reportUri = uriFrom.partition(',')[0] # first file reportDir = os.path.dirname(reportUri) if reportDir: reportDir += os.sep reportName = os.path.basename(reportUri).replace( "from.xsd", "report.xml") reportFile = reportDir + "report" + os.sep + reportName reportFullPath = self.webCache.normalizeUrl( reportFile, base) testcasesDir = os.path.dirname( os.path.dirname(reportFullPath)) if testcasesDir != priorTestcasesDir: # close prior report if priorTestcasesDir: for i, testcaseFile in enumerate(testcaseFiles): with open(testcaseFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8") testcaseName = os.path.basename(testcasesDir) testcaseFiles = [ testcasesDir + os.sep + testcaseName + "-creation-testcase.xml", testcasesDir + os.sep + testcaseName + "-consumption-testcase.xml" ] for i, testcaseFile in enumerate(testcaseFiles): XmlUtil.addChild( testcasesElements[i], None, "testcase", ("uri", testcaseFile[len(testGenDir) + 1:].replace("\\", "/"))) # start testcase file testcaseDocs = [ xml.dom.minidom.parseString( '<?xml version="1.0" encoding="UTF-8"?>' '<!-- Copyright 2011 XBRL International. All Rights Reserved. -->' '<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>' '<testcase name="XBRL Versioning 1.0 {1} Tests" date="{2}" ' ' xmlns="http://xbrl.org/2008/conformance"' ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' ' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">' '<creator>' '<name>Roland Hommes</name>' '<email>[email protected]</email>' '</creator>' '<name>{0}</name>' '<description>{0}</description>' '</testcase>'.format(testcaseName, purpose, today)) for purpose in ("Creation", "Consumption") ] testcaseElements = [ XmlUtil.child(testcaseDoc, conformanceNS, "testcase") for testcaseDoc in testcaseDocs ] priorTestcasesDir = testcasesDir variationID = 1 try: os.makedirs(os.path.dirname(reportFullPath)) except WindowsError: pass # dir already exists modelVersReport = ModelVersReport.ModelVersReport( modelTestcases) modelVersReport.diffDTSes(reportFullPath, modelDTSfrom, modelDTSto) # check for expected elements if expectedEvent and expectedEvent not in ("No change", "N.A."): if len( modelVersReport.xmlDocument. getElementsByTagNameNS('*', expectedEvent)) == 0: modelTestcases.error( "Generated test case {0} missing expected event {1}" .format(reportName, expectedEvent), "wrn", "missingEvent") modelVersReport.close([]) for i, testcaseElt in enumerate(testcaseElements): variationElement = XmlUtil.addChild( testcaseElt, conformanceNS, "variation", attributes=("id", "_{0:02n}".format(variationID))) XmlUtil.addChild(variationElement, conformanceNS, "name", text=intention) dataElement = XmlUtil.addChild(variationElement, conformanceNS, "data") for schemaURIs, dtsAttr in ((uriFrom, "from"), (uriTo, "to")): for schemaURI in schemaURIs.split(","): XmlUtil.addChild( dataElement, conformanceNS, "schema", attributes=((("dts", dtsAttr), ) + ((("readMeFirst", "true"), ) if i == 0 else ())), text=os.path.basename(schemaURI.strip())) resultElement = XmlUtil.addChild( variationElement, conformanceNS, "result") XmlUtil.addChild( resultElement if i == 0 else dataElement, conformanceNS, "versioningReport", attributes=(("readMeFirst", "true") if i == 1 else ()), text="report/" + reportName) variationID += 1 with open(logMessagesFile, "w") as fh: fh.writelines(self.logMessages) if priorTestcasesDir: for i, testcaseFile in enumerate(testcaseFiles): with open(testcaseFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8") for i, indexFile in enumerate(indexFiles): with open(indexFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def runFromExcel(self, options): testGenFileName = options.excelfilename #testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation\1000-2000-index.xls" testGenDir = os.path.dirname(testGenFileName) timeNow = XmlUtil.dateunionValue(datetime.datetime.now()) if options.testfiledate: today = options.testfiledate else: today = XmlUtil.dateunionValue(datetime.date.today()) startedAt = time.time() self.logMessages = [] logMessagesFile = testGenDir + os.sep + 'logGenerationMessages.txt' modelTestcases = ModelXbrl.create(self.modelManager) testcaseIndexBook = xlrd.open_workbook(testGenFileName) testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0) self.addToLog(_("[info] xls loaded in {0:.2} secs at {1}").format(time.time() - startedAt, timeNow)) # start index file indexFiles = [testGenDir + os.sep + 'creationTestcasesIndex.xml', testGenDir + os.sep + 'consumptionTestcasesIndex.xml'] indexDocs = [xml.dom.minidom.parseString( '<?xml version="1.0" encoding="UTF-8"?>' '<!-- XBRL Versioning 1.0 {0} Tests -->' '<!-- Copyright 2011 XBRL International. All Rights Reserved. -->' '<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>' '<testcases name="XBRL Versioning 1.0 Consumption Tests" date="{1}" ' ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' ' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">' '</testcases>'.format(purpose, today) ) for purpose in ("Creation","Consumption")] testcasesElements = [XmlUtil.child(indexDoc, None, "testcases") for indexDoc in indexDocs] priorTestcasesDir = None testcaseFiles = None testcaseDocs = None for iRow in range(1, testcaseIndexSheet.nrows): row = testcaseIndexSheet.row(iRow) if row[0].ctype == xlrd.XL_CELL_EMPTY or row[1].ctype == xlrd.XL_CELL_EMPTY or row[2].ctype == xlrd.XL_CELL_EMPTY: continue testDir = row[0].value uriFrom = row[1].value uriTo = row[2].value intention = row[3].value if intention is None or len(intention) == 0: continue # test not ready to run reason = row[4].value expectedEvent = row[5].value base = os.path.join(os.path.dirname(testGenFileName),testDir) + os.sep self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom)) if uriFrom and uriTo and reason.lower() not in ("n.a.", "error") and expectedEvent != "N.A.": for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"), True), (uriTo, _("loading to DTS"), False)): if ',' not in URIs: modelDTS = ModelXbrl.load(self.modelManager, URIs, msg, base=base) else: modelDTS = ModelXbrl.create(self.modelManager, ModelDocument.Type.DTSENTRIES, self.webCache.normalizeUrl(URIs.replace(", ","_") + ".dts", base), isEntry=True) DTSdoc = modelDTS.modelDocument DTSdoc.inDTS = True for uri in URIs.split(','): doc = ModelDocument.load(modelDTS, uri.strip(), base=base) DTSdoc.referencesDocument[doc] = "import" #fake import doc.inDTS = True if isFrom: modelDTSfrom = modelDTS else: modelDTSto = modelDTS if modelDTSfrom and modelDTSto: # generate differences report reportUri = uriFrom.partition(',')[0] # first file reportDir = os.path.dirname(reportUri) if reportDir: reportDir += os.sep reportName = os.path.basename(reportUri).replace("from.xsd","report.xml") reportFile = reportDir + "report" + os.sep + reportName reportFullPath = self.webCache.normalizeUrl( reportFile, base) testcasesDir = os.path.dirname(os.path.dirname(reportFullPath)) if testcasesDir != priorTestcasesDir: # close prior report if priorTestcasesDir: for i,testcaseFile in enumerate(testcaseFiles): with open(testcaseFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8") testcaseName = os.path.basename(testcasesDir) testcaseFiles = [testcasesDir + os.sep + testcaseName + "-creation-testcase.xml", testcasesDir + os.sep + testcaseName + "-consumption-testcase.xml"] for i,testcaseFile in enumerate(testcaseFiles): XmlUtil.addChild(testcasesElements[i], None, "testcase", ("uri", testcaseFile[len(testGenDir)+1:].replace("\\","/")) ) # start testcase file testcaseDocs = [xml.dom.minidom.parseString( '<?xml version="1.0" encoding="UTF-8"?>' '<!-- Copyright 2011 XBRL International. All Rights Reserved. -->' '<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>' '<testcase name="XBRL Versioning 1.0 {1} Tests" date="{2}" ' ' xmlns="http://xbrl.org/2008/conformance"' ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' ' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">' '<creator>' '<name>Roland Hommes</name>' '<email>[email protected]</email>' '</creator>' '<name>{0}</name>' '<description>{0}</description>' '</testcase>'.format(testcaseName,purpose,today) ) for purpose in ("Creation","Consumption")] testcaseElements = [XmlUtil.child(testcaseDoc, conformanceNS, "testcase") for testcaseDoc in testcaseDocs] priorTestcasesDir = testcasesDir variationID = 1 try: os.makedirs(os.path.dirname(reportFullPath)) except WindowsError: pass # dir already exists modelVersReport = ModelVersReport.ModelVersReport(modelTestcases) modelVersReport.diffDTSes(reportFullPath,modelDTSfrom, modelDTSto) # check for expected elements if expectedEvent and expectedEvent not in ( "No change", "N.A."): if len(modelVersReport.xmlDocument.getElementsByTagNameNS('*',expectedEvent)) == 0: modelTestcases.error( "Generated test case {0} missing expected event {1}".format( reportName, expectedEvent), "wrn", "missingEvent") modelVersReport.close([]) for i,testcaseElt in enumerate(testcaseElements): variationElement = XmlUtil.addChild(testcaseElt, conformanceNS, "variation", attributes=("id", "_{0:02n}".format(variationID))) XmlUtil.addChild(variationElement, conformanceNS, "name", text=intention) dataElement = XmlUtil.addChild(variationElement, conformanceNS, "data") for schemaURIs, dtsAttr in ((uriFrom,"from"), (uriTo,"to")): for schemaURI in schemaURIs.split(","): XmlUtil.addChild(dataElement, conformanceNS, "schema", attributes=((("dts",dtsAttr),) + ((("readMeFirst","true"),) if i == 0 else ())), text=os.path.basename(schemaURI.strip())) resultElement = XmlUtil.addChild(variationElement, conformanceNS, "result") XmlUtil.addChild(resultElement if i == 0 else dataElement, conformanceNS, "versioningReport", attributes=(("readMeFirst","true") if i == 1 else ()), text="report/" + reportName) variationID += 1 with open(logMessagesFile, "w") as fh: fh.writelines(self.logMessages) if priorTestcasesDir: for i,testcaseFile in enumerate(testcaseFiles): with open(testcaseFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8") for i,indexFile in enumerate(indexFiles): with open(indexFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def final(val): if not (val.validateEBA or val.validateEIOPA): return modelXbrl = val.modelXbrl modelDocument = modelXbrl.modelDocument _statusMsg = _("validating {0} filing rules").format(val.disclosureSystem.name) modelXbrl.profileActivity() modelXbrl.modelManager.showStatus(_statusMsg) if modelDocument.type == ModelDocument.Type.INSTANCE and (val.validateEBA or val.validateEIOPA): if not modelDocument.uri.endswith(".xbrl"): modelXbrl.warning("EBA.1.1", _('XBRL instance documents SHOULD use the extension ".xbrl" but it is "%(extension)s"'), modelObject=modelDocument, extension=os.path.splitext(modelDocument.basename)[1]) modelXbrl.error("EIOPA.S.1.1.a", _('XBRL instance documents MUST use the extension ".xbrl" but it is "%(extension)s"'), modelObject=modelDocument, extension=os.path.splitext(modelDocument.basename)[1]) if val.isEIOPA_2_0_1: _encodings = ("UTF-8", "utf-8-sig") else: _encodings = ("utf-8", "UTF-8", "utf-8-sig") if modelDocument.documentEncoding not in _encodings: modelXbrl.error(("EBA.1.4", "EIOPA.1.4"), _('XBRL instance documents MUST use "UTF-8" encoding but is "%(xmlEncoding)s"'), modelObject=modelDocument, xmlEncoding=modelDocument.documentEncoding) schemaRefElts = [] schemaRefFileNames = [] for doc, docRef in modelDocument.referencesDocument.items(): if docRef.referenceType == "href": if docRef.referringModelObject.localName == "schemaRef": schemaRefElts.append(docRef.referringModelObject) schemaRefFileNames.append(doc.basename) if not UrlUtil.isAbsolute(doc.uri): modelXbrl.error(("EBA.2.2", "EIOPA.S.1.5.a" if val.isEIOPAfullVersion else "EIOPA.S.1.5.b"), _('The link:schemaRef element in submitted instances MUST resolve to the full published entry point URL: %(url)s.'), modelObject=docRef.referringModelObject, url=doc.uri, messageCodes=("EBA.2.2", "EIOPA.S.1.5.a","EIOPA.S.1.5.b")) elif docRef.referringModelObject.localName == "linkbaseRef": modelXbrl.error(("EBA.2.3","EIOPA.S.1.5.a"), _('The link:linkbaseRef element is not allowed: %(fileName)s.'), modelObject=docRef.referringModelObject, fileName=doc.basename) _numSchemaRefs = len(XmlUtil.children(modelDocument.xmlRootElement, XbrlConst.link, "schemaRef")) if _numSchemaRefs > 1: modelXbrl.error(("EIOPA.S.1.5.a", "EBA.1.5"), _('XBRL instance documents MUST reference only one entry point schema but %(numEntryPoints)s were found: %(entryPointNames)s'), modelObject=modelDocument, numEntryPoints=_numSchemaRefs, entryPointNames=', '.join(sorted(schemaRefFileNames))) ### check entry point names appropriate for filing indicator (DPM DB?) if len(schemaRefElts) != 1: modelXbrl.error("EBA.2.3", _('Any reported XBRL instance document MUST contain only one xbrli:xbrl/link:schemaRef node, but %(entryPointCount)s.'), modelObject=schemaRefElts, entryPointCount=len(schemaRefElts)) # non-streaming EBA checks if not getattr(modelXbrl, "isStreamingMode", False): val.qnReportedCurrency = None if val.isEIOPA_2_0_1 and qnMetReportingCurrency in modelXbrl.factsByQname: for _multiCurrencyFact in modelXbrl.factsByQname[qnMetReportingCurrency]: # multi-currency fact val.qnReportedCurrency = _multiCurrencyFact.xValue break validateFacts(val, modelXbrl.facts) # check sum of fact md5s (otherwise checked in streaming process) xbrlFactsCheckVersion = None expectedSumOfFactMd5s = None for pi in modelDocument.xmlRootElement.getchildren(): if isinstance(pi, etree._ProcessingInstruction) and pi.target == "xbrl-facts-check": _match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']", pi.text) if _match: _matchGroups = _match.groups() if len(_matchGroups) == 2: if _matchGroups[0] == "version": xbrlFactsCheckVersion = _matchGroups[1] elif _matchGroups[0] == "sum-of-fact-md5s": try: expectedSumOfFactMd5s = Md5Sum(_matchGroups[1]) except ValueError: modelXbrl.error("EIOPA:xbrlFactsCheckError", _("Invalid sum-of-md5s %(sumOfMd5)s"), modelObject=modelXbrl, sumOfMd5=_matchGroups[1]) if xbrlFactsCheckVersion and expectedSumOfFactMd5s: sumOfFactMd5s = Md5Sum() for f in modelXbrl.factsInInstance: sumOfFactMd5s += f.md5sum if sumOfFactMd5s != expectedSumOfFactMd5s: modelXbrl.warning("EIOPA:xbrlFactsCheckWarning", _("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"), modelObject=modelXbrl, expectedMd5=expectedSumOfFactMd5s, actualMd5Sum=sumOfFactMd5s) else: modelXbrl.info("info", _("Successful XBRL facts sum of md5s."), modelObject=modelXbrl) if any(badError in modelXbrl.errors for badError in ("EBA.2.1", "EIOPA.2.1", "EIOPA.S.1.5.a/EIOPA.S.1.5.b")): pass # skip checking filingIndicators if bad errors elif not val.filingIndicators: modelXbrl.error(("EBA.1.6", "EIOPA.1.6.a"), _('Missing filing indicators. Reported XBRL instances MUST include appropriate (positive) filing indicator elements'), modelObject=modelDocument) elif all(filed == False for filed in val.filingIndicators.values()): modelXbrl.error(("EBA.1.6", "EIOPA.1.6.a"), _('All filing indicators are filed="false". Reported XBRL instances MUST include appropriate (positive) filing indicator elements'), modelObject=modelDocument) if val.numFilingIndicatorTuples > 1: modelXbrl.warning(("EBA.1.6.2", "EIOPA.1.6.2"), _('Multiple filing indicators tuples when not in streaming mode (info).'), modelObject=modelXbrl.factsByQname[qnFIndicators]) if len(val.cntxDates) > 1: modelXbrl.error(("EBA.2.13","EIOPA.2.13"), _('Contexts must have the same date: %(dates)s.'), # when streaming values are no longer available, but without streaming they can be logged modelObject=set(_cntx for _cntxs in val.cntxDates.values() for _cntx in _cntxs), dates=', '.join(XmlUtil.dateunionValue(_dt, subtractOneDay=True) for _dt in val.cntxDates.keys())) if val.unusedCntxIDs: if val.isEIOPA_2_0_1: modelXbrl.error("EIOPA.2.7", _('Unused xbrli:context nodes MUST NOT be present in the instance: %(unusedContextIDs)s.'), modelObject=[modelXbrl.contexts[unusedCntxID] for unusedCntxID in val.unusedCntxIDs if unusedCntxID in modelXbrl.contexts], unusedContextIDs=", ".join(sorted(val.unusedCntxIDs))) else: modelXbrl.warning(("EBA.2.7", "EIOPA.2.7"), _('Unused xbrli:context nodes SHOULD NOT be present in the instance: %(unusedContextIDs)s.'), modelObject=[modelXbrl.contexts[unusedCntxID] for unusedCntxID in val.unusedCntxIDs if unusedCntxID in modelXbrl.contexts], unusedContextIDs=", ".join(sorted(val.unusedCntxIDs))) if len(val.cntxEntities) > 1: modelXbrl.error(("EBA.2.9", "EIOPA.2.9"), _('All entity identifiers and schemes MUST be the same, %(count)s found: %(entities)s.'), modelObject=modelDocument, count=len(val.cntxEntities), entities=", ".join(sorted(str(cntxEntity) for cntxEntity in val.cntxEntities))) for _scheme, _LEI in val.cntxEntities: if (_scheme in ("http://standards.iso.org/iso/17442", "http://standard.iso.org/iso/17442", "LEI") or (not val.isEIOPAfullVersion and _scheme == "PRE-LEI")): if _scheme == "http://standard.iso.org/iso/17442": modelXbrl.warning(("EBA.3.6", "EIOPA.S.2.8.c"), _("Warning, context has entity scheme %(scheme)s should be plural: http://standards.iso.org/iso/17442."), modelObject=modelDocument, scheme=_scheme) result = LeiUtil.checkLei(_LEI) if result == LeiUtil.LEI_INVALID_LEXICAL: modelXbrl.error("EIOPA.S.2.8.c", _("Context has lexically invalid LEI %(lei)s."), modelObject=modelDocument, lei=_LEI) elif result == LeiUtil.LEI_INVALID_CHECKSUM: modelXbrl.error("EIOPA.S.2.8.c", _("Context has LEI checksum error in %(lei)s."), modelObject=modelDocument, lei=_LEI) elif _scheme == "SC": pass # anything is ok for Specific Code else: modelXbrl.error("EIOPA.S.2.8.c", _("Context has unrecognized entity scheme %(scheme)s."), modelObject=modelDocument, scheme=_scheme) if val.unusedUnitIDs: if val.isEIOPA_2_0_1: modelXbrl.error("EIOPA.2.22", _('Unused xbrli:unit nodes MUST NOT be present in the instance: %(unusedUnitIDs)s.'), modelObject=[modelXbrl.units[unusedUnitID] for unusedUnitID in val.unusedUnitIDs if unusedUnitID in modelXbrl.units], unusedUnitIDs=", ".join(sorted(val.unusedUnitIDs))) else: modelXbrl.warning(("EBA.2.22", "EIOPA.2.22"), _('Unused xbrli:unit nodes SHOULD NOT be present in the instance: %(unusedUnitIDs)s.'), modelObject=[modelXbrl.units[unusedUnitID] for unusedUnitID in val.unusedUnitIDs if unusedUnitID in modelXbrl.units], unusedUnitIDs=", ".join(sorted(val.unusedUnitIDs))) if len(val.currenciesUsed) > 1: modelXbrl.error(("EBA.3.1","EIOPA.3.1"), _("There MUST be only one currency but %(numCurrencies)s were found: %(currencies)s.'"), modelObject=val.currenciesUsed.values(), numCurrencies=len(val.currenciesUsed), currencies=", ".join(str(c) for c in val.currenciesUsed.keys())) elif val.isEIOPA_2_0_1 and any(_measure.localName != val.reportingCurrency for _measure in val.currenciesUsed.keys()): modelXbrl.error("EIOPA.3.1", _("There MUST be only one currency but reporting currency %(reportingCurrency)s differs from unit currencies: %(currencies)s.'"), modelObject=val.currenciesUsed.values(), reportingCurrency=val.reportingCurrency, currencies=", ".join(str(c) for c in val.currenciesUsed.keys())) if val.prefixesUnused: modelXbrl.warning(("EBA.3.4", "EIOPA.3.4"), _("There SHOULD be no unused prefixes but these were declared: %(unusedPrefixes)s.'"), modelObject=modelDocument, unusedPrefixes=', '.join(sorted(val.prefixesUnused))) for ns, prefixes in val.namespacePrefixesUsed.items(): nsDocs = modelXbrl.namespaceDocs.get(ns) if nsDocs: for nsDoc in nsDocs: nsDocPrefix = XmlUtil.xmlnsprefix(nsDoc.xmlRootElement, ns) if any(prefix != nsDocPrefix for prefix in prefixes if prefix is not None): modelXbrl.warning(("EBA.3.5", "EIOPA.3.5"), _("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s"), modelObject=modelDocument, namespace=ns, declaredPrefix=nsDocPrefix, foundPrefixes=', '.join(sorted(prefixes - {None}))) elif ns in CANONICAL_PREFIXES and any(prefix != CANONICAL_PREFIXES[ns] for prefix in prefixes if prefix is not None): modelXbrl.warning(("EBA.3.5", "EIOPA.3.5"), _("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s"), modelObject=modelDocument, namespace=ns, declaredPrefix=CANONICAL_PREFIXES[ns], foundPrefixes=', '.join(sorted(prefixes - {None}))) modelXbrl.profileActivity(_statusMsg, minTimeToShow=0.0) modelXbrl.modelManager.showStatus(None) del val.prefixNamespace, val.namespacePrefix, val.idObjects, val.typedDomainElements del val.utrValidator, val.firstFact, val.footnotesRelationshipSet
def run(self, options, sourceZipStream=None): self.entrypointFile = options.entrypointFile filesource = FileSource.openFileSource(self.entrypointFile, self, sourceZipStream) if options.validateEFM: if options.gfmName: self.addToLog(_("both --efm and --gfm validation are requested, proceeding with --efm only"), messageCode="info", file=self.entrypointFile) self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select("efm") elif options.gfmName: self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select(options.gfmName) elif options.validateHMRC: self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select("hmrc") else: self.modelManager.disclosureSystem.select(None) # just load ordinary mappings if options.calcDecimals: if options.calcPrecision: self.addToLog(_("both --calcDecimals and --calcPrecision validation are requested, proceeding with --calcDecimals only"), messageCode="info", file=self.entrypointFile) self.modelManager.validateInferDecimals = True self.modelManager.validateCalcLB = True elif options.calcPrecision: self.modelManager.validateInferDecimals = False self.modelManager.validateCalcLB = True if options.utrValidate: self.modelManager.validateUtr = True fo = FormulaOptions() if options.parameters: fo.parameterValues = dict(((qname(key, noPrefixIsNoNamespace=True),(None,value)) for param in options.parameters.split(',') for key,sep,value in (param.partition('='),) ) ) if options.formulaParamExprResult: fo.traceParameterExpressionResult = True if options.formulaParamInputValue: fo.traceParameterInputValue = True if options.formulaCallExprSource: fo.traceCallExpressionSource = True if options.formulaCallExprCode: fo.traceCallExpressionCode = True if options.formulaCallExprEval: fo.traceCallExpressionEvaluation = True if options.formulaCallExprResult: fo.traceCallExpressionResult = True if options.formulaVarSetExprEval: fo.traceVariableSetExpressionEvaluation = True if options.formulaVarSetExprResult: fo.traceVariableSetExpressionResult = True if options.formulaAsserResultCounts: fo.traceAssertionResultCounts = True if options.formulaFormulaRules: fo.traceFormulaRules = True if options.formulaVarsOrder: fo.traceVariablesOrder = True if options.formulaVarExpressionSource: fo.traceVariableExpressionSource = True if options.formulaVarExpressionCode: fo.traceVariableExpressionCode = True if options.formulaVarExpressionEvaluation: fo.traceVariableExpressionEvaluation = True if options.formulaVarExpressionResult: fo.traceVariableExpressionResult = True if options.formulaVarFilterWinnowing: fo.traceVariableFilterWinnowing = True if options.formulaVarFiltersResult: fo.traceVariableFiltersResult = True self.modelManager.formulaOptions = fo timeNow = XmlUtil.dateunionValue(datetime.datetime.now()) startedAt = time.time() modelDiffReport = None success = True modelXbrl = None try: modelXbrl = self.modelManager.load(filesource, _("views loading")) except Exception as err: self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format( err, traceback.format_tb(sys.exc_info()[2]))) success = False # loading errors, don't attempt to utilize loaded DTS if modelXbrl and modelXbrl.modelDocument: self.addToLog(format_string(self.modelManager.locale, _("loaded in %.2f secs at %s"), (time.time() - startedAt, timeNow)), messageCode="info", file=self.entrypointFile) if options.importFiles: for importFile in options.importFiles.split("|"): ModelDocument.load(modelXbrl, importFile.strip()) self.addToLog(format_string(self.modelManager.locale, _("imported in %.2f secs at %s"), (time.time() - startedAt, timeNow)), messageCode="info", file=importFile) if modelXbrl.errors: success = False # loading errors, don't attempt to utilize loaded DTS else: success = False if success and options.diffFile and options.versReportFile: diffFilesource = FileSource.FileSource(options.diffFile,self) startedAt = time.time() modelXbrl2 = self.modelManager.load(diffFilesource, _("views loading")) if modelXbrl2.errors: if not options.keepOpen: modelXbrl2.close() success = False else: self.addToLog(format_string(self.modelManager.locale, _("diff comparison DTS loaded in %.2f secs"), time.time() - startedAt), messageCode="info", file=self.entrypointFile) startedAt = time.time() modelDiffReport = self.modelManager.compareDTSes(options.versReportFile) self.addToLog(format_string(self.modelManager.locale, _("compared in %.2f secs"), time.time() - startedAt), messageCode="info", file=self.entrypointFile) if success: try: if options.validate: startedAt = time.time() self.modelManager.validate() self.addToLog(format_string(self.modelManager.locale, _("validated in %.2f secs"), time.time() - startedAt), messageCode="info", file=self.entrypointFile) if (options.testReport and self.modelManager.modelXbrl.modelDocument.type in (ModelDocument.Type.TESTCASESINDEX, ModelDocument.Type.TESTCASE, ModelDocument.Type.REGISTRY)): ViewFileTests.viewTests(self.modelManager.modelXbrl, options.testReport) if options.DTSFile: ViewFileDTS.viewDTS(modelXbrl, options.DTSFile) if options.factsFile: ViewFileFactList.viewFacts(modelXbrl, options.factsFile, labelrole=options.labelRole, lang=options.labelLang, cols=options.factListCols) if options.factTableFile: ViewFileFactTable.viewFacts(modelXbrl, options.factTableFile, labelrole=options.labelRole, lang=options.labelLang) if options.conceptsFile: ViewFileConcepts.viewConcepts(modelXbrl, options.conceptsFile, labelrole=options.labelRole, lang=options.labelLang) if options.preFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.preFile, "Presentation Linkbase", "http://www.xbrl.org/2003/arcrole/parent-child", labelrole=options.labelRole, lang=options.labelLang) if options.calFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.calFile, "Calculation Linkbase", "http://www.xbrl.org/2003/arcrole/summation-item", labelrole=options.labelRole, lang=options.labelLang) if options.dimFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.dimFile, "Dimensions", "XBRL-dimensions", labelrole=options.labelRole, lang=options.labelLang) if options.formulaeFile: ViewFileFormulae.viewFormulae(modelXbrl, options.formulaeFile, "Formulae", lang=options.labelLang) for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Run"): pluginXbrlMethod(self, options, modelXbrl) except (IOError, EnvironmentError) as err: self.addToLog(_("[IOError] Failed to save output:\n {0}").format(err)) success = False except Exception as err: self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format( err, traceback.format_tb(sys.exc_info()[2]))) success = False if not options.keepOpen: if modelDiffReport: modelDiffReport.close() elif modelXbrl: modelXbrl.close() return success
def produceOutputFact(xpCtx, formula, result): priorErrorCount = len(xpCtx.modelXbrl.errors) # assemble context conceptQname = aspectValue(xpCtx, formula, Aspect.CONCEPT, "xbrlfe:missingConceptRule") if isinstance(conceptQname, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} concept: {1}").format(formula, conceptQname.msg), "err", conceptQname.err) modelConcept = None else: modelConcept = xpCtx.modelXbrl.qnameConcepts[conceptQname] if modelConcept is None or not modelConcept.isItem: xpCtx.modelXbrl.error( _("Formula {0} concept {1} is not an item").format( formula, conceptQname), "err", "xbrlfe:missingConceptRule") # entity entityIdentScheme = aspectValue(xpCtx, formula, Aspect.SCHEME, "xbrlfe:missingEntityIdentifierRule") if isinstance(entityIdentScheme, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} entity identifier scheme: {1}").format( formula, entityIdentScheme.msg), "err", str(entityIdentScheme)) entityIdentValue = None else: entityIdentValue = aspectValue(xpCtx, formula, Aspect.VALUE, "xbrlfe:missingEntityIdentifierRule") if isinstance(entityIdentValue, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} entity identifier value: {1}").format( formula, entityIdentValue.msg), "err", str(entityIdentScheme)) # period periodType = aspectValue(xpCtx, formula, Aspect.PERIOD_TYPE, "xbrlfe:missingPeriodRule") periodStart = None periodEndInstant = None if isinstance(periodType, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} period type: {1}").format(formula, periodType.msg), "err", str(periodType)) elif periodType == "instant": periodEndInstant = aspectValue(xpCtx, formula, Aspect.INSTANT, "xbrlfe:missingPeriodRule") if isinstance(periodEndInstant, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} period start: {1}").format( formula, periodEndInstant.msg), "err", str(periodEndInstant)) elif periodType == "duration": periodStart = aspectValue(xpCtx, formula, Aspect.START, "xbrlfe:missingPeriodRule") if isinstance(periodStart, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} period start: {1}").format( formula, periodStart.msg), "err", str(periodStart)) periodEndInstant = aspectValue(xpCtx, formula, Aspect.END, "xbrlfe:missingPeriodRule") if isinstance(periodEndInstant, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} period end: {1}").format( formula, periodEndInstant.msg), "err", str(periodEndInstant)) # unit if modelConcept and modelConcept.isNumeric: unitSource = aspectValue(xpCtx, formula, Aspect.UNIT_MEASURES, None) multDivBy = aspectValue(xpCtx, formula, Aspect.MULTIPLY_BY, "xbrlfe:missingUnitRule") if isinstance(multDivBy, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} unit: {1}").format(formula, multDivBy.msg), "err", str(multDivBy) if isinstance(multDivBy, VariableBindingError) else "xbrlfe:missingUnitRule") multiplyBy = () divideBy = () # prevent errors later if bad else: divMultBy = aspectValue(xpCtx, formula, Aspect.DIVIDE_BY, "xbrlfe:missingUnitRule") if isinstance(divMultBy, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} unit: {1}").format(formula, divMultBy.msg), "err", str(multDivBy) if isinstance(divMultBy, VariableBindingError) else "xbrlfe:missingUnitRule") multiplyBy = () divideBy = () # prevent errors later if bad else: multiplyBy = unitSource[0] + multDivBy[0] + divMultBy[1] divideBy = unitSource[1] + multDivBy[1] + divMultBy[0] # remove cancelling mult/div units lookForCommonUnits = True while lookForCommonUnits: lookForCommonUnits = False for commonUnit in multiplyBy: if commonUnit in divideBy: multiplyBy.remove(commonUnit) divideBy.remove(commonUnit) lookForCommonUnits = True break if len(multiplyBy) == 0: # if no units add pure multiplyBy.append(XbrlConst.qnXbrliPure) # dimensions segOCCs = [] scenOCCs = [] if formula.aspectModel == "dimensional": dimAspects = {} dimQnames = aspectValue(xpCtx, formula, Aspect.DIMENSIONS, None) if dimQnames: for dimQname in dimQnames: dimConcept = xpCtx.modelXbrl.qnameConcepts[dimQname] dimErr = "xbrlfe:missing{0}DimensionRule".format( "typed" if dimConcept and dimConcept.isTypedDimension else "explicit") dimValue = aspectValue(xpCtx, formula, dimQname, dimErr) if isinstance(dimValue, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} dimension {1}: {2}").format( formula, dimQname, dimValue.msg), "err", dimErr) elif dimValue and xpCtx.modelXbrl.qnameDimensionDefaults.get( dimQname) != dimValue: dimAspects[dimQname] = dimValue segOCCs = aspectValue(xpCtx, formula, Aspect.NON_XDT_SEGMENT, None) scenOCCs = aspectValue(xpCtx, formula, Aspect.NON_XDT_SCENARIO, None) else: dimAspects = None # non-dimensional segOCCs = aspectValue(xpCtx, formula, Aspect.COMPLETE_SEGMENT, None) scenOCCs = aspectValue(xpCtx, formula, Aspect.COMPLETE_SCENARIO, None) if priorErrorCount < len(xpCtx.modelXbrl.errors): return None # had errors, don't produce output fact # does context exist in out instance document outputInstanceQname = formula.outputInstanceQname outputXbrlInstance = xpCtx.inScopeVars[outputInstanceQname] xbrlElt = outputXbrlInstance.modelDocument.xmlRootElement # in source instance document # add context prevCntx = outputXbrlInstance.matchContext(entityIdentScheme, entityIdentValue, periodType, periodStart, periodEndInstant, dimAspects, segOCCs, scenOCCs) if prevCntx: cntxId = prevCntx.id newCntxElt = prevCntx.element else: cntxId = 'c-{0:02n}'.format(len(outputXbrlInstance.contexts) + 1) newCntxElt = XmlUtil.addChild( xbrlElt, XbrlConst.xbrli, "context", attributes=("id", cntxId), afterSibling=xpCtx.outputLastContext.get(outputInstanceQname)) xpCtx.outputLastContext[outputInstanceQname] = newCntxElt entityElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "entity") XmlUtil.addChild(entityElt, XbrlConst.xbrli, "identifier", attributes=("scheme", entityIdentScheme), text=entityIdentValue) periodElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "period") if periodType == "forever": XmlUtil.addChild(periodElt, XbrlConst.xbrli, "forever") elif periodType == "instant": XmlUtil.addChild(periodElt, XbrlConst.xbrli, "instant", text=XmlUtil.dateunionValue(periodEndInstant, subtractOneDay=True)) elif periodType == "duration": XmlUtil.addChild(periodElt, XbrlConst.xbrli, "startDate", text=XmlUtil.dateunionValue(periodStart)) XmlUtil.addChild(periodElt, XbrlConst.xbrli, "endDate", text=XmlUtil.dateunionValue(periodEndInstant, subtractOneDay=True)) segmentElt = None scenarioElt = None from arelle.ModelObject import ModelDimensionValue if dimAspects: for dimQname in sorted(dimAspects.keys()): dimValue = dimAspects[dimQname] if isinstance(dimValue, ModelDimensionValue): if dimValue.isExplicit: dimMemberQname = dimValue.memberQname contextEltName = dimValue.contextElement else: # qname for explicit or node for typed dimMemberQname = dimValue contextEltName = xpCtx.modelXbrl.qnameDimensionContextElement.get( dimQname) if contextEltName == "segment": if not segmentElt: segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli, "segment") contextElt = segmentElt elif contextEltName == "scenario": if not scenarioElt: scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "scenario") contextElt = scenarioElt else: continue dimConcept = xpCtx.modelXbrl.qnameConcepts[dimQname] dimAttr = ("dimension", XmlUtil.addQnameValue(xbrlElt, dimConcept.qname)) if dimConcept.isTypedDimension: dimElt = XmlUtil.addChild(contextElt, XbrlConst.xbrldi, "typedMember", attributes=dimAttr) if isinstance(dimValue, ModelDimensionValue) and dimValue.isTyped: XmlUtil.copyChildren(dimElt, dimValue.typedMember) elif dimMemberQname: dimElt = XmlUtil.addChild(contextElt, XbrlConst.xbrldi, "explicitMember", attributes=dimAttr, text=XmlUtil.addQnameValue( xbrlElt, dimMemberQname)) if segOCCs: if not segmentElt: segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli, "segment") XmlUtil.copyNodes(segmentElt, segOCCs) if scenOCCs: if not scenarioElt: scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "scenario") XmlUtil.copyNodes(scenarioElt, scenOCCs) outputXbrlInstance.modelDocument.contextDiscover(newCntxElt) # does unit exist # add unit if modelConcept.isNumeric: prevUnit = outputXbrlInstance.matchUnit(multiplyBy, divideBy) if prevUnit: unitId = prevUnit.id newUnitElt = prevUnit.element else: unitId = 'u-{0:02n}'.format(len(outputXbrlInstance.units) + 1) newUnitElt = XmlUtil.addChild( xbrlElt, XbrlConst.xbrli, "unit", attributes=("id", unitId), afterSibling=xpCtx.outputLastUnit.get(outputInstanceQname)) xpCtx.outputLastUnit[outputInstanceQname] = newUnitElt if len(divideBy) == 0: for multiply in multiplyBy: XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue( xbrlElt, multiply)) else: divElt = XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "divide") numElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitNumerator") denElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitDenominator") for multiply in multiplyBy: XmlUtil.addChild(numElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue( xbrlElt, multiply)) for divide in divideBy: XmlUtil.addChild(denElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue( xbrlElt, divide)) outputXbrlInstance.modelDocument.unitDiscover(newUnitElt) # add fact attrs = [("contextRef", cntxId)] precision = None decimals = None if modelConcept.isNumeric: attrs.append(("unitRef", unitId)) value = formula.evaluate(xpCtx) valueSeqLen = len(value) if valueSeqLen > 1: xpCtx.modelXbrl.error( _("Formula {0} value is a sequence of length {1}").format( formula, valueSeqLen), "err", "xbrlfe:nonSingletonOutputValue") else: if valueSeqLen == 0: #xsi:nil if no value attrs.append((XbrlConst.qnXsiNil, "true")) v = None else: # add precision/decimals for non-fraction numerics if modelConcept.isNumeric and not modelConcept.isFraction: if formula.hasDecimals: decimals = formula.evaluateRule(xpCtx, Aspect.DECIMALS) attrs.append(("decimals", decimals)) else: if formula.hasPrecision: precision = formula.evaluateRule( xpCtx, Aspect.PRECISION) else: precision = 0 attrs.append(("precision", precision)) x = value[0] if isinstance(x, float): from math import (log10, isnan, isinf, fabs) if (isnan(x) or (precision and (isinf(precision) or precision == 0)) or (decimals and isinf(decimals))): v = string(xpCtx, x) elif decimals is not None: v = "%.*f" % (int(decimals), x) elif precision is not None: a = fabs(x) log = log10(a) if a != 0 else 0 v = "%.*f" % (int(precision) - int(log) - (1 if a >= 1 else 0), x) else: # no implicit precision yet v = string(xpCtx, x) elif isinstance(x, QName): v = XmlUtil.addQnameValue(xbrlElt, x) elif isinstance(x, datetime.datetime): v = XmlUtil.dateunionValue(x) else: v = string(xpCtx, x) itemElt = XmlUtil.addChild( xbrlElt, conceptQname, attributes=attrs, text=v, afterSibling=xpCtx.outputLastFact.get(outputInstanceQname)) xpCtx.outputLastFact[outputInstanceQname] = itemElt newFact = outputXbrlInstance.modelDocument.factDiscover( itemElt, outputXbrlInstance.facts) return newFact
def setArgForFactProperty(param, modelFact, propertyNameParts): propVal = None property = propertyNameParts[0] if property == "value": if isinstance(modelFact.xValue, Decimal): propVal = "{:,}".format(modelFact.xValue) else: propVal = modelFact.value elif property == "decimals": propVal = modelFact.decimals elif property == "label" and modelFact.concept is not None: propVal = modelFact.concept.label(labelrole, lang=lang, linkroleHint=XbrlConst.defaultLinkRole) elif property == "name": propVal = str(modelFact.qname) elif property == "localName": propVal = modelFact.qname.localName else: cntx = modelFact.context unit = modelFact.unit if cntx is not None: if property == "period": if len(propertyNameParts) == 1: if cntx.isForeverPeriod: propVal = "forever" elif cntx.isInstantPeriod: propVal = XmlUtil.dateunionValue(cntx.instantDatetime, subtractOneDay=True) else: propVal = "{} to {}".format(XmlUtil.dateunionValue(cntx.startDatetime), XmlUtil.dateunionValue(cntx.endDatetime, subtractOneDay=True)) else: dateSelection = propertyNameParts[1] if dateSelection == "startDate": propVal = XmlUtil.dateunionValue(cntx.startDatetime) elif dateSelection == "endDate": propVal = XmlUtil.dateunionValue(cntx.endDatetime, subtractOneDay=True) elif dateSelection == "instant": propVal = XmlUtil.dateunionValue(cntx.instant, subtractOneDay=True) elif dateSelection == "durationDays": propVal = str((cntx.endDatetime - cntx.startDatetime).days) elif property == "dimensions": if cntx.qnameDims: propVal = "\n".join("{} = {}".format(d.dimensionQname, d.memberQname if d.isExplicit else XmlUtil.xmlstring( XmlUtil.child(d), stripXmlns=True, prettyPrint=True )) for d in cntx.qnameDims.values()) else: propVal = "none" if property == "unit": if unit is None: propVal = "none" else: measures = unit.measures if measures[1]: propVal = 'mul {}\ndiv {} '.format( ', '.join(measureFormat(m) for m in measures[0]), ', '.join(measureFormat(m) for m in measures[1])) else: propVal = ', '.join(measureFormat(m) for m in measures[0]) if propVal is not None: fmtArgs[param] = propVal
def jsonDefaultEncoder(obj): if isinstance(obj, Decimal): return float(obj) elif isinstance(obj, (datetime.date, datetime.datetime)): return XmlUtil.dateunionValue(obj) raise TypeError("Type {} is not supported for json output".format(type(obj).__name__))
def __init__(self, modelXbrl, xfFile): self.modelXbrl = modelXbrl self.xfFile = xfFile self.xfLines = [] self.xmlns = {} self.eltTypeCount = {} for cfQnameArity in sorted(qnameArity for qnameArity in self.modelXbrl.modelCustomFunctionSignatures.keys() if isinstance(qnameArity, (tuple,list))): cfObject = self.modelXbrl.modelCustomFunctionSignatures[cfQnameArity] self.doObject(cfObject, None, "", set()) rootObjects = rootFormulaObjects(self) # sets var sets up # put parameters at root regardless of whether linked to for qn, param in sorted(self.modelXbrl.qnameParameters.items(), key=lambda i:i[0]): self.doObject(param, None, "", set()) for rootObject in sorted(rootObjects, key=formulaObjSortKey): self.doObject(rootObject, None, "", set()) if self.xmlns: self.xfLines.insert(0, "") for prefix, ns in sorted(self.xmlns.items(), reverse=True): self.xfLines.insert(0, "namespace \"{}\" = \"{}\";".format(prefix, ns)) self.xfLines.insert(0, "") self.xfLines.insert(0, "(: Generated from {} by Arelle on {} :)".format(modelXbrl.modelDocument.basename, XmlUtil.dateunionValue(datetime.datetime.now()))) with open(xfFile, "w", encoding="utf-8") as fh: fh.write("\n".join(self.xfLines)) modelXbrl.info("info", "saved formula file %(file)s", file=xfFile)
def insertDataPoints(self): # separate graph # document-> dataTypeSet -> dataType self.showStatus("insert DataPoints") # note these initial aspects Qnames used also must be in conceptsUsed above dimensions = [] # index by hash of dimension dimensionIds = {} # index for dimension if self.modelXbrl.modelDocument.type in (Type.INSTANCE, Type.INLINEXBRL): contextAspectValueSelections = {} # contexts processed already unitIDs = set() # units processed already periodProxies = {} entityIdentifierAspectProxies = {} dataPoints = self.report['dataPoints'] for fact in self.modelXbrl.factsInInstance: self.insertAspectProxies( (fact.qname,) ) factId = XmlUtil.elementFragmentIdentifier(fact) dataPoints[factId] = dataPoint = { 'document': modelObjectDocumentUri(fact), 'id': factId, 'sourceLine': fact.sourceline, 'dataPointUrl': modelObjectUri(fact), 'baseItem': self.aspectQnameProxyId(fact.qname) } context = fact.context concept = fact.concept if context is not None: if context.entityIdentifier not in entityIdentifierAspectProxies: entityScheme, entityIdentifier = context.entityIdentifier entityIdentifierAspectProxy = "{}/{}".format( qnamePrefix_Name(XbrlConst.qnXbrliIdentifier), entityIdentifier) e = self.insertAspectProxy(XbrlConst.qnXbrliIdentifier, entityIdentifierAspectProxy) e['scheme'] = entityScheme e['identifier'] = entityIdentifier entityIdentifierAspectProxies[context.entityIdentifier] = entityIdentifierAspectProxy else: entityIdentifierAspectProxy = entityIdentifierAspectProxies[context.entityIdentifier] dataPoint['entityIdentifier'] = entityIdentifierAspectProxy if context.isForeverPeriod: period = "forever" if context.isInstantPeriod: endDate = XmlUtil.dateunionValue(context.instantDatetime, subtractOneDay=True).replace(':','_') period = "instant/{}".format(endDate) else: startDate = XmlUtil.dateunionValue(context.startDatetime).replace(':','_') endDate = XmlUtil.dateunionValue(context.endDatetime, subtractOneDay=True).replace(':','_') period = "duration/{}/{}".format(startDate, endDate) if period not in periodProxies: periodProxy = "{}/{}".format( qnamePrefix_Name(XbrlConst.qnXbrliPeriod), period) p = self.insertAspectProxy(XbrlConst.qnXbrliPeriod, periodProxy) p['isForever'] = context.isForeverPeriod p['isInstant'] = context.isInstantPeriod if context.isStartEndPeriod: d = context.startDatetime if d.hour == 0 and d.minute == 0 and d.second == 0: d = d.date() p['startDate'] = d if context.isStartEndPeriod or context.isInstantPeriod: d = context.endDatetime if d.hour == 0 and d.minute == 0 and d.second == 0: d = (d - datetime.timedelta(1)).date() p['endDate'] = d periodProxies[period] = periodProxy else: periodProxy = periodProxies[period] dataPoint['period'] = periodProxy dataPoint['contextUrl'] = modelObjectUri(context) dataPoint['contextId'] = context.id if context.id not in contextAspectValueSelections: contextAspectValueSelections[context.id] = contextAspectValueSelection = [] for dimVal in context.qnameDims.values(): dim = qnamePrefix_Name(dimVal.dimensionQname) if dimVal.isExplicit: self.insertAspectProxies( (dimVal.memberQname,) ) # need imediate use of proxy v = self.aspectQnameProxyId(dimVal.memberQname) else: v = dimVal.typedMember.stringValue dimProxy = "{}/{}".format(dim, v) d = self.insertAspectProxy(dimVal.dimensionQname, dimProxy) contextAspectValueSelection.append(dimProxy) d['aspect'] = dim if dimVal.isExplicit: d['aspectValue'] = v else: d['typedValue'] = v else: contextAspectValueSelection = contextAspectValueSelections[context.id] dataPoint['aspectValueSelections'] = contextAspectValueSelection if fact.isNumeric: if fact.precision == "INF": dataPoint['precision'] = "INF" elif fact.precision is not None: dataPoint['precision'] = fact.precision if fact.decimals == "INF": dataPoint['decimals'] = "INF" elif fact.decimals is not None: dataPoint['decimals'] = fact.decimals if fact.unit is not None: unit = fact.unit unitProxy = "{}/{}".format( qnamePrefix_Name(XbrlConst.qnXbrliUnit), unit.id) dataPoint['unit'] = unitProxy if unit.id not in unitIDs: unitIDs.add(unit.id) u = self.insertAspectProxy(XbrlConst.qnXbrliUnit, unitProxy) u['unitId'] = unit.id mults, divs = unit.measures u['multiplyMeasures'] = [qnameUri(qn) for qn in mults] if divs: u['divideMeasures'] = [qnameUri(qn) for qn in divs] if fact.xmlLang is None and fact.concept is not None and fact.concept.baseXsdType is not None: dataPoint['value'] = fact.xValue # The insert with base XSD type but no language elif fact.xmlLang is not None: # assuming string type with language dataPoint['language'] = fact.xmlLang dataPoint['value'] = fact.value else: # Otherwise insert as plain liternal with no language or datatype dataPoint['value'] = fact.value if fact.modelTupleFacts: dataPoint['tuple'] = [XmlUtil.elementFragmentIdentifier(tupleFact) for tupleFact in fact.modelTupleFacts]
def createContext(self, entityIdentScheme, entityIdentValue, periodType, periodStart, periodEndInstant, priItem, dims, segOCCs, scenOCCs, afterSibling=None, beforeSibling=None): xbrlElt = self.modelDocument.xmlRootElement if afterSibling == AUTO_LOCATE_ELEMENT: afterSibling = XmlUtil.lastChild(xbrlElt, XbrlConst.xbrli, ("schemaLocation", "roleType", "arcroleType", "context")) cntxId = 'c-{0:02n}'.format( len(self.contexts) + 1) newCntxElt = XmlUtil.addChild(xbrlElt, XbrlConst.xbrli, "context", attributes=("id", cntxId), afterSibling=afterSibling, beforeSibling=beforeSibling) entityElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "entity") XmlUtil.addChild(entityElt, XbrlConst.xbrli, "identifier", attributes=("scheme", entityIdentScheme), text=entityIdentValue) periodElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "period") if periodType == "forever": XmlUtil.addChild(periodElt, XbrlConst.xbrli, "forever") elif periodType == "instant": XmlUtil.addChild(periodElt, XbrlConst.xbrli, "instant", text=XmlUtil.dateunionValue(periodEndInstant, subtractOneDay=True)) elif periodType == "duration": XmlUtil.addChild(periodElt, XbrlConst.xbrli, "startDate", text=XmlUtil.dateunionValue(periodStart)) XmlUtil.addChild(periodElt, XbrlConst.xbrli, "endDate", text=XmlUtil.dateunionValue(periodEndInstant, subtractOneDay=True)) segmentElt = None scenarioElt = None from arelle.ModelInstanceObject import ModelDimensionValue if dims: # requires primary item to determin ambiguous concepts ''' in theory we have to check full set of dimensions for validity in source or any other context element, but for shortcut will see if each dimension is already reported in an unambiguous valid contextElement ''' from arelle.PrototypeInstanceObject import FactPrototype, ContextPrototype, DimValuePrototype fp = FactPrototype(self, priItem, dims.items()) # force trying a valid prototype's context Elements if not isFactDimensionallyValid(self, fp, setPrototypeContextElements=True): self.info("arelleLinfo", _("Create context for %(priItem)s, cannot determine valid context elements, no suitable hypercubes"), modelObject=self, priItem=priItem) fpDims = fp.context.qnameDims for dimQname in sorted(fpDims.keys()): dimValue = fpDims[dimQname] if isinstance(dimValue, DimValuePrototype): dimMemberQname = dimValue.memberQname # None if typed dimension contextEltName = dimValue.contextElement else: # qname for explicit or node for typed dimMemberQname = None contextEltName = None if contextEltName == "segment": if segmentElt is None: segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli, "segment") contextElt = segmentElt elif contextEltName == "scenario": if scenarioElt is None: scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "scenario") contextElt = scenarioElt else: self.info("arelleLinfo", _("Create context, %(dimension)s, cannot determine context element, either no all relationship or validation issue"), modelObject=self, dimension=dimQname), continue dimConcept = self.qnameConcepts[dimQname] dimAttr = ("dimension", XmlUtil.addQnameValue(xbrlElt, dimConcept.qname)) if dimConcept.isTypedDimension: dimElt = XmlUtil.addChild(contextElt, XbrlConst.xbrldi, "xbrldi:typedMember", attributes=dimAttr) if isinstance(dimValue, (ModelDimensionValue, DimValuePrototype)) and dimValue.isTyped: XmlUtil.copyNodes(dimElt, dimValue.typedMember) elif dimMemberQname: dimElt = XmlUtil.addChild(contextElt, XbrlConst.xbrldi, "xbrldi:explicitMember", attributes=dimAttr, text=XmlUtil.addQnameValue(xbrlElt, dimMemberQname)) if segOCCs: if segmentElt is None: segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli, "segment") XmlUtil.copyNodes(segmentElt, segOCCs) if scenOCCs: if scenarioElt is None: scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "scenario") XmlUtil.copyNodes(scenarioElt, scenOCCs) self.modelDocument.contextDiscover(newCntxElt) XmlValidate.validate(self, newCntxElt) return newCntxElt
def produceOutputFact(xpCtx, formula, result): priorErrorCount = len(xpCtx.modelXbrl.errors) isTuple = isinstance(formula,ModelTuple) # assemble context conceptQname = aspectValue(xpCtx, formula, Aspect.CONCEPT, "xbrlfe:missingConceptRule") if isinstance(conceptQname, VariableBindingError): xpCtx.modelXbrl.error(conceptQname.err, _("Formula %(xlinkLabel)s concept: %(concept)s"), modelObject=formula, xlinkLabel=formula.xlinkLabel, concept=conceptQname.msg) modelConcept = None else: modelConcept = xpCtx.modelXbrl.qnameConcepts[conceptQname] if modelConcept is None or (not modelConcept.isTuple if isTuple else not modelConcept.isItem): xpCtx.modelXbrl.error("xbrlfe:missingConceptRule", _("Formula %(xlinkLabel)s concept %(concept)s is not a %(element)s"), modelObject=formula, xlinkLabel=formula.xlinkLabel, concept=conceptQname, element=formula.localName) outputLocation = aspectValue(xpCtx, formula, Aspect.LOCATION_RULE, None) if not isTuple: # entity entityIdentScheme = aspectValue(xpCtx, formula, Aspect.SCHEME, "xbrlfe:missingEntityIdentifierRule") if isinstance(entityIdentScheme, VariableBindingError): xpCtx.modelXbrl.error(str(entityIdentScheme), _("Formula %(xlinkLabel)s entity identifier scheme: %(scheme)s"), modelObject=formula, xlinkLabel=formula.xlinkLabel, scheme=entityIdentScheme.msg) entityIdentValue = None else: entityIdentValue = aspectValue(xpCtx, formula, Aspect.VALUE, "xbrlfe:missingEntityIdentifierRule") if isinstance(entityIdentValue, VariableBindingError): xpCtx.modelXbrl.error(str(entityIdentScheme), _("Formula %(xlinkLabel)s entity identifier value: %(entityIdentifier)s"), modelObject=formula, xlinkLabel=formula.xlinkLabel, entityIdentifier=entityIdentValue.msg) # period periodType = aspectValue(xpCtx, formula, Aspect.PERIOD_TYPE, "xbrlfe:missingPeriodRule") periodStart = None periodEndInstant = None if isinstance(periodType, VariableBindingError): xpCtx.modelXbrl.error(str(periodType), _("Formula %(xlinkLabel)s period type: %(periodType)s"), modelObject=formula, xlinkLabel=formula.xlinkLabel, periodType=periodType.msg) elif periodType == "instant": periodEndInstant = aspectValue(xpCtx, formula, Aspect.INSTANT, "xbrlfe:missingPeriodRule") if isinstance(periodEndInstant, VariableBindingError): xpCtx.modelXbrl.error(str(periodEndInstant), _("Formula %(xlinkLabel)s period end: %(period)s"), modelObject=formula, xlinkLabel=formula.xlinkLabel, period=periodEndInstant.msg) elif periodType == "duration": periodStart = aspectValue(xpCtx, formula, Aspect.START, "xbrlfe:missingPeriodRule") if isinstance(periodStart, VariableBindingError): xpCtx.modelXbrl.error(str(periodStart), _("Formula %(xlinkLabel)s period start: %(period)s"), modelObject=formula, xlinkLabel=formula.xlinkLabel, period=periodStart.msg) periodEndInstant = aspectValue(xpCtx, formula, Aspect.END, "xbrlfe:missingPeriodRule") if isinstance(periodEndInstant, VariableBindingError): xpCtx.modelXbrl.error(str(periodEndInstant), _("Formula %(xlinkLabel)s period end: %(period)s"), modelObject=formula, xlinkLabel=formula.xlinkLabel, period=periodEndInstant.msg) # unit if modelConcept is not None and modelConcept.isNumeric: unitSource = aspectValue(xpCtx, formula, Aspect.UNIT_MEASURES, None) multDivBy = aspectValue(xpCtx, formula, Aspect.MULTIPLY_BY, "xbrlfe:missingUnitRule") if isinstance(multDivBy, VariableBindingError): xpCtx.modelXbrl.error(str(multDivBy) if isinstance(multDivBy, VariableBindingError) else "xbrlfe:missingUnitRule", _("Formula %(xlinkLabel)s unit: %(unit)s"), modelObject=formula, xlinkLabel=formula.xlinkLabel, unit=multDivBy.msg) multiplyBy = (); divideBy = () # prevent errors later if bad else: divMultBy = aspectValue(xpCtx, formula, Aspect.DIVIDE_BY, "xbrlfe:missingUnitRule") if isinstance(divMultBy, VariableBindingError): xpCtx.modelXbrl.error(str(multDivBy) if isinstance(divMultBy, VariableBindingError) else "xbrlfe:missingUnitRule", _("Formula %(xlinkLabel)s unit: %(unit)s"), modelObject=formula, xlinkLabel=formula.xlinkLabel, unit=divMultBy.msg) multiplyBy = (); divideBy = () # prevent errors later if bad else: multiplyBy = unitSource[0] + multDivBy[0] + divMultBy[1] divideBy = unitSource[1] + multDivBy[1] + divMultBy[0] # remove cancelling mult/div units lookForCommonUnits = True while lookForCommonUnits: lookForCommonUnits = False for commonUnit in multiplyBy: if commonUnit in divideBy: multiplyBy.remove(commonUnit) divideBy.remove(commonUnit) lookForCommonUnits = True break if len(multiplyBy) == 0: # if no units add pure multiplyBy.append(XbrlConst.qnXbrliPure) # dimensions segOCCs = [] scenOCCs = [] if formula.aspectModel == "dimensional": dimAspects = {} dimQnames = aspectValue(xpCtx, formula, Aspect.DIMENSIONS, None) if dimQnames: for dimQname in dimQnames: dimConcept = xpCtx.modelXbrl.qnameConcepts[dimQname] dimErr = "xbrlfe:missing{0}DimensionRule".format("typed" if dimConcept is not None and dimConcept.isTypedDimension else "explicit") dimValue = aspectValue(xpCtx, formula, dimQname, dimErr) if isinstance(dimValue, VariableBindingError): xpCtx.modelXbrl.error(dimErr, _("Formula %(xlinkLabel)s dimension %(dimension)s: %(value)s"), modelObject=formula, xlinkLabel=formula.xlinkLabel, dimension=dimQname, value=dimValue.msg) elif dimConcept.isTypedDimension: if isinstance(dimValue, list): # result of flatten, always a list if len(dimValue) != 1 or not isinstance(dimValue[0], ModelObject): xpCtx.modelXbrl.error("xbrlfe:wrongXpathResultForTypedDimensionRule", _("Formula %(xlinkLabel)s dimension %(dimension)s value is not a node: %(value)s"), modelObject=formula, xlinkLabel=formula.xlinkLabel, dimension=dimQname, value=dimValue) continue dimValue = dimValue[0] dimAspects[dimQname] = dimValue elif dimValue is not None and xpCtx.modelXbrl.qnameDimensionDefaults.get(dimQname) != dimValue: dimAspects[dimQname] = dimValue segOCCs = aspectValue(xpCtx, formula, Aspect.NON_XDT_SEGMENT, None) scenOCCs = aspectValue(xpCtx, formula, Aspect.NON_XDT_SCENARIO, None) for occElt in xpCtx.flattenSequence((segOCCs, scenOCCs)): if isinstance(occElt, ModelObject) and occElt.namespaceURI == XbrlConst.xbrldi: xpCtx.modelXbrl.error("xbrlfe:badSubsequentOCCValue", _("Formula %(xlinkLabel)s OCC element %(occ)s covers a dimensional aspect"), modelObject=(formula,occElt), xlinkLabel=formula.xlinkLabel, occ=occElt.elementQname) else: dimAspects = None # non-dimensional segOCCs = aspectValue(xpCtx, formula, Aspect.COMPLETE_SEGMENT, None) scenOCCs = aspectValue(xpCtx, formula, Aspect.COMPLETE_SCENARIO, None) if priorErrorCount < len(xpCtx.modelXbrl.errors): return None # had errors, don't produce output fact # does context exist in out instance document outputInstanceQname = formula.outputInstanceQname outputXbrlInstance = xpCtx.inScopeVars[outputInstanceQname] xbrlElt = outputXbrlInstance.modelDocument.xmlRootElement # in source instance document newFact = None if isTuple: newFact = outputXbrlInstance.createFact(conceptQname, parent=outputLocation, afterSibling=xpCtx.outputLastFact.get(outputInstanceQname)) else: # add context prevCntx = outputXbrlInstance.matchContext( entityIdentScheme, entityIdentValue, periodType, periodStart, periodEndInstant, dimAspects, segOCCs, scenOCCs) if prevCntx is not None: cntxId = prevCntx.id newCntxElt = prevCntx else: newCntxElt = outputXbrlInstance.createContext(entityIdentScheme, entityIdentValue, periodType, periodStart, periodEndInstant, conceptQname, dimAspects, segOCCs, scenOCCs, afterSibling=xpCtx.outputLastContext.get(outputInstanceQname), beforeSibling=xpCtx.outputFirstFact.get(outputInstanceQname)) cntxId = newCntxElt.id xpCtx.outputLastContext[outputInstanceQname] = newCntxElt # does unit exist # add unit if modelConcept.isNumeric: prevUnit = outputXbrlInstance.matchUnit(multiplyBy, divideBy) if prevUnit is not None: unitId = prevUnit.id newUnitElt = prevUnit else: newUnitElt = outputXbrlInstance.createUnit(multiplyBy, divideBy, afterSibling=xpCtx.outputLastUnit.get(outputInstanceQname), beforeSibling=xpCtx.outputFirstFact.get(outputInstanceQname)) unitId = newUnitElt.id xpCtx.outputLastUnit[outputInstanceQname] = newUnitElt # add fact attrs = [("contextRef", cntxId)] precision = None decimals = None if modelConcept.isNumeric: attrs.append(("unitRef", unitId)) value = formula.evaluate(xpCtx) valueSeqLen = len(value) if valueSeqLen > 1: xpCtx.modelXbrl.error("xbrlfe:nonSingletonOutputValue", _("Formula %(xlinkLabel)s value is a sequence of length %(valueSequenceLength)s"), modelObject=formula, xlinkLabel=formula.xlinkLabel, valueSequenceLength=valueSeqLen) else: if valueSeqLen == 0: #xsi:nil if no value attrs.append((XbrlConst.qnXsiNil, "true")) v = None else: # add precision/decimals for non-fraction numerics if modelConcept.isNumeric and not modelConcept.isFraction: if formula.hasDecimals: decimals = formula.evaluateRule(xpCtx, Aspect.DECIMALS) attrs.append(("decimals", decimals)) else: if formula.hasPrecision: precision = formula.evaluateRule(xpCtx, Aspect.PRECISION) else: precision = 0 attrs.append(("precision", precision)) x = value[0] if isinstance(x,float): from math import (log10, isnan, isinf, fabs) if (isnan(x) or (precision and (isinf(precision) or precision == 0)) or (decimals and isinf(decimals))): v = xsString(xpCtx, None, x) elif decimals is not None: v = "%.*f" % ( int(decimals), x) elif precision is not None and precision != 0: a = fabs(x) log = log10(a) if a != 0 else 0 v = "%.*f" % ( int(precision) - int(log) - (1 if a >= 1 else 0), x) else: # no implicit precision yet v = xsString(xpCtx, None, x) elif isinstance(x,QName): v = XmlUtil.addQnameValue(xbrlElt, x) elif isinstance(x,datetime.datetime): v = XmlUtil.dateunionValue(x) else: v = xsString(xpCtx, None, x) newFact = outputXbrlInstance.createFact(conceptQname, attributes=attrs, text=v, parent=outputLocation, afterSibling=xpCtx.outputLastFact.get(outputInstanceQname)) if newFact is not None: xpCtx.outputLastFact[outputInstanceQname] = newFact if outputInstanceQname not in xpCtx.outputFirstFact: xpCtx.outputFirstFact[outputInstanceQname] = newFact return newFact
def final(val): if not (val.validateEBA or val.validateEIOPA): return modelXbrl = val.modelXbrl modelDocument = modelXbrl.modelDocument _statusMsg = _("validating {0} filing rules").format(val.disclosureSystem.name) modelXbrl.profileActivity() modelXbrl.modelManager.showStatus(_statusMsg) if modelDocument.type == ModelDocument.Type.INSTANCE and (val.validateEBA or val.validateEIOPA): if not modelDocument.uri.endswith(".xbrl"): modelXbrl.warning("EBA.1.1", _('XBRL instance documents SHOULD use the extension ".xbrl" but it is "%(extension)s"'), modelObject=modelDocument, extension=os.path.splitext(modelDocument.basename)[1]) if modelDocument.documentEncoding.lower() not in ("utf-8", "utf-8-sig"): modelXbrl.error("EBA.1.4", _('XBRL instance documents MUST use "UTF-8" encoding but is "%(xmlEncoding)s"'), modelObject=modelDocument, xmlEncoding=modelDocument.documentEncoding) schemaRefElts = [] schemaRefFileNames = [] for doc, docRef in modelDocument.referencesDocument.items(): if docRef.referenceType == "href": if docRef.referringModelObject.localName == "schemaRef": schemaRefElts.append(docRef.referringModelObject) schemaRefFileNames.append(doc.basename) if not UrlUtil.isAbsolute(doc.uri): modelXbrl.error("EBA.2.2", _('The link:schemaRef element in submitted instances MUST resolve to the full published entry point URL: %(url)s.'), modelObject=docRef.referringModelObject, url=doc.uri) elif docRef.referringModelObject.localName == "linkbaseRef": modelXbrl.error("EBA.2.3", _('The link:linkbaseRef element is not allowed: %(fileName)s.'), modelObject=docRef.referringModelObject, fileName=doc.basename) if len(schemaRefFileNames) > 1: modelXbrl.error("EBA.1.5", _('XBRL instance documents MUST reference only one entry point schema but %(numEntryPoints)s were found: %(entryPointNames)s'), modelObject=modelDocument, numEntryPoints=len(schemaRefFileNames), entryPointNames=', '.join(sorted(schemaRefFileNames))) ### check entry point names appropriate for filing indicator (DPM DB?) if len(schemaRefElts) != 1: modelXbrl.error("EBA.2.3", _('Any reported XBRL instance document MUST contain only one xbrli:xbrl/link:schemaRef node, but %(entryPointCount)s.'), modelObject=schemaRefElts, entryPointCount=len(schemaRefElts)) # non-streaming EBA checks if not getattr(modelXbrl, "isStreamingMode", False): validateFacts(val, modelXbrl.facts) # check sum of fact md5s (otherwise checked in streaming process) xbrlFactsCheckVersion = None expectedSumOfFactMd5s = None for pi in modelDocument.xmlRootElement.getchildren(): if isinstance(pi, etree._ProcessingInstruction) and pi.target == "xbrl-facts-check": _match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']", pi.text) if _match: _matchGroups = _match.groups() if len(_matchGroups) == 2: if _matchGroups[0] == "version": xbrlFactsCheckVersion = _matchGroups[1] elif _matchGroups[0] == "sum-of-fact-md5s": try: expectedSumOfFactMd5s = Md5Sum(_matchGroups[1]) except ValueError: modelXbrl.error("EIOPA:xbrlFactsCheckError", _("Invalid sum-of-md5s %(sumOfMd5)s"), modelObject=modelXbrl, sumOfMd5=_matchGroups[1]) if xbrlFactsCheckVersion and expectedSumOfFactMd5s: sumOfFactMd5s = Md5Sum() for f in modelXbrl.factsInInstance: sumOfFactMd5s += f.md5sum if sumOfFactMd5s != expectedSumOfFactMd5s: modelXbrl.warning("EIOPA:xbrlFactsCheckWarning", _("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"), modelObject=modelXbrl, expectedMd5=expectedSumOfFactMd5s, actualMd5Sum=sumOfFactMd5s) else: modelXbrl.info("info", _("Successful XBRL facts sum of md5s."), modelObject=modelXbrl) if not val.filingIndicators: modelXbrl.error("EBA.1.6", _('Missing filing indicators. Reported XBRL instances MUST include appropriate filing indicator elements'), modelObject=modelDocument) if val.numFilingIndicatorTuples > 1: modelXbrl.info("EBA.1.6.2", _('Multiple filing indicators tuples when not in streaming mode (info).'), modelObject=modelXbrl.factsByQname[qnFIndicators]) if len(val.cntxDates) > 1: modelXbrl.error("EBA.2.13", _('Contexts must have the same date: %(dates)s.'), # when streaming values are no longer available, but without streaming they can be logged modelObject=set(_cntx for _cntxs in val.cntxDates.values() for _cntx in _cntxs), dates=', '.join(XmlUtil.dateunionValue(_dt, subtractOneDay=True) for _dt in val.cntxDates.keys())) if val.unusedCntxIDs: modelXbrl.warning("EBA.2.7", _('Unused xbrli:context nodes SHOULD NOT be present in the instance: %(unusedContextIDs)s.'), modelObject=[modelXbrl.contexts[unusedCntxID] for unusedCntxID in val.unusedCntxIDs if unusedCntxID in modelXbrl.contexts], unusedContextIDs=", ".join(sorted(val.unusedCntxIDs))) if len(val.cntxEntities) > 1: modelXbrl.warning("EBA.2.9", _('All entity identifiers and schemes must be the same, %(count)s found: %(entities)s.'), modelObject=modelDocument, count=len(val.cntxEntities), entities=", ".join(sorted(str(cntxEntity) for cntxEntity in val.cntxEntities))) if val.unusedUnitIDs: modelXbrl.warning("EBA.2.21", _('Unused xbrli:unit nodes SHOULD NOT be present in the instance: %(unusedUnitIDs)s.'), modelObject=[modelXbrl.units[unusedUnitID] for unusedUnitID in val.unusedUnitIDs if unusedUnitID in modelXbrl.units], unusedUnitIDs=", ".join(sorted(val.unusedUnitIDs))) if len(val.currenciesUsed) > 1: modelXbrl.error("EBA.3.1", _("There MUST be only one currency but %(numCurrencies)s were found: %(currencies)s.'"), modelObject=val.currenciesUsed.values(), numCurrencies=len(val.currenciesUsed), currencies=", ".join(str(c) for c in val.currenciesUsed.keys())) if val.prefixesUnused: modelXbrl.warning("EBA.3.4", _("There SHOULD be no unused prefixes but these were declared: %(unusedPrefixes)s.'"), modelObject=modelDocument, unusedPrefixes=', '.join(sorted(val.prefixesUnused))) for ns, prefixes in val.namespacePrefixesUsed.items(): nsDocs = modelXbrl.namespaceDocs.get(ns) if nsDocs: for nsDoc in nsDocs: nsDocPrefix = XmlUtil.xmlnsprefix(nsDoc.xmlRootElement, ns) if any(prefix != nsDocPrefix for prefix in prefixes if prefix is not None): modelXbrl.warning("EBA.3.5", _("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s"), modelObject=modelDocument, namespace=ns, declaredPrefix=nsDocPrefix, foundPrefixes=', '.join(sorted(prefixes - {None}))) modelXbrl.profileActivity(_statusMsg, minTimeToShow=0.0) modelXbrl.modelManager.showStatus(None) del val.prefixNamespace, val.namespacePrefix, val.idObjects, val.typedDomainElements del val.utrValidator
def run(self, options): if options.logFile: self.messages = [] else: self.messages = None self.filename = options.filename filesource = FileSource.FileSource(self.filename, self) if options.validateEFM: if options.gfmName: self.addToLog( _("[info] both --efm and --gfm validation are requested, proceeding with --efm only" )) self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select("efm") elif options.gfmName: self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select(options.gfmName) else: self.modelManager.disclosureSystem.select( None) # just load ordinary mappings if options.calcDecimals: if options.calcPrecision: self.addToLog( _("[info] both --calcDecimals and --calcPrecision validation are requested, proceeding with --calcDecimals only" )) self.modelManager.validateInferDecimals = True self.modelManager.validateCalcLB = True elif options.calcPrecision: self.modelManager.validateInferDecimals = False self.modelManager.validateCalcLB = True if options.utrValidate: self.modelManager.validateUtr = True fo = FormulaOptions() if options.formulaParamExprResult: fo.traceParameterExpressionResult = True if options.formulaParamInputValue: fo.traceParameterInputValue = True if options.formulaCallExprSource: fo.traceCallExpressionSource = True if options.formulaCallExprCode: fo.traceCallExpressionCode = True if options.formulaCallExprEval: fo.traceCallExpressionEvaluation = True if options.formulaCallExprResult: fo.traceCallExpressionResult = True if options.formulaVarSetExprEval: fo.traceVariableSetExpressionEvaluation = True if options.formulaVarSetExprResult: fo.traceVariableSetExpressionResult = True if options.formulaAsserResultCounts: fo.traceAssertionResultCounts = True if options.formulaFormulaRules: fo.traceFormulaRules = True if options.formulaVarsOrder: fo.traceVariablesOrder = True if options.formulaVarExpressionSource: fo.traceVariableExpressionSource = True if options.formulaVarExpressionCode: fo.traceVariableExpressionCode = True if options.formulaVarExpressionEvaluation: fo.traceVariableExpressionEvaluation = True if options.formulaVarExpressionResult: fo.traceVariableExpressionResult = True if options.formulaVarFiltersResult: fo.traceVariableFiltersResult = True self.modelManager.formulaOptions = fo timeNow = XmlUtil.dateunionValue(datetime.datetime.now()) startedAt = time.time() modelXbrl = self.modelManager.load(filesource, _("views loading")) self.addToLog( format_string(self.modelManager.locale, _("[info] loaded in %.2f secs at %s"), (time.time() - startedAt, timeNow))) if options.diffFilename and options.versReportFilename: diffFilesource = FileSource.FileSource(self.diffFilename, self) startedAt = time.time() modelXbrl = self.modelManager.load(diffFilesource, _("views loading")) self.addToLog( format_string( self.modelManager.locale, _("[info] diff comparison DTS loaded in %.2f secs"), time.time() - startedAt)) startedAt = time.time() self.modelManager.compareDTSes(options.versReportFilename) self.addToLog( format_string(self.modelManager.locale, _("[info] compared in %.2f secs"), time.time() - startedAt)) try: if options.validate: startedAt = time.time() self.modelManager.validate() self.addToLog( format_string(self.modelManager.locale, _("[info] validated in %.2f secs"), time.time() - startedAt)) if (options.csvTestReport and self.modelManager.modelXbrl.modelDocument.type in (ModelDocument.Type.TESTCASESINDEX, ModelDocument.Type.REGISTRY)): ViewCsvTests.viewTests(self.modelManager.modelXbrl, options.csvTestReport) if options.csvDTS: ViewCsvDTS.viewDTS(modelXbrl, options.csvDTS) if options.csvFactList: ViewCsvFactList.viewFacts(modelXbrl, options.csvFactList) if options.csvConcepts: ViewCsvConcepts.viewConcepts(modelXbrl, options.csvConcepts) if options.csvPre: ViewCsvRelationshipSet.viewRelationshipSet( modelXbrl, options.csvPre, "Presentation", "http://www.xbrl.org/2003/arcrole/parent-child") if options.csvCal: ViewCsvRelationshipSet.viewRelationshipSet( modelXbrl, options.csvCal, "Calculation", "http://www.xbrl.org/2003/arcrole/summation-item") if options.csvDim: ViewCsvRelationshipSet.viewRelationshipSet( modelXbrl, options.csvDim, "Dimension", "XBRL-dimensions") except (IOError, EnvironmentError) as err: self.addToLog( _("[IOError] Failed to save output:\n {0}").format(err)) if self.messages: try: with open(options.logFile, "w", encoding="utf-8") as fh: fh.writelines(self.messages) except (IOError, EnvironmentError) as err: print("Unable to save log to file: " + err)
def final(val): if not (val.validateEBA or val.validateEIOPA): return modelXbrl = val.modelXbrl modelDocument = modelXbrl.modelDocument _statusMsg = _("validating {0} filing rules").format(val.disclosureSystem.name) modelXbrl.profileActivity() modelXbrl.modelManager.showStatus(_statusMsg) if modelDocument.type == ModelDocument.Type.INSTANCE and (val.validateEBA or val.validateEIOPA): if not modelDocument.uri.endswith(".xbrl"): modelXbrl.warning("EBA.1.1", _('XBRL instance documents SHOULD use the extension ".xbrl" but it is "%(extension)s"'), modelObject=modelDocument, extension=os.path.splitext(modelDocument.basename)[1]) modelXbrl.error("EIOPA.S.1.1.a", _('XBRL instance documents MUST use the extension ".xbrl" but it is "%(extension)s"'), modelObject=modelDocument, extension=os.path.splitext(modelDocument.basename)[1]) if val.isEIOPA_2_0_1: _encodings = ("UTF-8", "utf-8-sig") else: _encodings = ("utf-8", "UTF-8", "utf-8-sig") if modelDocument.documentEncoding not in _encodings: modelXbrl.error(("EBA.1.4", "EIOPA.1.4"), _('XBRL instance documents MUST use "UTF-8" encoding but is "%(xmlEncoding)s"'), modelObject=modelDocument, xmlEncoding=modelDocument.documentEncoding) schemaRefElts = [] schemaRefFileNames = [] for doc, docRef in modelDocument.referencesDocument.items(): if docRef.referenceType == "href": if docRef.referringModelObject.localName == "schemaRef": schemaRefElts.append(docRef.referringModelObject) schemaRefFileNames.append(doc.basename) if not UrlUtil.isAbsolute(doc.uri): modelXbrl.error(("EBA.2.2", "EIOPA.S.1.5.a" if val.isEIOPAfullVersion else "EIOPA.S.1.5.b"), _('The link:schemaRef element in submitted instances MUST resolve to the full published entry point URL: %(url)s.'), modelObject=docRef.referringModelObject, url=doc.uri, messageCodes=("EBA.2.2", "EIOPA.S.1.5.a","EIOPA.S.1.5.b")) elif docRef.referringModelObject.localName == "linkbaseRef": modelXbrl.error(("EBA.2.3","EIOPA.S.1.5.a"), _('The link:linkbaseRef element is not allowed: %(fileName)s.'), modelObject=docRef.referringModelObject, fileName=doc.basename) _numSchemaRefs = len(XmlUtil.children(modelDocument.xmlRootElement, XbrlConst.link, "schemaRef")) if _numSchemaRefs > 1: modelXbrl.error(("EIOPA.S.1.5.a", "EBA.1.5"), _('XBRL instance documents MUST reference only one entry point schema but %(numEntryPoints)s were found: %(entryPointNames)s'), modelObject=modelDocument, numEntryPoints=_numSchemaRefs, entryPointNames=', '.join(sorted(schemaRefFileNames))) ### check entry point names appropriate for filing indicator (DPM DB?) if len(schemaRefElts) != 1: modelXbrl.error("EBA.2.3", _('Any reported XBRL instance document MUST contain only one xbrli:xbrl/link:schemaRef node, but %(entryPointCount)s.'), modelObject=schemaRefElts, entryPointCount=len(schemaRefElts)) # non-streaming EBA checks if not getattr(modelXbrl, "isStreamingMode", False): val.qnReportedCurrency = None if val.isEIOPA_2_0_1 and qnMetReportingCurrency in modelXbrl.factsByQname: for _multiCurrencyFact in modelXbrl.factsByQname[qnMetReportingCurrency]: # multi-currency fact val.qnReportedCurrency = _multiCurrencyFact.xValue break validateFacts(val, modelXbrl.facts) # check sum of fact md5s (otherwise checked in streaming process) xbrlFactsCheckVersion = None expectedSumOfFactMd5s = None for pi in modelDocument.xmlRootElement.getchildren(): if isinstance(pi, etree._ProcessingInstruction) and pi.target == "xbrl-facts-check": _match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']", pi.text) if _match: _matchGroups = _match.groups() if len(_matchGroups) == 2: if _matchGroups[0] == "version": xbrlFactsCheckVersion = _matchGroups[1] elif _matchGroups[0] == "sum-of-fact-md5s": try: expectedSumOfFactMd5s = Md5Sum(_matchGroups[1]) except ValueError: modelXbrl.error("EIOPA:xbrlFactsCheckError", _("Invalid sum-of-md5s %(sumOfMd5)s"), modelObject=modelXbrl, sumOfMd5=_matchGroups[1]) if xbrlFactsCheckVersion and expectedSumOfFactMd5s: sumOfFactMd5s = Md5Sum() for f in modelXbrl.factsInInstance: sumOfFactMd5s += f.md5sum if sumOfFactMd5s != expectedSumOfFactMd5s: modelXbrl.warning("EIOPA:xbrlFactsCheckWarning", _("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"), modelObject=modelXbrl, expectedMd5=expectedSumOfFactMd5s, actualMd5Sum=sumOfFactMd5s) else: modelXbrl.info("info", _("Successful XBRL facts sum of md5s."), modelObject=modelXbrl) if any(badError in modelXbrl.errors for badError in ("EBA.2.1", "EIOPA.2.1", "EIOPA.S.1.5.a/EIOPA.S.1.5.b")): pass # skip checking filingIndicators if bad errors elif not val.filingIndicators: modelXbrl.error(("EBA.1.6", "EIOPA.1.6.a"), _('Missing filing indicators. Reported XBRL instances MUST include appropriate (positive) filing indicator elements'), modelObject=modelDocument) elif all(filed == False for filed in val.filingIndicators.values()): modelXbrl.error(("EBA.1.6", "EIOPA.1.6.a"), _('All filing indicators are filed="false". Reported XBRL instances MUST include appropriate (positive) filing indicator elements'), modelObject=modelDocument) if val.numFilingIndicatorTuples > 1: modelXbrl.warning(("EBA.1.6.2", "EIOPA.1.6.2"), _('Multiple filing indicators tuples when not in streaming mode (info).'), modelObject=modelXbrl.factsByQname[qnFIndicators]) if len(val.cntxDates) > 1: modelXbrl.error(("EBA.2.13","EIOPA.2.13"), _('Contexts must have the same date: %(dates)s.'), # when streaming values are no longer available, but without streaming they can be logged modelObject=set(_cntx for _cntxs in val.cntxDates.values() for _cntx in _cntxs), dates=', '.join(XmlUtil.dateunionValue(_dt, subtractOneDay=True) for _dt in val.cntxDates.keys())) if val.unusedCntxIDs: if val.isEIOPA_2_0_1: modelXbrl.error("EIOPA.2.7", _('Unused xbrli:context nodes MUST NOT be present in the instance: %(unusedContextIDs)s.'), modelObject=[modelXbrl.contexts[unusedCntxID] for unusedCntxID in val.unusedCntxIDs if unusedCntxID in modelXbrl.contexts], unusedContextIDs=", ".join(sorted(val.unusedCntxIDs))) else: modelXbrl.warning(("EBA.2.7", "EIOPA.2.7"), _('Unused xbrli:context nodes SHOULD NOT be present in the instance: %(unusedContextIDs)s.'), modelObject=[modelXbrl.contexts[unusedCntxID] for unusedCntxID in val.unusedCntxIDs if unusedCntxID in modelXbrl.contexts], unusedContextIDs=", ".join(sorted(val.unusedCntxIDs))) if len(val.cntxEntities) > 1: modelXbrl.error(("EBA.2.9", "EIOPA.2.9"), _('All entity identifiers and schemes MUST be the same, %(count)s found: %(entities)s.'), modelObject=modelDocument, count=len(val.cntxEntities), entities=", ".join(sorted(str(cntxEntity) for cntxEntity in val.cntxEntities))) for _scheme, _LEI in val.cntxEntities: if (_scheme in ("http://standards.iso.org/iso/17442", "http://standard.iso.org/iso/17442", "LEI") or (not val.isEIOPAfullVersion and _scheme == "PRE-LEI")): result = LeiUtil.checkLei(_LEI) if result == LeiUtil.LEI_INVALID_LEXICAL: modelXbrl.error("EIOPA.S.2.8.c", _("Context has lexically invalid LEI %(lei)s."), modelObject=modelDocument, lei=_LEI) elif result == LeiUtil.LEI_INVALID_CHECKSUM: modelXbrl.error("EIOPA.S.2.8.c", _("Context has LEI checksum error in %(lei)s."), modelObject=modelDocument, lei=_LEI) if _scheme == "http://standard.iso.org/iso/17442": modelXbrl.warning("EIOPA.S.2.8.c", _("Warning, context has entity scheme %(scheme)s should be plural: http://standards.iso.org/iso/17442."), modelObject=modelDocument, scheme=_scheme) elif _scheme == "SC": pass # anything is ok for Specific Code else: modelXbrl.error("EIOPA.S.2.8.c", _("Context has unrecognized entity scheme %(scheme)s."), modelObject=modelDocument, scheme=_scheme) if val.unusedUnitIDs: if val.isEIOPA_2_0_1: modelXbrl.error("EIOPA.2.22", _('Unused xbrli:unit nodes MUST NOT be present in the instance: %(unusedUnitIDs)s.'), modelObject=[modelXbrl.units[unusedUnitID] for unusedUnitID in val.unusedUnitIDs if unusedUnitID in modelXbrl.units], unusedUnitIDs=", ".join(sorted(val.unusedUnitIDs))) else: modelXbrl.warning(("EBA.2.22", "EIOPA.2.22"), _('Unused xbrli:unit nodes SHOULD NOT be present in the instance: %(unusedUnitIDs)s.'), modelObject=[modelXbrl.units[unusedUnitID] for unusedUnitID in val.unusedUnitIDs if unusedUnitID in modelXbrl.units], unusedUnitIDs=", ".join(sorted(val.unusedUnitIDs))) if len(val.currenciesUsed) > 1: modelXbrl.error(("EBA.3.1","EIOPA.3.1"), _("There MUST be only one currency but %(numCurrencies)s were found: %(currencies)s.'"), modelObject=val.currenciesUsed.values(), numCurrencies=len(val.currenciesUsed), currencies=", ".join(str(c) for c in val.currenciesUsed.keys())) elif val.isEIOPA_2_0_1 and any(_measure.localName != val.reportingCurrency for _measure in val.currenciesUsed.keys()): modelXbrl.error("EIOPA.3.1", _("There MUST be only one currency but reporting currency %(reportingCurrency)s differs from unit currencies: %(currencies)s.'"), modelObject=val.currenciesUsed.values(), reportingCurrency=val.reportingCurrency, currencies=", ".join(str(c) for c in val.currenciesUsed.keys())) if val.prefixesUnused: modelXbrl.warning(("EBA.3.4", "EIOPA.3.4"), _("There SHOULD be no unused prefixes but these were declared: %(unusedPrefixes)s.'"), modelObject=modelDocument, unusedPrefixes=', '.join(sorted(val.prefixesUnused))) for ns, prefixes in val.namespacePrefixesUsed.items(): nsDocs = modelXbrl.namespaceDocs.get(ns) if nsDocs: for nsDoc in nsDocs: nsDocPrefix = XmlUtil.xmlnsprefix(nsDoc.xmlRootElement, ns) if any(prefix != nsDocPrefix for prefix in prefixes if prefix is not None): modelXbrl.warning(("EBA.3.5", "EIOPA.3.5"), _("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s"), modelObject=modelDocument, namespace=ns, declaredPrefix=nsDocPrefix, foundPrefixes=', '.join(sorted(prefixes - {None}))) elif ns in CANONICAL_PREFIXES and any(prefix != CANONICAL_PREFIXES[ns] for prefix in prefixes if prefix is not None): modelXbrl.warning(("EBA.3.5", "EIOPA.3.5"), _("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s"), modelObject=modelDocument, namespace=ns, declaredPrefix=CANONICAL_PREFIXES[ns], foundPrefixes=', '.join(sorted(prefixes - {None}))) modelXbrl.profileActivity(_statusMsg, minTimeToShow=0.0) modelXbrl.modelManager.showStatus(None) del val.prefixNamespace, val.namespacePrefix, val.idObjects, val.typedDomainElements del val.utrValidator, val.firstFact, val.footnotesRelationshipSet
def final(val): if not (val.validateEBA or val.validateEIOPA): return modelXbrl = val.modelXbrl modelDocument = modelXbrl.modelDocument _statusMsg = _("validating {0} filing rules").format( val.disclosureSystem.name) modelXbrl.profileActivity() modelXbrl.modelManager.showStatus(_statusMsg) if modelDocument.type == ModelDocument.Type.INSTANCE and ( val.validateEBA or val.validateEIOPA): if not modelDocument.uri.endswith(".xbrl"): modelXbrl.warning( "EBA.1.1", _('XBRL instance documents SHOULD use the extension ".xbrl" but it is "%(extension)s"' ), modelObject=modelDocument, extension=os.path.splitext(modelDocument.basename)[1]) if modelDocument.documentEncoding.lower() not in ("utf-8", "utf-8-sig"): modelXbrl.error( "EBA.1.4", _('XBRL instance documents MUST use "UTF-8" encoding but is "%(xmlEncoding)s"' ), modelObject=modelDocument, xmlEncoding=modelDocument.documentEncoding) schemaRefElts = [] schemaRefFileNames = [] for doc, docRef in modelDocument.referencesDocument.items(): if docRef.referenceType == "href": if docRef.referringModelObject.localName == "schemaRef": schemaRefElts.append(docRef.referringModelObject) schemaRefFileNames.append(doc.basename) if not UrlUtil.isAbsolute(doc.uri): modelXbrl.error( "EBA.2.2", _('The link:schemaRef element in submitted instances MUST resolve to the full published entry point URL: %(url)s.' ), modelObject=docRef.referringModelObject, url=doc.uri) elif docRef.referringModelObject.localName == "linkbaseRef": modelXbrl.error( "EBA.2.3", _('The link:linkbaseRef element is not allowed: %(fileName)s.' ), modelObject=docRef.referringModelObject, fileName=doc.basename) if len(schemaRefFileNames) > 1: modelXbrl.error( "EBA.1.5", _('XBRL instance documents MUST reference only one entry point schema but %(numEntryPoints)s were found: %(entryPointNames)s' ), modelObject=modelDocument, numEntryPoints=len(schemaRefFileNames), entryPointNames=', '.join(sorted(schemaRefFileNames))) ### check entry point names appropriate for filing indicator (DPM DB?) if len(schemaRefElts) != 1: modelXbrl.error( "EBA.2.3", _('Any reported XBRL instance document MUST contain only one xbrli:xbrl/link:schemaRef node, but %(entryPointCount)s.' ), modelObject=schemaRefElts, entryPointCount=len(schemaRefElts)) # non-streaming EBA checks if not getattr(modelXbrl, "isStreamingMode", False): validateFacts(val, modelXbrl.facts) # check sum of fact md5s (otherwise checked in streaming process) xbrlFactsCheckVersion = None expectedSumOfFactMd5s = None for pi in modelDocument.xmlRootElement.getchildren(): if isinstance(pi, etree._ProcessingInstruction ) and pi.target == "xbrl-facts-check": _match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']", pi.text) if _match: _matchGroups = _match.groups() if len(_matchGroups) == 2: if _matchGroups[0] == "version": xbrlFactsCheckVersion = _matchGroups[1] elif _matchGroups[0] == "sum-of-fact-md5s": try: expectedSumOfFactMd5s = Md5Sum( _matchGroups[1]) except ValueError: modelXbrl.error( "EIOPA:xbrlFactsCheckError", _("Invalid sum-of-md5s %(sumOfMd5)s"), modelObject=modelXbrl, sumOfMd5=_matchGroups[1]) if xbrlFactsCheckVersion and expectedSumOfFactMd5s: sumOfFactMd5s = Md5Sum() for f in modelXbrl.factsInInstance: sumOfFactMd5s += f.md5sum if sumOfFactMd5s != expectedSumOfFactMd5s: modelXbrl.warning( "EIOPA:xbrlFactsCheckWarning", _("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s" ), modelObject=modelXbrl, expectedMd5=expectedSumOfFactMd5s, actualMd5Sum=sumOfFactMd5s) else: modelXbrl.info("info", _("Successful XBRL facts sum of md5s."), modelObject=modelXbrl) if not val.filingIndicators: modelXbrl.error( "EBA.1.6", _('Missing filing indicators. Reported XBRL instances MUST include appropriate filing indicator elements' ), modelObject=modelDocument) if val.numFilingIndicatorTuples > 1: modelXbrl.info( "EBA.1.6.2", _('Multiple filing indicators tuples when not in streaming mode (info).' ), modelObject=modelXbrl.factsByQname[qnFIndicators]) if len(val.cntxDates) > 1: modelXbrl.error( "EBA.2.13", _('Contexts must have the same date: %(dates)s.'), # when streaming values are no longer available, but without streaming they can be logged modelObject=set(_cntx for _cntxs in val.cntxDates.values() for _cntx in _cntxs), dates=', '.join( XmlUtil.dateunionValue(_dt, subtractOneDay=True) for _dt in val.cntxDates.keys())) if val.unusedCntxIDs: modelXbrl.warning( "EBA.2.7", _('Unused xbrli:context nodes SHOULD NOT be present in the instance: %(unusedContextIDs)s.' ), modelObject=[ modelXbrl.contexts[unusedCntxID] for unusedCntxID in val.unusedCntxIDs if unusedCntxID in modelXbrl.contexts ], unusedContextIDs=", ".join(sorted(val.unusedCntxIDs))) if len(val.cntxEntities) > 1: modelXbrl.warning( "EBA.2.9", _('All entity identifiers and schemes must be the same, %(count)s found: %(entities)s.' ), modelObject=modelDocument, count=len(val.cntxEntities), entities=", ".join( sorted(str(cntxEntity) for cntxEntity in val.cntxEntities))) if val.unusedUnitIDs: modelXbrl.warning( "EBA.2.21", _('Unused xbrli:unit nodes SHOULD NOT be present in the instance: %(unusedUnitIDs)s.' ), modelObject=[ modelXbrl.units[unusedUnitID] for unusedUnitID in val.unusedUnitIDs if unusedUnitID in modelXbrl.units ], unusedUnitIDs=", ".join(sorted(val.unusedUnitIDs))) if len(val.currenciesUsed) > 1: modelXbrl.error( "EBA.3.1", _("There MUST be only one currency but %(numCurrencies)s were found: %(currencies)s.'" ), modelObject=val.currenciesUsed.values(), numCurrencies=len(val.currenciesUsed), currencies=", ".join( str(c) for c in val.currenciesUsed.keys())) if val.prefixesUnused: modelXbrl.warning( "EBA.3.4", _("There SHOULD be no unused prefixes but these were declared: %(unusedPrefixes)s.'" ), modelObject=modelDocument, unusedPrefixes=', '.join(sorted(val.prefixesUnused))) for ns, prefixes in val.namespacePrefixesUsed.items(): nsDocs = modelXbrl.namespaceDocs.get(ns) if nsDocs: for nsDoc in nsDocs: nsDocPrefix = XmlUtil.xmlnsprefix(nsDoc.xmlRootElement, ns) if any(prefix != nsDocPrefix for prefix in prefixes if prefix is not None): modelXbrl.warning( "EBA.3.5", _("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s" ), modelObject=modelDocument, namespace=ns, declaredPrefix=nsDocPrefix, foundPrefixes=', '.join(sorted(prefixes - {None}))) modelXbrl.profileActivity(_statusMsg, minTimeToShow=0.0) modelXbrl.modelManager.showStatus(None) del val.prefixNamespace, val.namespacePrefix, val.idObjects, val.typedDomainElements del val.utrValidator
def run(self, options): self.filename = options.filename filesource = FileSource.openFileSource(self.filename,self) if options.validateEFM: if options.gfmName: self.addToLog(_("both --efm and --gfm validation are requested, proceeding with --efm only"), messageCode="info", file=self.filename) self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select("efm") elif options.gfmName: self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select(options.gfmName) else: self.modelManager.disclosureSystem.select(None) # just load ordinary mappings if options.calcDecimals: if options.calcPrecision: self.addToLog(_("both --calcDecimals and --calcPrecision validation are requested, proceeding with --calcDecimals only"), messageCode="info", file=self.filename) self.modelManager.validateInferDecimals = True self.modelManager.validateCalcLB = True elif options.calcPrecision: self.modelManager.validateInferDecimals = False self.modelManager.validateCalcLB = True if options.utrValidate: self.modelManager.validateUtr = True fo = FormulaOptions() if options.formulaParamExprResult: fo.traceParameterExpressionResult = True if options.formulaParamInputValue: fo.traceParameterInputValue = True if options.formulaCallExprSource: fo.traceCallExpressionSource = True if options.formulaCallExprCode: fo.traceCallExpressionCode = True if options.formulaCallExprEval: fo.traceCallExpressionEvaluation = True if options.formulaCallExprResult: fo.traceCallExpressionResult = True if options.formulaVarSetExprEval: fo.traceVariableSetExpressionEvaluation = True if options.formulaVarSetExprResult: fo.traceVariableSetExpressionResult = True if options.formulaAsserResultCounts: fo.traceAssertionResultCounts = True if options.formulaFormulaRules: fo.traceFormulaRules = True if options.formulaVarsOrder: fo.traceVariablesOrder = True if options.formulaVarExpressionSource: fo.traceVariableExpressionSource = True if options.formulaVarExpressionCode: fo.traceVariableExpressionCode = True if options.formulaVarExpressionEvaluation: fo.traceVariableExpressionEvaluation = True if options.formulaVarExpressionResult: fo.traceVariableExpressionResult = True if options.formulaVarFiltersResult: fo.traceVariableFiltersResult = True self.modelManager.formulaOptions = fo timeNow = XmlUtil.dateunionValue(datetime.datetime.now()) startedAt = time.time() modelXbrl = self.modelManager.load(filesource, _("views loading")) self.addToLog(format_string(self.modelManager.locale, _("loaded in %.2f secs at %s"), (time.time() - startedAt, timeNow)), messageCode="info", file=self.filename) if options.diffFilename and options.versReportFilename: diffFilesource = FileSource.FileSource(self.diffFilename,self) startedAt = time.time() modelXbrl = self.modelManager.load(diffFilesource, _("views loading")) self.addToLog(format_string(self.modelManager.locale, _("diff comparison DTS loaded in %.2f secs"), time.time() - startedAt), messageCode="info", file=self.filename) startedAt = time.time() self.modelManager.compareDTSes(options.versReportFilename) self.addToLog(format_string(self.modelManager.locale, _("compared in %.2f secs"), time.time() - startedAt), messageCode="info", file=self.filename) try: if options.validate: startedAt = time.time() self.modelManager.validate() self.addToLog(format_string(self.modelManager.locale, _("validated in %.2f secs"), time.time() - startedAt), messageCode="info", file=self.filename) if (options.csvTestReport and self.modelManager.modelXbrl.modelDocument.type in (ModelDocument.Type.TESTCASESINDEX, ModelDocument.Type.TESTCASE, ModelDocument.Type.REGISTRY)): ViewCsvTests.viewTests(self.modelManager.modelXbrl, options.csvTestReport) if options.csvDTS: ViewCsvDTS.viewDTS(modelXbrl, options.csvDTS) if options.csvFactList: ViewCsvFactList.viewFacts(modelXbrl, options.csvFactList, cols=options.csvFactListCols) if options.csvConcepts: ViewCsvConcepts.viewConcepts(modelXbrl, options.csvConcepts) if options.csvPre: ViewCsvRelationshipSet.viewRelationshipSet(modelXbrl, options.csvPre, "Presentation", "http://www.xbrl.org/2003/arcrole/parent-child") if options.csvCal: ViewCsvRelationshipSet.viewRelationshipSet(modelXbrl, options.csvCal, "Calculation", "http://www.xbrl.org/2003/arcrole/summation-item") if options.csvDim: ViewCsvRelationshipSet.viewRelationshipSet(modelXbrl, options.csvDim, "Dimension", "XBRL-dimensions") except (IOError, EnvironmentError) as err: self.addToLog(_("[IOError] Failed to save output:\n {0}").format(err)) except Exception as err: self.addToLog(_("[Exception] Failed to complete validation: \n{0} \n{1}").format( err, traceback.format_tb(sys.exc_info()[2])))
def produceOutputFact(xpCtx, formula, result): priorErrorCount = len(xpCtx.modelXbrl.errors) # assemble context conceptQname = aspectValue(xpCtx, formula, Aspect.CONCEPT, "xbrlfe:missingConceptRule") if isinstance(conceptQname, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} concept: {1}").format( formula, conceptQname.msg), "err", conceptQname.err) modelConcept = None else: modelConcept = xpCtx.modelXbrl.qnameConcepts[conceptQname] if modelConcept is None or not modelConcept.isItem: xpCtx.modelXbrl.error( _("Formula {0} concept {1} is not an item").format( formula, conceptQname), "err", "xbrlfe:missingConceptRule") # entity entityIdentScheme = aspectValue(xpCtx, formula, Aspect.SCHEME, "xbrlfe:missingEntityIdentifierRule") if isinstance(entityIdentScheme, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} entity identifier scheme: {1}").format( formula, entityIdentScheme.msg ), "err", str(entityIdentScheme)) entityIdentValue = None else: entityIdentValue = aspectValue(xpCtx, formula, Aspect.VALUE, "xbrlfe:missingEntityIdentifierRule") if isinstance(entityIdentValue, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} entity identifier value: {1}").format( formula, entityIdentValue.msg ), "err", str(entityIdentScheme)) # period periodType = aspectValue(xpCtx, formula, Aspect.PERIOD_TYPE, "xbrlfe:missingPeriodRule") periodStart = None periodEndInstant = None if isinstance(periodType, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} period type: {1}").format( formula, periodType.msg ), "err", str(periodType)) elif periodType == "instant": periodEndInstant = aspectValue(xpCtx, formula, Aspect.INSTANT, "xbrlfe:missingPeriodRule") if isinstance(periodEndInstant, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} period start: {1}").format( formula, periodEndInstant.msg ), "err", str(periodEndInstant)) elif periodType == "duration": periodStart = aspectValue(xpCtx, formula, Aspect.START, "xbrlfe:missingPeriodRule") if isinstance(periodStart, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} period start: {1}").format( formula, periodStart.msg ), "err", str(periodStart)) periodEndInstant = aspectValue(xpCtx, formula, Aspect.END, "xbrlfe:missingPeriodRule") if isinstance(periodEndInstant, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} period end: {1}").format( formula, periodEndInstant.msg ), "err", str(periodEndInstant)) # unit if modelConcept and modelConcept.isNumeric: unitSource = aspectValue(xpCtx, formula, Aspect.UNIT_MEASURES, None) multDivBy = aspectValue(xpCtx, formula, Aspect.MULTIPLY_BY, "xbrlfe:missingUnitRule") if isinstance(multDivBy, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} unit: {1}").format( formula, multDivBy.msg ), "err", str(multDivBy) if isinstance(multDivBy, VariableBindingError) else "xbrlfe:missingUnitRule") multiplyBy = (); divideBy = () # prevent errors later if bad else: divMultBy = aspectValue(xpCtx, formula, Aspect.DIVIDE_BY, "xbrlfe:missingUnitRule") if isinstance(divMultBy, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} unit: {1}").format( formula, divMultBy.msg ), "err", str(multDivBy) if isinstance(divMultBy, VariableBindingError) else "xbrlfe:missingUnitRule") multiplyBy = (); divideBy = () # prevent errors later if bad else: multiplyBy = unitSource[0] + multDivBy[0] + divMultBy[1] divideBy = unitSource[1] + multDivBy[1] + divMultBy[0] # remove cancelling mult/div units lookForCommonUnits = True while lookForCommonUnits: lookForCommonUnits = False for commonUnit in multiplyBy: if commonUnit in divideBy: multiplyBy.remove(commonUnit) divideBy.remove(commonUnit) lookForCommonUnits = True break if len(multiplyBy) == 0: # if no units add pure multiplyBy.append(XbrlConst.qnXbrliPure) # dimensions segOCCs = [] scenOCCs = [] if formula.aspectModel == "dimensional": dimAspects = {} dimQnames = aspectValue(xpCtx, formula, Aspect.DIMENSIONS, None) if dimQnames: for dimQname in dimQnames: dimConcept = xpCtx.modelXbrl.qnameConcepts[dimQname] dimErr = "xbrlfe:missing{0}DimensionRule".format("typed" if dimConcept and dimConcept.isTypedDimension else "explicit") dimValue = aspectValue(xpCtx, formula, dimQname, dimErr) if isinstance(dimValue, VariableBindingError): xpCtx.modelXbrl.error( _("Formula {0} dimension {1}: {2}").format( formula, dimQname, dimValue.msg ), "err", dimErr) elif dimValue and xpCtx.modelXbrl.qnameDimensionDefaults.get(dimQname) != dimValue: dimAspects[dimQname] = dimValue segOCCs = aspectValue(xpCtx, formula, Aspect.NON_XDT_SEGMENT, None) scenOCCs = aspectValue(xpCtx, formula, Aspect.NON_XDT_SCENARIO, None) else: dimAspects = None # non-dimensional segOCCs = aspectValue(xpCtx, formula, Aspect.COMPLETE_SEGMENT, None) scenOCCs = aspectValue(xpCtx, formula, Aspect.COMPLETE_SCENARIO, None) if priorErrorCount < len(xpCtx.modelXbrl.errors): return None # had errors, don't produce output fact # does context exist in out instance document outputInstanceQname = formula.outputInstanceQname outputXbrlInstance = xpCtx.inScopeVars[outputInstanceQname] xbrlElt = outputXbrlInstance.modelDocument.xmlRootElement # in source instance document # add context prevCntx = outputXbrlInstance.matchContext( entityIdentScheme, entityIdentValue, periodType, periodStart, periodEndInstant, dimAspects, segOCCs, scenOCCs) if prevCntx: cntxId = prevCntx.id newCntxElt = prevCntx.element else: cntxId = 'c-{0:02n}'.format( len(outputXbrlInstance.contexts) + 1) newCntxElt = XmlUtil.addChild(xbrlElt, XbrlConst.xbrli, "context", attributes=("id", cntxId), afterSibling=xpCtx.outputLastContext.get(outputInstanceQname)) xpCtx.outputLastContext[outputInstanceQname] = newCntxElt entityElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "entity") XmlUtil.addChild(entityElt, XbrlConst.xbrli, "identifier", attributes=("scheme", entityIdentScheme), text=entityIdentValue) periodElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "period") if periodType == "forever": XmlUtil.addChild(periodElt, XbrlConst.xbrli, "forever") elif periodType == "instant": XmlUtil.addChild(periodElt, XbrlConst.xbrli, "instant", text=XmlUtil.dateunionValue(periodEndInstant, subtractOneDay=True)) elif periodType == "duration": XmlUtil.addChild(periodElt, XbrlConst.xbrli, "startDate", text=XmlUtil.dateunionValue(periodStart)) XmlUtil.addChild(periodElt, XbrlConst.xbrli, "endDate", text=XmlUtil.dateunionValue(periodEndInstant, subtractOneDay=True)) segmentElt = None scenarioElt = None from arelle.ModelObject import ModelDimensionValue if dimAspects: for dimQname in sorted(dimAspects.keys()): dimValue = dimAspects[dimQname] if isinstance(dimValue, ModelDimensionValue): if dimValue.isExplicit: dimMemberQname = dimValue.memberQname contextEltName = dimValue.contextElement else: # qname for explicit or node for typed dimMemberQname = dimValue contextEltName = xpCtx.modelXbrl.qnameDimensionContextElement.get(dimQname) if contextEltName == "segment": if not segmentElt: segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli, "segment") contextElt = segmentElt elif contextEltName == "scenario": if not scenarioElt: scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "scenario") contextElt = scenarioElt else: continue dimConcept = xpCtx.modelXbrl.qnameConcepts[dimQname] dimAttr = ("dimension", XmlUtil.addQnameValue(xbrlElt, dimConcept.qname)) if dimConcept.isTypedDimension: dimElt = XmlUtil.addChild(contextElt, XbrlConst.xbrldi, "typedMember", attributes=dimAttr) if isinstance(dimValue, ModelDimensionValue) and dimValue.isTyped: XmlUtil.copyChildren(dimElt, dimValue.typedMember) elif dimMemberQname: dimElt = XmlUtil.addChild(contextElt, XbrlConst.xbrldi, "explicitMember", attributes=dimAttr, text=XmlUtil.addQnameValue(xbrlElt, dimMemberQname)) if segOCCs: if not segmentElt: segmentElt = XmlUtil.addChild(entityElt, XbrlConst.xbrli, "segment") XmlUtil.copyNodes(segmentElt, segOCCs) if scenOCCs: if not scenarioElt: scenarioElt = XmlUtil.addChild(newCntxElt, XbrlConst.xbrli, "scenario") XmlUtil.copyNodes(scenarioElt, scenOCCs) outputXbrlInstance.modelDocument.contextDiscover(newCntxElt) # does unit exist # add unit if modelConcept.isNumeric: prevUnit = outputXbrlInstance.matchUnit(multiplyBy, divideBy) if prevUnit: unitId = prevUnit.id newUnitElt = prevUnit.element else: unitId = 'u-{0:02n}'.format( len(outputXbrlInstance.units) + 1) newUnitElt = XmlUtil.addChild(xbrlElt, XbrlConst.xbrli, "unit", attributes=("id", unitId), afterSibling=xpCtx.outputLastUnit.get(outputInstanceQname)) xpCtx.outputLastUnit[outputInstanceQname] = newUnitElt if len(divideBy) == 0: for multiply in multiplyBy: XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, multiply)) else: divElt = XmlUtil.addChild(newUnitElt, XbrlConst.xbrli, "divide") numElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitNumerator") denElt = XmlUtil.addChild(divElt, XbrlConst.xbrli, "unitDenominator") for multiply in multiplyBy: XmlUtil.addChild(numElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, multiply)) for divide in divideBy: XmlUtil.addChild(denElt, XbrlConst.xbrli, "measure", text=XmlUtil.addQnameValue(xbrlElt, divide)) outputXbrlInstance.modelDocument.unitDiscover(newUnitElt) # add fact attrs = [("contextRef", cntxId)] precision = None decimals = None if modelConcept.isNumeric: attrs.append(("unitRef", unitId)) value = formula.evaluate(xpCtx) valueSeqLen = len(value) if valueSeqLen > 1: xpCtx.modelXbrl.error( _("Formula {0} value is a sequence of length {1}").format( formula, valueSeqLen ), "err", "xbrlfe:nonSingletonOutputValue") else: if valueSeqLen == 0: #xsi:nil if no value attrs.append((XbrlConst.qnXsiNil, "true")) v = None else: # add precision/decimals for non-fraction numerics if modelConcept.isNumeric and not modelConcept.isFraction: if formula.hasDecimals: decimals = formula.evaluateRule(xpCtx, Aspect.DECIMALS) attrs.append(("decimals", decimals)) else: if formula.hasPrecision: precision = formula.evaluateRule(xpCtx, Aspect.PRECISION) else: precision = 0 attrs.append(("precision", precision)) x = value[0] if isinstance(x,float): from math import (log10, isnan, isinf, fabs) if (isnan(x) or (precision and (isinf(precision) or precision == 0)) or (decimals and isinf(decimals))): v = string(xpCtx, x) elif decimals is not None: v = "%.*f" % ( int(decimals), x) elif precision is not None: a = fabs(x) log = log10(a) if a != 0 else 0 v = "%.*f" % ( int(precision) - int(log) - (1 if a >= 1 else 0), x) else: # no implicit precision yet v = string(xpCtx, x) elif isinstance(x,QName): v = XmlUtil.addQnameValue(xbrlElt, x) elif isinstance(x,datetime.datetime): v = XmlUtil.dateunionValue(x) else: v = string(xpCtx, x) itemElt = XmlUtil.addChild(xbrlElt, conceptQname, attributes=attrs, text=v, afterSibling=xpCtx.outputLastFact.get(outputInstanceQname)) xpCtx.outputLastFact[outputInstanceQname] = itemElt newFact = outputXbrlInstance.modelDocument.factDiscover(itemElt, outputXbrlInstance.facts) return newFact