def resolveHtmlUri(elt, name, value): if name == "archive": # URILIST return " ".join( resolveHtmlUri(elt, "archiveListElement", v) for v in value.split(" ")) if not UrlUtil.isAbsolute(value): if elt.localName == "object" and name in ( "classid", "data", "archiveListElement") and elt.get("codebase"): base = elt.get("codebase") + "/" else: base = getattr( elt.modelDocument, "htmlBase" ) # None if no htmlBase, empty string if it's not set if base: if value.startswith("/"): # add to authority value = UrlUtil.authority(base) + value elif value.startswith("#"): # add anchor to base document value = base + value else: value = os.path.dirname(base) + "/" + value # canonicalize ../ and ./ scheme, sep, pathpart = value.rpartition("://") if sep: pathpart = pathpart.replace('\\', '/') endingSep = '/' if pathpart[ -1] == '/' else '' # normpath drops ending directory separator _uri = scheme + "://" + posixpath.normpath(pathpart) + endingSep else: _uri = posixpath.normpath(value) return _uri # .replace(" ", "%20") requirement for this is not yet clear
def resolveHtmlUri(elt, name, value): if name == "archive": # URILIST return " ".join(resolveHtmlUri(elt, None, v) for v in value.split(" ")) if not UrlUtil.isAbsolute(value) and not value.startswith("/"): if elt.modelDocument.htmlBase is not None: value = elt.modelDocument.htmlBase + value # canonicalize ../ and ./ scheme, sep, pathpart = value.rpartition("://") if sep: pathpart = pathpart.replace('\\','/') endingSep = '/' if pathpart[-1] == '/' else '' # normpath drops ending directory separator _uri = scheme + "://" + posixpath.normpath(pathpart) + endingSep else: _uri = posixpath.normpath(value) return _uri # .replace(" ", "%20") requirement for this is not yet clear
def validateTestcase(self, testcase): self.modelXbrl.info("info", "Testcase", modelDocument=testcase) self.modelXbrl.viewModelObject(testcase.objectId()) if testcase.type in (Type.TESTCASESINDEX, Type.REGISTRY): for doc in sorted(testcase.referencesDocument.keys(), key=lambda doc: doc.uri): self.validateTestcase(doc) # testcases doc's are sorted by their uri (file names), e.g., for formula elif hasattr(testcase, "testcaseVariations"): for modelTestcaseVariation in testcaseVariationsByTarget(testcase.testcaseVariations): # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) # is this a versioning report? resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance resultIsTaxonomyPackage = modelTestcaseVariation.resultIsTaxonomyPackage formulaOutputInstance = None inputDTSes = defaultdict(list) baseForElement = testcase.baseForElement(modelTestcaseVariation) # try to load instance document self.modelXbrl.info("info", _("Variation %(id)s%(name)s%(target)s: %(expected)s - %(description)s"), modelObject=modelTestcaseVariation, id=modelTestcaseVariation.id, name=(" {}".format(modelTestcaseVariation.name) if modelTestcaseVariation.name else ""), target=(" target {}".format(modelTestcaseVariation.ixdsTarget) if modelTestcaseVariation.ixdsTarget else ""), expected=modelTestcaseVariation.expected, description=modelTestcaseVariation.description) if self.modelXbrl.modelManager.formulaOptions.testcaseResultsCaptureWarnings: errorCaptureLevel = logging._checkLevel("WARNING") else: errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY parameters = modelTestcaseVariation.parameters.copy() for readMeFirstUri in modelTestcaseVariation.readMeFirstUris: if isinstance(readMeFirstUri,tuple): # dtsName is for formula instances, but is from/to dts if versioning dtsName, readMeFirstUri = readMeFirstUri elif resultIsVersioningReport: if inputDTSes: dtsName = "to" else: dtsName = "from" else: dtsName = None if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document if dtsName in inputDTSes: dtsName = inputDTSes[dtsName] else: modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager, Type.DTSENTRIES, self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement), isEntry=True, errorCaptureLevel=errorCaptureLevel) DTSdoc = modelXbrl.modelDocument DTSdoc.inDTS = True doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement) if doc is not None: DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement) #fake import doc.inDTS = True elif resultIsTaxonomyPackage: from arelle import PackageManager, PrototypeInstanceObject dtsName = readMeFirstUri modelXbrl = PrototypeInstanceObject.XbrlPrototype(self.modelXbrl.modelManager, readMeFirstUri) PackageManager.packageInfo(self.modelXbrl.modelManager.cntlr, readMeFirstUri, reload=True, errors=modelXbrl.errors) else: # not a multi-schemaRef versioning report if self.useFileSource.isArchive: modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, readMeFirstUri, _("validating"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel, ixdsTarget=modelTestcaseVariation.ixdsTarget) else: # need own file source, may need instance discovery filesource = FileSource.openFileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr, base=baseForElement) if filesource and not filesource.selection and filesource.isArchive: try: if filesource.isTaxonomyPackage: _rptPkgIxdsOptions = {} for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ReportPackageIxdsOptions"): pluginXbrlMethod(self, _rptPkgIxdsOptions) filesource.loadTaxonomyPackageMappings() for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ReportPackageIxds"): filesource.select(pluginXbrlMethod(filesource, **_rptPkgIxdsOptions)) else: from arelle.CntlrCmdLine import filesourceEntrypointFiles entrypoints = filesourceEntrypointFiles(filesource) if entrypoints: # resolve an IXDS in entrypoints for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ArchiveIxds"): pluginXbrlMethod(self, filesource,entrypoints) filesource.select(entrypoints[0].get("file", None) ) except Exception as err: self.modelXbrl.error("exception:" + type(err).__name__, _("Testcase variation validation exception: %(error)s, entry URL: %(instance)s"), modelXbrl=self.modelXbrl, instance=readMeFirstUri, error=err) continue # don't try to load this entry URL modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, filesource, _("validating"), base=baseForElement, errorCaptureLevel=errorCaptureLevel, ixdsTarget=modelTestcaseVariation.ixdsTarget) modelXbrl.isTestcaseVariation = True if modelXbrl.modelDocument is None: modelXbrl.info("arelle:notLoaded", _("Variation %(id)s %(name)s readMeFirst document not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) self.determineNotLoadedTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif resultIsVersioningReport or resultIsTaxonomyPackage: inputDTSes[dtsName] = modelXbrl elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl) self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif testcase.type == Type.REGISTRYTESTCASE: self.instValidator.validate(modelXbrl) # required to set up dimensions, etc self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id, modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest) self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) self.instValidator.close() modelXbrl.close() else: inputDTSes[dtsName].append(modelXbrl) # validate except for formulas _hasFormulae = modelXbrl.hasFormulae modelXbrl.hasFormulae = False try: for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"): pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation) self.instValidator.validate(modelXbrl, parameters) for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"): pluginXbrlMethod(self.modelXbrl, modelXbrl) except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase variation validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=(type(err) is not AssertionError)) modelXbrl.hasFormulae = _hasFormulae if resultIsVersioningReport and modelXbrl.modelDocument: versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl( modelTestcaseVariation.versioningReportUri, baseForElement) if os.path.exists(versReportFile): #validate existing modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report")) if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport) self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors) modelVersReport.close() elif len(inputDTSes) == 2: ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes( versReportFile, inputDTSes["from"], inputDTSes["to"]) modelTestcaseVariation.status = "generated" else: modelXbrl.error("arelle:notLoaded", _("Variation %(id)s %(name)s input DTSes not loaded, unable to generate versioning report: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) modelTestcaseVariation.status = "failed" for inputDTS in inputDTSes.values(): inputDTS.close() del inputDTSes # dereference elif resultIsTaxonomyPackage: self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif inputDTSes: # validate schema, linkbase, or instance modelXbrl = inputDTSes[None][0] expectedDataFiles = set(modelXbrl.modelManager.cntlr.webCache.normalizeUrl(uri, baseForElement) for d in modelTestcaseVariation.dataUris.values() for uri in d if not UrlUtil.isAbsolute(uri)) foundDataFiles = set() variationBase = os.path.dirname(baseForElement) for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters if dtsName: # named instance parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes) elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula for _inputDTS in inputDTS: for docUrl, doc in _inputDTS.urlDocs.items(): if docUrl.startswith(variationBase) and not doc.type == Type.INLINEXBRLDOCUMENTSET: if getattr(doc,"loadedFromXbrlFormula", False): # may have been sourced from xf file if docUrl.replace("-formula.xml", ".xf") in expectedDataFiles: docUrl = docUrl.replace("-formula.xml", ".xf") foundDataFiles.add(docUrl) if expectedDataFiles - foundDataFiles: modelXbrl.info("arelle:testcaseDataNotUsed", _("Variation %(id)s %(name)s data files not used: %(missingDataFiles)s"), modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id, missingDataFiles=", ".join(sorted(os.path.basename(f) for f in expectedDataFiles - foundDataFiles))) if foundDataFiles - expectedDataFiles: modelXbrl.info("arelle:testcaseDataUnexpected", _("Variation %(id)s %(name)s files not in variation data: %(unexpectedDataFiles)s"), modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id, unexpectedDataFiles=", ".join(sorted(os.path.basename(f) for f in foundDataFiles - expectedDataFiles))) if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable: try: RenderingEvaluator.init(modelXbrl) except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase RenderingEvaluator.init exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) modelXbrlHasFormulae = modelXbrl.hasFormulae if modelXbrlHasFormulae and self.modelXbrl.modelManager.formulaOptions.formulaAction != "none": try: # validate only formulae self.instValidator.parameters = parameters ValidateFormula.validate(self.instValidator) except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=(type(err) is not AssertionError)) if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset: for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"): pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri) infoset = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultInfosetUri, _("loading result infoset"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) if infoset.modelDocument is None: modelXbrl.error("arelle:notLoaded", _("Variation %(id)s %(name)s result infoset not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstance)) modelTestcaseVariation.status = "result infoset not loadable" else: # check infoset ValidateInfoset.validate(self.instValidator, modelXbrl, infoset) infoset.close() if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset: # diff (or generate) table infoset resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement) if not any(alternativeValidation(modelXbrl, resultTableUri) for alternativeValidation in pluginClassMethods("Validate.TableInfoset")): try: ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase table linkbase validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) self.instValidator.close() extraErrors = [] for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"): pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors) self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual): # if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost) self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters) self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors) if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance formulaOutputInstance = modelXbrl.formulaOutputInstance modelXbrl.formulaOutputInstance = None # prevent it from being closed now self.instValidator.close() compareIxResultInstance = (modelXbrl.modelDocument.type in (Type.INLINEXBRL, Type.INLINEXBRLDOCUMENTSET) and modelTestcaseVariation.resultXbrlInstanceUri is not None) if compareIxResultInstance: formulaOutputInstance = modelXbrl # compare modelXbrl to generated output instance errMsgPrefix = "ix" else: # delete input instances before formula output comparision for inputDTSlist in inputDTSes.values(): for inputDTS in inputDTSlist: inputDTS.close() del inputDTSes # dereference errMsgPrefix = "formula" if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument: _matchExpectedResultIDs = not modelXbrlHasFormulae # formula restuls have inconsistent IDs expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultXbrlInstanceUri, _("loading expected result XBRL instance"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) if expectedInstance.modelDocument is None: self.modelXbrl.error("{}:expectedResultNotLoaded".format(errMsgPrefix), _("Testcase \"%(name)s\" %(id)s expected result instance not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstanceUri), messageCodes=("formula:expectedResultNotLoaded","ix:expectedResultNotLoaded")) modelTestcaseVariation.status = "result not loadable" else: # compare facts for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.ExpectedInstance.Loaded"): pluginXbrlMethod(expectedInstance, formulaOutputInstance) if len(expectedInstance.facts) != len(formulaOutputInstance.facts): formulaOutputInstance.error("{}:resultFactCounts".format(errMsgPrefix), _("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"), modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts), expectedFacts=len(expectedInstance.facts), messageCodes=("formula:resultFactCounts","ix:resultFactCounts")) else: formulaOutputFootnotesRelSet = ModelRelationshipSet(formulaOutputInstance, "XBRL-footnotes") expectedFootnotesRelSet = ModelRelationshipSet(expectedInstance, "XBRL-footnotes") def factFootnotes(fact, footnotesRelSet): footnotes = {} footnoteRels = footnotesRelSet.fromModelObject(fact) if footnoteRels: # most process rels in same order between two instances, use labels to sort for i, footnoteRel in enumerate(sorted(footnoteRels, key=lambda r: (r.fromLabel,r.toLabel))): modelObject = footnoteRel.toModelObject if isinstance(modelObject, ModelResource): xml = collapseWhitespace(modelObject.viewText().strip()) footnotes["Footnote {}".format(i+1)] = xml #re.sub(r'\s+', ' ', collapseWhitespace(modelObject.stringValue)) elif isinstance(modelObject, ModelFact): footnotes["Footnoted fact {}".format(i+1)] = \ "{} context: {} value: {}".format( modelObject.qname, modelObject.contextID, collapseWhitespace(modelObject.value)) return footnotes for expectedInstanceFact in expectedInstance.facts: unmatchedFactsStack = [] formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=_matchExpectedResultIDs, matchLang=False) #formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=True, matchLang=True) if formulaOutputFact is None: if unmatchedFactsStack: # get missing nested tuple fact, if possible missingFact = unmatchedFactsStack[-1] else: missingFact = expectedInstanceFact # is it possible to show value mismatches? expectedFacts = formulaOutputInstance.factsByQname.get(missingFact.qname) if len(expectedFacts) == 1: formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix), _("Output missing expected fact %(fact)s, extracted value \"%(value1)s\", expected value \"%(value2)s\""), modelXbrl=missingFact, fact=missingFact.qname, value1=missingFact.xValue, value2=next(iter(expectedFacts)).xValue, messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing")) else: formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix), _("Output missing expected fact %(fact)s"), modelXbrl=missingFact, fact=missingFact.qname, messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing")) else: # compare footnotes expectedInstanceFactFootnotes = factFootnotes(expectedInstanceFact, expectedFootnotesRelSet) formulaOutputFactFootnotes = factFootnotes(formulaOutputFact, formulaOutputFootnotesRelSet) if (len(expectedInstanceFactFootnotes) != len(formulaOutputFactFootnotes) or set(expectedInstanceFactFootnotes.values()) != set(formulaOutputFactFootnotes.values())): formulaOutputInstance.error("{}:expectedFactFootnoteDifference".format(errMsgPrefix), _("Output expected fact %(fact)s expected footnotes %(footnotes1)s produced footnotes %(footnotes2)s"), modelXbrl=(formulaOutputFact,expectedInstanceFact), fact=expectedInstanceFact.qname, footnotes1=sorted(expectedInstanceFactFootnotes.items()), footnotes2=sorted(formulaOutputFactFootnotes.items()), messageCodes=("formula:expectedFactFootnoteDifference","ix:expectedFactFootnoteDifference")) # for debugging uncomment next line to save generated instance document # formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml") expectedInstance.close() del expectedInstance # dereference self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors) formulaOutputInstance.close() del formulaOutputInstance if compareIxResultInstance: for inputDTSlist in inputDTSes.values(): for inputDTS in inputDTSlist: inputDTS.close() del inputDTSes # dereference # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) _statusCounts = OrderedDict((("pass",0),("fail",0))) for tv in getattr(testcase, "testcaseVariations", ()): _statusCounts[tv.status] = _statusCounts.get(tv.status, 0) + 1 self.modelXbrl.info("arelle:testCaseResults", ", ".join("{}={}".format(k,c) for k, c in _statusCounts.items() if k)) self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def final(val): if not (val.validateEBA or val.validateEIOPA): return modelXbrl = val.modelXbrl modelDocument = modelXbrl.modelDocument _statusMsg = _("validating {0} filing rules").format(val.disclosureSystem.name) modelXbrl.profileActivity() modelXbrl.modelManager.showStatus(_statusMsg) if modelDocument.type == ModelDocument.Type.INSTANCE and (val.validateEBA or val.validateEIOPA): if not modelDocument.uri.endswith(".xbrl"): modelXbrl.warning("EBA.1.1", _('XBRL instance documents SHOULD use the extension ".xbrl" but it is "%(extension)s"'), modelObject=modelDocument, extension=os.path.splitext(modelDocument.basename)[1]) if modelDocument.documentEncoding.lower() not in ("utf-8", "utf-8-sig"): modelXbrl.error("EBA.1.4", _('XBRL instance documents MUST use "UTF-8" encoding but is "%(xmlEncoding)s"'), modelObject=modelDocument, xmlEncoding=modelDocument.documentEncoding) schemaRefElts = [] schemaRefFileNames = [] for doc, docRef in modelDocument.referencesDocument.items(): if docRef.referenceType == "href": if docRef.referringModelObject.localName == "schemaRef": schemaRefElts.append(docRef.referringModelObject) schemaRefFileNames.append(doc.basename) if not UrlUtil.isAbsolute(doc.uri): modelXbrl.error("EBA.2.2", _('The link:schemaRef element in submitted instances MUST resolve to the full published entry point URL: %(url)s.'), modelObject=docRef.referringModelObject, url=doc.uri) elif docRef.referringModelObject.localName == "linkbaseRef": modelXbrl.error("EBA.2.3", _('The link:linkbaseRef element is not allowed: %(fileName)s.'), modelObject=docRef.referringModelObject, fileName=doc.basename) if len(schemaRefFileNames) > 1: modelXbrl.error("EBA.1.5", _('XBRL instance documents MUST reference only one entry point schema but %(numEntryPoints)s were found: %(entryPointNames)s'), modelObject=modelDocument, numEntryPoints=len(schemaRefFileNames), entryPointNames=', '.join(sorted(schemaRefFileNames))) ### check entry point names appropriate for filing indicator (DPM DB?) if len(schemaRefElts) != 1: modelXbrl.error("EBA.2.3", _('Any reported XBRL instance document MUST contain only one xbrli:xbrl/link:schemaRef node, but %(entryPointCount)s.'), modelObject=schemaRefElts, entryPointCount=len(schemaRefElts)) # non-streaming EBA checks if not getattr(modelXbrl, "isStreamingMode", False): validateFacts(val, modelXbrl.facts) # check sum of fact md5s (otherwise checked in streaming process) xbrlFactsCheckVersion = None expectedSumOfFactMd5s = None for pi in modelDocument.xmlRootElement.getchildren(): if isinstance(pi, etree._ProcessingInstruction) and pi.target == "xbrl-facts-check": _match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']", pi.text) if _match: _matchGroups = _match.groups() if len(_matchGroups) == 2: if _matchGroups[0] == "version": xbrlFactsCheckVersion = _matchGroups[1] elif _matchGroups[0] == "sum-of-fact-md5s": try: expectedSumOfFactMd5s = Md5Sum(_matchGroups[1]) except ValueError: modelXbrl.error("EIOPA:xbrlFactsCheckError", _("Invalid sum-of-md5s %(sumOfMd5)s"), modelObject=modelXbrl, sumOfMd5=_matchGroups[1]) if xbrlFactsCheckVersion and expectedSumOfFactMd5s: sumOfFactMd5s = Md5Sum() for f in modelXbrl.factsInInstance: sumOfFactMd5s += f.md5sum if sumOfFactMd5s != expectedSumOfFactMd5s: modelXbrl.warning("EIOPA:xbrlFactsCheckWarning", _("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"), modelObject=modelXbrl, expectedMd5=expectedSumOfFactMd5s, actualMd5Sum=sumOfFactMd5s) else: modelXbrl.info("info", _("Successful XBRL facts sum of md5s."), modelObject=modelXbrl) if not val.filingIndicators: modelXbrl.error("EBA.1.6", _('Missing filing indicators. Reported XBRL instances MUST include appropriate filing indicator elements'), modelObject=modelDocument) if val.numFilingIndicatorTuples > 1: modelXbrl.info("EBA.1.6.2", _('Multiple filing indicators tuples when not in streaming mode (info).'), modelObject=modelXbrl.factsByQname[qnFIndicators]) if len(val.cntxDates) > 1: modelXbrl.error("EBA.2.13", _('Contexts must have the same date: %(dates)s.'), # when streaming values are no longer available, but without streaming they can be logged modelObject=set(_cntx for _cntxs in val.cntxDates.values() for _cntx in _cntxs), dates=', '.join(XmlUtil.dateunionValue(_dt, subtractOneDay=True) for _dt in val.cntxDates.keys())) if val.unusedCntxIDs: modelXbrl.warning("EBA.2.7", _('Unused xbrli:context nodes SHOULD NOT be present in the instance: %(unusedContextIDs)s.'), modelObject=[modelXbrl.contexts[unusedCntxID] for unusedCntxID in val.unusedCntxIDs if unusedCntxID in modelXbrl.contexts], unusedContextIDs=", ".join(sorted(val.unusedCntxIDs))) if len(val.cntxEntities) > 1: modelXbrl.warning("EBA.2.9", _('All entity identifiers and schemes must be the same, %(count)s found: %(entities)s.'), modelObject=modelDocument, count=len(val.cntxEntities), entities=", ".join(sorted(str(cntxEntity) for cntxEntity in val.cntxEntities))) if val.unusedUnitIDs: modelXbrl.warning("EBA.2.21", _('Unused xbrli:unit nodes SHOULD NOT be present in the instance: %(unusedUnitIDs)s.'), modelObject=[modelXbrl.units[unusedUnitID] for unusedUnitID in val.unusedUnitIDs if unusedUnitID in modelXbrl.units], unusedUnitIDs=", ".join(sorted(val.unusedUnitIDs))) if len(val.currenciesUsed) > 1: modelXbrl.error("EBA.3.1", _("There MUST be only one currency but %(numCurrencies)s were found: %(currencies)s.'"), modelObject=val.currenciesUsed.values(), numCurrencies=len(val.currenciesUsed), currencies=", ".join(str(c) for c in val.currenciesUsed.keys())) if val.prefixesUnused: modelXbrl.warning("EBA.3.4", _("There SHOULD be no unused prefixes but these were declared: %(unusedPrefixes)s.'"), modelObject=modelDocument, unusedPrefixes=', '.join(sorted(val.prefixesUnused))) for ns, prefixes in val.namespacePrefixesUsed.items(): nsDocs = modelXbrl.namespaceDocs.get(ns) if nsDocs: for nsDoc in nsDocs: nsDocPrefix = XmlUtil.xmlnsprefix(nsDoc.xmlRootElement, ns) if any(prefix != nsDocPrefix for prefix in prefixes if prefix is not None): modelXbrl.warning("EBA.3.5", _("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s"), modelObject=modelDocument, namespace=ns, declaredPrefix=nsDocPrefix, foundPrefixes=', '.join(sorted(prefixes - {None}))) modelXbrl.profileActivity(_statusMsg, minTimeToShow=0.0) modelXbrl.modelManager.showStatus(None) del val.prefixNamespace, val.namespacePrefix, val.idObjects, val.typedDomainElements del val.utrValidator
def final(val): if not (val.validateEBA or val.validateEIOPA): return modelXbrl = val.modelXbrl modelDocument = modelXbrl.modelDocument _statusMsg = _("validating {0} filing rules").format( val.disclosureSystem.name) modelXbrl.profileActivity() modelXbrl.modelManager.showStatus(_statusMsg) if modelDocument.type == ModelDocument.Type.INSTANCE and ( val.validateEBA or val.validateEIOPA): if not modelDocument.uri.endswith(".xbrl"): modelXbrl.warning( "EBA.1.1", _('XBRL instance documents SHOULD use the extension ".xbrl" but it is "%(extension)s"' ), modelObject=modelDocument, extension=os.path.splitext(modelDocument.basename)[1]) if modelDocument.documentEncoding.lower() not in ("utf-8", "utf-8-sig"): modelXbrl.error( "EBA.1.4", _('XBRL instance documents MUST use "UTF-8" encoding but is "%(xmlEncoding)s"' ), modelObject=modelDocument, xmlEncoding=modelDocument.documentEncoding) schemaRefElts = [] schemaRefFileNames = [] for doc, docRef in modelDocument.referencesDocument.items(): if docRef.referenceType == "href": if docRef.referringModelObject.localName == "schemaRef": schemaRefElts.append(docRef.referringModelObject) schemaRefFileNames.append(doc.basename) if not UrlUtil.isAbsolute(doc.uri): modelXbrl.error( "EBA.2.2", _('The link:schemaRef element in submitted instances MUST resolve to the full published entry point URL: %(url)s.' ), modelObject=docRef.referringModelObject, url=doc.uri) elif docRef.referringModelObject.localName == "linkbaseRef": modelXbrl.error( "EBA.2.3", _('The link:linkbaseRef element is not allowed: %(fileName)s.' ), modelObject=docRef.referringModelObject, fileName=doc.basename) if len(schemaRefFileNames) > 1: modelXbrl.error( "EBA.1.5", _('XBRL instance documents MUST reference only one entry point schema but %(numEntryPoints)s were found: %(entryPointNames)s' ), modelObject=modelDocument, numEntryPoints=len(schemaRefFileNames), entryPointNames=', '.join(sorted(schemaRefFileNames))) ### check entry point names appropriate for filing indicator (DPM DB?) if len(schemaRefElts) != 1: modelXbrl.error( "EBA.2.3", _('Any reported XBRL instance document MUST contain only one xbrli:xbrl/link:schemaRef node, but %(entryPointCount)s.' ), modelObject=schemaRefElts, entryPointCount=len(schemaRefElts)) # non-streaming EBA checks if not getattr(modelXbrl, "isStreamingMode", False): validateFacts(val, modelXbrl.facts) # check sum of fact md5s (otherwise checked in streaming process) xbrlFactsCheckVersion = None expectedSumOfFactMd5s = None for pi in modelDocument.xmlRootElement.getchildren(): if isinstance(pi, etree._ProcessingInstruction ) and pi.target == "xbrl-facts-check": _match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']", pi.text) if _match: _matchGroups = _match.groups() if len(_matchGroups) == 2: if _matchGroups[0] == "version": xbrlFactsCheckVersion = _matchGroups[1] elif _matchGroups[0] == "sum-of-fact-md5s": try: expectedSumOfFactMd5s = Md5Sum( _matchGroups[1]) except ValueError: modelXbrl.error( "EIOPA:xbrlFactsCheckError", _("Invalid sum-of-md5s %(sumOfMd5)s"), modelObject=modelXbrl, sumOfMd5=_matchGroups[1]) if xbrlFactsCheckVersion and expectedSumOfFactMd5s: sumOfFactMd5s = Md5Sum() for f in modelXbrl.factsInInstance: sumOfFactMd5s += f.md5sum if sumOfFactMd5s != expectedSumOfFactMd5s: modelXbrl.warning( "EIOPA:xbrlFactsCheckWarning", _("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s" ), modelObject=modelXbrl, expectedMd5=expectedSumOfFactMd5s, actualMd5Sum=sumOfFactMd5s) else: modelXbrl.info("info", _("Successful XBRL facts sum of md5s."), modelObject=modelXbrl) if not val.filingIndicators: modelXbrl.error( "EBA.1.6", _('Missing filing indicators. Reported XBRL instances MUST include appropriate filing indicator elements' ), modelObject=modelDocument) if val.numFilingIndicatorTuples > 1: modelXbrl.info( "EBA.1.6.2", _('Multiple filing indicators tuples when not in streaming mode (info).' ), modelObject=modelXbrl.factsByQname[qnFIndicators]) if len(val.cntxDates) > 1: modelXbrl.error( "EBA.2.13", _('Contexts must have the same date: %(dates)s.'), # when streaming values are no longer available, but without streaming they can be logged modelObject=set(_cntx for _cntxs in val.cntxDates.values() for _cntx in _cntxs), dates=', '.join( XmlUtil.dateunionValue(_dt, subtractOneDay=True) for _dt in val.cntxDates.keys())) if val.unusedCntxIDs: modelXbrl.warning( "EBA.2.7", _('Unused xbrli:context nodes SHOULD NOT be present in the instance: %(unusedContextIDs)s.' ), modelObject=[ modelXbrl.contexts[unusedCntxID] for unusedCntxID in val.unusedCntxIDs if unusedCntxID in modelXbrl.contexts ], unusedContextIDs=", ".join(sorted(val.unusedCntxIDs))) if len(val.cntxEntities) > 1: modelXbrl.warning( "EBA.2.9", _('All entity identifiers and schemes must be the same, %(count)s found: %(entities)s.' ), modelObject=modelDocument, count=len(val.cntxEntities), entities=", ".join( sorted(str(cntxEntity) for cntxEntity in val.cntxEntities))) if val.unusedUnitIDs: modelXbrl.warning( "EBA.2.21", _('Unused xbrli:unit nodes SHOULD NOT be present in the instance: %(unusedUnitIDs)s.' ), modelObject=[ modelXbrl.units[unusedUnitID] for unusedUnitID in val.unusedUnitIDs if unusedUnitID in modelXbrl.units ], unusedUnitIDs=", ".join(sorted(val.unusedUnitIDs))) if len(val.currenciesUsed) > 1: modelXbrl.error( "EBA.3.1", _("There MUST be only one currency but %(numCurrencies)s were found: %(currencies)s.'" ), modelObject=val.currenciesUsed.values(), numCurrencies=len(val.currenciesUsed), currencies=", ".join( str(c) for c in val.currenciesUsed.keys())) if val.prefixesUnused: modelXbrl.warning( "EBA.3.4", _("There SHOULD be no unused prefixes but these were declared: %(unusedPrefixes)s.'" ), modelObject=modelDocument, unusedPrefixes=', '.join(sorted(val.prefixesUnused))) for ns, prefixes in val.namespacePrefixesUsed.items(): nsDocs = modelXbrl.namespaceDocs.get(ns) if nsDocs: for nsDoc in nsDocs: nsDocPrefix = XmlUtil.xmlnsprefix(nsDoc.xmlRootElement, ns) if any(prefix != nsDocPrefix for prefix in prefixes if prefix is not None): modelXbrl.warning( "EBA.3.5", _("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s" ), modelObject=modelDocument, namespace=ns, declaredPrefix=nsDocPrefix, foundPrefixes=', '.join(sorted(prefixes - {None}))) modelXbrl.profileActivity(_statusMsg, minTimeToShow=0.0) modelXbrl.modelManager.showStatus(None) del val.prefixNamespace, val.namespacePrefix, val.idObjects, val.typedDomainElements del val.utrValidator
def final(val): if not (val.validateEBA or val.validateEIOPA): return modelXbrl = val.modelXbrl modelDocument = modelXbrl.modelDocument _statusMsg = _("validating {0} filing rules").format(val.disclosureSystem.name) modelXbrl.profileActivity() modelXbrl.modelManager.showStatus(_statusMsg) if modelDocument.type == ModelDocument.Type.INSTANCE and (val.validateEBA or val.validateEIOPA): if not modelDocument.uri.endswith(".xbrl"): modelXbrl.warning("EBA.1.1", _('XBRL instance documents SHOULD use the extension ".xbrl" but it is "%(extension)s"'), modelObject=modelDocument, extension=os.path.splitext(modelDocument.basename)[1]) modelXbrl.error("EIOPA.S.1.1.a", _('XBRL instance documents MUST use the extension ".xbrl" but it is "%(extension)s"'), modelObject=modelDocument, extension=os.path.splitext(modelDocument.basename)[1]) if val.isEIOPA_2_0_1: _encodings = ("UTF-8", "utf-8-sig") else: _encodings = ("utf-8", "UTF-8", "utf-8-sig") if modelDocument.documentEncoding not in _encodings: modelXbrl.error(("EBA.1.4", "EIOPA.1.4"), _('XBRL instance documents MUST use "UTF-8" encoding but is "%(xmlEncoding)s"'), modelObject=modelDocument, xmlEncoding=modelDocument.documentEncoding) schemaRefElts = [] schemaRefFileNames = [] for doc, docRef in modelDocument.referencesDocument.items(): if docRef.referenceType == "href": if docRef.referringModelObject.localName == "schemaRef": schemaRefElts.append(docRef.referringModelObject) schemaRefFileNames.append(doc.basename) if not UrlUtil.isAbsolute(doc.uri): modelXbrl.error(("EBA.2.2", "EIOPA.S.1.5.a" if val.isEIOPAfullVersion else "EIOPA.S.1.5.b"), _('The link:schemaRef element in submitted instances MUST resolve to the full published entry point URL: %(url)s.'), modelObject=docRef.referringModelObject, url=doc.uri, messageCodes=("EBA.2.2", "EIOPA.S.1.5.a","EIOPA.S.1.5.b")) elif docRef.referringModelObject.localName == "linkbaseRef": modelXbrl.error(("EBA.2.3","EIOPA.S.1.5.a"), _('The link:linkbaseRef element is not allowed: %(fileName)s.'), modelObject=docRef.referringModelObject, fileName=doc.basename) _numSchemaRefs = len(XmlUtil.children(modelDocument.xmlRootElement, XbrlConst.link, "schemaRef")) if _numSchemaRefs > 1: modelXbrl.error(("EIOPA.S.1.5.a", "EBA.1.5"), _('XBRL instance documents MUST reference only one entry point schema but %(numEntryPoints)s were found: %(entryPointNames)s'), modelObject=modelDocument, numEntryPoints=_numSchemaRefs, entryPointNames=', '.join(sorted(schemaRefFileNames))) ### check entry point names appropriate for filing indicator (DPM DB?) if len(schemaRefElts) != 1: modelXbrl.error("EBA.2.3", _('Any reported XBRL instance document MUST contain only one xbrli:xbrl/link:schemaRef node, but %(entryPointCount)s.'), modelObject=schemaRefElts, entryPointCount=len(schemaRefElts)) # non-streaming EBA checks if not getattr(modelXbrl, "isStreamingMode", False): val.qnReportedCurrency = None if val.isEIOPA_2_0_1 and qnMetReportingCurrency in modelXbrl.factsByQname: for _multiCurrencyFact in modelXbrl.factsByQname[qnMetReportingCurrency]: # multi-currency fact val.qnReportedCurrency = _multiCurrencyFact.xValue break validateFacts(val, modelXbrl.facts) # check sum of fact md5s (otherwise checked in streaming process) xbrlFactsCheckVersion = None expectedSumOfFactMd5s = None for pi in modelDocument.xmlRootElement.getchildren(): if isinstance(pi, etree._ProcessingInstruction) and pi.target == "xbrl-facts-check": _match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']", pi.text) if _match: _matchGroups = _match.groups() if len(_matchGroups) == 2: if _matchGroups[0] == "version": xbrlFactsCheckVersion = _matchGroups[1] elif _matchGroups[0] == "sum-of-fact-md5s": try: expectedSumOfFactMd5s = Md5Sum(_matchGroups[1]) except ValueError: modelXbrl.error("EIOPA:xbrlFactsCheckError", _("Invalid sum-of-md5s %(sumOfMd5)s"), modelObject=modelXbrl, sumOfMd5=_matchGroups[1]) if xbrlFactsCheckVersion and expectedSumOfFactMd5s: sumOfFactMd5s = Md5Sum() for f in modelXbrl.factsInInstance: sumOfFactMd5s += f.md5sum if sumOfFactMd5s != expectedSumOfFactMd5s: modelXbrl.warning("EIOPA:xbrlFactsCheckWarning", _("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"), modelObject=modelXbrl, expectedMd5=expectedSumOfFactMd5s, actualMd5Sum=sumOfFactMd5s) else: modelXbrl.info("info", _("Successful XBRL facts sum of md5s."), modelObject=modelXbrl) if any(badError in modelXbrl.errors for badError in ("EBA.2.1", "EIOPA.2.1", "EIOPA.S.1.5.a/EIOPA.S.1.5.b")): pass # skip checking filingIndicators if bad errors elif not val.filingIndicators: modelXbrl.error(("EBA.1.6", "EIOPA.1.6.a"), _('Missing filing indicators. Reported XBRL instances MUST include appropriate (positive) filing indicator elements'), modelObject=modelDocument) elif all(filed == False for filed in val.filingIndicators.values()): modelXbrl.error(("EBA.1.6", "EIOPA.1.6.a"), _('All filing indicators are filed="false". Reported XBRL instances MUST include appropriate (positive) filing indicator elements'), modelObject=modelDocument) if val.numFilingIndicatorTuples > 1: modelXbrl.warning(("EBA.1.6.2", "EIOPA.1.6.2"), _('Multiple filing indicators tuples when not in streaming mode (info).'), modelObject=modelXbrl.factsByQname[qnFIndicators]) if len(val.cntxDates) > 1: modelXbrl.error(("EBA.2.13","EIOPA.2.13"), _('Contexts must have the same date: %(dates)s.'), # when streaming values are no longer available, but without streaming they can be logged modelObject=set(_cntx for _cntxs in val.cntxDates.values() for _cntx in _cntxs), dates=', '.join(XmlUtil.dateunionValue(_dt, subtractOneDay=True) for _dt in val.cntxDates.keys())) if val.unusedCntxIDs: if val.isEIOPA_2_0_1: modelXbrl.error("EIOPA.2.7", _('Unused xbrli:context nodes MUST NOT be present in the instance: %(unusedContextIDs)s.'), modelObject=[modelXbrl.contexts[unusedCntxID] for unusedCntxID in val.unusedCntxIDs if unusedCntxID in modelXbrl.contexts], unusedContextIDs=", ".join(sorted(val.unusedCntxIDs))) else: modelXbrl.warning(("EBA.2.7", "EIOPA.2.7"), _('Unused xbrli:context nodes SHOULD NOT be present in the instance: %(unusedContextIDs)s.'), modelObject=[modelXbrl.contexts[unusedCntxID] for unusedCntxID in val.unusedCntxIDs if unusedCntxID in modelXbrl.contexts], unusedContextIDs=", ".join(sorted(val.unusedCntxIDs))) if len(val.cntxEntities) > 1: modelXbrl.error(("EBA.2.9", "EIOPA.2.9"), _('All entity identifiers and schemes MUST be the same, %(count)s found: %(entities)s.'), modelObject=modelDocument, count=len(val.cntxEntities), entities=", ".join(sorted(str(cntxEntity) for cntxEntity in val.cntxEntities))) for _scheme, _LEI in val.cntxEntities: if (_scheme in ("http://standards.iso.org/iso/17442", "http://standard.iso.org/iso/17442", "LEI") or (not val.isEIOPAfullVersion and _scheme == "PRE-LEI")): if _scheme == "http://standard.iso.org/iso/17442": modelXbrl.warning(("EBA.3.6", "EIOPA.S.2.8.c"), _("Warning, context has entity scheme %(scheme)s should be plural: http://standards.iso.org/iso/17442."), modelObject=modelDocument, scheme=_scheme) result = LeiUtil.checkLei(_LEI) if result == LeiUtil.LEI_INVALID_LEXICAL: modelXbrl.error("EIOPA.S.2.8.c", _("Context has lexically invalid LEI %(lei)s."), modelObject=modelDocument, lei=_LEI) elif result == LeiUtil.LEI_INVALID_CHECKSUM: modelXbrl.error("EIOPA.S.2.8.c", _("Context has LEI checksum error in %(lei)s."), modelObject=modelDocument, lei=_LEI) elif _scheme == "SC": pass # anything is ok for Specific Code else: modelXbrl.error("EIOPA.S.2.8.c", _("Context has unrecognized entity scheme %(scheme)s."), modelObject=modelDocument, scheme=_scheme) if val.unusedUnitIDs: if val.isEIOPA_2_0_1: modelXbrl.error("EIOPA.2.22", _('Unused xbrli:unit nodes MUST NOT be present in the instance: %(unusedUnitIDs)s.'), modelObject=[modelXbrl.units[unusedUnitID] for unusedUnitID in val.unusedUnitIDs if unusedUnitID in modelXbrl.units], unusedUnitIDs=", ".join(sorted(val.unusedUnitIDs))) else: modelXbrl.warning(("EBA.2.22", "EIOPA.2.22"), _('Unused xbrli:unit nodes SHOULD NOT be present in the instance: %(unusedUnitIDs)s.'), modelObject=[modelXbrl.units[unusedUnitID] for unusedUnitID in val.unusedUnitIDs if unusedUnitID in modelXbrl.units], unusedUnitIDs=", ".join(sorted(val.unusedUnitIDs))) if len(val.currenciesUsed) > 1: modelXbrl.error(("EBA.3.1","EIOPA.3.1"), _("There MUST be only one currency but %(numCurrencies)s were found: %(currencies)s.'"), modelObject=val.currenciesUsed.values(), numCurrencies=len(val.currenciesUsed), currencies=", ".join(str(c) for c in val.currenciesUsed.keys())) elif val.isEIOPA_2_0_1 and any(_measure.localName != val.reportingCurrency for _measure in val.currenciesUsed.keys()): modelXbrl.error("EIOPA.3.1", _("There MUST be only one currency but reporting currency %(reportingCurrency)s differs from unit currencies: %(currencies)s.'"), modelObject=val.currenciesUsed.values(), reportingCurrency=val.reportingCurrency, currencies=", ".join(str(c) for c in val.currenciesUsed.keys())) if val.prefixesUnused: modelXbrl.warning(("EBA.3.4", "EIOPA.3.4"), _("There SHOULD be no unused prefixes but these were declared: %(unusedPrefixes)s.'"), modelObject=modelDocument, unusedPrefixes=', '.join(sorted(val.prefixesUnused))) for ns, prefixes in val.namespacePrefixesUsed.items(): nsDocs = modelXbrl.namespaceDocs.get(ns) if nsDocs: for nsDoc in nsDocs: nsDocPrefix = XmlUtil.xmlnsprefix(nsDoc.xmlRootElement, ns) if any(prefix != nsDocPrefix for prefix in prefixes if prefix is not None): modelXbrl.warning(("EBA.3.5", "EIOPA.3.5"), _("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s"), modelObject=modelDocument, namespace=ns, declaredPrefix=nsDocPrefix, foundPrefixes=', '.join(sorted(prefixes - {None}))) elif ns in CANONICAL_PREFIXES and any(prefix != CANONICAL_PREFIXES[ns] for prefix in prefixes if prefix is not None): modelXbrl.warning(("EBA.3.5", "EIOPA.3.5"), _("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s"), modelObject=modelDocument, namespace=ns, declaredPrefix=CANONICAL_PREFIXES[ns], foundPrefixes=', '.join(sorted(prefixes - {None}))) modelXbrl.profileActivity(_statusMsg, minTimeToShow=0.0) modelXbrl.modelManager.showStatus(None) del val.prefixNamespace, val.namespacePrefix, val.idObjects, val.typedDomainElements del val.utrValidator, val.firstFact, val.footnotesRelationshipSet
def final(val): if not (val.validateEBA or val.validateEIOPA): return modelXbrl = val.modelXbrl modelDocument = modelXbrl.modelDocument _statusMsg = _("validating {0} filing rules").format(val.disclosureSystem.name) modelXbrl.profileActivity() val.showStatus(_statusMsg) if modelDocument.type == ModelDocument.Type.INSTANCE and (val.validateEBA or val.validateEIOPA): if not modelDocument.uri.endswith(".xbrl"): modelXbrl.warning("EBA.1.1", _('XBRL instance documents SHOULD use the extension ".xbrl" but it is "%(extension)s"'), modelObject=modelDocument, xmlEncoding=os.path.splitext(modelDocument.basename)[1]) if modelDocument.documentEncoding.lower() not in ("utf-8", "utf-8-sig"): modelXbrl.error("EBA.1.4", _('XBRL instance documents MUST use "UTF-8" encoding but is "%(xmlEncoding)s"'), modelObject=modelDocument, xmlEncoding=modelDocument.documentEncoding) schemaRefElts = [] schemaRefFileNames = [] for doc, docRef in modelDocument.referencesDocument.items(): if docRef.referenceType == "href": if docRef.referringModelObject.localName == "schemaRef": schemaRefElts.append(docRef.referringModelObject) schemaRefFileNames.append(doc.basename) if not UrlUtil.isAbsolute(doc.uri): modelXbrl.error("EBA.2.2", _('The link:schemaRef element in submitted instances MUST resolve to the full published entry point URL: %(url)s.'), modelObject=docRef.referringModelObject, url=doc.uri) elif docRef.referringModelObject.localName == "linkbaseRef": modelXbrl.error("EBA.2.3", _('The link:linkbaseRef element is not allowed: %(fileName)s.'), modelObject=docRef.referringModelObject, fileName=doc.basename) if len(schemaRefFileNames) > 1: modelXbrl.error("EBA.1.5", _('XBRL instance documents MUST reference only one entry point schema but %(numEntryPoints)s were found: %(entryPointNames)s'), modelObject=modelDocument, numEntryPoints=len(schemaRefFileNames), entryPointNames=', '.join(sorted(schemaRefFileNames))) ### check entry point names appropriate for filing indicator (DPM DB?) if len(schemaRefElts) != 1: modelXbrl.error("EBA.2.3", _('Any reported XBRL instance document MUST contain only one xbrli:xbrl/link:schemaRef node, but %(entryPointCount)s.'), modelObject=schemaRefElts, entryPointCount=len(schemaRefElts)) filingIndicators = {} for fIndicator in modelXbrl.factsByQname[qnFilingIndicator]: _value = (fIndicator.xValue or fIndicator.value) # use validated xValue if DTS else value for skipDTS if _value in filingIndicators: modelXbrl.error("EBA.1.6.1", _('Multiple filing indicators facts for indicator %(filingIndicator)s.'), modelObject=(fIndicator, filingIndicators[_value]), filingIndicator=_value) filingIndicators[_value] = fIndicator if not filingIndicators: modelXbrl.error("EBA.1.6", _('Missing filing indicators. Reported XBRL instances MUST include appropriate filing indicator elements'), modelObject=modelDocument) numFilingIndicatorTuples = len(modelXbrl.factsByQname[qnFIndicators]) if numFilingIndicatorTuples > 1 and not getattr(modelXbrl, "isStreamingMode", False): modelXbrl.info("EBA.1.6.2", _('Multiple filing indicators tuples when not in streaming mode (info).'), modelObject=modelXbrl.factsByQname[qnFIndicators]) # note EBA 2.1 is in ModelDocument.py cntxIDs = set() cntxEntities = set() cntxDates = defaultdict(list) timelessDatePattern = re.compile(r"\s*([0-9]{4})-([0-9]{2})-([0-9]{2})\s*$") for cntx in modelXbrl.contexts.values(): cntxIDs.add(cntx.id) cntxEntities.add(cntx.entityIdentifier) dateElts = XmlUtil.descendants(cntx, XbrlConst.xbrli, ("startDate","endDate","instant")) if any(not timelessDatePattern.match(e.textValue) for e in dateElts): modelXbrl.error("EBA.2.10", _('Period dates must be whole dates without time or timezone: %(dates)s.'), modelObject=cntx, dates=", ".join(e.text for e in dateElts)) if cntx.isForeverPeriod: modelXbrl.error("EBA.2.11", _('Forever context period is not allowed.'), modelObject=cntx) elif cntx.isStartEndPeriod: modelXbrl.error("EBA.2.13", _('Start-End (flow) context period is not allowed.'), modelObject=cntx) elif cntx.isInstantPeriod: cntxDates[cntx.instantDatetime].append(cntx) if XmlUtil.hasChild(cntx, XbrlConst.xbrli, "segment"): modelXbrl.error("EBA.2.14", _("The segment element not allowed in context Id: %(context)s"), modelObject=cntx, context=cntx.contextID) for scenElt in XmlUtil.descendants(cntx, XbrlConst.xbrli, "scenario"): childTags = ", ".join([child.prefixedName for child in scenElt.iterchildren() if isinstance(child,ModelObject) and child.tag != "{http://xbrl.org/2006/xbrldi}explicitMember" and child.tag != "{http://xbrl.org/2006/xbrldi}typedMember"]) if len(childTags) > 0: modelXbrl.error("EBA.2.15", _("Scenario of context Id %(context)s has disallowed content: %(content)s"), modelObject=cntx, context=cntx.id, content=childTags) if len(cntxDates) > 1: modelXbrl.error("EBA.2.13", _('Contexts must have the same date: %(dates)s.'), modelObject=[_cntx for _cntxs in cntxDates.values() for _cntx in _cntxs], dates=', '.join(XmlUtil.dateunionValue(_dt, subtractOneDay=True) for _dt in cntxDates.keys())) unusedCntxIDs = cntxIDs - {fact.contextID for fact in modelXbrl.factsInInstance if fact.contextID} # skip tuples if unusedCntxIDs: modelXbrl.warning("EBA.2.7", _('Unused xbrli:context nodes SHOULD NOT be present in the instance: %(unusedContextIDs)s.'), modelObject=[modelXbrl.contexts[unusedCntxID] for unusedCntxID in unusedCntxIDs], unusedContextIDs=", ".join(sorted(unusedCntxIDs))) if len(cntxEntities) > 1: modelXbrl.warning("EBA.2.9", _('All entity identifiers and schemes must be the same, %(count)s found: %(entities)s.'), modelObject=modelDocument, count=len(cntxEntities), entities=", ".join(sorted(str(cntxEntity) for cntxEntity in cntxEntities))) otherFacts = {} # (contextHash, unitHash, xmlLangHash) : fact nilFacts = [] stringFactsWithoutXmlLang = [] nonMonetaryNonPureFacts = [] unitIDsUsed = set() currenciesUsed = {} for qname, facts in modelXbrl.factsByQname.items(): for f in facts: if modelXbrl.skipDTS: c = f.qname.localName[0] isNumeric = c in ('m', 'p', 'i') isMonetary = c == 'm' isInteger = c == 'i' isPercent = c == 'p' isString = c == 's' else: concept = f.concept if concept is not None: isNumeric = concept.isNumeric isMonetary = concept.isMonetary isInteger = concept.baseXbrliType in integerItemTypes isPercent = concept.typeQname == qnPercentItemType isString = concept.baseXbrliType in ("stringItemType", "normalizedStringItemType") else: isNumeric = isString = False # error situation k = (f.getparent().objectIndex, f.qname, f.context.contextDimAwareHash if f.context is not None else None, f.unit.hash if f.unit is not None else None, hash(f.xmlLang)) if f.qname == qnFIndicators and val.validateEIOPA: pass elif k not in otherFacts: otherFacts[k] = {f} else: matches = [o for o in otherFacts[k] if (f.getparent().objectIndex == o.getparent().objectIndex and f.qname == o.qname and f.context.isEqualTo(o.context) if f.context is not None and o.context is not None else True) and (f.unit.isEqualTo(o.unit) if f.unit is not None and o.unit is not None else True) and (f.xmlLang == o.xmlLang)] if matches: contexts = [f.contextID] + [o.contextID for o in matches] modelXbrl.error("EBA.2.16", _('Facts are duplicates %(fact)s contexts %(contexts)s.'), modelObject=[f] + matches, fact=f.qname, contexts=', '.join(contexts)) else: otherFacts[k].add(f) if isNumeric: if f.precision: modelXbrl.error("EBA.2.17", _("Numeric fact %(fact)s of context %(contextID)s has a precision attribute '%(precision)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, precision=f.precision) if f.decimals and f.decimals != "INF": try: dec = int(f.decimals) if isMonetary: if dec < -3: modelXbrl.error("EBA.2.17", _("Monetary fact %(fact)s of context %(contextID)s has a decimal attribute < -3: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) elif isInteger: if dec != 0: modelXbrl.error("EBA.2.17", _("Integer fact %(fact)s of context %(contextID)s has a decimal attribute \u2260 0: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) elif isPercent: if dec < 4: modelXbrl.error("EBA.2.17", _("Percent fact %(fact)s of context %(contextID)s has a decimal attribute < 4: '%(decimals)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, decimals=f.decimals) except ValueError: pass # should have been reported as a schema error by loader '''' (not intended by EBA 2.18, paste here is from EFM) if not f.isNil and getattr(f,"xValid", 0) == 4: try: insignificance = insignificantDigits(f.xValue, decimals=f.decimals) if insignificance: # if not None, returns (truncatedDigits, insiginficantDigits) modelXbrl.error(("EFM.6.05.37", "GFM.1.02.26"), _("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s has nonzero digits in insignificant portion %(insignificantDigits)s."), modelObject=f1, fact=f1.qname, contextID=f1.contextID, decimals=f1.decimals, value=f1.xValue, truncatedDigits=insignificance[0], insignificantDigits=insignificance[1]) except (ValueError,TypeError): modelXbrl.error(("EBA.2.18"), _("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s causes Value Error exception."), modelObject=f1, fact=f1.qname, contextID=f1.contextID, decimals=f1.decimals, value=f1.value) ''' unit = f.unit if unit is not None: if isMonetary: if unit.measures[0]: currenciesUsed[unit.measures[0][0]] = unit elif not unit.isSingleMeasure or unit.measures[0][0] != XbrlConst.qnXbrliPure: nonMonetaryNonPureFacts.append(f) elif isString: if not f.xmlLang: stringFactsWithoutXmlLang.append(f) if f.unitID is not None: unitIDsUsed.add(f.unitID) if f.isNil: nilFacts.append(f) if nilFacts: modelXbrl.error("EBA.2.19", _('Nil facts MUST NOT be present in the instance: %(nilFacts)s.'), modelObject=nilFacts, nilFacts=", ".join(str(f.qname) for f in nilFacts)) if stringFactsWithoutXmlLang: modelXbrl.error("EBA.2.20", _("String facts need to report xml:lang: '%(langLessFacts)s'"), modelObject=stringFactsWithoutXmlLang, langLEssFacts=", ".join(str(f.qname) for f in stringFactsWithoutXmlLang)) if nonMonetaryNonPureFacts: modelXbrl.error("EBA.3.2", _("Non monetary (numeric) facts MUST use the pure unit: '%(langLessFacts)s'"), modelObject=nonMonetaryNonPureFacts, langLessFacts=", ".join(str(f.qname) for f in nonMonetaryNonPureFacts)) unusedUnitIDs = modelXbrl.units.keys() - unitIDsUsed if unusedUnitIDs: modelXbrl.warning("EBA.2.21", _('Unused xbrli:unit nodes SHOULD NOT be present in the instance: %(unusedUnitIDs)s.'), modelObject=[modelXbrl.units[unusedUnitID] for unusedUnitID in unusedUnitIDs], unusedUnitIDs=", ".join(sorted(unusedUnitIDs))) unitHashes = {} for unit in modelXbrl.units.values(): h = hash(unit) if h in unitHashes and unit.isEqualTo(unitHashes[h]): modelXbrl.warning("EBA.2.32", _("Duplicate units SHOULD NOT be reported, units %(unit1)s and %(unit2)s have same measures.'"), modelObject=(unit, unitHashes[h]), unit1=unit.id, unit2=unitHashes[h].id) else: unitHashes[h] = unit if len(currenciesUsed) > 1: modelXbrl.error("EBA.3.1", _("There MUST be only one currency but %(numCurrencies)s were found: %(currencies)s.'"), modelObject=currenciesUsed.values(), numCurrencies=len(currenciesUsed), currencies=", ".join(str(c) for c in currenciesUsed.keys())) namespacePrefixesUsed = defaultdict(set) prefixesUnused = set(modelDocument.xmlRootElement.keys()).copy() for elt in modelDocument.xmlRootElement.iter(): if isinstance(elt, ModelObject): # skip comments and processing instructions namespacePrefixesUsed[elt.qname.namespaceURI].add(elt.qname.prefix) prefixesUnused.discard(elt.qname.prefix) if prefixesUnused: modelXbrl.warning("EBA.3.4", _("There SHOULD be no unused prefixes but these were declared: %(unusedPrefixes)s.'"), modelObject=modelDocument, unusedPrefixes=', '.join(sorted(prefixesUnused))) for ns, prefixes in namespacePrefixesUsed.items(): nsDocs = modelXbrl.namespaceDocs.get(ns) if nsDocs: for nsDoc in nsDocs: nsDocPrefix = XmlUtil.xmlnsprefix(nsDoc.xmlRootElement, ns) if any(prefix != nsDocPrefix for prefix in prefixes if prefix is not None): modelXbrl.warning("EBA.3.5", _("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s"), modelObject=modelDocument, namespace=ns, declaredPrefix=nsDocPrefix, foundPrefixes=', '.join(sorted(prefixes - {None}))) modelXbrl.profileActivity(_statusMsg, minTimeToShow=0.0) val.showStatus(None) del val.prefixNamespace, val.namespacePrefix, val.idObjects, val.typedDomainElements
def final(val): if not (val.validateEBA or val.validateEIOPA): return modelXbrl = val.modelXbrl modelDocument = modelXbrl.modelDocument _statusMsg = _("validating {0} filing rules").format(val.disclosureSystem.name) modelXbrl.profileActivity() modelXbrl.modelManager.showStatus(_statusMsg) if modelDocument.type == ModelDocument.Type.INSTANCE and (val.validateEBA or val.validateEIOPA): if not modelDocument.uri.endswith(".xbrl"): modelXbrl.warning("EBA.1.1", _('XBRL instance documents SHOULD use the extension ".xbrl" but it is "%(extension)s"'), modelObject=modelDocument, extension=os.path.splitext(modelDocument.basename)[1]) modelXbrl.error("EIOPA.S.1.1.a", _('XBRL instance documents MUST use the extension ".xbrl" but it is "%(extension)s"'), modelObject=modelDocument, extension=os.path.splitext(modelDocument.basename)[1]) if val.isEIOPA_2_0_1: _encodings = ("UTF-8", "utf-8-sig") else: _encodings = ("utf-8", "UTF-8", "utf-8-sig") if modelDocument.documentEncoding not in _encodings: modelXbrl.error(("EBA.1.4", "EIOPA.1.4"), _('XBRL instance documents MUST use "UTF-8" encoding but is "%(xmlEncoding)s"'), modelObject=modelDocument, xmlEncoding=modelDocument.documentEncoding) schemaRefElts = [] schemaRefFileNames = [] for doc, docRef in modelDocument.referencesDocument.items(): if docRef.referenceType == "href": if docRef.referringModelObject.localName == "schemaRef": schemaRefElts.append(docRef.referringModelObject) schemaRefFileNames.append(doc.basename) if not UrlUtil.isAbsolute(doc.uri): modelXbrl.error(("EBA.2.2", "EIOPA.S.1.5.a" if val.isEIOPAfullVersion else "EIOPA.S.1.5.b"), _('The link:schemaRef element in submitted instances MUST resolve to the full published entry point URL: %(url)s.'), modelObject=docRef.referringModelObject, url=doc.uri, messageCodes=("EBA.2.2", "EIOPA.S.1.5.a","EIOPA.S.1.5.b")) elif docRef.referringModelObject.localName == "linkbaseRef": modelXbrl.error(("EBA.2.3","EIOPA.S.1.5.a"), _('The link:linkbaseRef element is not allowed: %(fileName)s.'), modelObject=docRef.referringModelObject, fileName=doc.basename) _numSchemaRefs = len(XmlUtil.children(modelDocument.xmlRootElement, XbrlConst.link, "schemaRef")) if _numSchemaRefs > 1: modelXbrl.error(("EIOPA.S.1.5.a", "EBA.1.5"), _('XBRL instance documents MUST reference only one entry point schema but %(numEntryPoints)s were found: %(entryPointNames)s'), modelObject=modelDocument, numEntryPoints=_numSchemaRefs, entryPointNames=', '.join(sorted(schemaRefFileNames))) ### check entry point names appropriate for filing indicator (DPM DB?) if len(schemaRefElts) != 1: modelXbrl.error("EBA.2.3", _('Any reported XBRL instance document MUST contain only one xbrli:xbrl/link:schemaRef node, but %(entryPointCount)s.'), modelObject=schemaRefElts, entryPointCount=len(schemaRefElts)) # non-streaming EBA checks if not getattr(modelXbrl, "isStreamingMode", False): val.qnReportedCurrency = None if val.isEIOPA_2_0_1 and qnMetReportingCurrency in modelXbrl.factsByQname: for _multiCurrencyFact in modelXbrl.factsByQname[qnMetReportingCurrency]: # multi-currency fact val.qnReportedCurrency = _multiCurrencyFact.xValue break validateFacts(val, modelXbrl.facts) # check sum of fact md5s (otherwise checked in streaming process) xbrlFactsCheckVersion = None expectedSumOfFactMd5s = None for pi in modelDocument.xmlRootElement.getchildren(): if isinstance(pi, etree._ProcessingInstruction) and pi.target == "xbrl-facts-check": _match = re.search("([\\w-]+)=[\"']([^\"']+)[\"']", pi.text) if _match: _matchGroups = _match.groups() if len(_matchGroups) == 2: if _matchGroups[0] == "version": xbrlFactsCheckVersion = _matchGroups[1] elif _matchGroups[0] == "sum-of-fact-md5s": try: expectedSumOfFactMd5s = Md5Sum(_matchGroups[1]) except ValueError: modelXbrl.error("EIOPA:xbrlFactsCheckError", _("Invalid sum-of-md5s %(sumOfMd5)s"), modelObject=modelXbrl, sumOfMd5=_matchGroups[1]) if xbrlFactsCheckVersion and expectedSumOfFactMd5s: sumOfFactMd5s = Md5Sum() for f in modelXbrl.factsInInstance: sumOfFactMd5s += f.md5sum if sumOfFactMd5s != expectedSumOfFactMd5s: modelXbrl.warning("EIOPA:xbrlFactsCheckWarning", _("XBRL facts sum of md5s expected %(expectedMd5)s not matched to actual sum %(actualMd5Sum)s"), modelObject=modelXbrl, expectedMd5=expectedSumOfFactMd5s, actualMd5Sum=sumOfFactMd5s) else: modelXbrl.info("info", _("Successful XBRL facts sum of md5s."), modelObject=modelXbrl) if any(badError in modelXbrl.errors for badError in ("EBA.2.1", "EIOPA.2.1", "EIOPA.S.1.5.a/EIOPA.S.1.5.b")): pass # skip checking filingIndicators if bad errors elif not val.filingIndicators: modelXbrl.error(("EBA.1.6", "EIOPA.1.6.a"), _('Missing filing indicators. Reported XBRL instances MUST include appropriate (positive) filing indicator elements'), modelObject=modelDocument) elif all(filed == False for filed in val.filingIndicators.values()): modelXbrl.error(("EBA.1.6", "EIOPA.1.6.a"), _('All filing indicators are filed="false". Reported XBRL instances MUST include appropriate (positive) filing indicator elements'), modelObject=modelDocument) if val.numFilingIndicatorTuples > 1: modelXbrl.warning(("EBA.1.6.2", "EIOPA.1.6.2"), _('Multiple filing indicators tuples when not in streaming mode (info).'), modelObject=modelXbrl.factsByQname[qnFIndicators]) if len(val.cntxDates) > 1: modelXbrl.error(("EBA.2.13","EIOPA.2.13"), _('Contexts must have the same date: %(dates)s.'), # when streaming values are no longer available, but without streaming they can be logged modelObject=set(_cntx for _cntxs in val.cntxDates.values() for _cntx in _cntxs), dates=', '.join(XmlUtil.dateunionValue(_dt, subtractOneDay=True) for _dt in val.cntxDates.keys())) if val.unusedCntxIDs: if val.isEIOPA_2_0_1: modelXbrl.error("EIOPA.2.7", _('Unused xbrli:context nodes MUST NOT be present in the instance: %(unusedContextIDs)s.'), modelObject=[modelXbrl.contexts[unusedCntxID] for unusedCntxID in val.unusedCntxIDs if unusedCntxID in modelXbrl.contexts], unusedContextIDs=", ".join(sorted(val.unusedCntxIDs))) else: modelXbrl.warning(("EBA.2.7", "EIOPA.2.7"), _('Unused xbrli:context nodes SHOULD NOT be present in the instance: %(unusedContextIDs)s.'), modelObject=[modelXbrl.contexts[unusedCntxID] for unusedCntxID in val.unusedCntxIDs if unusedCntxID in modelXbrl.contexts], unusedContextIDs=", ".join(sorted(val.unusedCntxIDs))) if len(val.cntxEntities) > 1: modelXbrl.error(("EBA.2.9", "EIOPA.2.9"), _('All entity identifiers and schemes MUST be the same, %(count)s found: %(entities)s.'), modelObject=modelDocument, count=len(val.cntxEntities), entities=", ".join(sorted(str(cntxEntity) for cntxEntity in val.cntxEntities))) for _scheme, _LEI in val.cntxEntities: if (_scheme in ("http://standards.iso.org/iso/17442", "http://standard.iso.org/iso/17442", "LEI") or (not val.isEIOPAfullVersion and _scheme == "PRE-LEI")): result = LeiUtil.checkLei(_LEI) if result == LeiUtil.LEI_INVALID_LEXICAL: modelXbrl.error("EIOPA.S.2.8.c", _("Context has lexically invalid LEI %(lei)s."), modelObject=modelDocument, lei=_LEI) elif result == LeiUtil.LEI_INVALID_CHECKSUM: modelXbrl.error("EIOPA.S.2.8.c", _("Context has LEI checksum error in %(lei)s."), modelObject=modelDocument, lei=_LEI) if _scheme == "http://standard.iso.org/iso/17442": modelXbrl.warning("EIOPA.S.2.8.c", _("Warning, context has entity scheme %(scheme)s should be plural: http://standards.iso.org/iso/17442."), modelObject=modelDocument, scheme=_scheme) elif _scheme == "SC": pass # anything is ok for Specific Code else: modelXbrl.error("EIOPA.S.2.8.c", _("Context has unrecognized entity scheme %(scheme)s."), modelObject=modelDocument, scheme=_scheme) if val.unusedUnitIDs: if val.isEIOPA_2_0_1: modelXbrl.error("EIOPA.2.22", _('Unused xbrli:unit nodes MUST NOT be present in the instance: %(unusedUnitIDs)s.'), modelObject=[modelXbrl.units[unusedUnitID] for unusedUnitID in val.unusedUnitIDs if unusedUnitID in modelXbrl.units], unusedUnitIDs=", ".join(sorted(val.unusedUnitIDs))) else: modelXbrl.warning(("EBA.2.22", "EIOPA.2.22"), _('Unused xbrli:unit nodes SHOULD NOT be present in the instance: %(unusedUnitIDs)s.'), modelObject=[modelXbrl.units[unusedUnitID] for unusedUnitID in val.unusedUnitIDs if unusedUnitID in modelXbrl.units], unusedUnitIDs=", ".join(sorted(val.unusedUnitIDs))) if len(val.currenciesUsed) > 1: modelXbrl.error(("EBA.3.1","EIOPA.3.1"), _("There MUST be only one currency but %(numCurrencies)s were found: %(currencies)s.'"), modelObject=val.currenciesUsed.values(), numCurrencies=len(val.currenciesUsed), currencies=", ".join(str(c) for c in val.currenciesUsed.keys())) elif val.isEIOPA_2_0_1 and any(_measure.localName != val.reportingCurrency for _measure in val.currenciesUsed.keys()): modelXbrl.error("EIOPA.3.1", _("There MUST be only one currency but reporting currency %(reportingCurrency)s differs from unit currencies: %(currencies)s.'"), modelObject=val.currenciesUsed.values(), reportingCurrency=val.reportingCurrency, currencies=", ".join(str(c) for c in val.currenciesUsed.keys())) if val.prefixesUnused: modelXbrl.warning(("EBA.3.4", "EIOPA.3.4"), _("There SHOULD be no unused prefixes but these were declared: %(unusedPrefixes)s.'"), modelObject=modelDocument, unusedPrefixes=', '.join(sorted(val.prefixesUnused))) for ns, prefixes in val.namespacePrefixesUsed.items(): nsDocs = modelXbrl.namespaceDocs.get(ns) if nsDocs: for nsDoc in nsDocs: nsDocPrefix = XmlUtil.xmlnsprefix(nsDoc.xmlRootElement, ns) if any(prefix != nsDocPrefix for prefix in prefixes if prefix is not None): modelXbrl.warning(("EBA.3.5", "EIOPA.3.5"), _("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s"), modelObject=modelDocument, namespace=ns, declaredPrefix=nsDocPrefix, foundPrefixes=', '.join(sorted(prefixes - {None}))) elif ns in CANONICAL_PREFIXES and any(prefix != CANONICAL_PREFIXES[ns] for prefix in prefixes if prefix is not None): modelXbrl.warning(("EBA.3.5", "EIOPA.3.5"), _("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s"), modelObject=modelDocument, namespace=ns, declaredPrefix=CANONICAL_PREFIXES[ns], foundPrefixes=', '.join(sorted(prefixes - {None}))) modelXbrl.profileActivity(_statusMsg, minTimeToShow=0.0) modelXbrl.modelManager.showStatus(None) del val.prefixNamespace, val.namespacePrefix, val.idObjects, val.typedDomainElements del val.utrValidator, val.firstFact, val.footnotesRelationshipSet
def validateTestcase(self, testcase): self.modelXbrl.info("info", "Testcase", modelDocument=testcase) self.modelXbrl.viewModelObject(testcase.objectId()) if testcase.type in (Type.TESTCASESINDEX, Type.REGISTRY): for doc in sorted(testcase.referencesDocument.keys(), key=lambda doc: doc.uri): self.validateTestcase(doc) # testcases doc's are sorted by their uri (file names), e.g., for formula elif hasattr(testcase, "testcaseVariations"): for modelTestcaseVariation in testcase.testcaseVariations: # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) # is this a versioning report? resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance resultIsTaxonomyPackage = modelTestcaseVariation.resultIsTaxonomyPackage formulaOutputInstance = None inputDTSes = defaultdict(list) baseForElement = testcase.baseForElement(modelTestcaseVariation) # try to load instance document self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s - %(description)s"), modelObject=modelTestcaseVariation, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, expected=modelTestcaseVariation.expected, description=modelTestcaseVariation.description) if self.modelXbrl.modelManager.formulaOptions.testcaseResultsCaptureWarnings: errorCaptureLevel = logging._checkLevel("WARNING") else: errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY parameters = modelTestcaseVariation.parameters.copy() for readMeFirstUri in modelTestcaseVariation.readMeFirstUris: if isinstance(readMeFirstUri,tuple): # dtsName is for formula instances, but is from/to dts if versioning dtsName, readMeFirstUri = readMeFirstUri elif resultIsVersioningReport: if inputDTSes: dtsName = "to" else: dtsName = "from" else: dtsName = None if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document if dtsName in inputDTSes: dtsName = inputDTSes[dtsName] else: modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager, Type.DTSENTRIES, self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement), isEntry=True, errorCaptureLevel=errorCaptureLevel) DTSdoc = modelXbrl.modelDocument DTSdoc.inDTS = True doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement) if doc is not None: DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement) #fake import doc.inDTS = True elif resultIsTaxonomyPackage: from arelle import PackageManager, PrototypeInstanceObject dtsName = readMeFirstUri modelXbrl = PrototypeInstanceObject.XbrlPrototype(self.modelXbrl.modelManager, readMeFirstUri) PackageManager.packageInfo(self.modelXbrl.modelManager.cntlr, readMeFirstUri, reload=True, errors=modelXbrl.errors) else: # not a multi-schemaRef versioning report if self.useFileSource.isArchive: modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, readMeFirstUri, _("validating"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) else: # need own file source, may need instance discovery filesource = FileSource.FileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr) if filesource and not filesource.selection and filesource.isArchive: for _archiveFile in filesource.dir or (): # find instance document in archive filesource.select(_archiveFile) if ModelDocument.Type.identify(filesource, filesource.url) in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL): break # use this selection modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, filesource, _("validating"), base=baseForElement, errorCaptureLevel=errorCaptureLevel) modelXbrl.isTestcaseVariation = True if modelXbrl.modelDocument is None: modelXbrl.error("arelle:notLoaded", _("Variation %(id)s %(name)s readMeFirst document not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) self.determineNotLoadedTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif resultIsVersioningReport or resultIsTaxonomyPackage: inputDTSes[dtsName] = modelXbrl elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl) self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif testcase.type == Type.REGISTRYTESTCASE: self.instValidator.validate(modelXbrl) # required to set up dimensions, etc self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id, modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest) self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) self.instValidator.close() modelXbrl.close() else: inputDTSes[dtsName].append(modelXbrl) # validate except for formulas _hasFormulae = modelXbrl.hasFormulae modelXbrl.hasFormulae = False try: for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"): pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation) self.instValidator.validate(modelXbrl, parameters) for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"): pluginXbrlMethod(self.modelXbrl, modelXbrl) except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase variation validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) modelXbrl.hasFormulae = _hasFormulae if resultIsVersioningReport and modelXbrl.modelDocument: versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl( modelTestcaseVariation.versioningReportUri, baseForElement) if os.path.exists(versReportFile): #validate existing modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report")) if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport) self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors) modelVersReport.close() elif len(inputDTSes) == 2: ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes( versReportFile, inputDTSes["from"], inputDTSes["to"]) modelTestcaseVariation.status = "generated" else: modelXbrl.error("arelle:notLoaded", _("Variation %(id)s %(name)s input DTSes not loaded, unable to generate versioning report: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) modelTestcaseVariation.status = "failed" for inputDTS in inputDTSes.values(): inputDTS.close() del inputDTSes # dereference elif resultIsTaxonomyPackage: self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif inputDTSes: # validate schema, linkbase, or instance modelXbrl = inputDTSes[None][0] expectedDataFiles = set(modelXbrl.modelManager.cntlr.webCache.normalizeUrl(uri, baseForElement) for d in modelTestcaseVariation.dataUris.values() for uri in d if not UrlUtil.isAbsolute(uri)) foundDataFiles = set() variationBase = os.path.dirname(baseForElement) for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters if dtsName: # named instance parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes) elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula for _inputDTS in inputDTS: for docUrl, doc in _inputDTS.urlDocs.items(): if docUrl.startswith(variationBase) and not doc.type == Type.INLINEXBRLDOCUMENTSET: if getattr(doc,"loadedFromXbrlFormula", False): # may have been sourced from xf file if docUrl.replace("-formula.xml", ".xf") in expectedDataFiles: docUrl = docUrl.replace("-formula.xml", ".xf") foundDataFiles.add(docUrl) if expectedDataFiles - foundDataFiles: modelXbrl.info("arelle:testcaseDataNotUsed", _("Variation %(id)s %(name)s data files not used: %(missingDataFiles)s"), modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id, missingDataFiles=", ".join(sorted(os.path.basename(f) for f in expectedDataFiles - foundDataFiles))) if foundDataFiles - expectedDataFiles: modelXbrl.info("arelle:testcaseDataUnexpected", _("Variation %(id)s %(name)s files not in variation data: %(unexpectedDataFiles)s"), modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id, unexpectedDataFiles=", ".join(sorted(os.path.basename(f) for f in foundDataFiles - expectedDataFiles))) if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable: try: RenderingEvaluator.init(modelXbrl) except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase RenderingEvaluator.init exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) modelXbrlHasFormulae = modelXbrl.hasFormulae if modelXbrlHasFormulae: try: # validate only formulae self.instValidator.parameters = parameters ValidateFormula.validate(self.instValidator) except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset: for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"): pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri) infoset = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultInfosetUri, _("loading result infoset"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) if infoset.modelDocument is None: modelXbrl.error("arelle:notLoaded", _("Variation %(id)s %(name)s result infoset not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstance)) modelTestcaseVariation.status = "result infoset not loadable" else: # check infoset ValidateInfoset.validate(self.instValidator, modelXbrl, infoset) infoset.close() if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset: # diff (or generate) table infoset resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement) if not any(alternativeValidation(modelXbrl, resultTableUri) for alternativeValidation in pluginClassMethods("Validate.TableInfoset")): try: ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase table linkbase validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) self.instValidator.close() extraErrors = [] for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"): pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors) self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual): # if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost) self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters) self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors) if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance formulaOutputInstance = modelXbrl.formulaOutputInstance modelXbrl.formulaOutputInstance = None # prevent it from being closed now self.instValidator.close() compareIxResultInstance = (modelXbrl.modelDocument.type in (Type.INLINEXBRL, Type.INLINEXBRLDOCUMENTSET) and modelTestcaseVariation.resultXbrlInstanceUri is not None) if compareIxResultInstance: formulaOutputInstance = modelXbrl # compare modelXbrl to generated output instance errMsgPrefix = "ix" else: # delete input instances before formula output comparision for inputDTSlist in inputDTSes.values(): for inputDTS in inputDTSlist: inputDTS.close() del inputDTSes # dereference errMsgPrefix = "formula" if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument: _matchExpectedResultIDs = not modelXbrlHasFormulae # formula restuls have inconsistent IDs expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultXbrlInstanceUri, _("loading expected result XBRL instance"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) if expectedInstance.modelDocument is None: self.modelXbrl.error("{}:expectedResultNotLoaded".format(errMsgPrefix), _("Testcase \"%(name)s\" %(id)s expected result instance not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstanceUri), messageCodes=("formula:expectedResultNotLoaded","ix:expectedResultNotLoaded")) modelTestcaseVariation.status = "result not loadable" else: # compare facts if len(expectedInstance.facts) != len(formulaOutputInstance.facts): formulaOutputInstance.error("{}:resultFactCounts".format(errMsgPrefix), _("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"), modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts), expectedFacts=len(expectedInstance.facts), messageCodes=("formula:resultFactCounts","ix:resultFactCounts")) else: formulaOutputFootnotesRelSet = ModelRelationshipSet(formulaOutputInstance, "XBRL-footnotes") expectedFootnotesRelSet = ModelRelationshipSet(expectedInstance, "XBRL-footnotes") def factFootnotes(fact, footnotesRelSet): footnotes = {} footnoteRels = footnotesRelSet.fromModelObject(fact) if footnoteRels: # most process rels in same order between two instances, use labels to sort for i, footnoteRel in enumerate(sorted(footnoteRels, key=lambda r: (r.fromLabel,r.toLabel))): modelObject = footnoteRel.toModelObject if isinstance(modelObject, ModelResource): xml = modelObject.viewText().strip() footnotes["Footnote {}".format(i+1)] = xml #re.sub(r'\s+', ' ', collapseWhitespace(modelObject.stringValue)) elif isinstance(modelObject, ModelFact): footnotes["Footnoted fact {}".format(i+1)] = \ "{} context: {} value: {}".format( modelObject.qname, modelObject.contextID, collapseWhitespace(modelObject.value)) return footnotes for expectedInstanceFact in expectedInstance.facts: unmatchedFactsStack = [] formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=_matchExpectedResultIDs, matchLang=False) #formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=True, matchLang=True) if formulaOutputFact is None: if unmatchedFactsStack: # get missing nested tuple fact, if possible missingFact = unmatchedFactsStack[-1] else: missingFact = expectedInstanceFact formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix), _("Output missing expected fact %(fact)s"), modelXbrl=missingFact, fact=missingFact.qname, messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing")) else: # compare footnotes expectedInstanceFactFootnotes = factFootnotes(expectedInstanceFact, expectedFootnotesRelSet) formulaOutputFactFootnotes = factFootnotes(formulaOutputFact, formulaOutputFootnotesRelSet) if (len(expectedInstanceFactFootnotes) != len(formulaOutputFactFootnotes) or set(expectedInstanceFactFootnotes.values()) != set(formulaOutputFactFootnotes.values())): formulaOutputInstance.error("{}:expectedFactFootnoteDifference".format(errMsgPrefix), _("Output expected fact %(fact)s expected footnotes %(footnotes1)s produced footnotes %(footnotes2)s"), modelXbrl=(formulaOutputFact,expectedInstanceFact), fact=expectedInstanceFact.qname, footnotes1=sorted(expectedInstanceFactFootnotes.items()), footnotes2=sorted(formulaOutputFactFootnotes.items()), messageCodes=("formula:expectedFactFootnoteDifference","ix:expectedFactFootnoteDifference")) # for debugging uncomment next line to save generated instance document # formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml") expectedInstance.close() del expectedInstance # dereference self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors) formulaOutputInstance.close() del formulaOutputInstance if compareIxResultInstance: for inputDTSlist in inputDTSes.values(): for inputDTS in inputDTSlist: inputDTS.close() del inputDTSes # dereference # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) _statusCounts = OrderedDict((("pass",0),("fail",0))) for tv in getattr(testcase, "testcaseVariations", ()): _statusCounts[tv.status] = _statusCounts.get(tv.status, 0) + 1 self.modelXbrl.info("arelle:testCaseResults", ", ".join("{}={}".format(k,c) for k, c in _statusCounts.items() if k)) self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def checkDTSdocument(val, modelDocument): modelXbrl = val.modelXbrl if modelDocument.type == ModelDocument.Type.INSTANCE: if not modelDocument.uri.endswith(".xbrl"): modelXbrl.warning("EBA.1.1", _('XBRL instance documents SHOULD use the extension ".xbrl" encoding but it is "%(extension)s"'), modelObject=modelDocument, xmlEncoding=os.path.splitext(modelDocument.basename)[1]) if modelDocument.documentEncoding.lower() not in ("utf-8", "utf-8-sig"): modelXbrl.error("EBA.1.4", _('XBRL instance documents MUST use "UTF-8" encoding but is "%(xmlEncoding)s"'), modelObject=modelDocument, xmlEncoding=modelDocument.documentEncoding) schemaRefElts = [] schemaRefFileNames = [] for doc, docRef in modelDocument.referencesDocument.items(): if docRef.referenceType == "href": if docRef.referringModelObject.localName == "schemaRef": schemaRefElts.append(docRef.referringModelObject) schemaRefFileNames.append(doc.basename) if not UrlUtil.isAbsolute(doc.uri): modelXbrl.error("EBA.2.2", _('The link:schemaRef element in submitted instances MUST resolve to the full published entry point URL: %(url)s.'), modelObject=docRef.referringModelObject, url=doc.uri) elif docRef.referringModelObject.localName == "linkbaseRef": modelXbrl.error("EBA.2.3", _('The link:linkbaseRef element is not allowed: %(fileName)s.'), modelObject=docRef.referringModelObject, fileName=doc.basename) if len(schemaRefFileNames) > 1: modelXbrl.error("EBA.1.5", _('XBRL instance documents MUST reference only one entry point schema but %(numEntryPoints)s were found: %(entryPointNames)s'), modelObject=modelDocument, numEntryPoints=len(schemaRefFileNames), entryPointNames=', '.join(sorted(schemaRefFileNames))) ### check entry point names appropriate for filing indicator (DPM DB?) if len(schemaRefElts) != 1: modelXbrl.error("EBA.2.3", _('Any reported XBRL instance document MUST contain only one xbrli:xbrl/link:schemaRef node, but %(entryPointCount)s.'), modelObject=schemaRefElts, entryPointCount=len(schemaRefElts)) filingIndicators = {} for fIndicator in modelXbrl.factsByQname[qnFilingIndicator]: if fIndicator.xValue in filingIndicators: modelXbrl.error("EBA.1.6.1", _('Multiple filing indicators facts for indicator %(filingIndicator)s.'), modelObject=(fIndicator, filingIndicators[filingIndicators]), filingIndicator=fIndicator.xValue) filingIndicators[fIndicator.xValue] = fIndicator if not filingIndicators: modelXbrl.error("EBA.1.6", _('Missing filing indicators. Reported XBRL instances MUST include appropriate filing indicator elements'), modelObject=modelDocument) numFilingIndicatorTuples = len(modelXbrl.factsByQname[qnFilingIndicators]) if numFilingIndicatorTuples > 1 and not getattr(modelXbrl, "isStreamingMode", False): modelXbrl.info("EBA.1.6.2", _('Multiple filing indicators tuples when not in streaming mode (info).'), modelObject=modelXbrl.factsByQname[qnFilingIndicators]) # note EBA 2.1 is in ModelDocument.py cntxIDs = set() cntxEntities = set() timelessDatePattern = re.compile(r"\s*([0-9]{4})-([0-9]{2})-([0-9]{2})\s*$") for cntx in modelXbrl.contexts.values(): cntxIDs.add(cntx.id) cntxEntities.add(cntx.entityIdentifier) dateElts = XmlUtil.descendants(cntx, XbrlConst.xbrli, ("startDate","endDate","instant")) if any(not timelessDatePattern.match(e.textValue) for e in dateElts): modelXbrl.error("EBA.2.10", _('Period dates must be whole dates without time or timezone: %(dates)s.'), modelObject=cntx, dates=", ".join(e.text for e in dateElts)) if cntx.isForeverPeriod: modelXbrl.error("EBA.2.11", _('Forever context period is not allowed.'), modelObject=cntx) if XmlUtil.hasChild(cntx, XbrlConst.xbrli, "segment"): modelXbrl.error("EBA.2.14", _("The segment element not allowed in context Id: %(context)s"), modelObject=cntx, context=cntx.contextID) for scenElt in XmlUtil.descendants(cntx, XbrlConst.xbrli, "scenario"): childTags = ", ".join([child.prefixedName for child in scenElt.iterchildren() if isinstance(child,ModelObject) and child.tag != "{http://xbrl.org/2006/xbrldi}explicitMember" and child.tag != "{http://xbrl.org/2006/xbrldi}typedMember"]) if len(childTags) > 0: modelXbrl.error("EBA.2.15", _("Scenario of context Id %(context)s has disallowed content: %(content)s"), modelObject=cntx, context=cntx.id, content=childTags) unusedCntxIDs = cntxIDs - {fact.contextID for fact in modelXbrl.facts} if unusedCntxIDs: modelXbrl.warning("EBA.2.7", _('Unused xbrli:context nodes SHOULD NOT be present in the instance: %(unusedContextIDs)s.'), modelObject=[modelXbrl.contexts[unusedCntxID] for unusedCntxID in unusedCntxIDs], unusedContextIDs=", ".join(sorted(unusedCntxIDs))) if len(cntxEntities) > 1: modelXbrl.warning("EBA.2.9", _('All entity identifiers and schemes must be the same, %(count)s found: %(entities)s.'), modelObject=modelDocument, count=len(cntxEntities), entities=", ".join(sorted(str(cntxEntity) for cntxEntity in cntxEntities))) otherFacts = {} # (contextHash, unitHash, xmlLangHash) : fact nilFacts = [] unitIDsUsed = set() currenciesUsed = {} for qname, facts in modelXbrl.factsByQname.items(): for f in facts: concept = f.concept k = (f.context.contextDimAwareHash if f.context is not None else None, f.unit.hash if f.unit is not None else None, hash(f.xmlLang)) if k not in otherFacts: otherFacts[k] = {f} else: matches = [o for o in otherFacts[k] if (f.context.isEqualTo(o.context) if f.context is not None and o.context is not None else True) and (f.unit.isEqualTo(o.unit) if f.unit is not None and o.unit is not None else True) and (f.xmlLang == o.xmlLang)] if matches: modelXbrl.error("EBA.2.16", _('Facts are duplicates %(fact)s and %(otherFacts)s.'), modelObject=[f] + matches, fact=f.qname, otherFacts=', '.join(str(f.qname) for f in matches)) else: otherFacts[k].add(f) if concept is not None and concept.isNumeric: if f.precision: modelXbrl.error("EBA.2.17", _("Numeric fact %(fact)s of context %(contextID)s has a precision attribute '%(precision)s'"), modelObject=f, fact=f.qname, contextID=f.contextID, precision=f.precision) '''' (not intended by EBA 2.18) if f.decimals and f.decimals != "INF" and not f.isNil and getattr(f,"xValid", 0) == 4: try: insignificance = insignificantDigits(f.xValue, decimals=f.decimals) if insignificance: # if not None, returns (truncatedDigits, insiginficantDigits) modelXbrl.error(("EFM.6.05.37", "GFM.1.02.26"), _("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s has nonzero digits in insignificant portion %(insignificantDigits)s."), modelObject=f1, fact=f1.qname, contextID=f1.contextID, decimals=f1.decimals, value=f1.xValue, truncatedDigits=insignificance[0], insignificantDigits=insignificance[1]) except (ValueError,TypeError): modelXbrl.error(("EFM.6.05.37", "GFM.1.02.26"), _("Fact %(fact)s of context %(contextID)s decimals %(decimals)s value %(value)s causes Value Error exception."), modelObject=f1, fact=f1.qname, contextID=f1.contextID, decimals=f1.decimals, value=f1.value) ''' unit = f.unit if concept.isMonetary and unit is not None and unit.measures[0]: currenciesUsed[unit.measures[0][0]] = unit if f.unitID is not None: unitIDsUsed.add(f.unitID) if f.isNil: nilFacts.append(f) if nilFacts: modelXbrl.warning("EBA.2.19", _('Nil facts SHOULD NOT be present in the instance: %(nilFacts)s.'), modelObject=nilFacts, nilFacts=", ".join(str(f.qname) for f in nilFacts)) unusedUnitIDs = modelXbrl.units.keys() - unitIDsUsed if unusedUnitIDs: modelXbrl.warning("EBA.2.21", _('Unused xbrli:unit nodes SHOULD NOT be present in the instance: %(unusedUnitIDs)s.'), modelObject=[modelXbrl.units[unusedUnitID] for unusedUnitID in unusedUnitIDs], unusedUnitIDs=", ".join(sorted(unusedUnitIDs))) unitHashes = {} for unit in modelXbrl.units.values(): h = hash(unit) if h in unitHashes and unit.isEqualTo(unitHashes[h]): modelXbrl.error("EBA.2.32", _("Units %(unit1)s and %(unit2)s have same measures.'"), modelObject=(unit, unitHashes[h]), unit1=unit.id, unit2=unitHashes[h].id) else: unitHashes[h] = unit if len(currenciesUsed) > 1: modelXbrl.error("EBA.3.1", _("There MUST be only one currency but %(numCurrencies)s were found: %(currencies)s.'"), modelObject=currenciesUsed.values(), numCurrencies=len(currenciesUsed), currencies=", ".join(str(c) for c in currenciesUsed.keys())) namespacePrefixesUsed = defaultdict(set) prefixesUnused = set(modelDocument.xmlRootElement.keys()).copy() for elt in modelDocument.xmlRootElement.iter(): if isinstance(elt, ModelObject): # skip comments and processing instructions namespacePrefixesUsed[elt.qname.namespaceURI].add(elt.qname.prefix) prefixesUnused.discard(elt.qname.prefix) if prefixesUnused: modelXbrl.warning("EBA.3.4", _("There SHOULD be no unused prefixes but these were declared: %(unusedPrefixes)s.'"), modelObject=modelDocument, unusedPrefixes=', '.join(sorted(prefixesUnused))) for ns, prefixes in namespacePrefixesUsed.items(): nsDocs = modelXbrl.namespaceDocs.get(ns) if nsDocs: for nsDoc in nsDocs: nsDocPrefix = XmlUtil.xmlnsprefix(nsDoc.xmlRootElement, ns) if any(prefix != nsDocPrefix for prefix in prefixes if prefix is not None): modelXbrl.warning("EBA.3.5", _("Prefix for namespace %(namespace)s is %(declaredPrefix)s but these were found %(foundPrefixes)s"), modelObject=modelDocument, namespace=ns, declaredPrefix=nsDocPrefix, foundPrefixes=', '.join(sorted(prefixes - {None})))