def backgroundProfileFormula(cntlr, profileReportFile, maxRunTime, excludeCompileTime): from arelle import Locale, XPathParser, ValidateXbrlDimensions, ValidateFormula # build grammar before profiling (if this is the first pass, so it doesn't count in profile statistics) XPathParser.initializeParser(cntlr.modelManager) # load dimension defaults ValidateXbrlDimensions.loadDimensionDefaults(cntlr.modelManager) import cProfile, pstats, sys, time # a minimal validation class for formula validator parameters that are needed class Validate: def __init__(self, modelXbrl, maxRunTime): self.modelXbrl = modelXbrl self.parameters = None self.validateSBRNL = False self.maxFormulaRunTime = maxRunTime def close(self): self.__dict__.clear() val = Validate(cntlr.modelManager.modelXbrl, maxRunTime) formulaOptions = val.modelXbrl.modelManager.formulaOptions if excludeCompileTime: startedAt = time.time() cntlr.addToLog(_("pre-compiling formulas before profiling")) val.validateFormulaCompileOnly = True ValidateFormula.validate(val) del val.validateFormulaCompileOnly cntlr.addToLog(Locale.format_string(cntlr.modelManager.locale, _("formula pre-compiling completed in %.2f secs"), time.time() - startedAt)) cntlr.addToLog(_("executing formulas for profiling")) else: cntlr.addToLog(_("compiling and executing formulas for profiling")) startedAt = time.time() statsFile = profileReportFile + ".bin" cProfile.runctx("ValidateFormula.validate(val)", globals(), locals(), statsFile) cntlr.addToLog(Locale.format_string(cntlr.modelManager.locale, _("formula profiling completed in %.2f secs"), time.time() - startedAt)) # dereference val val.close() # specify a file for log priorStdOut = sys.stdout sys.stdout = open(profileReportFile, "w") statObj = pstats.Stats(statsFile) statObj.strip_dirs() statObj.sort_stats("time") statObj.print_stats() statObj.print_callees() statObj.print_callers() sys.stdout.flush() sys.stdout.close() del statObj sys.stdout = priorStdOut os.remove(statsFile)
def validateTestcase(self, testcase): self.modelXbrl.info("info", "Testcase", modelDocument=testcase) self.modelXbrl.viewModelObject(testcase.objectId()) if hasattr(testcase, "testcaseVariations"): for modelTestcaseVariation in testcase.testcaseVariations: # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) # is this a versioning report? resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance resultIsTaxonomyPackage = modelTestcaseVariation.resultIsTaxonomyPackage formulaOutputInstance = None inputDTSes = defaultdict(list) baseForElement = testcase.baseForElement(modelTestcaseVariation) # try to load instance document self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s - %(description)s"), modelObject=modelTestcaseVariation, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, expected=modelTestcaseVariation.expected, description=modelTestcaseVariation.description) errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY parameters = modelTestcaseVariation.parameters.copy() for readMeFirstUri in modelTestcaseVariation.readMeFirstUris: if isinstance(readMeFirstUri,tuple): # dtsName is for formula instances, but is from/to dts if versioning dtsName, readMeFirstUri = readMeFirstUri elif resultIsVersioningReport: if inputDTSes: dtsName = "to" else: dtsName = "from" else: dtsName = None if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document if dtsName in inputDTSes: dtsName = inputDTSes[dtsName] else: modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager, Type.DTSENTRIES, self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement), isEntry=True, errorCaptureLevel=errorCaptureLevel) DTSdoc = modelXbrl.modelDocument DTSdoc.inDTS = True doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement) if doc is not None: DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement) #fake import doc.inDTS = True elif resultIsTaxonomyPackage: from arelle import PackageManager, PrototypeInstanceObject dtsName = readMeFirstUri modelXbrl = PrototypeInstanceObject.XbrlPrototype(self.modelXbrl.modelManager, readMeFirstUri) PackageManager.packageInfo(self.modelXbrl.modelManager.cntlr, readMeFirstUri, reload=True, errors=modelXbrl.errors) else: # not a multi-schemaRef versioning report if self.useFileSource.isArchive: modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, readMeFirstUri, _("validating"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) else: # need own file source, may need instance discovery filesource = FileSource.FileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr) if filesource and not filesource.selection and filesource.isArchive: for _archiveFile in filesource.dir: # find instance document in archive filesource.select(_archiveFile) if ModelDocument.Type.identify(filesource, filesource.url) in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL): break # use this selection modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, filesource, _("validating"), base=baseForElement, errorCaptureLevel=errorCaptureLevel) modelXbrl.isTestcaseVariation = True if modelXbrl.modelDocument is None: modelXbrl.error("arelle:notLoaded", _("Testcase %(id)s %(name)s document not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) self.determineNotLoadedTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif resultIsVersioningReport or resultIsTaxonomyPackage: inputDTSes[dtsName] = modelXbrl elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl) self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif testcase.type == Type.REGISTRYTESTCASE: self.instValidator.validate(modelXbrl) # required to set up dimensions, etc self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id, modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest) self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) self.instValidator.close() modelXbrl.close() else: inputDTSes[dtsName].append(modelXbrl) # validate except for formulas _hasFormulae = modelXbrl.hasFormulae modelXbrl.hasFormulae = False try: for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"): pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation) self.instValidator.validate(modelXbrl, parameters) for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"): pluginXbrlMethod(self.modelXbrl, modelXbrl) except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase variation validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) modelXbrl.hasFormulae = _hasFormulae if resultIsVersioningReport and modelXbrl.modelDocument: versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl( modelTestcaseVariation.versioningReportUri, baseForElement) if os.path.exists(versReportFile): #validate existing modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report")) if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport) self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors) modelVersReport.close() elif len(inputDTSes) == 2: ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes( versReportFile, inputDTSes["from"], inputDTSes["to"]) modelTestcaseVariation.status = "generated" else: modelXbrl.error("arelle:notLoaded", _("Testcase %(id)s %(name)s DTSes not loaded, unable to generate versioning report: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) modelTestcaseVariation.status = "failed" for inputDTS in inputDTSes.values(): inputDTS.close() del inputDTSes # dereference elif resultIsTaxonomyPackage: self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif inputDTSes: # validate schema, linkbase, or instance modelXbrl = inputDTSes[None][0] for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters if dtsName: # named instance parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes) elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable: RenderingEvaluator.init(modelXbrl) if modelXbrl.hasFormulae: try: # validate only formulae self.instValidator.parameters = parameters ValidateFormula.validate(self.instValidator) except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset: for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"): pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri) infoset = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultInfosetUri, _("loading result infoset"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) if infoset.modelDocument is None: modelXbrl.error("arelle:notLoaded", _("Testcase %(id)s %(name)s result infoset not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstance)) modelTestcaseVariation.status = "result infoset not loadable" else: # check infoset ValidateInfoset.validate(self.instValidator, modelXbrl, infoset) infoset.close() if modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset: # diff (or generate) table infoset resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement) if not any(alternativeValidation(modelXbrl, resultTableUri) for alternativeValidation in pluginClassMethods("Validate.TableInfoset")): ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files self.instValidator.close() extraErrors = [] for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"): pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors) self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual): # if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost) self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters) self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors) if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance formulaOutputInstance = modelXbrl.formulaOutputInstance modelXbrl.formulaOutputInstance = None # prevent it from being closed now self.instValidator.close() compareIxResultInstance = getattr(modelXbrl, "extractedInlineInstance", False) and modelTestcaseVariation.resultXbrlInstanceUri if compareIxResultInstance: formulaOutputInstance = modelXbrl # compare modelXbrl to generated output instance errMsgPrefix = "ix" else: # delete input instances before formula output comparision for inputDTSlist in inputDTSes.values(): for inputDTS in inputDTSlist: inputDTS.close() del inputDTSes # dereference errMsgPrefix = "formula" if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument: expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultXbrlInstanceUri, _("loading expected result XBRL instance"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) if expectedInstance.modelDocument is None: self.modelXbrl.error("{}:expectedResultNotLoaded".format(errMsgPrefix), _("Testcase %(id)s %(name)s expected result instance not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstanceUri), messageCodes=("formula:expectedResultNotLoaded","ix:expectedResultNotLoaded")) modelTestcaseVariation.status = "result not loadable" else: # compare facts if len(expectedInstance.facts) != len(formulaOutputInstance.facts): formulaOutputInstance.error("{}:resultFactCounts".format(errMsgPrefix), _("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"), modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts), expectedFacts=len(expectedInstance.facts), messageCodes=("formula:resultFactCounts","ix:resultFactCounts")) else: formulaOutputFootnotesRelSet = ModelRelationshipSet(formulaOutputInstance, "XBRL-footnotes") expectedFootnotesRelSet = ModelRelationshipSet(expectedInstance, "XBRL-footnotes") def factFootnotes(fact, footnotesRelSet): footnotes = [] footnoteRels = footnotesRelSet.fromModelObject(fact) if footnoteRels: # most process rels in same order between two instances, use labels to sort for i, footnoteRel in enumerate(sorted(footnoteRels, key=lambda r: (r.fromLabel,r.toLabel))): modelObject = footnoteRel.toModelObject if isinstance(modelObject, ModelResource): xml = modelObject.viewText().strip() footnotes.append("Footnote {}: {}".format( i+1, # compare footnote with HTML serialized xml, #re.sub(r'\s+', ' ', collapseWhitespace(modelObject.stringValue)) )) elif isinstance(modelObject, ModelFact): footnotes.append("Footnoted fact {}: {} context: {} value: {}".format( i+1, modelObject.qname, modelObject.contextID, collapseWhitespace(modelObject.value))) return footnotes for expectedInstanceFact in expectedInstance.facts: unmatchedFactsStack = [] formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True) if formulaOutputFact is None: if unmatchedFactsStack: # get missing nested tuple fact, if possible missingFact = unmatchedFactsStack[-1] else: missingFact = expectedInstanceFact formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix), _("Output missing expected fact %(fact)s"), modelXbrl=missingFact, fact=missingFact.qname, messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing")) else: # compare footnotes expectedInstanceFactFootnotes = factFootnotes(expectedInstanceFact, expectedFootnotesRelSet) formulaOutputFactFootnotes = factFootnotes(formulaOutputFact, formulaOutputFootnotesRelSet) if expectedInstanceFactFootnotes != formulaOutputFactFootnotes: formulaOutputInstance.error("{}:expectedFactFootnoteDifference".format(errMsgPrefix), _("Output expected fact %(fact)s expected footnotes %(footnotes1)s produced footnotes %(footnotes2)s"), modelXbrl=(formulaOutputFact,expectedInstanceFact), fact=expectedInstanceFact.qname, footnotes1=expectedInstanceFactFootnotes, footnotes2=formulaOutputFactFootnotes, messageCodes=("formula:expectedFactFootnoteDifference","ix:expectedFactFootnoteDifference")) # for debugging uncomment next line to save generated instance document # formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml") expectedInstance.close() del expectedInstance # dereference self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors) formulaOutputInstance.close() del formulaOutputInstance if compareIxResultInstance: for inputDTSlist in inputDTSes.values(): for inputDTS in inputDTSlist: inputDTS.close() del inputDTSes # dereference # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def validateTestcase(self, testcase): self.modelXbrl.info("info", "Testcase", modelDocument=testcase) self.modelXbrl.viewModelObject(testcase.objectId()) if hasattr(testcase, "testcaseVariations"): for modelTestcaseVariation in testcase.testcaseVariations: # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) # is this a versioning report? resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance formulaOutputInstance = None inputDTSes = defaultdict(list) baseForElement = testcase.baseForElement(modelTestcaseVariation) # try to load instance document self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s - %(description)s"), modelObject=modelTestcaseVariation, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, expected=modelTestcaseVariation.expected, description=modelTestcaseVariation.description) errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY parameters = modelTestcaseVariation.parameters.copy() for readMeFirstUri in modelTestcaseVariation.readMeFirstUris: if isinstance(readMeFirstUri,tuple): # dtsName is for formula instances, but is from/to dts if versioning dtsName, readMeFirstUri = readMeFirstUri elif resultIsVersioningReport: if inputDTSes: dtsName = "to" else: dtsName = "from" else: dtsName = None if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document if dtsName in inputDTSes: dtsName = inputDTSes[dtsName] else: modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager, Type.DTSENTRIES, self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement), isEntry=True, errorCaptureLevel=errorCaptureLevel) DTSdoc = modelXbrl.modelDocument DTSdoc.inDTS = True doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement) if doc is not None: DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement) #fake import doc.inDTS = True else: # not a multi-schemaRef versioning report if self.useFileSource.isArchive: modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, readMeFirstUri, _("validating"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) else: # need own file source, may need instance discovery filesource = FileSource.FileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr) if filesource and not filesource.selection and filesource.isArchive: for _archiveFile in filesource.dir: # find instance document in archive filesource.select(_archiveFile) if ModelDocument.Type.identify(filesource, filesource.url) in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL): break # use this selection modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, filesource, _("validating"), base=baseForElement, errorCaptureLevel=errorCaptureLevel) if modelXbrl.modelDocument is None: self.modelXbrl.error("arelle:notLoaded", _("Testcase %(id)s %(name)s document not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) modelXbrl.close() self.determineNotLoadedTestStatus(modelTestcaseVariation) elif resultIsVersioningReport: inputDTSes[dtsName] = modelXbrl elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl) self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif testcase.type == Type.REGISTRYTESTCASE: self.instValidator.validate(modelXbrl) # required to set up dimensions, etc self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id, modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest) self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) self.instValidator.close() modelXbrl.close() else: inputDTSes[dtsName].append(modelXbrl) # validate except for formulas _hasFormulae = modelXbrl.hasFormulae modelXbrl.hasFormulae = False try: for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"): pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation) self.instValidator.validate(modelXbrl, parameters) for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"): pluginXbrlMethod(self.modelXbrl, modelXbrl) except Exception as err: self.modelXbrl.error("exception", _("Testcase variation validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) modelXbrl.hasFormulae = _hasFormulae if resultIsVersioningReport and modelXbrl.modelDocument: versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl( modelTestcaseVariation.versioningReportUri, baseForElement) if os.path.exists(versReportFile): #validate existing modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report")) if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport) self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors) modelVersReport.close() elif len(inputDTSes) == 2: ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes( versReportFile, inputDTSes["from"], inputDTSes["to"]) modelTestcaseVariation.status = "generated" else: self.modelXbrl.error("arelle:notLoaded", _("Testcase %(id)s %(name)s DTSes not loaded, unable to generate versioning report: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) modelTestcaseVariation.status = "failed" for inputDTS in inputDTSes.values(): inputDTS.close() del inputDTSes # dereference elif inputDTSes: # validate schema, linkbase, or instance modelXbrl = inputDTSes[None][0] for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters if dtsName: # named instance parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes) elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable: RenderingEvaluator.init(modelXbrl) if modelXbrl.hasFormulae: try: # validate only formulae self.instValidator.parameters = parameters ValidateFormula.validate(self.instValidator) except Exception as err: self.modelXbrl.error("exception", _("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset: for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"): pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri) infoset = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultInfosetUri, _("loading result infoset"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) if infoset.modelDocument is None: self.modelXbrl.error("arelle:notLoaded", _("Testcase %(id)s %(name)s result infoset not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstance)) modelTestcaseVariation.status = "result infoset not loadable" else: # check infoset ValidateInfoset.validate(self.instValidator, modelXbrl, infoset) infoset.close() if modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset: # diff (or generate) table infoset resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement) if not any(alternativeValidation(modelXbrl, resultTableUri) for alternativeValidation in pluginClassMethods("Validate.TableInfoset")): ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files self.instValidator.close() extraErrors = [] for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"): pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors) self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual): # if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost) self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters) self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors) if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance formulaOutputInstance = modelXbrl.formulaOutputInstance modelXbrl.formulaOutputInstance = None # prevent it from being closed now self.instValidator.close() for inputDTSlist in inputDTSes.values(): for inputDTS in inputDTSlist: inputDTS.close() del inputDTSes # dereference if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument: expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultXbrlInstanceUri, _("loading expected result XBRL instance"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) if expectedInstance.modelDocument is None: self.modelXbrl.error("arelle:notLoaded", _("Testcase %(id)s %(name)s expected result instance not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstance)) modelTestcaseVariation.status = "result not loadable" else: # compare facts if len(expectedInstance.facts) != len(formulaOutputInstance.facts): formulaOutputInstance.error("formula:resultFactCounts", _("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"), modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts), expectedFacts=len(expectedInstance.facts)) else: for fact in expectedInstance.facts: unmatchedFactsStack = [] if formulaOutputInstance.matchFact(fact, unmatchedFactsStack, deemP0inf=True) is None: if unmatchedFactsStack: # get missing nested tuple fact, if possible missingFact = unmatchedFactsStack[-1] else: missingFact = fact formulaOutputInstance.error("formula:expectedFactMissing", _("Formula output missing expected fact %(fact)s"), modelXbrl=missingFact, fact=missingFact.qname) # for debugging uncomment next line to save generated instance document # formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml") expectedInstance.close() del expectedInstance # dereference self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors) formulaOutputInstance.close() del formulaOutputInstance # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def init(modelXbrl): # setup modelXbrl for rendering evaluation # dimension defaults required in advance of validation from arelle import ValidateXbrlDimensions, ValidateFormula, ModelDocument ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) hasXbrlTables = False # validate table linkbase dimensions for baseSetKey in modelXbrl.baseSets.keys(): arcrole, ELR, linkqname, arcqname = baseSetKey if ELR and linkqname and arcqname and XbrlConst.isTableRenderingArcrole( arcrole): ValidateFormula.checkBaseSet( modelXbrl, arcrole, ELR, modelXbrl.relationshipSet(arcrole, ELR, linkqname, arcqname)) if arcrole in (XbrlConst.tableBreakdown, XbrlConst.tableBreakdownMMDD, XbrlConst.tableBreakdown201305, XbrlConst.tableBreakdown201301, XbrlConst.tableAxis2011): hasXbrlTables = True # provide context for view if modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE: instance = None # use instance of the entry pont else: # need dummy instance instance = ModelDocument.create( modelXbrl, ModelDocument.Type.INSTANCE, "dummy.xml", # fake URI and fake schemaRef ("http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd", )) if hasXbrlTables: # formula processor is needed for 2011 XBRL tables but not for 2010 Eurofiling tables modelXbrl.rendrCntx = XPathContext.create(modelXbrl, instance) modelXbrl.profileStat(None) # setup fresh parameters from formula options modelXbrl.parameters = modelXbrl.modelManager.formulaOptions.typedParameters( ) # validate parameters and custom function signatures ValidateFormula.validate(modelXbrl, xpathContext=modelXbrl.rendrCntx, parametersOnly=True, statusMsg=_("compiling rendering tables")) # deprecated as of 2013-05-17 # check and extract message expressions into compilable programs for msgArcrole in (XbrlConst.tableDefinitionNodeMessage201301, XbrlConst.tableDefinitionNodeSelectionMessage201301, XbrlConst.tableAxisMessage2011, XbrlConst.tableAxisSelectionMessage2011): for msgRel in modelXbrl.relationshipSet( msgArcrole).modelRelationships: ValidateFormula.checkMessageExpressions( modelXbrl, msgRel.toModelObject) # compile and validate tables for modelTable in modelXbrl.modelRenderingTables: modelTable.fromInstanceQnames = None # required if referred to by variables scope chaining modelTable.compile() hasNsWithAspectModel = modelTable.namespaceURI in ( XbrlConst.euRend, XbrlConst.table2011, XbrlConst.table201301, XbrlConst.table201305) # check aspectModel (attribute removed 2013-06, now always dimensional) if modelTable.aspectModel not in ( "non-dimensional", "dimensional") and hasNsWithAspectModel: modelXbrl.error( "xbrlte:unknownAspectModel", _("Table %(xlinkLabel)s, aspect model %(aspectModel)s not recognized" ), modelObject=modelTable, xlinkLabel=modelTable.xlinkLabel, aspectModel=modelTable.aspectModel) else: modelTable.priorAspectAxisDisposition = {} # check ordinate aspects against aspectModel oppositeAspectModel = ( _DICT_SET({'dimensional', 'non-dimensional'}) - _DICT_SET({modelTable.aspectModel})).pop() if hasNsWithAspectModel: uncoverableAspects = aspectModels[ oppositeAspectModel] - aspectModels[ modelTable.aspectModel] else: uncoverableAspects = () aspectsCovered = set() for tblAxisRel in modelXbrl.relationshipSet( (XbrlConst.tableBreakdown, XbrlConst.tableBreakdownMMDD, XbrlConst.tableBreakdown201305, XbrlConst.tableBreakdown201301, XbrlConst.tableAxis2011)).fromModelObject(modelTable): breakdownAspectsCovered = set() hasCoveredAspect = checkBreakdownDefinitionNode( modelXbrl, modelTable, tblAxisRel, tblAxisRel.axisDisposition, uncoverableAspects, breakdownAspectsCovered) ''' removed 2013-10 if not hasCoveredAspect: definitionNode = tblAxisRel.toModelObject modelXbrl.error("xbrlte:breakdownDefinesNoAspects", _("Breakdown %(xlinkLabel)s has no participating aspects"), modelObject=(modelTable,definitionNode), xlinkLabel=definitionNode.xlinkLabel, axis=definitionNode.localName) ''' aspectsCovered |= breakdownAspectsCovered checkBreakdownLeafNodeAspects(modelXbrl, modelTable, tblAxisRel, set(), breakdownAspectsCovered) if Aspect.CONCEPT not in aspectsCovered and not hasNsWithAspectModel: modelXbrl.error( "xbrlte:tableMissingConceptAspect", _("Table %(xlinkLabel)s does not include the concept aspect as one of its participating aspects" ), modelObject=modelTable, xlinkLabel=modelTable.xlinkLabel) del modelTable.priorAspectAxisDisposition # check for table-parameter name clash parameterNames = {} for tblParamRel in modelXbrl.relationshipSet( (XbrlConst.tableParameter, XbrlConst.tableParameterMMDD )).fromModelObject(modelTable): parameterName = tblParamRel.variableQname if parameterName in parameterNames: modelXbrl.error( "xbrlte:tableParameterNameClash ", _("Table %(xlinkLabel)s has parameter name clash for variable %(name)s" ), modelObject=(modelTable, tblParamRel, parameterNames[parameterName]), xlinkLabel=modelTable.xlinkLabel, name=parameterName) else: parameterNames[parameterName] = tblParamRel modelXbrl.profileStat(_("compileTables"))
def validateTestcase(self, testcase): self.modelXbrl.info("info", "Testcase", modelDocument=testcase) self.modelXbrl.viewModelObject(testcase.objectId()) if testcase.type in (Type.TESTCASESINDEX, Type.REGISTRY): for doc in sorted(testcase.referencesDocument.keys(), key=lambda doc: doc.uri): self.validateTestcase(doc) # testcases doc's are sorted by their uri (file names), e.g., for formula elif hasattr(testcase, "testcaseVariations"): for modelTestcaseVariation in testcaseVariationsByTarget(testcase.testcaseVariations): # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) # is this a versioning report? resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance resultIsTaxonomyPackage = modelTestcaseVariation.resultIsTaxonomyPackage formulaOutputInstance = None inputDTSes = defaultdict(list) baseForElement = testcase.baseForElement(modelTestcaseVariation) # try to load instance document self.modelXbrl.info("info", _("Variation %(id)s%(name)s%(target)s: %(expected)s - %(description)s"), modelObject=modelTestcaseVariation, id=modelTestcaseVariation.id, name=(" {}".format(modelTestcaseVariation.name) if modelTestcaseVariation.name else ""), target=(" target {}".format(modelTestcaseVariation.ixdsTarget) if modelTestcaseVariation.ixdsTarget else ""), expected=modelTestcaseVariation.expected, description=modelTestcaseVariation.description) if self.modelXbrl.modelManager.formulaOptions.testcaseResultsCaptureWarnings: errorCaptureLevel = logging._checkLevel("WARNING") else: errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY parameters = modelTestcaseVariation.parameters.copy() for readMeFirstUri in modelTestcaseVariation.readMeFirstUris: if isinstance(readMeFirstUri,tuple): # dtsName is for formula instances, but is from/to dts if versioning dtsName, readMeFirstUri = readMeFirstUri elif resultIsVersioningReport: if inputDTSes: dtsName = "to" else: dtsName = "from" else: dtsName = None if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document if dtsName in inputDTSes: dtsName = inputDTSes[dtsName] else: modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager, Type.DTSENTRIES, self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement), isEntry=True, errorCaptureLevel=errorCaptureLevel) DTSdoc = modelXbrl.modelDocument DTSdoc.inDTS = True doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement) if doc is not None: DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement) #fake import doc.inDTS = True elif resultIsTaxonomyPackage: from arelle import PackageManager, PrototypeInstanceObject dtsName = readMeFirstUri modelXbrl = PrototypeInstanceObject.XbrlPrototype(self.modelXbrl.modelManager, readMeFirstUri) PackageManager.packageInfo(self.modelXbrl.modelManager.cntlr, readMeFirstUri, reload=True, errors=modelXbrl.errors) else: # not a multi-schemaRef versioning report if self.useFileSource.isArchive: modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, readMeFirstUri, _("validating"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel, ixdsTarget=modelTestcaseVariation.ixdsTarget) else: # need own file source, may need instance discovery filesource = FileSource.openFileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr, base=baseForElement) if filesource and not filesource.selection and filesource.isArchive: try: if filesource.isTaxonomyPackage: _rptPkgIxdsOptions = {} for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ReportPackageIxdsOptions"): pluginXbrlMethod(self, _rptPkgIxdsOptions) filesource.loadTaxonomyPackageMappings() for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ReportPackageIxds"): filesource.select(pluginXbrlMethod(filesource, **_rptPkgIxdsOptions)) else: from arelle.CntlrCmdLine import filesourceEntrypointFiles entrypoints = filesourceEntrypointFiles(filesource) if entrypoints: # resolve an IXDS in entrypoints for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ArchiveIxds"): pluginXbrlMethod(self, filesource,entrypoints) filesource.select(entrypoints[0].get("file", None) ) except Exception as err: self.modelXbrl.error("exception:" + type(err).__name__, _("Testcase variation validation exception: %(error)s, entry URL: %(instance)s"), modelXbrl=self.modelXbrl, instance=readMeFirstUri, error=err) continue # don't try to load this entry URL modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, filesource, _("validating"), base=baseForElement, errorCaptureLevel=errorCaptureLevel, ixdsTarget=modelTestcaseVariation.ixdsTarget) modelXbrl.isTestcaseVariation = True if modelXbrl.modelDocument is None: modelXbrl.info("arelle:notLoaded", _("Variation %(id)s %(name)s readMeFirst document not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) self.determineNotLoadedTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif resultIsVersioningReport or resultIsTaxonomyPackage: inputDTSes[dtsName] = modelXbrl elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl) self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif testcase.type == Type.REGISTRYTESTCASE: self.instValidator.validate(modelXbrl) # required to set up dimensions, etc self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id, modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest) self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) self.instValidator.close() modelXbrl.close() else: inputDTSes[dtsName].append(modelXbrl) # validate except for formulas _hasFormulae = modelXbrl.hasFormulae modelXbrl.hasFormulae = False try: for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"): pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation) self.instValidator.validate(modelXbrl, parameters) for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"): pluginXbrlMethod(self.modelXbrl, modelXbrl) except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase variation validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=(type(err) is not AssertionError)) modelXbrl.hasFormulae = _hasFormulae if resultIsVersioningReport and modelXbrl.modelDocument: versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl( modelTestcaseVariation.versioningReportUri, baseForElement) if os.path.exists(versReportFile): #validate existing modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report")) if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport) self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors) modelVersReport.close() elif len(inputDTSes) == 2: ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes( versReportFile, inputDTSes["from"], inputDTSes["to"]) modelTestcaseVariation.status = "generated" else: modelXbrl.error("arelle:notLoaded", _("Variation %(id)s %(name)s input DTSes not loaded, unable to generate versioning report: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) modelTestcaseVariation.status = "failed" for inputDTS in inputDTSes.values(): inputDTS.close() del inputDTSes # dereference elif resultIsTaxonomyPackage: self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif inputDTSes: # validate schema, linkbase, or instance modelXbrl = inputDTSes[None][0] expectedDataFiles = set(modelXbrl.modelManager.cntlr.webCache.normalizeUrl(uri, baseForElement) for d in modelTestcaseVariation.dataUris.values() for uri in d if not UrlUtil.isAbsolute(uri)) foundDataFiles = set() variationBase = os.path.dirname(baseForElement) for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters if dtsName: # named instance parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes) elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula for _inputDTS in inputDTS: for docUrl, doc in _inputDTS.urlDocs.items(): if docUrl.startswith(variationBase) and not doc.type == Type.INLINEXBRLDOCUMENTSET: if getattr(doc,"loadedFromXbrlFormula", False): # may have been sourced from xf file if docUrl.replace("-formula.xml", ".xf") in expectedDataFiles: docUrl = docUrl.replace("-formula.xml", ".xf") foundDataFiles.add(docUrl) if expectedDataFiles - foundDataFiles: modelXbrl.info("arelle:testcaseDataNotUsed", _("Variation %(id)s %(name)s data files not used: %(missingDataFiles)s"), modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id, missingDataFiles=", ".join(sorted(os.path.basename(f) for f in expectedDataFiles - foundDataFiles))) if foundDataFiles - expectedDataFiles: modelXbrl.info("arelle:testcaseDataUnexpected", _("Variation %(id)s %(name)s files not in variation data: %(unexpectedDataFiles)s"), modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id, unexpectedDataFiles=", ".join(sorted(os.path.basename(f) for f in foundDataFiles - expectedDataFiles))) if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable: try: RenderingEvaluator.init(modelXbrl) except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase RenderingEvaluator.init exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) modelXbrlHasFormulae = modelXbrl.hasFormulae if modelXbrlHasFormulae and self.modelXbrl.modelManager.formulaOptions.formulaAction != "none": try: # validate only formulae self.instValidator.parameters = parameters ValidateFormula.validate(self.instValidator) except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=(type(err) is not AssertionError)) if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset: for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"): pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri) infoset = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultInfosetUri, _("loading result infoset"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) if infoset.modelDocument is None: modelXbrl.error("arelle:notLoaded", _("Variation %(id)s %(name)s result infoset not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstance)) modelTestcaseVariation.status = "result infoset not loadable" else: # check infoset ValidateInfoset.validate(self.instValidator, modelXbrl, infoset) infoset.close() if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset: # diff (or generate) table infoset resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement) if not any(alternativeValidation(modelXbrl, resultTableUri) for alternativeValidation in pluginClassMethods("Validate.TableInfoset")): try: ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase table linkbase validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) self.instValidator.close() extraErrors = [] for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"): pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors) self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual): # if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost) self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters) self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors) if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance formulaOutputInstance = modelXbrl.formulaOutputInstance modelXbrl.formulaOutputInstance = None # prevent it from being closed now self.instValidator.close() compareIxResultInstance = (modelXbrl.modelDocument.type in (Type.INLINEXBRL, Type.INLINEXBRLDOCUMENTSET) and modelTestcaseVariation.resultXbrlInstanceUri is not None) if compareIxResultInstance: formulaOutputInstance = modelXbrl # compare modelXbrl to generated output instance errMsgPrefix = "ix" else: # delete input instances before formula output comparision for inputDTSlist in inputDTSes.values(): for inputDTS in inputDTSlist: inputDTS.close() del inputDTSes # dereference errMsgPrefix = "formula" if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument: _matchExpectedResultIDs = not modelXbrlHasFormulae # formula restuls have inconsistent IDs expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultXbrlInstanceUri, _("loading expected result XBRL instance"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) if expectedInstance.modelDocument is None: self.modelXbrl.error("{}:expectedResultNotLoaded".format(errMsgPrefix), _("Testcase \"%(name)s\" %(id)s expected result instance not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstanceUri), messageCodes=("formula:expectedResultNotLoaded","ix:expectedResultNotLoaded")) modelTestcaseVariation.status = "result not loadable" else: # compare facts for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.ExpectedInstance.Loaded"): pluginXbrlMethod(expectedInstance, formulaOutputInstance) if len(expectedInstance.facts) != len(formulaOutputInstance.facts): formulaOutputInstance.error("{}:resultFactCounts".format(errMsgPrefix), _("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"), modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts), expectedFacts=len(expectedInstance.facts), messageCodes=("formula:resultFactCounts","ix:resultFactCounts")) else: formulaOutputFootnotesRelSet = ModelRelationshipSet(formulaOutputInstance, "XBRL-footnotes") expectedFootnotesRelSet = ModelRelationshipSet(expectedInstance, "XBRL-footnotes") def factFootnotes(fact, footnotesRelSet): footnotes = {} footnoteRels = footnotesRelSet.fromModelObject(fact) if footnoteRels: # most process rels in same order between two instances, use labels to sort for i, footnoteRel in enumerate(sorted(footnoteRels, key=lambda r: (r.fromLabel,r.toLabel))): modelObject = footnoteRel.toModelObject if isinstance(modelObject, ModelResource): xml = collapseWhitespace(modelObject.viewText().strip()) footnotes["Footnote {}".format(i+1)] = xml #re.sub(r'\s+', ' ', collapseWhitespace(modelObject.stringValue)) elif isinstance(modelObject, ModelFact): footnotes["Footnoted fact {}".format(i+1)] = \ "{} context: {} value: {}".format( modelObject.qname, modelObject.contextID, collapseWhitespace(modelObject.value)) return footnotes for expectedInstanceFact in expectedInstance.facts: unmatchedFactsStack = [] formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=_matchExpectedResultIDs, matchLang=False) #formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=True, matchLang=True) if formulaOutputFact is None: if unmatchedFactsStack: # get missing nested tuple fact, if possible missingFact = unmatchedFactsStack[-1] else: missingFact = expectedInstanceFact # is it possible to show value mismatches? expectedFacts = formulaOutputInstance.factsByQname.get(missingFact.qname) if len(expectedFacts) == 1: formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix), _("Output missing expected fact %(fact)s, extracted value \"%(value1)s\", expected value \"%(value2)s\""), modelXbrl=missingFact, fact=missingFact.qname, value1=missingFact.xValue, value2=next(iter(expectedFacts)).xValue, messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing")) else: formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix), _("Output missing expected fact %(fact)s"), modelXbrl=missingFact, fact=missingFact.qname, messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing")) else: # compare footnotes expectedInstanceFactFootnotes = factFootnotes(expectedInstanceFact, expectedFootnotesRelSet) formulaOutputFactFootnotes = factFootnotes(formulaOutputFact, formulaOutputFootnotesRelSet) if (len(expectedInstanceFactFootnotes) != len(formulaOutputFactFootnotes) or set(expectedInstanceFactFootnotes.values()) != set(formulaOutputFactFootnotes.values())): formulaOutputInstance.error("{}:expectedFactFootnoteDifference".format(errMsgPrefix), _("Output expected fact %(fact)s expected footnotes %(footnotes1)s produced footnotes %(footnotes2)s"), modelXbrl=(formulaOutputFact,expectedInstanceFact), fact=expectedInstanceFact.qname, footnotes1=sorted(expectedInstanceFactFootnotes.items()), footnotes2=sorted(formulaOutputFactFootnotes.items()), messageCodes=("formula:expectedFactFootnoteDifference","ix:expectedFactFootnoteDifference")) # for debugging uncomment next line to save generated instance document # formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml") expectedInstance.close() del expectedInstance # dereference self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors) formulaOutputInstance.close() del formulaOutputInstance if compareIxResultInstance: for inputDTSlist in inputDTSes.values(): for inputDTS in inputDTSlist: inputDTS.close() del inputDTSes # dereference # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) _statusCounts = OrderedDict((("pass",0),("fail",0))) for tv in getattr(testcase, "testcaseVariations", ()): _statusCounts[tv.status] = _statusCounts.get(tv.status, 0) + 1 self.modelXbrl.info("arelle:testCaseResults", ", ".join("{}={}".format(k,c) for k, c in _statusCounts.items() if k)) self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def init(modelXbrl): # setup modelXbrl for rendering evaluation # dimension defaults required in advance of validation from arelle import ValidateXbrlDimensions, ValidateFormula, ModelDocument ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) hasXbrlTables = False # validate table linkbase dimensions for baseSetKey in modelXbrl.baseSets.keys(): arcrole, ELR, linkqname, arcqname = baseSetKey if ELR and linkqname and arcqname and XbrlConst.isTableRenderingArcrole(arcrole): ValidateFormula.checkBaseSet(modelXbrl, arcrole, ELR, modelXbrl.relationshipSet(arcrole,ELR,linkqname,arcqname)) if arcrole in (XbrlConst.tableBreakdown, XbrlConst.tableBreakdownMMDD, XbrlConst.tableBreakdown201305, XbrlConst.tableBreakdown201301, XbrlConst.tableAxis2011): hasXbrlTables = True # provide context for view if modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE: instance = None # use instance of the entry pont else: # need dummy instance instance = ModelDocument.create(modelXbrl, ModelDocument.Type.INSTANCE, "dummy.xml", # fake URI and fake schemaRef ("http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd",)) if hasXbrlTables: # formula processor is needed for 2011 XBRL tables but not for 2010 Eurofiling tables modelXbrl.rendrCntx = XPathContext.create(modelXbrl, instance) modelXbrl.profileStat(None) # setup fresh parameters from formula options modelXbrl.parameters = modelXbrl.modelManager.formulaOptions.typedParameters() # validate parameters and custom function signatures ValidateFormula.validate(modelXbrl, xpathContext=modelXbrl.rendrCntx, parametersOnly=True, statusMsg=_("compiling rendering tables")) # deprecated as of 2013-05-17 # check and extract message expressions into compilable programs for msgArcrole in (XbrlConst.tableDefinitionNodeMessage201301, XbrlConst.tableDefinitionNodeSelectionMessage201301, XbrlConst.tableAxisMessage2011, XbrlConst.tableAxisSelectionMessage2011): for msgRel in modelXbrl.relationshipSet(msgArcrole).modelRelationships: ValidateFormula.checkMessageExpressions(modelXbrl, msgRel.toModelObject) # compile and validate tables for modelTable in modelXbrl.modelRenderingTables: modelTable.fromInstanceQnames = None # required if referred to by variables scope chaining modelTable.compile() hasNsWithAspectModel = modelTable.namespaceURI in (XbrlConst.euRend, XbrlConst.table2011, XbrlConst.table201301, XbrlConst.table201305) # check aspectModel (attribute removed 2013-06, now always dimensional) if modelTable.aspectModel not in ("non-dimensional", "dimensional") and hasNsWithAspectModel: modelXbrl.error("xbrlte:unknownAspectModel", _("Table %(xlinkLabel)s, aspect model %(aspectModel)s not recognized"), modelObject=modelTable, xlinkLabel=modelTable.xlinkLabel, aspectModel=modelTable.aspectModel) else: modelTable.priorAspectAxisDisposition = {} # check ordinate aspects against aspectModel oppositeAspectModel = (_DICT_SET({'dimensional','non-dimensional'}) - _DICT_SET({modelTable.aspectModel})).pop() if hasNsWithAspectModel: uncoverableAspects = aspectModels[oppositeAspectModel] - aspectModels[modelTable.aspectModel] else: uncoverableAspects = () aspectsCovered = set() for tblAxisRel in modelXbrl.relationshipSet((XbrlConst.tableBreakdown, XbrlConst.tableBreakdownMMDD, XbrlConst.tableBreakdown201305, XbrlConst.tableBreakdown201301,XbrlConst.tableAxis2011)).fromModelObject(modelTable): breakdownAspectsCovered = set() hasCoveredAspect = checkBreakdownDefinitionNode(modelXbrl, modelTable, tblAxisRel, tblAxisRel.axisDisposition, uncoverableAspects, breakdownAspectsCovered) ''' removed 2013-10 if not hasCoveredAspect: definitionNode = tblAxisRel.toModelObject modelXbrl.error("xbrlte:breakdownDefinesNoAspects", _("Breakdown %(xlinkLabel)s has no participating aspects"), modelObject=(modelTable,definitionNode), xlinkLabel=definitionNode.xlinkLabel, axis=definitionNode.localName) ''' aspectsCovered |= breakdownAspectsCovered checkBreakdownLeafNodeAspects(modelXbrl, modelTable, tblAxisRel, set(), breakdownAspectsCovered) if Aspect.CONCEPT not in aspectsCovered and not hasNsWithAspectModel: modelXbrl.error("xbrlte:tableMissingConceptAspect", _("Table %(xlinkLabel)s does not include the concept aspect as one of its participating aspects"), modelObject=modelTable, xlinkLabel=modelTable.xlinkLabel) del modelTable.priorAspectAxisDisposition # check for table-parameter name clash parameterNames = {} for tblParamRel in modelXbrl.relationshipSet((XbrlConst.tableParameter, XbrlConst.tableParameterMMDD)).fromModelObject(modelTable): parameterName = tblParamRel.variableQname if parameterName in parameterNames: modelXbrl.error("xbrlte:tableParameterNameClash ", _("Table %(xlinkLabel)s has parameter name clash for variable %(name)s"), modelObject=(modelTable,tblParamRel,parameterNames[parameterName]), xlinkLabel=modelTable.xlinkLabel, name=parameterName) else: parameterNames[parameterName] = tblParamRel modelXbrl.profileStat(_("compileTables"))
def run(self, options, sourceZipStream=None): """Process command line arguments or web service request, such as to load and validate an XBRL document, or start web server. When a web server has been requested, this method may be called multiple times, once for each web service (REST) request that requires processing. Otherwise (when called for a command line request) this method is called only once for the command line arguments request. :param options: OptionParser options from parse_args of main argv arguments (when called from command line) or corresponding arguments from web service (REST) request. :type options: optparse.Values """ if options.showOptions: # debug options for optName, optValue in sorted(options.__dict__.items(), key=lambda optItem: optItem[0]): self.addToLog("Option {0}={1}".format(optName, optValue), messageCode="info") self.addToLog("sys.argv {0}".format(sys.argv), messageCode="info") if options.uiLang: # set current UI Lang (but not config setting) self.setUiLanguage(options.uiLang) if options.proxy: if options.proxy != "show": proxySettings = proxyTuple(options.proxy) self.webCache.resetProxies(proxySettings) self.config["proxySettings"] = proxySettings self.saveConfig() self.addToLog(_("Proxy configuration has been set."), messageCode="info") useOsProxy, urlAddr, urlPort, user, password = self.config.get("proxySettings", proxyTuple("none")) if useOsProxy: self.addToLog(_("Proxy configured to use {0}.").format( _('Microsoft Windows Internet Settings') if sys.platform.startswith("win") else (_('Mac OS X System Configuration') if sys.platform in ("darwin", "macos") else _('environment variables'))), messageCode="info") elif urlAddr: self.addToLog(_("Proxy setting: http://{0}{1}{2}{3}{4}").format( user if user else "", ":****" if password else "", "@" if (user or password) else "", urlAddr, ":{0}".format(urlPort) if urlPort else ""), messageCode="info") else: self.addToLog(_("Proxy is disabled."), messageCode="info") if options.plugins: from arelle import PluginManager resetPlugins = False savePluginChanges = True showPluginModules = False for pluginCmd in options.plugins.split('|'): cmd = pluginCmd.strip() if cmd == "show": showPluginModules = True elif cmd == "temp": savePluginChanges = False elif cmd.startswith("+"): moduleInfo = PluginManager.addPluginModule(cmd[1:]) if moduleInfo: self.addToLog(_("Addition of plug-in {0} successful.").format(moduleInfo.get("name")), messageCode="info", file=moduleInfo.get("moduleURL")) resetPlugins = True else: self.addToLog(_("Unable to load plug-in."), messageCode="info", file=cmd[1:]) elif cmd.startswith("~"): if PluginManager.reloadPluginModule(cmd[1:]): self.addToLog(_("Reload of plug-in successful."), messageCode="info", file=cmd[1:]) resetPlugins = True else: self.addToLog(_("Unable to reload plug-in."), messageCode="info", file=cmd[1:]) elif cmd.startswith("-"): if PluginManager.removePluginModule(cmd[1:]): self.addToLog(_("Deletion of plug-in successful."), messageCode="info", file=cmd[1:]) resetPlugins = True else: self.addToLog(_("Unable to delete plug-in."), messageCode="info", file=cmd[1:]) else: # assume it is a module or package savePluginChanges = False moduleInfo = PluginManager.addPluginModule(cmd) if moduleInfo: self.addToLog(_("Activation of plug-in {0} successful.").format(moduleInfo.get("name")), messageCode="info", file=moduleInfo.get("moduleURL")) resetPlugins = True else: self.addToLog(_("Unable to load {0} as a plug-in or {0} is not recognized as a command. ").format(cmd), messageCode="info", file=cmd) if resetPlugins: PluginManager.reset() if savePluginChanges: PluginManager.save(self) if showPluginModules: self.addToLog(_("Plug-in modules:"), messageCode="info") for i, moduleItem in enumerate(sorted(PluginManager.pluginConfig.get("modules", {}).items())): moduleInfo = moduleItem[1] self.addToLog(_("Plug-in: {0}; author: {1}; version: {2}; status: {3}; date: {4}; description: {5}; license {6}.").format( moduleItem[0], moduleInfo.get("author"), moduleInfo.get("version"), moduleInfo.get("status"), moduleInfo.get("fileDate"), moduleInfo.get("description"), moduleInfo.get("license")), messageCode="info", file=moduleInfo.get("moduleURL")) # run utility command line options that don't depend on entrypoint Files hasUtilityPlugin = False for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Utility.Run"): hasUtilityPlugin = True pluginXbrlMethod(self, options) # if no entrypointFile is applicable, quit now if options.proxy or options.plugins or hasUtilityPlugin: if not options.entrypointFile: return True # success self.username = options.username self.password = options.password self.entrypointFile = options.entrypointFile if self.entrypointFile: filesource = FileSource.openFileSource(self.entrypointFile, self, sourceZipStream) else: filesource = None if options.validateEFM: if options.disclosureSystemName: self.addToLog(_("both --efm and --disclosureSystem validation are requested, proceeding with --efm only"), messageCode="info", file=self.entrypointFile) self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select("efm") elif options.disclosureSystemName: self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select(options.disclosureSystemName) elif options.validateHMRC: self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select("hmrc") else: self.modelManager.disclosureSystem.select(None) # just load ordinary mappings self.modelManager.validateDisclosureSystem = False if options.utrUrl: # override disclosureSystem utrUrl self.modelManager.disclosureSystem.utrUrl = options.utrUrl # can be set now because the utr is first loaded at validation time # disclosure system sets logging filters, override disclosure filters, if specified by command line if options.logLevelFilter: self.setLogLevelFilter(options.logLevelFilter) if options.logCodeFilter: self.setLogCodeFilter(options.logCodeFilter) if options.calcDecimals: if options.calcPrecision: self.addToLog(_("both --calcDecimals and --calcPrecision validation are requested, proceeding with --calcDecimals only"), messageCode="info", file=self.entrypointFile) self.modelManager.validateInferDecimals = True self.modelManager.validateCalcLB = True elif options.calcPrecision: self.modelManager.validateInferDecimals = False self.modelManager.validateCalcLB = True if options.utrValidate: self.modelManager.validateUtr = True if options.infosetValidate: self.modelManager.validateInfoset = True if options.abortOnMajorError: self.modelManager.abortOnMajorError = True if options.collectProfileStats: self.modelManager.collectProfileStats = True if options.internetConnectivity == "offline": self.webCache.workOffline = True elif options.internetConnectivity == "online": self.webCache.workOffline = False if options.internetTimeout is not None: self.webCache.timeout = (options.internetTimeout or None) # use None if zero specified to disable timeout fo = FormulaOptions() if options.parameters: parameterSeparator = (options.parameterSeparator or ',') fo.parameterValues = dict(((qname(key, noPrefixIsNoNamespace=True),(None,value)) for param in options.parameters.split(parameterSeparator) for key,sep,value in (param.partition('='),) ) ) if options.formulaParamExprResult: fo.traceParameterExpressionResult = True if options.formulaParamInputValue: fo.traceParameterInputValue = True if options.formulaCallExprSource: fo.traceCallExpressionSource = True if options.formulaCallExprCode: fo.traceCallExpressionCode = True if options.formulaCallExprEval: fo.traceCallExpressionEvaluation = True if options.formulaCallExprResult: fo.traceCallExpressionResult = True if options.formulaVarSetExprEval: fo.traceVariableSetExpressionEvaluation = True if options.formulaVarSetExprResult: fo.traceVariableSetExpressionResult = True if options.formulaAsserResultCounts: fo.traceAssertionResultCounts = True if options.formulaFormulaRules: fo.traceFormulaRules = True if options.formulaVarsOrder: fo.traceVariablesOrder = True if options.formulaVarExpressionSource: fo.traceVariableExpressionSource = True if options.formulaVarExpressionCode: fo.traceVariableExpressionCode = True if options.formulaVarExpressionEvaluation: fo.traceVariableExpressionEvaluation = True if options.formulaVarExpressionResult: fo.traceVariableExpressionResult = True if options.timeVariableSetEvaluation: fo.timeVariableSetEvaluation = True if options.formulaVarFilterWinnowing: fo.traceVariableFilterWinnowing = True if options.formulaVarFiltersResult: fo.traceVariableFiltersResult = True if options.formulaVarFiltersResult: fo.traceVariableFiltersResult = True self.modelManager.formulaOptions = fo timeNow = XmlUtil.dateunionValue(datetime.datetime.now()) firstStartedAt = startedAt = time.time() modelDiffReport = None success = True modelXbrl = None try: if filesource: modelXbrl = self.modelManager.load(filesource, _("views loading")) except ModelDocument.LoadingException: pass except Exception as err: self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format( err, traceback.format_tb(sys.exc_info()[2]))) success = False # loading errors, don't attempt to utilize loaded DTS if modelXbrl and modelXbrl.modelDocument: loadTime = time.time() - startedAt modelXbrl.profileStat(_("load"), loadTime) self.addToLog(format_string(self.modelManager.locale, _("loaded in %.2f secs at %s"), (loadTime, timeNow)), messageCode="info", file=self.entrypointFile) if options.importFiles: for importFile in options.importFiles.split("|"): fileName = importFile.strip() if sourceZipStream is not None and not (fileName.startswith('http://') or os.path.isabs(fileName)): fileName = os.path.dirname(modelXbrl.uri) + os.sep + fileName # make relative to sourceZipStream ModelDocument.load(modelXbrl, fileName) loadTime = time.time() - startedAt self.addToLog(format_string(self.modelManager.locale, _("import in %.2f secs at %s"), (loadTime, timeNow)), messageCode="info", file=importFile) modelXbrl.profileStat(_("import"), loadTime) if modelXbrl.errors: success = False # loading errors, don't attempt to utilize loaded DTS if modelXbrl.modelDocument.type in ModelDocument.Type.TESTCASETYPES: for pluginXbrlMethod in pluginClassMethods("Testcases.Start"): pluginXbrlMethod(self, options, modelXbrl) else: # not a test case, probably instance or DTS for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Loaded"): pluginXbrlMethod(self, options, modelXbrl) else: success = False if success and options.diffFile and options.versReportFile: try: diffFilesource = FileSource.FileSource(options.diffFile,self) startedAt = time.time() modelXbrl2 = self.modelManager.load(diffFilesource, _("views loading")) if modelXbrl2.errors: if not options.keepOpen: modelXbrl2.close() success = False else: loadTime = time.time() - startedAt modelXbrl.profileStat(_("load"), loadTime) self.addToLog(format_string(self.modelManager.locale, _("diff comparison DTS loaded in %.2f secs"), loadTime), messageCode="info", file=self.entrypointFile) startedAt = time.time() modelDiffReport = self.modelManager.compareDTSes(options.versReportFile) diffTime = time.time() - startedAt modelXbrl.profileStat(_("diff"), diffTime) self.addToLog(format_string(self.modelManager.locale, _("compared in %.2f secs"), diffTime), messageCode="info", file=self.entrypointFile) except ModelDocument.LoadingException: success = False except Exception as err: success = False self.addToLog(_("[Exception] Failed to doad diff file: \n{0} \n{1}").format( err, traceback.format_tb(sys.exc_info()[2]))) if success: try: modelXbrl = self.modelManager.modelXbrl hasFormulae = modelXbrl.hasFormulae if options.validate: startedAt = time.time() if options.formulaAction: # don't automatically run formulas modelXbrl.hasFormulae = False self.modelManager.validate() if options.formulaAction: # restore setting modelXbrl.hasFormulae = hasFormulae self.addToLog(format_string(self.modelManager.locale, _("validated in %.2f secs"), time.time() - startedAt), messageCode="info", file=self.entrypointFile) if options.formulaAction in ("validate", "run"): # do nothing here if "none" from arelle import ValidateXbrlDimensions, ValidateFormula startedAt = time.time() if not options.validate: ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # setup fresh parameters from formula optoins modelXbrl.parameters = fo.typedParameters() ValidateFormula.validate(modelXbrl, compileOnly=(options.formulaAction != "run")) self.addToLog(format_string(self.modelManager.locale, _("formula validation and execution in %.2f secs") if options.formulaAction == "run" else _("formula validation only in %.2f secs"), time.time() - startedAt), messageCode="info", file=self.entrypointFile) if options.testReport: ViewFileTests.viewTests(self.modelManager.modelXbrl, options.testReport, options.testReportCols) if options.rssReport: ViewFileRssFeed.viewRssFeed(self.modelManager.modelXbrl, options.rssReport, options.rssReportCols) if options.DTSFile: ViewFileDTS.viewDTS(modelXbrl, options.DTSFile) if options.factsFile: ViewFileFactList.viewFacts(modelXbrl, options.factsFile, labelrole=options.labelRole, lang=options.labelLang, cols=options.factListCols) if options.factTableFile: ViewFileFactTable.viewFacts(modelXbrl, options.factTableFile, labelrole=options.labelRole, lang=options.labelLang) if options.conceptsFile: ViewFileConcepts.viewConcepts(modelXbrl, options.conceptsFile, labelrole=options.labelRole, lang=options.labelLang) if options.preFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.preFile, "Presentation Linkbase", "http://www.xbrl.org/2003/arcrole/parent-child", labelrole=options.labelRole, lang=options.labelLang) if options.calFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.calFile, "Calculation Linkbase", "http://www.xbrl.org/2003/arcrole/summation-item", labelrole=options.labelRole, lang=options.labelLang) if options.dimFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.dimFile, "Dimensions", "XBRL-dimensions", labelrole=options.labelRole, lang=options.labelLang) if options.formulaeFile: ViewFileFormulae.viewFormulae(modelXbrl, options.formulaeFile, "Formulae", lang=options.labelLang) if options.viewArcrole and options.viewFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.viewFile, os.path.basename(options.viewArcrole), options.viewArcrole, labelrole=options.labelRole, lang=options.labelLang) for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Run"): pluginXbrlMethod(self, options, modelXbrl) except (IOError, EnvironmentError) as err: self.addToLog(_("[IOError] Failed to save output:\n {0}").format(err)) success = False except Exception as err: self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format( err, traceback.format_tb(sys.exc_info()[2]))) success = False if modelXbrl: modelXbrl.profileStat(_("total"), time.time() - firstStartedAt) if options.collectProfileStats and modelXbrl: modelXbrl.logProfileStats() if not options.keepOpen: if modelDiffReport: self.modelManager.close(modelDiffReport) elif modelXbrl: self.modelManager.close(modelXbrl) self.username = self.password = None #dereference password return success
def watchCycle(self): while not self.stopRequested: rssWatchOptions = self.rssModelXbrl.modelManager.rssWatchOptions # check rss expiration rssHeaders = self.cntlr.webCache.getheaders(self.rssModelXbrl.modelManager.rssWatchOptions.get("feedSourceUri")) expires = parseRfcDatetime(rssHeaders.get("expires")) reloadNow = True # texpires and expires > datetime.datetime.now() # reload rss feed self.rssModelXbrl.reload('checking RSS items', reloadCache=reloadNow) if self.stopRequested: break # setup validator postLoadActions = [] if rssWatchOptions.get("validateDisclosureSystemRules"): self.instValidator = ValidateFiling.ValidateFiling(self.rssModelXbrl) postLoadActions.append(_("validating")) elif rssWatchOptions.get("validateXbrlRules") or rssWatchOptions.get("validateFormulaAssertions"): self.instValidator = ValidateXbrl.ValidateXbrl(self.rssModelXbrl) postLoadActions.append(_("validating")) if (rssWatchOptions.get("validateFormulaAssertions")): postLoadActions.append(_("running formulas")) else: self.instValidator = None matchTextExpr = rssWatchOptions.get("matchTextExpr") if matchTextExpr: matchPattern = re.compile(matchTextExpr) postLoadActions.append(_("matching text")) else: matchPattern= None postLoadAction = ', '.join(postLoadActions) # anything to check new filings for if (rssWatchOptions.get("validateDisclosureSystemRules") or rssWatchOptions.get("validateXbrlRules") or rssWatchOptions.get("validateCalcLinkbase") or rssWatchOptions.get("validateFormulaAssertions") or rssWatchOptions.get("alertMatchedFactText") or any(pluginXbrlMethod(rssWatchOptions) for pluginXbrlMethod in pluginClassMethods("RssWatch.HasWatchAction")) ): # form keys in ascending order of pubdate pubDateRssItems = [] for rssItem in self.rssModelXbrl.modelDocument.rssItems: pubDateRssItems.append((rssItem.pubDate,rssItem.objectId())) for pubDate, rssItemObjectId in sorted(pubDateRssItems): rssItem = self.rssModelXbrl.modelObject(rssItemObjectId) # update ui thread via modelManager (running in background here) self.rssModelXbrl.modelManager.viewModelObject(self.rssModelXbrl, rssItem.objectId()) if self.stopRequested: break latestPubDate = XmlUtil.datetimeValue(rssWatchOptions.get("latestPubDate")) if (latestPubDate and rssItem.pubDate < latestPubDate): continue try: # try zipped URL if possible, else expanded instance document modelXbrl = ModelXbrl.load(self.rssModelXbrl.modelManager, openFileSource(rssItem.zippedUrl, self.cntlr), postLoadAction) if self.stopRequested: modelXbrl.close() break emailAlert = False if modelXbrl.modelDocument is None: modelXbrl.error("arelle.rssWatch", _("RSS item %(company)s %(form)s document not loaded: %(date)s"), modelXbrl=modelXbrl, company=rssItem.companyName, form=rssItem.formType, date=rssItem.filingDate) rssItem.status = "not loadable" else: # validate schema, linkbase, or instance if self.stopRequested: modelXbrl.close() break if self.instValidator: self.instValidator.validate(modelXbrl) if modelXbrl.errors and rssWatchOptions.get("alertValiditionError"): emailAlert = True for pluginXbrlMethod in pluginClassMethods("RssWatch.DoWatchAction"): pluginXbrlMethod(modelXbrl, rssWatchOptions, rssItem) # check match expression if matchPattern: for fact in modelXbrl.factsInInstance: v = fact.value if v is not None: m = matchPattern.search(v) if m: fr, to = m.span() msg = _("Fact Variable {0}\n context {1}\n matched text: {2}").format( fact.qname, fact.contextID, v[max(0,fr-20):to+20]) modelXbrl.info("arelle.rssInfo", msg, modelXbrl=modelXbrl) # msg as code passes it through to the status if rssWatchOptions.get("alertMatchedFactText"): emailAlert = True if (rssWatchOptions.get("formulaFileUri") and rssWatchOptions.get("validateFormulaAssertions") and self.instValidator): # attach formulas ModelDocument.load(modelXbrl, rssWatchOptions["formulaFileUri"]) ValidateFormula.validate(self.instValidator) rssItem.setResults(modelXbrl) modelXbrl.close() del modelXbrl # completely dereference self.rssModelXbrl.modelManager.viewModelObject(self.rssModelXbrl, rssItem.objectId()) if rssItem.assertionUnsuccessful and rssWatchOptions.get("alertAssertionUnsuccessful"): emailAlert = True msg = _("Filing CIK {0}\n " "company {1}\n " "published {2}\n " "form type {3}\n " "filing date {4}\n " "period {5}\n " "year end {6}\n " "results: {7}").format( rssItem.cikNumber, rssItem.companyName, rssItem.pubDate, rssItem.formType, rssItem.filingDate, rssItem.period, rssItem.fiscalYearEnd, rssItem.status) self.rssModelXbrl.info("arelle:rssWatch", msg, modelXbrl=self.rssModelXbrl) emailAddress = rssWatchOptions.get("emailAddress") if emailAlert and emailAddress: self.rssModelXbrl.modelManager.showStatus(_("sending e-mail alert")) import smtplib from email.mime.text import MIMEText emailMsg = MIMEText(msg) emailMsg["Subject"] = _("Arelle RSS Watch alert on {0}").format(rssItem.companyName) emailMsg["From"] = emailAddress emailMsg["To"] = emailAddress smtp = smtplib.SMTP() smtp.sendmail(emailAddress, [emailAddress], emailMsg.as_string()) smtp.quit() self.rssModelXbrl.modelManager.showStatus(_("RSS item {0}, {1} completed, status {2}").format(rssItem.companyName, rssItem.formType, rssItem.status), 3500) self.rssModelXbrl.modelManager.cntlr.rssWatchUpdateOption(rssItem.pubDate.strftime('%Y-%m-%dT%H:%M:%S')) except Exception as err: self.rssModelXbrl.error("arelle.rssError", _("RSS item %(company)s, %(form)s, %(date)s, exception: %(error)s"), modelXbrl=self.rssModelXbrl, company=rssItem.companyName, form=rssItem.formType, date=rssItem.filingDate, error=err, exc_info=True) if self.stopRequested: break if self.stopRequested: self.cntlr.showStatus(_("RSS watch, stop requested"), 10000) else: import time time.sleep(600) self.thread = None # close thread self.stopRequested = False
def init(modelXbrl): # setup modelXbrl for rendering evaluation # dimension defaults required in advance of validation from arelle import ValidateXbrlDimensions, ValidateFormula, ModelDocument ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) hasXbrlTables = False # validate table linkbase dimensions for baseSetKey in modelXbrl.baseSets.keys(): arcrole, ELR, linkqname, arcqname = baseSetKey if ELR and linkqname and arcqname and XbrlConst.isTableRenderingArcrole(arcrole): ValidateFormula.checkBaseSet(modelXbrl, arcrole, ELR, modelXbrl.relationshipSet(arcrole,ELR,linkqname,arcqname)) if arcrole in (XbrlConst.tableBreakdown, XbrlConst.tableAxis2011): hasXbrlTables = True # provide context for view if modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE: instance = None # use instance of the entry pont else: # need dummy instance instance = ModelDocument.create(modelXbrl, ModelDocument.Type.INSTANCE, "dummy.xml", # fake URI and fake schemaRef ("http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd",)) if hasXbrlTables: # formula processor is needed for 2011 XBRL tables but not for 2010 Eurofiling tables modelXbrl.rendrCntx = XPathContext.create(modelXbrl, instance) modelXbrl.profileStat(None) # setup fresh parameters from formula optoins modelXbrl.parameters = modelXbrl.modelManager.formulaOptions.typedParameters() # validate parameters and custom function signatures ValidateFormula.validate(modelXbrl, xpathContext=modelXbrl.rendrCntx, parametersOnly=True, statusMsg=_("compiling rendering tables")) # check and extract message expressions into compilable programs for msgArcrole in (XbrlConst.tableDefinitionNodeMessage, XbrlConst.tableDefinitionNodeSelectionMessage, XbrlConst.tableAxisMessage2011, XbrlConst.tableAxisSelectionMessage2011): for msgRel in modelXbrl.relationshipSet(msgArcrole).modelRelationships: ValidateFormula.checkMessageExpressions(modelXbrl, msgRel.toModelObject) # compile and validate tables for modelTable in modelXbrl.modelRenderingTables: modelTable.fromInstanceQnames = None # required if referred to by variables scope chaining modelTable.compile() # check aspectModel if modelTable.aspectModel not in ("non-dimensional", "dimensional"): modelXbrl.error("xbrlte:unknownAspectModel", _("Table %(xlinkLabel)s, aspect model %(aspectModel)s not recognized"), modelObject=modelTable, xlinkLabel=modelTable.xlinkLabel, aspectModel=modelTable.aspectModel) else: modelTable.priorAspectAxisDisposition = {} # check ordinate aspects against aspectModel oppositeAspectModel = (_DICT_SET({'dimensional','non-dimensional'}) - _DICT_SET({modelTable.aspectModel})).pop() uncoverableAspects = aspectModels[oppositeAspectModel] - aspectModels[modelTable.aspectModel] for tblAxisRel in modelXbrl.relationshipSet((XbrlConst.tableBreakdown,XbrlConst.tableAxis2011)).fromModelObject(modelTable): checkDefinitionNodeAspectModel(modelXbrl, modelTable, tblAxisRel, uncoverableAspects) del modelTable.priorAspectAxisDisposition modelXbrl.profileStat(_("compileTables"))
def init(modelXbrl): # setup modelXbrl for rendering evaluation # dimension defaults required in advance of validation from arelle import ValidateXbrlDimensions, ValidateFormula, ModelDocument ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) hasXbrlTables = False # validate table linkbase dimensions for baseSetKey in modelXbrl.baseSets.keys(): arcrole, ELR, linkqname, arcqname = baseSetKey if ELR and linkqname and arcqname and XbrlConst.isTableRenderingArcrole( arcrole): ValidateFormula.checkBaseSet( modelXbrl, arcrole, ELR, modelXbrl.relationshipSet(arcrole, ELR, linkqname, arcqname)) if arcrole in (XbrlConst.tableBreakdown, XbrlConst.tableBreakdown201301, XbrlConst.tableAxis2011): hasXbrlTables = True # provide context for view if modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE: instance = None # use instance of the entry pont else: # need dummy instance instance = ModelDocument.create( modelXbrl, ModelDocument.Type.INSTANCE, "dummy.xml", # fake URI and fake schemaRef ("http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd", )) if hasXbrlTables: # formula processor is needed for 2011 XBRL tables but not for 2010 Eurofiling tables modelXbrl.rendrCntx = XPathContext.create(modelXbrl, instance) modelXbrl.profileStat(None) # setup fresh parameters from formula optoins modelXbrl.parameters = modelXbrl.modelManager.formulaOptions.typedParameters( ) # validate parameters and custom function signatures ValidateFormula.validate(modelXbrl, xpathContext=modelXbrl.rendrCntx, parametersOnly=True, statusMsg=_("compiling rendering tables")) # deprecated as of 2013-05-17 # check and extract message expressions into compilable programs for msgArcrole in (XbrlConst.tableDefinitionNodeMessage201301, XbrlConst.tableDefinitionNodeSelectionMessage201301, XbrlConst.tableAxisMessage2011, XbrlConst.tableAxisSelectionMessage2011): for msgRel in modelXbrl.relationshipSet( msgArcrole).modelRelationships: ValidateFormula.checkMessageExpressions( modelXbrl, msgRel.toModelObject) # compile and validate tables for modelTable in modelXbrl.modelRenderingTables: modelTable.fromInstanceQnames = None # required if referred to by variables scope chaining modelTable.compile() # check aspectModel if modelTable.aspectModel not in ("non-dimensional", "dimensional"): modelXbrl.error( "xbrlte:unknownAspectModel", _("Table %(xlinkLabel)s, aspect model %(aspectModel)s not recognized" ), modelObject=modelTable, xlinkLabel=modelTable.xlinkLabel, aspectModel=modelTable.aspectModel) else: modelTable.priorAspectAxisDisposition = {} # check ordinate aspects against aspectModel oppositeAspectModel = ( _DICT_SET({'dimensional', 'non-dimensional'}) - _DICT_SET({modelTable.aspectModel})).pop() uncoverableAspects = aspectModels[ oppositeAspectModel] - aspectModels[modelTable.aspectModel] for tblAxisRel in modelXbrl.relationshipSet( (XbrlConst.tableBreakdown, XbrlConst.tableBreakdown201301, XbrlConst.tableAxis2011)).fromModelObject(modelTable): checkDefinitionNodeAspectModel(modelXbrl, modelTable, tblAxisRel, uncoverableAspects) del modelTable.priorAspectAxisDisposition modelXbrl.profileStat(_("compileTables"))
def executeCallTest(self, modelXbrl, name, callTuple, testTuple): self.modelXbrl = modelXbrl ValidateFormula.executeCallTest(self, name, callTuple, testTuple)
def validate(self, modelXbrl, parameters=None): self.parameters = parameters self.precisionPattern = re.compile("^([0-9]+|INF)$") self.decimalsPattern = re.compile("^(-?[0-9]+|INF)$") self.isoCurrencyPattern = re.compile(r"^[A-Z]{3}$") self.modelXbrl = modelXbrl self.validateDisclosureSystem = modelXbrl.modelManager.validateDisclosureSystem self.disclosureSystem = modelXbrl.modelManager.disclosureSystem self.validateEFM = self.validateDisclosureSystem and self.disclosureSystem.EFM self.validateGFM = self.validateDisclosureSystem and self.disclosureSystem.GFM self.validateEFMorGFM = self.validateDisclosureSystem and self.disclosureSystem.EFMorGFM self.validateHMRC = self.validateDisclosureSystem and self.disclosureSystem.HMRC self.validateSBRNL = self.validateDisclosureSystem and self.disclosureSystem.SBRNL self.validateXmlLang = self.validateDisclosureSystem and self.disclosureSystem.xmlLangPattern self.validateCalcLB = modelXbrl.modelManager.validateCalcLB self.validateInferDecimals = modelXbrl.modelManager.validateInferDecimals # xlink validation modelXbrl.profileStat(None) modelXbrl.modelManager.showStatus(_("validating links")) modelLinks = set() self.remoteResourceLocElements = set() self.genericArcArcroles = set() for baseSetExtLinks in modelXbrl.baseSets.values(): for baseSetExtLink in baseSetExtLinks: modelLinks.add(baseSetExtLink) # ext links are unique (no dups) for modelLink in modelLinks: fromToArcs = {} locLabels = {} resourceLabels = {} resourceArcTos = [] for arcElt in modelLink.iterchildren(): if isinstance(arcElt,ModelObject): xlinkType = arcElt.get("{http://www.w3.org/1999/xlink}type") # locator must have an href if xlinkType == "locator": if arcElt.get("{http://www.w3.org/1999/xlink}href") is None: modelXbrl.error("xlink:locatorHref", _("Xlink locator %(xlinkLabel)s missing href in extended link %(linkrole)s"), modelObject=arcElt, linkrole=modelLink.role, xlinkLabel=arcElt.get("{http://www.w3.org/1999/xlink}label")) locLabels[arcElt.get("{http://www.w3.org/1999/xlink}label")] = arcElt elif xlinkType == "resource": resourceLabels[arcElt.get("{http://www.w3.org/1999/xlink}label")] = arcElt # can be no duplicated arcs between same from and to elif xlinkType == "arc": fromLabel = arcElt.get("{http://www.w3.org/1999/xlink}from") toLabel = arcElt.get("{http://www.w3.org/1999/xlink}to") fromTo = (fromLabel,toLabel) if fromTo in fromToArcs: modelXbrl.error("xlink:dupArcs", _("Duplicate xlink arcs in extended link %(linkrole)s from %(xlinkLabelFrom)s to %(xlinkLabelTo)s"), modelObject=arcElt, linkrole=modelLink.role, xlinkLabelFrom=fromLabel, xlinkLabelTo=toLabel) else: fromToArcs[fromTo] = arcElt if arcElt.namespaceURI == XbrlConst.link: if arcElt.localName in arcNamesTo21Resource: #("labelArc","referenceArc"): resourceArcTos.append((toLabel, arcElt.get("use"), arcElt)) elif self.isGenericArc(arcElt): arcrole = arcElt.get("{http://www.w3.org/1999/xlink}arcrole") self.genericArcArcroles.add(arcrole) if arcrole in (XbrlConst.elementLabel, XbrlConst.elementReference): resourceArcTos.append((toLabel, arcrole, arcElt)) # values of type (not needed for validating parsers) if xlinkType not in xlinkTypeValues: # ("", "simple", "extended", "locator", "arc", "resource", "title", "none"): modelXbrl.error("xlink:type", _("Xlink type %(xlinkType)s invalid in extended link %(linkrole)s"), modelObject=arcElt, linkrole=modelLink.role, xlinkType=xlinkType) # values of actuate (not needed for validating parsers) xlinkActuate = arcElt.get("{http://www.w3.org/1999/xlink}actuate") if xlinkActuate not in xlinkActuateValues: # ("", "onLoad", "onRequest", "other", "none"): modelXbrl.error("xlink:actuate", _("Actuate %(xlinkActuate)s invalid in extended link %(linkrole)s"), modelObject=arcElt, linkrole=modelLink.role, xlinkActuate=xlinkActuate) # values of show (not needed for validating parsers) xlinkShow = arcElt.get("{http://www.w3.org/1999/xlink}show") if xlinkShow not in xlinkShowValues: # ("", "new", "replace", "embed", "other", "none"): modelXbrl.error("xlink:show", _("Show %(xlinkShow)s invalid in extended link %(linkrole)s"), modelObject=arcElt, linkrole=modelLink.role, xlinkShow=xlinkShow) # check from, to of arcs have a resource or loc for fromTo, arcElt in fromToArcs.items(): fromLabel, toLabel = fromTo for name, value, sect in (("from", fromLabel, "3.5.3.9.2"),("to",toLabel, "3.5.3.9.3")): if value not in locLabels and value not in resourceLabels: modelXbrl.error("xbrl.{0}:arcResource".format(sect), _("Arc in extended link %(linkrole)s from %(xlinkLabelFrom)s to %(xlinkLabelTo)s attribute '%(attribute)s' has no matching loc or resource label"), modelObject=arcElt, linkrole=modelLink.role, xlinkLabelFrom=fromLabel, xlinkLabelTo=toLabel, attribute=name) if arcElt.localName == "footnoteArc" and arcElt.namespaceURI == XbrlConst.link and \ arcElt.get("{http://www.w3.org/1999/xlink}arcrole") == XbrlConst.factFootnote: if fromLabel not in locLabels: modelXbrl.error("xbrl.4.11.1.3.1:factFootnoteArcFrom", _("Footnote arc in extended link %(linkrole)s from %(xlinkLabelFrom)s to %(xlinkLabelTo)s \"from\" is not a loc"), modelObject=arcElt, linkrole=modelLink.role, xlinkLabelFrom=fromLabel, xlinkLabelTo=toLabel) if toLabel not in resourceLabels or qname(resourceLabels[toLabel]) != XbrlConst.qnLinkFootnote: modelXbrl.error("xbrl.4.11.1.3.1:factFootnoteArcTo", _("Footnote arc in extended link %(linkrole)s from %(xlinkLabelFrom)s to %(xlinkLabelTo)s \"to\" is not a footnote resource"), modelObject=arcElt, linkrole=modelLink.role, xlinkLabelFrom=fromLabel, xlinkLabelTo=toLabel) # check unprohibited label arcs to remote locs for resourceArcTo in resourceArcTos: resourceArcToLabel, resourceArcUse, arcElt = resourceArcTo if resourceArcToLabel in locLabels: toLabel = locLabels[resourceArcToLabel] if resourceArcUse == "prohibited": self.remoteResourceLocElements.add(toLabel) else: modelXbrl.error("xbrl.5.2.2.3:labelArcRemoteResource", _("Unprohibited labelArc in extended link %(linkrole)s has illegal remote resource loc labeled %(xlinkLabel)s href %(xlinkHref)s"), modelObject=arcElt, linkrole=modelLink.role, xlinkLabel=resourceArcToLabel, xlinkHref=toLabel.get("{http://www.w3.org/1999/xlink}href")) elif resourceArcToLabel in resourceLabels: toResource = resourceLabels[resourceArcToLabel] if resourceArcUse == XbrlConst.elementLabel: if not self.isGenericLabel(toResource): modelXbrl.error("xbrlle.2.1.1:genericLabelTarget", _("Generic label arc in extended link %(linkrole)s to %(xlinkLabel)s must target a generic label"), modelObject=arcElt, linkrole=modelLink.role, xlinkLabel=resourceArcToLabel) elif resourceArcUse == XbrlConst.elementReference: if not self.isGenericReference(toResource): modelXbrl.error("xbrlre.2.1.1:genericReferenceTarget", _("Generic reference arc in extended link %(linkrole)s to %(xlinkLabel)s must target a generic reference"), modelObject=arcElt, linkrole=modelLink.role, xlinkLabel=resourceArcToLabel) resourceArcTos = None # dereference arcs modelXbrl.profileStat(_("validateLinks")) modelXbrl.dimensionDefaultConcepts = {} modelXbrl.qnameDimensionDefaults = {} modelXbrl.qnameDimensionContextElement = {} # check base set cycles, dimensions modelXbrl.modelManager.showStatus(_("validating relationship sets")) for baseSetKey in modelXbrl.baseSets.keys(): arcrole, ELR, linkqname, arcqname = baseSetKey if arcrole.startswith("XBRL-") or ELR is None or \ linkqname is None or arcqname is None: continue elif arcrole in XbrlConst.standardArcroleCyclesAllowed: # TODO: table should be in this module, where it is used cyclesAllowed, specSect = XbrlConst.standardArcroleCyclesAllowed[arcrole] elif arcrole in self.modelXbrl.arcroleTypes and len(self.modelXbrl.arcroleTypes[arcrole]) > 0: cyclesAllowed = self.modelXbrl.arcroleTypes[arcrole][0].cyclesAllowed if arcrole in self.genericArcArcroles: specSect = "xbrlgene:violatedCyclesConstraint" else: specSect = "xbrl.5.1.4.3:cycles" else: cyclesAllowed = "any" specSect = None if cyclesAllowed != "any" or arcrole in (XbrlConst.summationItem,) \ or arcrole in self.genericArcArcroles \ or arcrole.startswith(XbrlConst.formulaStartsWith): relsSet = modelXbrl.relationshipSet(arcrole,ELR,linkqname,arcqname) if cyclesAllowed != "any" and \ (XbrlConst.isStandardExtLinkQname(linkqname) and XbrlConst.isStandardArcQname(arcqname)) \ or arcrole in self.genericArcArcroles: noUndirected = cyclesAllowed == "none" fromRelationships = relsSet.fromModelObjects() for relFrom, rels in fromRelationships.items(): cycleFound = self.fwdCycle(relsSet, rels, noUndirected, {relFrom}) if cycleFound is not None: path = str(relFrom.qname) + " " + " - ".join( "{0}:{1} {2}".format(rel.modelDocument.basename, rel.sourceline, rel.toModelObject.qname) for rel in reversed(cycleFound[1:])) modelXbrl.error(specSect, _("Relationships have a %(cycle)s cycle in arcrole %(arcrole)s \nlink role %(linkrole)s \nlink %(linkname)s, \narc %(arcname)s, \npath %(path)s"), modelObject=cycleFound[1:], cycle=cycleFound[0], path=path, arcrole=arcrole, linkrole=ELR, linkname=linkqname, arcname=arcqname), break # check calculation arcs for weight issues (note calc arc is an "any" cycles) if arcrole == XbrlConst.summationItem: for modelRel in relsSet.modelRelationships: weight = modelRel.weight fromConcept = modelRel.fromModelObject toConcept = modelRel.toModelObject if fromConcept is not None and toConcept is not None: if weight == 0: modelXbrl.error("xbrl.5.2.5.2.1:zeroWeight", _("Calculation relationship has zero weight from %(source)s to %(target)s in link role %(linkrole)s"), modelObject=modelRel, source=fromConcept.qname, target=toConcept.qname, linkrole=ELR), fromBalance = fromConcept.balance toBalance = toConcept.balance if fromBalance and toBalance: if (fromBalance == toBalance and weight < 0) or \ (fromBalance != toBalance and weight > 0): modelXbrl.error("xbrl.5.1.1.2:balanceCalcWeightIllegal" + ("Negative" if weight < 0 else "Positive"), _("Calculation relationship has illegal weight %(weight)s from %(source)s, %(sourceBalance)s, to %(target)s, %(targetBalance)s, in link role %(linkrole)s (per 5.1.1.2 Table 6)"), modelObject=modelRel, weight=weight, source=fromConcept.qname, target=toConcept.qname, linkrole=ELR, sourceBalance=fromBalance, targetBalance=toBalance) if not fromConcept.isNumeric or not toConcept.isNumeric: modelXbrl.error("xbrl.5.2.5.2:nonNumericCalc", _("Calculation relationship has illegal concept from %(source)s%(sourceNumericDecorator)s to %(target)s%(targetNumericDecorator)s in link role %(linkrole)s"), modelObject=modelRel, source=fromConcept.qname, target=toConcept.qname, linkrole=ELR, sourceNumericDecorator="" if fromConcept.isNumeric else _(" (non-numeric)"), targetNumericDecorator="" if toConcept.isNumeric else _(" (non-numeric)")) # check presentation relationships for preferredLabel issues elif arcrole == XbrlConst.parentChild: for modelRel in relsSet.modelRelationships: preferredLabel = modelRel.preferredLabel toConcept = modelRel.toModelObject if preferredLabel is not None and toConcept is not None and \ toConcept.label(preferredLabel=preferredLabel,fallbackToQname=False) is None: modelXbrl.error("xbrl.5.2.4.2.1:preferredLabelMissing", _("Presentation relationship from %(source)s to %(target)s in link role %(linkrole)s missing preferredLabel %(preferredLabel)s"), modelObject=modelRel, source=modelRel.fromModelObject.qname, target=toConcept.qname, linkrole=ELR, preferredLabel=preferredLabel) # check essence-alias relationships elif arcrole == XbrlConst.essenceAlias: for modelRel in relsSet.modelRelationships: fromConcept = modelRel.fromModelObject toConcept = modelRel.toModelObject if fromConcept is not None and toConcept is not None: if fromConcept.type != toConcept.type or fromConcept.periodType != toConcept.periodType: modelXbrl.error("xbrl.5.2.6.2.2:essenceAliasTypes", _("Essence-alias relationship from %(source)s to %(target)s in link role %(linkrole)s has different types or periodTypes"), modelObject=modelRel, source=fromConcept.qname, target=toConcept.qname, linkrole=ELR) fromBalance = fromConcept.balance toBalance = toConcept.balance if fromBalance and toBalance: if fromBalance and toBalance and fromBalance != toBalance: modelXbrl.error("xbrl.5.2.6.2.2:essenceAliasBalance", _("Essence-alias relationship from %(source)s to %(target)s in link role %(linkrole)s has different balances")).format( modelObject=modelRel, source=fromConcept.qname, target=toConcept.qname, linkrole=ELR) elif modelXbrl.hasXDT and arcrole.startswith(XbrlConst.dimStartsWith): ValidateXbrlDimensions.checkBaseSet(self, arcrole, ELR, relsSet) elif (modelXbrl.hasFormulae or modelXbrl.hasTableRendering) and arcrole.startswith(XbrlConst.formulaStartsWith): ValidateFormula.checkBaseSet(self, arcrole, ELR, relsSet) modelXbrl.isDimensionsValidated = True modelXbrl.profileStat(_("validateRelationships")) # instance checks modelXbrl.modelManager.showStatus(_("validating instance")) self.footnoteRefs = set() if modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE or \ modelXbrl.modelDocument.type == ModelDocument.Type.INLINEXBRL: self.factsWithDeprecatedIxNamespace = [] self.checkFacts(modelXbrl.facts) if self.factsWithDeprecatedIxNamespace: self.modelXbrl.info("arelle:info", _("%(count)s facts have deprecated transformation namespace %(namespace)s"), modelObject=self.factsWithDeprecatedIxNamespace, count=len(self.factsWithDeprecatedIxNamespace), namespace=FunctionIxt.deprecatedNamespaceURI) del self.factsWithDeprecatedIxNamespace #instance checks for cntx in modelXbrl.contexts.values(): if cntx.isStartEndPeriod: try: if cntx.endDatetime <= cntx.startDatetime: self.modelXbrl.error("xbrl.4.7.2:periodStartBeforeEnd", _("Context %(contextID)s must have startDate less than endDate"), modelObject=cntx, contextID=cntx.id) except (TypeError, ValueError) as err: self.modelXbrl.error("xbrl.4.7.2:contextDateError", _("Context %(contextID) startDate or endDate: %(error)s"), modelObject=cntx, contextID=cntx.id, error=err) elif cntx.isInstantPeriod: try: cntx.instantDatetime #parse field except ValueError as err: self.modelXbrl.error("xbrl.4.7.2:contextDateError", _("Context %(contextID)s instant date: %(error)s"), modelObject=cntx, contextID=cntx.id, error=err) self.segmentScenario(cntx.segment, cntx.id, "segment", "4.7.3.2") self.segmentScenario(cntx.scenario, cntx.id, "scenario", "4.7.4") for unit in modelXbrl.units.values(): mulDivMeasures = unit.measures if mulDivMeasures: for measures in mulDivMeasures: for measure in measures: if measure.namespaceURI == XbrlConst.xbrli and not \ measure in (XbrlConst.qnXbrliPure, XbrlConst.qnXbrliShares): self.modelXbrl.error("xbrl.4.8.2:measureElement", _("Unit %(unitID)s illegal measure: %(measure)s"), modelObject=unit, unitID=unit.id, measure=measure) for numeratorMeasure in mulDivMeasures[0]: if numeratorMeasure in mulDivMeasures[1]: self.modelXbrl.error("xbrl.4.8.4:measureBothNumDenom", _("Unit %(unitID)s numerator measure: %(measure)s also appears as denominator measure"), modelObject=unit, unitID=unit.id, measure=numeratorMeasure) modelXbrl.profileStat(_("validateInstance")) if modelXbrl.hasXDT: modelXbrl.modelManager.showStatus(_("validating dimensions")) ''' uncomment if using otherFacts in checkFact dimCheckableFacts = set(f for f in modelXbrl.factsInInstance if f.concept.isItem and f.context is not None) while (dimCheckableFacts): # check one and all of its compatible family members f = dimCheckableFacts.pop() ValidateXbrlDimensions.checkFact(self, f, dimCheckableFacts) del dimCheckableFacts ''' self.checkFactDimensions(modelXbrl.facts) # check fact dimensions in document order for cntx in modelXbrl.contexts.values(): ValidateXbrlDimensions.checkContext(self,cntx) modelXbrl.profileStat(_("validateDimensions")) # dimensional validity #concepts checks modelXbrl.modelManager.showStatus(_("validating concepts")) for concept in modelXbrl.qnameConcepts.values(): conceptType = concept.type if XbrlConst.isStandardNamespace(concept.qname.namespaceURI) or \ not concept.modelDocument.inDTS: continue if concept.isTuple: # must be global if not concept.getparent().localName == "schema": self.modelXbrl.error("xbrl.4.9:tupleGloballyDeclared", _("Tuple %(concept)s must be declared globally"), modelObject=concept, concept=concept.qname) if concept.periodType: self.modelXbrl.error("xbrl.4.9:tuplePeriodType", _("Tuple %(concept)s must not have periodType"), modelObject=concept, concept=concept.qname) if concept.balance: self.modelXbrl.error("xbrl.4.9:tupleBalance", _("Tuple %(concept)s must not have balance"), modelObject=concept, concept=concept.qname) if conceptType is not None: # check attribute declarations for attribute in conceptType.attributes.values(): if attribute.qname.namespaceURI in (XbrlConst.xbrli, XbrlConst.link, XbrlConst.xlink, XbrlConst.xl): self.modelXbrl.error("xbrl.4.9:tupleAttribute", _("Tuple %(concept)s must not have attribute in this namespace %(attribute)s"), modelObject=concept, concept=concept.qname, attribute=attribute.qname) # check for mixed="true" or simple content if XmlUtil.descendantAttr(conceptType, XbrlConst.xsd, ("complexType", "complexContent"), "mixed") == "true": self.modelXbrl.error("xbrl.4.9:tupleMixedContent", _("Tuple %(concept)s must not have mixed content"), modelObject=concept, concept=concept.qname) if XmlUtil.descendant(conceptType, XbrlConst.xsd, "simpleContent"): self.modelXbrl.error("xbrl.4.9:tupleSimpleContent", _("Tuple %(concept)s must not have simple content"), modelObject=concept, concept=concept.qname) # child elements must be item or tuple for elementQname in conceptType.elements: childConcept = self.modelXbrl.qnameConcepts.get(elementQname) if childConcept is None: self.modelXbrl.error("xbrl.4.9:tupleElementUndefined", _("Tuple %(concept)s element %(tupleElement)s not defined"), modelObject=concept, concept=str(concept.qname), tupleElement=elementQname) elif not (childConcept.isItem or childConcept.isTuple or # isItem/isTuple do not include item or tuple itself childConcept.qname == XbrlConst.qnXbrliItem or # subs group includes item as member childConcept.qname == XbrlConst.qnXbrliTuple): self.modelXbrl.error("xbrl.4.9:tupleElementItemOrTuple", _("Tuple %(concept)s must not have element %(tupleElement)s not an item or tuple"), modelObject=concept, concept=concept.qname, tupleElement=elementQname) elif concept.isItem: if concept.periodType not in periodTypeValues: #("instant","duration"): self.modelXbrl.error("xbrl.5.1.1.1:itemPeriodType", _("Item %(concept)s must have a valid periodType"), modelObject=concept, concept=concept.qname) if concept.isMonetary: if concept.balance not in balanceValues: #(None, "credit","debit"): self.modelXbrl.error("xbrl.5.1.1.2:itemBalance", _("Item %(concept)s must have a valid balance %(balance)s"), modelObject=concept, concept=concept.qname, balance=concept.balance) else: if concept.balance: self.modelXbrl.error("xbrl.5.1.1.2:itemBalance", _("Item %(concept)s may not have a balance"), modelObject=concept, concept=concept.qname) if concept.baseXbrliType not in baseXbrliTypes: self.modelXbrl.error("xbrl.5.1.1.3:itemType", _("Item %(concept)s type %(itemType)s invalid"), modelObject=concept, concept=concept.qname, itemType=concept.baseXbrliType) if modelXbrl.hasXDT: if concept.isHypercubeItem and not concept.abstract == "true": self.modelXbrl.error("xbrldte:HypercubeElementIsNotAbstractError", _("Hypercube item %(concept)s must be abstract"), modelObject=concept, concept=concept.qname) elif concept.isDimensionItem and not concept.abstract == "true": self.modelXbrl.error("xbrldte:DimensionElementIsNotAbstractError", _("Dimension item %(concept)s must be abstract"), modelObject=concept, concept=concept.qname) if modelXbrl.hasXDT: ValidateXbrlDimensions.checkConcept(self, concept) modelXbrl.profileStat(_("validateConcepts")) modelXbrl.modelManager.showStatus(_("validating DTS")) self.DTSreferenceResourceIDs = {} ValidateXbrlDTS.checkDTS(self, modelXbrl.modelDocument, []) del self.DTSreferenceResourceIDs global validateUniqueParticleAttribution if validateUniqueParticleAttribution is None: from arelle.XmlValidateParticles import validateUniqueParticleAttribution for modelType in modelXbrl.qnameTypes.values(): validateUniqueParticleAttribution(modelXbrl, modelType.particlesList, modelType) modelXbrl.profileStat(_("validateDTS")) if self.validateCalcLB: modelXbrl.modelManager.showStatus(_("Validating instance calculations")) ValidateXbrlCalcs.validate(modelXbrl, inferDecimals=self.validateInferDecimals) modelXbrl.profileStat(_("validateCalculations")) if (modelXbrl.modelManager.validateUtr or (self.parameters and self.parameters.get(qname("forceUtrValidation",noPrefixIsNoNamespace=True),(None,"false"))[1] == "true") or #(self.validateEFM and #any((concept.namespaceURI in self.disclosureSystem.standardTaxonomiesDict) # for concept in self.modelXbrl.nameConcepts.get("UTR",())))): (self.validateEFM and any(modelDoc.definesUTR for modelDoc in self.modelXbrl.urlDocs.values()))): ValidateUtr.validate(modelXbrl) modelXbrl.profileStat(_("validateUTR")) if modelXbrl.hasFormulae or modelXbrl.modelRenderingTables: ValidateFormula.validate(self, statusMsg=_("compiling formulae and rendering tables") if (modelXbrl.hasFormulae and modelXbrl.modelRenderingTables) else (_("compiling formulae") if modelXbrl.hasFormulae else _("compiling rendering tables"))) modelXbrl.modelManager.showStatus(_("ready"), 2000)
def validate(self, modelXbrl, parameters=None): self.parameters = parameters self.NCnamePattern = re.compile( "^[_A-Za-z\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]" r"[_\-\." "\xB7A-Za-z0-9\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\u0300-\u036F\u203F-\u2040]*$" ) self.precisionPattern = re.compile("^([0-9]+|INF)$") self.decimalsPattern = re.compile("^(-?[0-9]+|INF)$") self.isoCurrencyPattern = re.compile(r"^[A-Z]{3}$") self.modelXbrl = modelXbrl self.validateDisclosureSystem = modelXbrl.modelManager.validateDisclosureSystem self.disclosureSystem = modelXbrl.modelManager.disclosureSystem self.validateEFM = self.validateDisclosureSystem and self.disclosureSystem.EFM self.validateGFM = self.validateDisclosureSystem and self.disclosureSystem.GFM self.validateEFMorGFM = self.validateDisclosureSystem and self.disclosureSystem.EFMorGFM self.validateHMRC = self.validateDisclosureSystem and self.disclosureSystem.HMRC self.validateSBRNL = self.validateDisclosureSystem and self.disclosureSystem.SBRNL self.validateXmlLang = self.validateDisclosureSystem and self.disclosureSystem.xmlLangPattern self.validateCalcLB = modelXbrl.modelManager.validateCalcLB self.validateInferDecimals = modelXbrl.modelManager.validateInferDecimals # xlink validation modelXbrl.modelManager.showStatus(_("validating links")) modelLinks = set() self.remoteResourceLocElements = set() self.genericArcArcroles = set() for baseSetExtLinks in modelXbrl.baseSets.values(): for baseSetExtLink in baseSetExtLinks: modelLinks.add( baseSetExtLink) # ext links are unique (no dups) for modelLink in modelLinks: fromToArcs = {} locLabels = {} resourceLabels = {} resourceArcTos = [] for arcElt in modelLink.element.childNodes: if arcElt.nodeType == 1: xlinkType = arcElt.getAttributeNS(XbrlConst.xlink, "type") # locator must have an href if xlinkType == "locator": if not arcElt.hasAttributeNS(XbrlConst.xlink, "href"): modelXbrl.error( _("Linkbase {0} extended link {1} locator {2} missing href" ).format( modelLink.modelDocument.basename, modelLink.role, arcElt.hasAttributeNS( XbrlConst.xlink, "label")), "err", "xlink:locatorHref") locLabels[arcElt.getAttributeNS( XbrlConst.xlink, "label")] = arcElt elif xlinkType == "resource": resourceLabels[arcElt.getAttributeNS( XbrlConst.xlink, "label")] = arcElt # can be no duplicated arcs between same from and to elif xlinkType == "arc": fromLabel = arcElt.getAttributeNS( XbrlConst.xlink, "from") toLabel = arcElt.getAttributeNS(XbrlConst.xlink, "to") fromTo = (fromLabel, toLabel) if fromTo in fromToArcs: modelXbrl.error( _("Linkbase {0} extended link {1} duplicate arcs from {2} to {3}" ).format(modelLink.modelDocument.basename, modelLink.role, fromLabel, toLabel), "err", "xlink:dupArcs") else: fromToArcs[fromTo] = arcElt if arcElt.namespaceURI == XbrlConst.link: if arcElt.localName in arcNamesTo21Resource: #("labelArc","referenceArc"): resourceArcTos.append( (toLabel, arcElt.getAttribute("use"))) elif self.isGenericArc(arcElt): arcrole = arcElt.getAttributeNS( XbrlConst.xlink, "arcrole") self.genericArcArcroles.add(arcrole) if arcrole in (XbrlConst.elementLabel, XbrlConst.elementReference): resourceArcTos.append((toLabel, arcrole)) # values of type (not needed for validating parsers) if xlinkType not in xlinkTypeValues: # ("", "simple", "extended", "locator", "arc", "resource", "title", "none"): modelXbrl.error( _("Linkbase {0} extended link {1} type {2} invalid" ).format(modelLink.modelDocument.basename, modelLink.role, xlinkType), "err", "xlink:type") # values of actuate (not needed for validating parsers) xlinkActuate = arcElt.getAttributeNS( XbrlConst.xlink, "actuate") if xlinkActuate not in xlinkActuateValues: # ("", "onLoad", "onRequest", "other", "none"): modelXbrl.error( _("Linkbase {0} extended link {1} actuate {2} invalid" ).format(modelLink.modelDocument.basename, modelLink.role, xlinkActuate), "err", "xlink:actuate") # values of show (not needed for validating parsers) xlinkShow = arcElt.getAttributeNS(XbrlConst.xlink, "show") if xlinkShow not in xlinkShowValues: # ("", "new", "replace", "embed", "other", "none"): modelXbrl.error( _("Linkbase {0} extended link {1} show {2} invalid" ).format(modelLink.modelDocument.basename, modelLink.role, xlinkShow), "err", "xlink:show") # values of label, from, to (not needed for validating parsers) for name in xlinkLabelAttributes: # ("label", "from", "to"): value = arcElt.getAttributeNS(XbrlConst.xlink, name) if value != "" and not self.NCnamePattern.match(value): modelXbrl.error( _("Linkbase {0} extended link {1} element {2} {3} '{4}' not an NCname" ).format(modelLink.modelDocument.basename, modelLink.role, arcElt.tagName, name, value), "err", "xlink:{0}".format(name)) # check from, to of arcs have a resource or loc for fromTo, arcElt in fromToArcs.items(): fromLabel, toLabel in fromTo for name, value, sect in (("from", fromLabel, "3.5.3.9.2"), ("to", toLabel, "3.5.3.9.3")): if value not in locLabels and value not in resourceLabels: modelXbrl.error( _("Arc in linkbase {0} extended link {1} from {2} to {3} attribute \"{4}\" has no matching loc or resource label" ).format(modelLink.modelDocument.basename, modelLink.role, fromLabel, toLabel, name), "err", "xbrl.{0}:arcResource".format(sect)) if arcElt.localName == "footnoteArc" and arcElt.namespaceURI == XbrlConst.link and \ arcElt.getAttributeNS(XbrlConst.xlink,"arcrole") == XbrlConst.factFootnote: if fromLabel not in locLabels: modelXbrl.error( _("FootnoteArc in {0} extended link {1} from {2} to {3} \"from\" is not a loc" ).format(modelLink.modelDocument.basename, modelLink.role, fromLabel, toLabel), "err", "xbrl.4.11.1.3.1:factFootnoteArcFrom") if toLabel not in resourceLabels or qname( resourceLabels[toLabel] ) != XbrlConst.qnLinkFootnote: modelXbrl.error( _("FootnoteArc in {0} extended link {1} from {2} to {3} \"to\" is not a footnote resource" ).format(modelLink.modelDocument.basename, modelLink.role, fromLabel, toLabel), "err", "xbrl.4.11.1.3.1:factFootnoteArcTo") # check unprohibited label arcs to remote locs for resourceArcTo in resourceArcTos: resourceArcToLabel, resourceArcUse = resourceArcTo if resourceArcToLabel in locLabels: toLabel = locLabels[resourceArcToLabel] if resourceArcUse == "prohibited": self.remoteResourceLocElements.add(toLabel) else: modelXbrl.error( _("Unprohibited labelArc in linkbase {0} extended link {1} has illegal remote resource loc labeled {2} href {3}" ).format( modelLink.modelDocument.basename, modelLink.role, resourceArcToLabel, toLabel.getAttributeNS( XbrlConst.xlink, "href")), "err", "xbrl.5.2.2.3:labelArcRemoteResource") elif resourceArcToLabel in resourceLabels: toResource = resourceLabels[resourceArcToLabel] if resourceArcUse == XbrlConst.elementLabel: if not self.isGenericLabel(toResource): modelXbrl.error( _("Generic label arc in linkbase {0} extended link {1} to {2} must target a generic label" ).format(modelLink.modelDocument.basename, modelLink.role, resourceArcToLabel), "err", "xbrlle.2.1.1:genericLabelTarget") elif resourceArcUse == XbrlConst.elementReference: if not self.isGenericReference(toResource): modelXbrl.error( _("Generic reference arc in linkbase {0} extended link {1} to {2} must target a generic reference" ).format(modelLink.modelDocument.basename, modelLink.role, resourceArcToLabel), "err", "xbrlre.2.1.1:genericReferenceTarget") self.dimensionDefaults = {} modelXbrl.qnameDimensionDefaults = {} modelXbrl.qnameDimensionContextElement = {} # check base set cycles, dimensions modelXbrl.modelManager.showStatus(_("validating relationship sets")) for baseSetKey in modelXbrl.baseSets.keys(): arcrole, ELR, linkqname, arcqname = baseSetKey if arcrole.startswith("XBRL-") or ELR is None or \ linkqname is None or arcqname is None: continue elif arcrole in XbrlConst.standardArcroleCyclesAllowed: cyclesAllowed, specSect = XbrlConst.standardArcroleCyclesAllowed[ arcrole] elif arcrole in self.modelXbrl.arcroleTypes and len( self.modelXbrl.arcroleTypes[arcrole]) > 0: cyclesAllowed = self.modelXbrl.arcroleTypes[arcrole][ 0].cyclesAllowed if arcrole in self.genericArcArcroles: specSect = "xbrlgene:violatedCyclesConstraint" else: specSect = "xbrl.5.1.4.3:cycles" else: cyclesAllowed = "any" specSect = None if cyclesAllowed != "any" or arcrole in (XbrlConst.summationItem,) \ or arcrole in self.genericArcArcroles \ or arcrole.startswith(XbrlConst.formulaStartsWith): relsSet = modelXbrl.relationshipSet(arcrole, ELR, linkqname, arcqname) if cyclesAllowed != "any" and \ (XbrlConst.isStandardExtLinkQname(linkqname) and XbrlConst.isStandardArcQname(arcqname)) \ or arcrole in self.genericArcArcroles: noUndirected = cyclesAllowed == "none" fromRelationships = relsSet.fromModelObjects() for relFrom, rels in fromRelationships.items(): cycleFound = self.fwdCycle(relsSet, rels, noUndirected, {relFrom}) if cycleFound: modelXbrl.error( _("Relationships have a {0} cycle in arcrole {1} link role {2} link {3}, arc {4} starting from {5}" ).format(cycleFound, arcrole, ELR, linkqname, arcqname, relFrom.qname), "err", "{0}".format(specSect)) break # check calculation arcs for weight issues (note calc arc is an "any" cycles) if arcrole == XbrlConst.summationItem: for modelRel in relsSet.modelRelationships: weight = modelRel.weight fromConcept = modelRel.fromModelObject toConcept = modelRel.toModelObject if fromConcept and toConcept: if weight == 0: modelXbrl.error( _("Calculation relationship has zero weight from {0} to {1} in link role {2}" ).format(fromConcept.qname, toConcept.qname, ELR), "err", "xbrl.5.2.5.2.1:zeroWeight") fromBalance = fromConcept.balance toBalance = toConcept.balance if fromBalance and toBalance: if (fromBalance == toBalance and weight < 0) or \ (fromBalance != toBalance and weight > 0): modelXbrl.error( _("Calculation relationship has illegal weight {0} from {1}, {2}, to {3}, {4} in link role {5} (per 5.1.1.2 Table 6)" ).format(weight, fromConcept.qname, fromBalance, toConcept.qname, toBalance, ELR), "err", "xbrl.5.1.1.2:balanceCalcWeight") if not fromConcept.isNumeric or not toConcept.isNumeric: modelXbrl.error( _("Calculation relationship has illegal concept from {0}{1} to {2}{3} in link role {4}" ).format( fromConcept.qname, "" if fromConcept.isNumeric else " (non-numeric)", toConcept.qname, "" if fromConcept.isNumeric else " (non-numeric)", ELR), "err", "xbrl.5.2.5.2:nonNumericCalc") # check presentation relationships for preferredLabel issues elif arcrole == XbrlConst.parentChild: for modelRel in relsSet.modelRelationships: preferredLabel = modelRel.preferredLabel toConcept = modelRel.toModelObject if preferredLabel and toConcept and \ toConcept.label(preferredLabel=preferredLabel,fallbackToQname=False) is None: modelXbrl.error( _("Presentation relationship from {0} to {1} in link role {2} missing preferredLabel {3}" ).format(modelRel.fromModelObject.qname, toConcept.qname, ELR, preferredLabel), "err", "xbrl.5.2.4.2.1:preferredLabelMissing") # check essence-alias relationships elif arcrole == XbrlConst.essenceAlias: for modelRel in relsSet.modelRelationships: fromConcept = modelRel.fromModelObject toConcept = modelRel.toModelObject if fromConcept and toConcept: if fromConcept.type != toConcept.type or fromConcept.periodType != toConcept.periodType: modelXbrl.error( _("Essence-alias relationship from {0} to {1} in link role {2} has different types or periodTypes" ).format(fromConcept.qname, toConcept.qname, ELR), "err", "xbrl.5.2.6.2.2:essenceAliasTypes") fromBalance = fromConcept.balance toBalance = toConcept.balance if fromBalance and toBalance: if fromBalance and toBalance and fromBalance != toBalance: modelXbrl.error( _("Essence-alias relationship from {0} to {1} in link role {2} has different balances" ).format(fromConcept.qname, toConcept.qname, ELR), "err", "xbrl.5.2.6.2.2:essenceAliasBalance") elif modelXbrl.hasXDT and arcrole.startswith( XbrlConst.dimStartsWith): ValidateXbrlDimensions.checkBaseSet(self, arcrole, ELR, relsSet) elif modelXbrl.hasFormulae and arcrole.startswith( XbrlConst.formulaStartsWith): ValidateFormula.checkBaseSet(self, arcrole, ELR, relsSet) # instance checks modelXbrl.modelManager.showStatus(_("validating instance")) self.footnoteRefs = set() if modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE or \ modelXbrl.modelDocument.type == ModelDocument.Type.INLINEXBRL: for f in modelXbrl.facts: concept = f.concept if concept: if concept.isNumeric: unit = f.unit if f.unitID is None or unit is None: self.modelXbrl.error( _("Fact {0} context {1} is numeric and must have a unit" ).format(modelXbrl.modelDocument.basename, f.qname, f.contextID), "err", "xbrl.4.6.2:numericUnit") else: if concept.isMonetary: measures = unit.measures if not measures or len(measures[0]) != 1 or len(measures[1]) != 0 or \ measures[0][0].namespaceURI != XbrlConst.iso4217 or \ not self.isoCurrencyPattern.match(measures[0][0].localName): self.modelXbrl.error( _("Fact {0} context {1} must have a monetary unit {2}" ).format( modelXbrl.modelDocument.basename, f.qname, f.contextID, f.unitID), "err", "xbrl.4.8.2:monetaryFactUnit") elif concept.isShares: measures = unit.measures if not measures or len(measures[0]) != 1 or len(measures[1]) != 0 or \ measures[0][0] != XbrlConst.qnXbrliShares: self.modelXbrl.error( _("Fact {0} context {1} must have a xbrli:shares unit {2}" ).format(f.qname, f.contextID, f.unitID), "err", "xbrl.4.8.2:sharesFactUnit") precision = f.precision hasPrecision = precision is not None if hasPrecision and precision != "INF" and not precision.isdigit( ): self.modelXbrl.error( _("Fact {0} context {1} precision {2} is invalid" ).format(f.qname, f.contextID, precision), "err", "xbrl.4.6.4:precision") decimals = f.decimals hasDecimals = decimals is not None if hasPrecision and not self.precisionPattern.match( precision): self.modelXbrl.error( _("Fact {0} context {1} precision {2} is invalid" ).format(f.qname, f.contextID, precision), "err", "xbrl.4.6.4:precision") if hasPrecision and hasDecimals: self.modelXbrl.error( _("Fact {0} context {1} can not have both precision and decimals" ).format(f.qname, f.contextID), "err", "xbrl.4.6.3:bothPrecisionAndDecimals") if hasDecimals and not self.decimalsPattern.match( decimals): self.modelXbrl.error( _("Fact {0} context {1} decimals {2} is invalid" ).format(f.qname, f.contextID, decimals), "err", "xbrl.4.6.5:decimals") if concept.isItem: context = f.context if context is None: self.modelXbrl.error( _("Item {0} must have a context").format( f.qname), "err", "xbrl.4.6.1:itemContextRef") else: periodType = concept.periodType if (periodType == "instant" and not context.isInstantPeriod) or \ (periodType == "duration" and not (context.isStartEndPeriod or context.isForeverPeriod)): self.modelXbrl.error( _("Fact {0} context {1} has period type {2} conflict with context" ).format(f.qname, f.contextID, periodType), "err", "xbrl.4.7.2:contextPeriodType") if modelXbrl.hasXDT: ValidateXbrlDimensions.checkFact(self, f) # check precision and decimals if f.xsiNil == "true": if hasPrecision or hasDecimals: self.modelXbrl.error( _("Fact {0} context {1} can not be nil and have either precision or decimals" ).format(f.qname, f.contextID), "err", "xbrl.4.6.3:nilPrecisionDecimals") elif concept.isFraction: if hasPrecision or hasDecimals: self.modelXbrl.error( _("Fact {0} context {1} is a fraction concept and cannot have either precision or decimals" ).format(f.qname, f.contextID), "err", "xbrl.4.6.3:fractionPrecisionDecimals") numerator, denominator = f.fractionValue if not (numerator == "INF" or numerator.isnumeric()): self.modelXbrl.error( _("Fact {0} context {1} is a fraction with invalid numerator {2}" ).format(f.qname, f.contextID, numerator), "err", "xbrl.5.1.1:fractionPrecisionDecimals") if not denominator.isnumeric() or int( denominator) == 0: self.modelXbrl.error( _("Fact {0} context {1} is a fraction with invalid denominator {2}" ).format(f.qname, f.contextID, denominator), "err", "xbrl.5.1.1:fractionPrecisionDecimals") else: if modelXbrl.modelDocument.type != ModelDocument.Type.INLINEXBRL: for child in f.element.childNodes: if child.nodeType == 1: self.modelXbrl.error( _("Fact {0} context {1} may not have child elements {2}" ).format(f.qname, f.contextID, child.tagName), "err", "xbrl.5.1.1:itemMixedContent") break if concept.isNumeric and not hasPrecision and not hasDecimals: self.modelXbrl.error( _("Fact {0} context {1} is a numeric concept and must have either precision or decimals" ).format(f.qname, f.contextID), "err", "xbrl.4.6.3:missingPrecisionDecimals") elif concept.isTuple: if f.contextID: self.modelXbrl.error( _("Tuple {0} must not have a context").format( f.qname), "err", "xbrl.4.6.1:tupleContextRef") if hasPrecision or hasDecimals: self.modelXbrl.error( _("Fact {0} is a tuple and cannot have either precision or decimals" ).format(f.qname), "err", "xbrl.4.6.3:tuplePrecisionDecimals") # custom attributes may be allowed by anyAttribute but not by 2.1 for attrQname, attrValue in XbrlUtil.attributes( self.modelXbrl, concept, f.element): if attrQname.namespaceURI in (XbrlConst.xbrli, XbrlConst.link, XbrlConst.xlink, XbrlConst.xl): self.modelXbrl.error( _("Fact {0} is a tuple and must not have attribute in this namespace {1}" ).format(f.qname, attrQname), "err", "xbrl.4.9:tupleAttribute") else: self.modelXbrl.error( _("Fact {0} must be an item or tuple").format( f.qname), "err", "xbrl.4.6:notItemOrTuple") from arelle.ModelObject import ModelInlineFact if isinstance(f, ModelInlineFact): self.footnoteRefs.update(f.footnoteRefs) #instance checks for cntx in modelXbrl.contexts.values(): if cntx.isStartEndPeriod: try: if cntx.endDatetime <= cntx.startDatetime: self.modelXbrl.error( _("Context {0} must have startDate less than endDate" ).format(cntx.id), "err", "xbrl.4.7.2:periodStartBeforeEnd") except ValueError as err: self.modelXbrl.error( _("Context {0} startDate or endDate: {1}").format( cntx.id, err), "err", "xbrl.4.7.2:contextDateError") elif cntx.isInstantPeriod: try: cntx.instantDatetime #parse field except ValueError as err: self.modelXbrl.error( _("Context {0} instant date: {1}").format( cntx.id, err), "err", "xbrl.4.7.2:contextDateError") self.segmentScenario(cntx.segment, cntx.id, "segment", "4.7.3.2") self.segmentScenario(cntx.scenario, cntx.id, "scenario", "4.7.4") if modelXbrl.hasXDT: ValidateXbrlDimensions.checkContext(self, cntx) for unit in modelXbrl.units.values(): mulDivMeasures = unit.measures if mulDivMeasures: for measures in mulDivMeasures: for measure in measures: if measure.namespaceURI == XbrlConst.xbrli and not \ measure in (XbrlConst.qnXbrliPure, XbrlConst.qnXbrliShares): self.modelXbrl.error( _("Unit {0} illegal measure: {1}").format( unit.id, measure), "err", "xbrl.4.8.2:measureElement") for numeratorMeasure in mulDivMeasures[0]: if numeratorMeasure in mulDivMeasures[1]: self.modelXbrl.error( _("Unit {0} numerator measure: {1} also appears as denominator measure" ).format(unit.id, numeratorMeasure), "err", "xbrl.4.8.4:measureBothNumDenom") #concepts checks modelXbrl.modelManager.showStatus(_("validating concepts")) for concept in modelXbrl.qnameConcepts.values(): conceptType = concept.type if XbrlConst.isStandardNamespace(concept.namespaceURI) or \ not concept.modelDocument.inDTS: continue if concept.isTuple: # must be global if not concept.element.parentNode.localName == "schema": self.modelXbrl.error( _("Tuple {0} must be declared globally").format( concept.qname), "err", "xbrl.4.9:tupleGloballyDeclared") if concept.periodType: self.modelXbrl.error( _("Tuple {0} must not have periodType").format( concept.qname), "err", "xbrl.4.9:tuplePeriodType") if concept.balance: self.modelXbrl.error( _("Tuple {0} must not have balance").format( concept.qname), "err", "xbrl.4.9:tupleBalance") # check attribute declarations for attributeQname in conceptType.attributes: if attributeQname.namespaceURI in (XbrlConst.xbrli, XbrlConst.link, XbrlConst.xlink, XbrlConst.xl): self.modelXbrl.error( _("Tuple {0} must not have attribute in this namespace {1}" ).format(concept.qname, attributeQname), "err", "xbrl.4.9:tupleAttribute") # check for mixed="true" or simple content if XmlUtil.descendantAttr(conceptType.element, XbrlConst.xsd, ("complexType", "complexContent"), "mixed") == "true": self.modelXbrl.error( _("Tuple {0} must not have mixed content").format( concept.qname), "err", "xbrl.4.9:tupleMixedContent") if XmlUtil.descendant(conceptType.element, XbrlConst.xsd, "simpleContent"): self.modelXbrl.error( _("Tuple {0} must not have simple content").format( concept.qname), "err", "xbrl.4.9:tupleSimpleContent") # child elements must be item or tuple for elementQname in conceptType.elements: childConcept = self.modelXbrl.qnameConcepts.get( elementQname) if childConcept is None: self.modelXbrl.error( _("Tuple {0} element {1} not defined").format( concept.qname, elementQname), "err", "xbrl.4.9:tupleElementUndefined") elif not ( childConcept.isItem or childConcept.isTuple or # isItem/isTuple do not include item or tuple itself childConcept.qname == XbrlConst.qnXbrliItem or # subs group includes item as member childConcept.qname == XbrlConst.qnXbrliTuple): self.modelXbrl.error( _("Tuple {0} must not have element {1} not an item or tuple" ).format(concept.qname, elementQname), "err", "xbrl.4.9:tupleElementItemOrTuple") elif concept.isItem: if concept.periodType not in periodTypeValues: #("instant","duration"): self.modelXbrl.error( _("Item {0} must have a valid periodType").format( concept.qname), "err", "xbrl.5.1.1.1:itemPeriodType") if concept.isMonetary: if concept.balance not in balanceValues: #(None, "credit","debit"): self.modelXbrl.error( _("Item {0} must have a valid balance {1}").format( concept.qname, concept.balance), "err", "xbrl.5.1.1.2:itemBalance") else: if concept.balance: self.modelXbrl.error( _("Item {0} may not have a balance").format( concept.qname), "err", "xbrl.5.1.1.2:itemBalance") if concept.baseXbrliType not in baseXbrliTypes: self.modelXbrl.error( _("Item {0} type {1} invalid").format( concept.qname, concept.baseXbrliType), "err", "xbrl.5.1.1.3:itemType") if modelXbrl.hasXDT: if concept.isHypercubeItem and not concept.abstract == "true": self.modelXbrl.error( _("Hypercube item {0} must be abstract").format( concept.qname), "err", "xbrldte:HypercubeElementIsNotAbstractError") elif concept.isDimensionItem and not concept.abstract == "true": self.modelXbrl.error( _("Dimension item {0} must be abstract").format( concept.qname), "err", "xbrldte:DimensionElementIsNotAbstractError") if modelXbrl.hasXDT: ValidateXbrlDimensions.checkConcept(self, concept) modelXbrl.modelManager.showStatus(_("validating DTS")) self.DTSreferenceResourceIDs = {} ValidateXbrlDTS.checkDTS(self, modelXbrl.modelDocument, []) del self.DTSreferenceResourceIDs if self.validateCalcLB: modelXbrl.modelManager.showStatus( _("Validating instance calculations")) ValidateXbrlCalcs.validate( modelXbrl, inferPrecision=(not self.validateInferDecimals)) if modelXbrl.modelManager.validateUtr: ValidateUtr.validate(modelXbrl) if modelXbrl.hasFormulae: ValidateFormula.validate(self) modelXbrl.modelManager.showStatus(_("ready"), 2000)
def validateTestcase(self, testcase): self.modelXbrl.info("info", "Testcase", modelDocument=testcase) self.modelXbrl.viewModelObject(testcase.objectId()) if testcase.type in (Type.TESTCASESINDEX, Type.REGISTRY): for doc in sorted(testcase.referencesDocument.keys(), key=lambda doc: doc.uri): self.validateTestcase(doc) # testcases doc's are sorted by their uri (file names), e.g., for formula elif hasattr(testcase, "testcaseVariations"): for modelTestcaseVariation in testcase.testcaseVariations: # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) # is this a versioning report? resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance resultIsTaxonomyPackage = modelTestcaseVariation.resultIsTaxonomyPackage formulaOutputInstance = None inputDTSes = defaultdict(list) baseForElement = testcase.baseForElement(modelTestcaseVariation) # try to load instance document self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s - %(description)s"), modelObject=modelTestcaseVariation, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, expected=modelTestcaseVariation.expected, description=modelTestcaseVariation.description) if self.modelXbrl.modelManager.formulaOptions.testcaseResultsCaptureWarnings: errorCaptureLevel = logging._checkLevel("WARNING") else: errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY parameters = modelTestcaseVariation.parameters.copy() for readMeFirstUri in modelTestcaseVariation.readMeFirstUris: if isinstance(readMeFirstUri,tuple): # dtsName is for formula instances, but is from/to dts if versioning dtsName, readMeFirstUri = readMeFirstUri elif resultIsVersioningReport: if inputDTSes: dtsName = "to" else: dtsName = "from" else: dtsName = None if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document if dtsName in inputDTSes: dtsName = inputDTSes[dtsName] else: modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager, Type.DTSENTRIES, self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement), isEntry=True, errorCaptureLevel=errorCaptureLevel) DTSdoc = modelXbrl.modelDocument DTSdoc.inDTS = True doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement) if doc is not None: DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement) #fake import doc.inDTS = True elif resultIsTaxonomyPackage: from arelle import PackageManager, PrototypeInstanceObject dtsName = readMeFirstUri modelXbrl = PrototypeInstanceObject.XbrlPrototype(self.modelXbrl.modelManager, readMeFirstUri) PackageManager.packageInfo(self.modelXbrl.modelManager.cntlr, readMeFirstUri, reload=True, errors=modelXbrl.errors) else: # not a multi-schemaRef versioning report if self.useFileSource.isArchive: modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, readMeFirstUri, _("validating"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) else: # need own file source, may need instance discovery filesource = FileSource.FileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr) if filesource and not filesource.selection and filesource.isArchive: for _archiveFile in filesource.dir or (): # find instance document in archive filesource.select(_archiveFile) if ModelDocument.Type.identify(filesource, filesource.url) in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL): break # use this selection modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, filesource, _("validating"), base=baseForElement, errorCaptureLevel=errorCaptureLevel) modelXbrl.isTestcaseVariation = True if modelXbrl.modelDocument is None: modelXbrl.error("arelle:notLoaded", _("Variation %(id)s %(name)s readMeFirst document not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) self.determineNotLoadedTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif resultIsVersioningReport or resultIsTaxonomyPackage: inputDTSes[dtsName] = modelXbrl elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl) self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif testcase.type == Type.REGISTRYTESTCASE: self.instValidator.validate(modelXbrl) # required to set up dimensions, etc self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id, modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest) self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) self.instValidator.close() modelXbrl.close() else: inputDTSes[dtsName].append(modelXbrl) # validate except for formulas _hasFormulae = modelXbrl.hasFormulae modelXbrl.hasFormulae = False try: for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"): pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation) self.instValidator.validate(modelXbrl, parameters) for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"): pluginXbrlMethod(self.modelXbrl, modelXbrl) except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase variation validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) modelXbrl.hasFormulae = _hasFormulae if resultIsVersioningReport and modelXbrl.modelDocument: versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl( modelTestcaseVariation.versioningReportUri, baseForElement) if os.path.exists(versReportFile): #validate existing modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report")) if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport) self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors) modelVersReport.close() elif len(inputDTSes) == 2: ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes( versReportFile, inputDTSes["from"], inputDTSes["to"]) modelTestcaseVariation.status = "generated" else: modelXbrl.error("arelle:notLoaded", _("Variation %(id)s %(name)s input DTSes not loaded, unable to generate versioning report: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) modelTestcaseVariation.status = "failed" for inputDTS in inputDTSes.values(): inputDTS.close() del inputDTSes # dereference elif resultIsTaxonomyPackage: self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif inputDTSes: # validate schema, linkbase, or instance modelXbrl = inputDTSes[None][0] expectedDataFiles = set(modelXbrl.modelManager.cntlr.webCache.normalizeUrl(uri, baseForElement) for d in modelTestcaseVariation.dataUris.values() for uri in d if not UrlUtil.isAbsolute(uri)) foundDataFiles = set() variationBase = os.path.dirname(baseForElement) for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters if dtsName: # named instance parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes) elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula for _inputDTS in inputDTS: for docUrl, doc in _inputDTS.urlDocs.items(): if docUrl.startswith(variationBase) and not doc.type == Type.INLINEXBRLDOCUMENTSET: if getattr(doc,"loadedFromXbrlFormula", False): # may have been sourced from xf file if docUrl.replace("-formula.xml", ".xf") in expectedDataFiles: docUrl = docUrl.replace("-formula.xml", ".xf") foundDataFiles.add(docUrl) if expectedDataFiles - foundDataFiles: modelXbrl.info("arelle:testcaseDataNotUsed", _("Variation %(id)s %(name)s data files not used: %(missingDataFiles)s"), modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id, missingDataFiles=", ".join(sorted(os.path.basename(f) for f in expectedDataFiles - foundDataFiles))) if foundDataFiles - expectedDataFiles: modelXbrl.info("arelle:testcaseDataUnexpected", _("Variation %(id)s %(name)s files not in variation data: %(unexpectedDataFiles)s"), modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id, unexpectedDataFiles=", ".join(sorted(os.path.basename(f) for f in foundDataFiles - expectedDataFiles))) if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable: try: RenderingEvaluator.init(modelXbrl) except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase RenderingEvaluator.init exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) modelXbrlHasFormulae = modelXbrl.hasFormulae if modelXbrlHasFormulae: try: # validate only formulae self.instValidator.parameters = parameters ValidateFormula.validate(self.instValidator) except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset: for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"): pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri) infoset = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultInfosetUri, _("loading result infoset"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) if infoset.modelDocument is None: modelXbrl.error("arelle:notLoaded", _("Variation %(id)s %(name)s result infoset not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstance)) modelTestcaseVariation.status = "result infoset not loadable" else: # check infoset ValidateInfoset.validate(self.instValidator, modelXbrl, infoset) infoset.close() if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset: # diff (or generate) table infoset resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement) if not any(alternativeValidation(modelXbrl, resultTableUri) for alternativeValidation in pluginClassMethods("Validate.TableInfoset")): try: ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase table linkbase validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) self.instValidator.close() extraErrors = [] for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"): pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors) self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual): # if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost) self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters) self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors) if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance formulaOutputInstance = modelXbrl.formulaOutputInstance modelXbrl.formulaOutputInstance = None # prevent it from being closed now self.instValidator.close() compareIxResultInstance = (modelXbrl.modelDocument.type in (Type.INLINEXBRL, Type.INLINEXBRLDOCUMENTSET) and modelTestcaseVariation.resultXbrlInstanceUri is not None) if compareIxResultInstance: formulaOutputInstance = modelXbrl # compare modelXbrl to generated output instance errMsgPrefix = "ix" else: # delete input instances before formula output comparision for inputDTSlist in inputDTSes.values(): for inputDTS in inputDTSlist: inputDTS.close() del inputDTSes # dereference errMsgPrefix = "formula" if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument: _matchExpectedResultIDs = not modelXbrlHasFormulae # formula restuls have inconsistent IDs expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultXbrlInstanceUri, _("loading expected result XBRL instance"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) if expectedInstance.modelDocument is None: self.modelXbrl.error("{}:expectedResultNotLoaded".format(errMsgPrefix), _("Testcase \"%(name)s\" %(id)s expected result instance not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstanceUri), messageCodes=("formula:expectedResultNotLoaded","ix:expectedResultNotLoaded")) modelTestcaseVariation.status = "result not loadable" else: # compare facts if len(expectedInstance.facts) != len(formulaOutputInstance.facts): formulaOutputInstance.error("{}:resultFactCounts".format(errMsgPrefix), _("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"), modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts), expectedFacts=len(expectedInstance.facts), messageCodes=("formula:resultFactCounts","ix:resultFactCounts")) else: formulaOutputFootnotesRelSet = ModelRelationshipSet(formulaOutputInstance, "XBRL-footnotes") expectedFootnotesRelSet = ModelRelationshipSet(expectedInstance, "XBRL-footnotes") def factFootnotes(fact, footnotesRelSet): footnotes = {} footnoteRels = footnotesRelSet.fromModelObject(fact) if footnoteRels: # most process rels in same order between two instances, use labels to sort for i, footnoteRel in enumerate(sorted(footnoteRels, key=lambda r: (r.fromLabel,r.toLabel))): modelObject = footnoteRel.toModelObject if isinstance(modelObject, ModelResource): xml = modelObject.viewText().strip() footnotes["Footnote {}".format(i+1)] = xml #re.sub(r'\s+', ' ', collapseWhitespace(modelObject.stringValue)) elif isinstance(modelObject, ModelFact): footnotes["Footnoted fact {}".format(i+1)] = \ "{} context: {} value: {}".format( modelObject.qname, modelObject.contextID, collapseWhitespace(modelObject.value)) return footnotes for expectedInstanceFact in expectedInstance.facts: unmatchedFactsStack = [] formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=_matchExpectedResultIDs, matchLang=False) #formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=True, matchLang=True) if formulaOutputFact is None: if unmatchedFactsStack: # get missing nested tuple fact, if possible missingFact = unmatchedFactsStack[-1] else: missingFact = expectedInstanceFact formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix), _("Output missing expected fact %(fact)s"), modelXbrl=missingFact, fact=missingFact.qname, messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing")) else: # compare footnotes expectedInstanceFactFootnotes = factFootnotes(expectedInstanceFact, expectedFootnotesRelSet) formulaOutputFactFootnotes = factFootnotes(formulaOutputFact, formulaOutputFootnotesRelSet) if (len(expectedInstanceFactFootnotes) != len(formulaOutputFactFootnotes) or set(expectedInstanceFactFootnotes.values()) != set(formulaOutputFactFootnotes.values())): formulaOutputInstance.error("{}:expectedFactFootnoteDifference".format(errMsgPrefix), _("Output expected fact %(fact)s expected footnotes %(footnotes1)s produced footnotes %(footnotes2)s"), modelXbrl=(formulaOutputFact,expectedInstanceFact), fact=expectedInstanceFact.qname, footnotes1=sorted(expectedInstanceFactFootnotes.items()), footnotes2=sorted(formulaOutputFactFootnotes.items()), messageCodes=("formula:expectedFactFootnoteDifference","ix:expectedFactFootnoteDifference")) # for debugging uncomment next line to save generated instance document # formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml") expectedInstance.close() del expectedInstance # dereference self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors) formulaOutputInstance.close() del formulaOutputInstance if compareIxResultInstance: for inputDTSlist in inputDTSes.values(): for inputDTS in inputDTSlist: inputDTS.close() del inputDTSes # dereference # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) _statusCounts = OrderedDict((("pass",0),("fail",0))) for tv in getattr(testcase, "testcaseVariations", ()): _statusCounts[tv.status] = _statusCounts.get(tv.status, 0) + 1 self.modelXbrl.info("arelle:testCaseResults", ", ".join("{}={}".format(k,c) for k, c in _statusCounts.items() if k)) self.modelXbrl.modelManager.showStatus(_("ready"), 2000)