Example #1
0
def backgroundProfileFormula(cntlr, profileReportFile, maxRunTime, excludeCompileTime):
    from arelle import Locale, XPathParser, ValidateXbrlDimensions, ValidateFormula

    # build grammar before profiling (if this is the first pass, so it doesn't count in profile statistics)
    XPathParser.initializeParser(cntlr.modelManager)
    
    # load dimension defaults
    ValidateXbrlDimensions.loadDimensionDefaults(cntlr.modelManager)
    
    import cProfile, pstats, sys, time
    
    # a minimal validation class for formula validator parameters that are needed
    class Validate:
        def __init__(self, modelXbrl, maxRunTime):
            self.modelXbrl = modelXbrl
            self.parameters = None
            self.validateSBRNL = False
            self.maxFormulaRunTime = maxRunTime
        def close(self):
            self.__dict__.clear()
            
    val = Validate(cntlr.modelManager.modelXbrl, maxRunTime)
    formulaOptions = val.modelXbrl.modelManager.formulaOptions
    if excludeCompileTime:
        startedAt = time.time()
        cntlr.addToLog(_("pre-compiling formulas before profiling"))
        val.validateFormulaCompileOnly = True
        ValidateFormula.validate(val)
        del val.validateFormulaCompileOnly
        cntlr.addToLog(Locale.format_string(cntlr.modelManager.locale, 
                                            _("formula pre-compiling completed in %.2f secs"), 
                                            time.time() - startedAt))
        cntlr.addToLog(_("executing formulas for profiling"))
    else:
        cntlr.addToLog(_("compiling and executing formulas for profiling"))
    startedAt = time.time()
            
    statsFile = profileReportFile + ".bin"
    cProfile.runctx("ValidateFormula.validate(val)", globals(), locals(), statsFile)
    cntlr.addToLog(Locale.format_string(cntlr.modelManager.locale, 
                                        _("formula profiling completed in %.2f secs"), 
                                        time.time() - startedAt))
    # dereference val
    val.close()
    
    # specify a file for log
    priorStdOut = sys.stdout
    sys.stdout = open(profileReportFile, "w")

    statObj = pstats.Stats(statsFile)
    statObj.strip_dirs()
    statObj.sort_stats("time")
    statObj.print_stats()
    statObj.print_callees()
    statObj.print_callers()
    sys.stdout.flush()
    sys.stdout.close()
    del statObj
    sys.stdout = priorStdOut
    os.remove(statsFile)
Example #2
0
 def validateTestcase(self, testcase):
     self.modelXbrl.info("info", "Testcase", modelDocument=testcase)
     self.modelXbrl.viewModelObject(testcase.objectId())
     if hasattr(testcase, "testcaseVariations"):
         for modelTestcaseVariation in testcase.testcaseVariations:
             # update ui thread via modelManager (running in background here)
             self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
             # is this a versioning report?
             resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
             resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
             formulaOutputInstance = None
             inputDTSes = defaultdict(list)
             baseForElement = testcase.baseForElement(modelTestcaseVariation)
             # try to load instance document
             self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s - %(description)s"),
                                 modelObject=modelTestcaseVariation, 
                                 id=modelTestcaseVariation.id, 
                                 name=modelTestcaseVariation.name, 
                                 expected=modelTestcaseVariation.expected, 
                                 description=modelTestcaseVariation.description)
             errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY
             parameters = modelTestcaseVariation.parameters.copy()
             for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
                 if isinstance(readMeFirstUri,tuple):
                     # dtsName is for formula instances, but is from/to dts if versioning
                     dtsName, readMeFirstUri = readMeFirstUri
                 elif resultIsVersioningReport:
                     if inputDTSes: dtsName = "to"
                     else: dtsName = "from"
                 else:
                     dtsName = None
                 if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
                     if dtsName in inputDTSes:
                         dtsName = inputDTSes[dtsName]
                     else:
                         modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager, 
                                      Type.DTSENTRIES,
                                      self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
                                      isEntry=True,
                                      errorCaptureLevel=errorCaptureLevel)
                     DTSdoc = modelXbrl.modelDocument
                     DTSdoc.inDTS = True
                     doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement)
                     if doc is not None:
                         DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement)  #fake import
                         doc.inDTS = True
                 else: # not a multi-schemaRef versioning report
                     if self.useFileSource.isArchive:
                         modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, 
                                                    readMeFirstUri,
                                                    _("validating"), 
                                                    base=baseForElement,
                                                    useFileSource=self.useFileSource,
                                                    errorCaptureLevel=errorCaptureLevel)
                     else: # need own file source, may need instance discovery
                         filesource = FileSource.FileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr)
                         if filesource and not filesource.selection and filesource.isArchive:
                             for _archiveFile in filesource.dir: # find instance document in archive
                                 filesource.select(_archiveFile)
                                 if ModelDocument.Type.identify(filesource, filesource.url) in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL):
                                     break # use this selection
                         modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, 
                                                    filesource,
                                                    _("validating"), 
                                                    base=baseForElement,
                                                    errorCaptureLevel=errorCaptureLevel)
                 if modelXbrl.modelDocument is None:
                     self.modelXbrl.error("arelle:notLoaded",
                          _("Testcase %(id)s %(name)s document not loaded: %(file)s"),
                          modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
                     modelXbrl.close()
                     self.determineNotLoadedTestStatus(modelTestcaseVariation)
                 elif resultIsVersioningReport:
                     inputDTSes[dtsName] = modelXbrl
                 elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT:
                     ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl)
                     self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
                     modelXbrl.close()
                 elif testcase.type == Type.REGISTRYTESTCASE:
                     self.instValidator.validate(modelXbrl)  # required to set up dimensions, etc
                     self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id, 
                                modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
                     self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
                     self.instValidator.close()
                     modelXbrl.close()
                 else:
                     inputDTSes[dtsName].append(modelXbrl)
                     # validate except for formulas
                     _hasFormulae = modelXbrl.hasFormulae
                     modelXbrl.hasFormulae = False
                     try:
                         for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"):
                             pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation)
                         self.instValidator.validate(modelXbrl, parameters)
                         for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"):
                             pluginXbrlMethod(self.modelXbrl, modelXbrl)
                     except Exception as err:
                         self.modelXbrl.error("exception",
                             _("Testcase variation validation exception: %(error)s, instance: %(instance)s"),
                             modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
                     modelXbrl.hasFormulae = _hasFormulae
             if resultIsVersioningReport and modelXbrl.modelDocument:
                 versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
                     modelTestcaseVariation.versioningReportUri, baseForElement)
                 if os.path.exists(versReportFile): #validate existing
                     modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
                     if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT:
                         ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
                         self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors)
                         modelVersReport.close()
                 elif len(inputDTSes) == 2:
                     ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
                           versReportFile, inputDTSes["from"], inputDTSes["to"])
                     modelTestcaseVariation.status = "generated"
                 else:
                     self.modelXbrl.error("arelle:notLoaded",
                          _("Testcase %(id)s %(name)s DTSes not loaded, unable to generate versioning report: %(file)s"),
                          modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
                     modelTestcaseVariation.status = "failed"
                 for inputDTS in inputDTSes.values():
                     inputDTS.close()
                 del inputDTSes # dereference
             elif inputDTSes:
                 # validate schema, linkbase, or instance
                 modelXbrl = inputDTSes[None][0]
                 for dtsName, inputDTS in inputDTSes.items():  # input instances are also parameters
                     if dtsName: # named instance
                         parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes)
                     elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents
                         parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula
                 if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable:
                     RenderingEvaluator.init(modelXbrl)
                 if modelXbrl.hasFormulae:
                     try:
                         # validate only formulae
                         self.instValidator.parameters = parameters
                         ValidateFormula.validate(self.instValidator)
                     except Exception as err:
                         self.modelXbrl.error("exception",
                             _("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"),
                             modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
                 if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset:
                     for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"):
                         pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri)
                     infoset = ModelXbrl.load(self.modelXbrl.modelManager, 
                                              modelTestcaseVariation.resultInfosetUri,
                                                _("loading result infoset"), 
                                                base=baseForElement,
                                                useFileSource=self.useFileSource,
                                                errorCaptureLevel=errorCaptureLevel)
                     if infoset.modelDocument is None:
                         self.modelXbrl.error("arelle:notLoaded",
                             _("Testcase %(id)s %(name)s result infoset not loaded: %(file)s"),
                             modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, 
                             file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
                         modelTestcaseVariation.status = "result infoset not loadable"
                     else:   # check infoset
                         ValidateInfoset.validate(self.instValidator, modelXbrl, infoset)
                     infoset.close()
                 if modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset:
                     # diff (or generate) table infoset
                     resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement)
                     if not any(alternativeValidation(modelXbrl, resultTableUri)
                                for alternativeValidation in pluginClassMethods("Validate.TableInfoset")):
                         ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True)  # false to save infoset files
                 self.instValidator.close()
                 extraErrors = []
                 for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"):
                     pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors)
                 self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status
                 if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual): 
                     # if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it
                     modelXbrl.formulaOutputInstance.hasFormulae = False #  block formulae on output instance (so assertion of input is not lost)
                     self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
                     self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors)
                     if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance
                         formulaOutputInstance = modelXbrl.formulaOutputInstance
                         modelXbrl.formulaOutputInstance = None # prevent it from being closed now
                     self.instValidator.close()
                 for inputDTSlist in inputDTSes.values():
                     for inputDTS in inputDTSlist:
                         inputDTS.close()
                 del inputDTSes # dereference
                 if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
                     expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager, 
                                                modelTestcaseVariation.resultXbrlInstanceUri,
                                                _("loading expected result XBRL instance"), 
                                                base=baseForElement,
                                                useFileSource=self.useFileSource,
                                                errorCaptureLevel=errorCaptureLevel)
                     if expectedInstance.modelDocument is None:
                         self.modelXbrl.error("arelle:notLoaded",
                             _("Testcase %(id)s %(name)s expected result instance not loaded: %(file)s"),
                             modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, 
                             file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
                         modelTestcaseVariation.status = "result not loadable"
                     else:   # compare facts
                         if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
                             formulaOutputInstance.error("formula:resultFactCounts",
                                 _("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"),
                                 modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts),
                                      expectedFacts=len(expectedInstance.facts))
                         else:
                             for fact in expectedInstance.facts:
                                 unmatchedFactsStack = []
                                 if formulaOutputInstance.matchFact(fact, unmatchedFactsStack, deemP0inf=True) is None:
                                     if unmatchedFactsStack: # get missing nested tuple fact, if possible
                                         missingFact = unmatchedFactsStack[-1]
                                     else:
                                         missingFact = fact
                                     formulaOutputInstance.error("formula:expectedFactMissing",
                                         _("Formula output missing expected fact %(fact)s"),
                                         modelXbrl=missingFact, fact=missingFact.qname)
                         # for debugging uncomment next line to save generated instance document
                         # formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml")
                     expectedInstance.close()
                     del expectedInstance # dereference
                     self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors)
                     formulaOutputInstance.close()
                     del formulaOutputInstance
             # update ui thread via modelManager (running in background here)
             self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
                 
         self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
Example #3
0
def init(modelXbrl):
    # setup modelXbrl for rendering evaluation

    # dimension defaults required in advance of validation
    from arelle import ValidateXbrlDimensions, ValidateFormula, ModelDocument
    ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
    
    hasXbrlTables = False
    
    # validate table linkbase dimensions
    for baseSetKey in modelXbrl.baseSets.keys():
        arcrole, ELR, linkqname, arcqname = baseSetKey
        if ELR and linkqname and arcqname and XbrlConst.isTableRenderingArcrole(arcrole):
            ValidateFormula.checkBaseSet(modelXbrl, arcrole, ELR, modelXbrl.relationshipSet(arcrole,ELR,linkqname,arcqname))
            if arcrole in (XbrlConst.tableBreakdown, XbrlConst.tableBreakdownMMDD, XbrlConst.tableBreakdown201305, XbrlConst.tableBreakdown201301, XbrlConst.tableAxis2011):
                hasXbrlTables = True

    # provide context for view
    if modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE:
        instance = None # use instance of the entry pont
    else: # need dummy instance
        instance = ModelDocument.create(modelXbrl, ModelDocument.Type.INSTANCE, 
                                        "dummy.xml",  # fake URI and fake schemaRef 
                                        ("http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd",))
        
    if hasXbrlTables:
        # formula processor is needed for 2011 XBRL tables but not for 2010 Eurofiling tables
        modelXbrl.rendrCntx = XPathContext.create(modelXbrl, instance)
        
        modelXbrl.profileStat(None)
        
        # setup fresh parameters from formula options
        modelXbrl.parameters = modelXbrl.modelManager.formulaOptions.typedParameters()
        
        # validate parameters and custom function signatures
        ValidateFormula.validate(modelXbrl, xpathContext=modelXbrl.rendrCntx, parametersOnly=True, statusMsg=_("compiling rendering tables"))
        
        # deprecated as of 2013-05-17
        # check and extract message expressions into compilable programs
        for msgArcrole in (XbrlConst.tableDefinitionNodeMessage201301, XbrlConst.tableDefinitionNodeSelectionMessage201301,
                           XbrlConst.tableAxisMessage2011, XbrlConst.tableAxisSelectionMessage2011):
            for msgRel in modelXbrl.relationshipSet(msgArcrole).modelRelationships:
                ValidateFormula.checkMessageExpressions(modelXbrl, msgRel.toModelObject)
                
        # compile and validate tables
        for modelTable in modelXbrl.modelRenderingTables:
            modelTable.fromInstanceQnames = None # required if referred to by variables scope chaining
            modelTable.compile()

            hasNsWithAspectModel = modelTable.namespaceURI in (XbrlConst.euRend, XbrlConst.table2011, XbrlConst.table201301, XbrlConst.table201305) 
        
            # check aspectModel  (attribute removed 2013-06, now always dimensional)
            if modelTable.aspectModel not in ("non-dimensional", "dimensional") and hasNsWithAspectModel:
                modelXbrl.error("xbrlte:unknownAspectModel",
                    _("Table %(xlinkLabel)s, aspect model %(aspectModel)s not recognized"),
                    modelObject=modelTable, xlinkLabel=modelTable.xlinkLabel, aspectModel=modelTable.aspectModel)
            else:
                modelTable.priorAspectAxisDisposition = {}
                # check ordinate aspects against aspectModel
                oppositeAspectModel = (_DICT_SET({'dimensional','non-dimensional'}) - _DICT_SET({modelTable.aspectModel})).pop()
                if hasNsWithAspectModel:
                    uncoverableAspects = aspectModels[oppositeAspectModel] - aspectModels[modelTable.aspectModel]
                else:
                    uncoverableAspects = ()
                aspectsCovered = set()
                for tblAxisRel in modelXbrl.relationshipSet((XbrlConst.tableBreakdown, XbrlConst.tableBreakdownMMDD, XbrlConst.tableBreakdown201305, XbrlConst.tableBreakdown201301,XbrlConst.tableAxis2011)).fromModelObject(modelTable):
                    breakdownAspectsCovered = set()
                    hasCoveredAspect = checkBreakdownDefinitionNode(modelXbrl, modelTable, tblAxisRel, tblAxisRel.axisDisposition, uncoverableAspects, breakdownAspectsCovered)
                    ''' removed 2013-10
                    if not hasCoveredAspect:
                        definitionNode = tblAxisRel.toModelObject
                        modelXbrl.error("xbrlte:breakdownDefinesNoAspects",
                            _("Breakdown %(xlinkLabel)s has no participating aspects"),
                            modelObject=(modelTable,definitionNode), xlinkLabel=definitionNode.xlinkLabel, axis=definitionNode.localName)
                    '''
                    aspectsCovered |= breakdownAspectsCovered
                    checkBreakdownLeafNodeAspects(modelXbrl, modelTable, tblAxisRel, set(), breakdownAspectsCovered)
                if Aspect.CONCEPT not in aspectsCovered and not hasNsWithAspectModel:
                    modelXbrl.error("xbrlte:tableMissingConceptAspect",
                        _("Table %(xlinkLabel)s does not include the concept aspect as one of its participating aspects"),
                        modelObject=modelTable, xlinkLabel=modelTable.xlinkLabel)
                del modelTable.priorAspectAxisDisposition
                # check for table-parameter name clash
                parameterNames = {}
                for tblParamRel in modelXbrl.relationshipSet((XbrlConst.tableParameter, XbrlConst.tableParameterMMDD)).fromModelObject(modelTable):
                    parameterName = tblParamRel.variableQname
                    if parameterName in parameterNames:
                        modelXbrl.error("xbrlte:tableParameterNameClash ",
                            _("Table %(xlinkLabel)s has parameter name clash for variable %(name)s"),
                            modelObject=(modelTable,tblParamRel,parameterNames[parameterName]), xlinkLabel=modelTable.xlinkLabel, name=parameterName)
                    else:
                        parameterNames[parameterName] = tblParamRel
    
        modelXbrl.profileStat(_("compileTables"))
Example #4
0
    def run(self, options, sourceZipStream=None):
        """Process command line arguments or web service request, such as to load and validate an XBRL document, or start web server.
        
        When a web server has been requested, this method may be called multiple times, once for each web service (REST) request that requires processing.
        Otherwise (when called for a command line request) this method is called only once for the command line arguments request.
           
        :param options: OptionParser options from parse_args of main argv arguments (when called from command line) or corresponding arguments from web service (REST) request.
        :type options: optparse.Values
        """
        if options.showOptions: # debug options
            for optName, optValue in sorted(options.__dict__.items(), key=lambda optItem: optItem[0]):
                self.addToLog("Option {0}={1}".format(optName, optValue), messageCode="info")
            self.addToLog("sys.argv {0}".format(sys.argv), messageCode="info")
        if options.uiLang: # set current UI Lang (but not config setting)
            self.setUiLanguage(options.uiLang)
        if options.proxy:
            if options.proxy != "show":
                proxySettings = proxyTuple(options.proxy)
                self.webCache.resetProxies(proxySettings)
                self.config["proxySettings"] = proxySettings
                self.saveConfig()
                self.addToLog(_("Proxy configuration has been set."), messageCode="info")
            useOsProxy, urlAddr, urlPort, user, password = self.config.get("proxySettings", proxyTuple("none"))
            if useOsProxy:
                self.addToLog(_("Proxy configured to use {0}.").format(
                    _('Microsoft Windows Internet Settings') if sys.platform.startswith("win")
                    else (_('Mac OS X System Configuration') if sys.platform in ("darwin", "macos")
                          else _('environment variables'))), messageCode="info")
            elif urlAddr:
                self.addToLog(_("Proxy setting: http://{0}{1}{2}{3}{4}").format(
                    user if user else "",
                    ":****" if password else "",
                    "@" if (user or password) else "",
                    urlAddr,
                    ":{0}".format(urlPort) if urlPort else ""), messageCode="info")
            else:
                self.addToLog(_("Proxy is disabled."), messageCode="info")
        if options.plugins:
            from arelle import PluginManager
            resetPlugins = False
            savePluginChanges = True
            showPluginModules = False
            for pluginCmd in options.plugins.split('|'):
                cmd = pluginCmd.strip()
                if cmd == "show":
                    showPluginModules = True
                elif cmd == "temp":
                    savePluginChanges = False
                elif cmd.startswith("+"):
                    moduleInfo = PluginManager.addPluginModule(cmd[1:])
                    if moduleInfo:
                        self.addToLog(_("Addition of plug-in {0} successful.").format(moduleInfo.get("name")), 
                                      messageCode="info", file=moduleInfo.get("moduleURL"))
                        resetPlugins = True
                    else:
                        self.addToLog(_("Unable to load plug-in."), messageCode="info", file=cmd[1:])
                elif cmd.startswith("~"):
                    if PluginManager.reloadPluginModule(cmd[1:]):
                        self.addToLog(_("Reload of plug-in successful."), messageCode="info", file=cmd[1:])
                        resetPlugins = True
                    else:
                        self.addToLog(_("Unable to reload plug-in."), messageCode="info", file=cmd[1:])
                elif cmd.startswith("-"):
                    if PluginManager.removePluginModule(cmd[1:]):
                        self.addToLog(_("Deletion of plug-in successful."), messageCode="info", file=cmd[1:])
                        resetPlugins = True
                    else:
                        self.addToLog(_("Unable to delete plug-in."), messageCode="info", file=cmd[1:])
                else: # assume it is a module or package
                    savePluginChanges = False
                    moduleInfo = PluginManager.addPluginModule(cmd)
                    if moduleInfo:
                        self.addToLog(_("Activation of plug-in {0} successful.").format(moduleInfo.get("name")), 
                                      messageCode="info", file=moduleInfo.get("moduleURL"))
                        resetPlugins = True
                    else:
                        self.addToLog(_("Unable to load {0} as a plug-in or {0} is not recognized as a command. ").format(cmd), messageCode="info", file=cmd)
                if resetPlugins:
                    PluginManager.reset()
                    if savePluginChanges:
                        PluginManager.save(self)
            if showPluginModules:
                self.addToLog(_("Plug-in modules:"), messageCode="info")
                for i, moduleItem in enumerate(sorted(PluginManager.pluginConfig.get("modules", {}).items())):
                    moduleInfo = moduleItem[1]
                    self.addToLog(_("Plug-in: {0}; author: {1}; version: {2}; status: {3}; date: {4}; description: {5}; license {6}.").format(
                                  moduleItem[0], moduleInfo.get("author"), moduleInfo.get("version"), moduleInfo.get("status"),
                                  moduleInfo.get("fileDate"), moduleInfo.get("description"), moduleInfo.get("license")),
                                  messageCode="info", file=moduleInfo.get("moduleURL"))
                
        # run utility command line options that don't depend on entrypoint Files
        hasUtilityPlugin = False
        for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Utility.Run"):
            hasUtilityPlugin = True
            pluginXbrlMethod(self, options)
            
        # if no entrypointFile is applicable, quit now
        if options.proxy or options.plugins or hasUtilityPlugin:
            if not options.entrypointFile:
                return True # success
        self.username = options.username
        self.password = options.password
        self.entrypointFile = options.entrypointFile
        if self.entrypointFile:
            filesource = FileSource.openFileSource(self.entrypointFile, self, sourceZipStream)
        else:
            filesource = None
        if options.validateEFM:
            if options.disclosureSystemName:
                self.addToLog(_("both --efm and --disclosureSystem validation are requested, proceeding with --efm only"),
                              messageCode="info", file=self.entrypointFile)
            self.modelManager.validateDisclosureSystem = True
            self.modelManager.disclosureSystem.select("efm")
        elif options.disclosureSystemName:
            self.modelManager.validateDisclosureSystem = True
            self.modelManager.disclosureSystem.select(options.disclosureSystemName)
        elif options.validateHMRC:
            self.modelManager.validateDisclosureSystem = True
            self.modelManager.disclosureSystem.select("hmrc")
        else:
            self.modelManager.disclosureSystem.select(None) # just load ordinary mappings
            self.modelManager.validateDisclosureSystem = False
        if options.utrUrl:  # override disclosureSystem utrUrl
            self.modelManager.disclosureSystem.utrUrl = options.utrUrl
            # can be set now because the utr is first loaded at validation time 
            
        # disclosure system sets logging filters, override disclosure filters, if specified by command line
        if options.logLevelFilter:
            self.setLogLevelFilter(options.logLevelFilter)
        if options.logCodeFilter:
            self.setLogCodeFilter(options.logCodeFilter)
        if options.calcDecimals:
            if options.calcPrecision:
                self.addToLog(_("both --calcDecimals and --calcPrecision validation are requested, proceeding with --calcDecimals only"),
                              messageCode="info", file=self.entrypointFile)
            self.modelManager.validateInferDecimals = True
            self.modelManager.validateCalcLB = True
        elif options.calcPrecision:
            self.modelManager.validateInferDecimals = False
            self.modelManager.validateCalcLB = True
        if options.utrValidate:
            self.modelManager.validateUtr = True
        if options.infosetValidate:
            self.modelManager.validateInfoset = True
        if options.abortOnMajorError:
            self.modelManager.abortOnMajorError = True
        if options.collectProfileStats:
            self.modelManager.collectProfileStats = True
        if options.internetConnectivity == "offline":
            self.webCache.workOffline = True
        elif options.internetConnectivity == "online":
            self.webCache.workOffline = False
        if options.internetTimeout is not None:
            self.webCache.timeout = (options.internetTimeout or None)  # use None if zero specified to disable timeout
        fo = FormulaOptions()
        if options.parameters:
            parameterSeparator = (options.parameterSeparator or ',')
            fo.parameterValues = dict(((qname(key, noPrefixIsNoNamespace=True),(None,value)) 
                                       for param in options.parameters.split(parameterSeparator) 
                                       for key,sep,value in (param.partition('='),) ) )   
        if options.formulaParamExprResult:
            fo.traceParameterExpressionResult = True
        if options.formulaParamInputValue:
            fo.traceParameterInputValue = True
        if options.formulaCallExprSource:
            fo.traceCallExpressionSource = True
        if options.formulaCallExprCode:
            fo.traceCallExpressionCode = True
        if options.formulaCallExprEval:
            fo.traceCallExpressionEvaluation = True
        if options.formulaCallExprResult:
            fo.traceCallExpressionResult = True
        if options.formulaVarSetExprEval:
            fo.traceVariableSetExpressionEvaluation = True
        if options.formulaVarSetExprResult:
            fo.traceVariableSetExpressionResult = True
        if options.formulaAsserResultCounts:
            fo.traceAssertionResultCounts = True
        if options.formulaFormulaRules:
            fo.traceFormulaRules = True
        if options.formulaVarsOrder:
            fo.traceVariablesOrder = True
        if options.formulaVarExpressionSource:
            fo.traceVariableExpressionSource = True
        if options.formulaVarExpressionCode:
            fo.traceVariableExpressionCode = True
        if options.formulaVarExpressionEvaluation:
            fo.traceVariableExpressionEvaluation = True
        if options.formulaVarExpressionResult:
            fo.traceVariableExpressionResult = True
        if options.timeVariableSetEvaluation:
            fo.timeVariableSetEvaluation = True
        if options.formulaVarFilterWinnowing:
            fo.traceVariableFilterWinnowing = True
        if options.formulaVarFiltersResult:
            fo.traceVariableFiltersResult = True
        if options.formulaVarFiltersResult:
            fo.traceVariableFiltersResult = True
        self.modelManager.formulaOptions = fo
        timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
        firstStartedAt = startedAt = time.time()
        modelDiffReport = None
        success = True
        modelXbrl = None
        try:
            if filesource:
                modelXbrl = self.modelManager.load(filesource, _("views loading"))
        except ModelDocument.LoadingException:
            pass
        except Exception as err:
            self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format(
                        err,
                        traceback.format_tb(sys.exc_info()[2])))
            success = False    # loading errors, don't attempt to utilize loaded DTS
        if modelXbrl and modelXbrl.modelDocument:
            loadTime = time.time() - startedAt
            modelXbrl.profileStat(_("load"), loadTime)
            self.addToLog(format_string(self.modelManager.locale, 
                                        _("loaded in %.2f secs at %s"), 
                                        (loadTime, timeNow)), 
                                        messageCode="info", file=self.entrypointFile)
            if options.importFiles:
                for importFile in options.importFiles.split("|"):
                    fileName = importFile.strip()
                    if sourceZipStream is not None and not (fileName.startswith('http://') or os.path.isabs(fileName)):
                        fileName = os.path.dirname(modelXbrl.uri) + os.sep + fileName # make relative to sourceZipStream
                    ModelDocument.load(modelXbrl, fileName)
                    loadTime = time.time() - startedAt
                    self.addToLog(format_string(self.modelManager.locale, 
                                                _("import in %.2f secs at %s"), 
                                                (loadTime, timeNow)), 
                                                messageCode="info", file=importFile)
                    modelXbrl.profileStat(_("import"), loadTime)
                if modelXbrl.errors:
                    success = False    # loading errors, don't attempt to utilize loaded DTS
            if modelXbrl.modelDocument.type in ModelDocument.Type.TESTCASETYPES:
                for pluginXbrlMethod in pluginClassMethods("Testcases.Start"):
                    pluginXbrlMethod(self, options, modelXbrl)
            else: # not a test case, probably instance or DTS
                for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Loaded"):
                    pluginXbrlMethod(self, options, modelXbrl)
        else:
            success = False
        if success and options.diffFile and options.versReportFile:
            try:
                diffFilesource = FileSource.FileSource(options.diffFile,self)
                startedAt = time.time()
                modelXbrl2 = self.modelManager.load(diffFilesource, _("views loading"))
                if modelXbrl2.errors:
                    if not options.keepOpen:
                        modelXbrl2.close()
                    success = False
                else:
                    loadTime = time.time() - startedAt
                    modelXbrl.profileStat(_("load"), loadTime)
                    self.addToLog(format_string(self.modelManager.locale, 
                                                _("diff comparison DTS loaded in %.2f secs"), 
                                                loadTime), 
                                                messageCode="info", file=self.entrypointFile)
                    startedAt = time.time()
                    modelDiffReport = self.modelManager.compareDTSes(options.versReportFile)
                    diffTime = time.time() - startedAt
                    modelXbrl.profileStat(_("diff"), diffTime)
                    self.addToLog(format_string(self.modelManager.locale, 
                                                _("compared in %.2f secs"), 
                                                diffTime), 
                                                messageCode="info", file=self.entrypointFile)
            except ModelDocument.LoadingException:
                success = False
            except Exception as err:
                success = False
                self.addToLog(_("[Exception] Failed to doad diff file: \n{0} \n{1}").format(
                            err,
                            traceback.format_tb(sys.exc_info()[2])))
        if success:
            try:
                modelXbrl = self.modelManager.modelXbrl
                hasFormulae = modelXbrl.hasFormulae
                if options.validate:
                    startedAt = time.time()
                    if options.formulaAction: # don't automatically run formulas
                        modelXbrl.hasFormulae = False
                    self.modelManager.validate()
                    if options.formulaAction: # restore setting
                        modelXbrl.hasFormulae = hasFormulae
                    self.addToLog(format_string(self.modelManager.locale, 
                                                _("validated in %.2f secs"), 
                                                time.time() - startedAt),
                                                messageCode="info", file=self.entrypointFile)
                if options.formulaAction in ("validate", "run"):  # do nothing here if "none"
                    from arelle import ValidateXbrlDimensions, ValidateFormula
                    startedAt = time.time()
                    if not options.validate:
                        ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
                    # setup fresh parameters from formula optoins
                    modelXbrl.parameters = fo.typedParameters()
                    ValidateFormula.validate(modelXbrl, compileOnly=(options.formulaAction != "run"))
                    self.addToLog(format_string(self.modelManager.locale, 
                                                _("formula validation and execution in %.2f secs")
                                                if options.formulaAction == "run"
                                                else _("formula validation only in %.2f secs"), 
                                                time.time() - startedAt),
                                                messageCode="info", file=self.entrypointFile)
                    

                if options.testReport:
                    ViewFileTests.viewTests(self.modelManager.modelXbrl, options.testReport, options.testReportCols)
                    
                if options.rssReport:
                    ViewFileRssFeed.viewRssFeed(self.modelManager.modelXbrl, options.rssReport, options.rssReportCols)
                    
                if options.DTSFile:
                    ViewFileDTS.viewDTS(modelXbrl, options.DTSFile)
                if options.factsFile:
                    ViewFileFactList.viewFacts(modelXbrl, options.factsFile, labelrole=options.labelRole, lang=options.labelLang, cols=options.factListCols)
                if options.factTableFile:
                    ViewFileFactTable.viewFacts(modelXbrl, options.factTableFile, labelrole=options.labelRole, lang=options.labelLang)
                if options.conceptsFile:
                    ViewFileConcepts.viewConcepts(modelXbrl, options.conceptsFile, labelrole=options.labelRole, lang=options.labelLang)
                if options.preFile:
                    ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.preFile, "Presentation Linkbase", "http://www.xbrl.org/2003/arcrole/parent-child", labelrole=options.labelRole, lang=options.labelLang)
                if options.calFile:
                    ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.calFile, "Calculation Linkbase", "http://www.xbrl.org/2003/arcrole/summation-item", labelrole=options.labelRole, lang=options.labelLang)
                if options.dimFile:
                    ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.dimFile, "Dimensions", "XBRL-dimensions", labelrole=options.labelRole, lang=options.labelLang)
                if options.formulaeFile:
                    ViewFileFormulae.viewFormulae(modelXbrl, options.formulaeFile, "Formulae", lang=options.labelLang)
                if options.viewArcrole and options.viewFile:
                    ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.viewFile, os.path.basename(options.viewArcrole), options.viewArcrole, labelrole=options.labelRole, lang=options.labelLang)
                for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Run"):
                    pluginXbrlMethod(self, options, modelXbrl)
                                        
            except (IOError, EnvironmentError) as err:
                self.addToLog(_("[IOError] Failed to save output:\n {0}").format(err))
                success = False
            except Exception as err:
                self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format(
                            err,
                            traceback.format_tb(sys.exc_info()[2])))
                success = False
        if modelXbrl:
            modelXbrl.profileStat(_("total"), time.time() - firstStartedAt)
            if options.collectProfileStats and modelXbrl:
                modelXbrl.logProfileStats()
            if not options.keepOpen:
                if modelDiffReport:
                    self.modelManager.close(modelDiffReport)
                elif modelXbrl:
                    self.modelManager.close(modelXbrl)
        self.username = self.password = None #dereference password
        return success
Example #5
0
 def watchCycle(self):
     while not self.stopRequested:
         rssWatchOptions = self.rssModelXbrl.modelManager.rssWatchOptions
         
         # check rss expiration
         rssHeaders = self.cntlr.webCache.getheaders(self.rssModelXbrl.modelManager.rssWatchOptions.get("feedSourceUri"))
         expires = parseRfcDatetime(rssHeaders.get("expires"))
         reloadNow = True # texpires and expires > datetime.datetime.now()
         
         # reload rss feed
         self.rssModelXbrl.reload('checking RSS items', reloadCache=reloadNow)
         if self.stopRequested: break
         # setup validator
         postLoadActions = []
         if rssWatchOptions.get("validateDisclosureSystemRules"):
             self.instValidator = ValidateFiling.ValidateFiling(self.rssModelXbrl)
             postLoadActions.append(_("validating"))
         elif rssWatchOptions.get("validateXbrlRules") or rssWatchOptions.get("validateFormulaAssertions"):
             self.instValidator = ValidateXbrl.ValidateXbrl(self.rssModelXbrl)
             postLoadActions.append(_("validating"))
             if (rssWatchOptions.get("validateFormulaAssertions")):
                 postLoadActions.append(_("running formulas"))
         else:
             self.instValidator = None
            
         matchTextExpr = rssWatchOptions.get("matchTextExpr") 
         if matchTextExpr:
             matchPattern = re.compile(matchTextExpr)
             postLoadActions.append(_("matching text"))
         else:
             matchPattern= None
         postLoadAction = ', '.join(postLoadActions)
         
         # anything to check new filings for
         if (rssWatchOptions.get("validateDisclosureSystemRules") or
             rssWatchOptions.get("validateXbrlRules") or
             rssWatchOptions.get("validateCalcLinkbase") or
             rssWatchOptions.get("validateFormulaAssertions") or
             rssWatchOptions.get("alertMatchedFactText") or
             any(pluginXbrlMethod(rssWatchOptions)
                 for pluginXbrlMethod in pluginClassMethods("RssWatch.HasWatchAction"))
             ):
             # form keys in ascending order of pubdate
             pubDateRssItems = []
             for rssItem in self.rssModelXbrl.modelDocument.rssItems:
                 pubDateRssItems.append((rssItem.pubDate,rssItem.objectId()))
             
             for pubDate, rssItemObjectId in sorted(pubDateRssItems):
                 rssItem = self.rssModelXbrl.modelObject(rssItemObjectId)
                 # update ui thread via modelManager (running in background here)
                 self.rssModelXbrl.modelManager.viewModelObject(self.rssModelXbrl, rssItem.objectId())
                 if self.stopRequested:
                     break
                 latestPubDate = XmlUtil.datetimeValue(rssWatchOptions.get("latestPubDate"))
                 if (latestPubDate and 
                     rssItem.pubDate < latestPubDate):
                     continue
                 try:
                     # try zipped URL if possible, else expanded instance document
                     modelXbrl = ModelXbrl.load(self.rssModelXbrl.modelManager, 
                                                openFileSource(rssItem.zippedUrl, self.cntlr),
                                                postLoadAction)
                     if self.stopRequested:
                         modelXbrl.close()
                         break
                     
                     emailAlert = False
                     if modelXbrl.modelDocument is None:
                         modelXbrl.error("arelle.rssWatch",
                                         _("RSS item %(company)s %(form)s document not loaded: %(date)s"),
                                         modelXbrl=modelXbrl, company=rssItem.companyName, 
                                         form=rssItem.formType, date=rssItem.filingDate)
                         rssItem.status = "not loadable"
                     else:
                         # validate schema, linkbase, or instance
                         if self.stopRequested:
                             modelXbrl.close()
                             break
                         if self.instValidator:
                             self.instValidator.validate(modelXbrl)
                             if modelXbrl.errors and rssWatchOptions.get("alertValiditionError"):
                                 emailAlert = True
                         for pluginXbrlMethod in pluginClassMethods("RssWatch.DoWatchAction"):  
                             pluginXbrlMethod(modelXbrl, rssWatchOptions, rssItem)      
                         # check match expression
                         if matchPattern:
                             for fact in modelXbrl.factsInInstance:
                                 v = fact.value
                                 if v is not None:
                                     m = matchPattern.search(v)
                                     if m:
                                         fr, to = m.span()
                                         msg = _("Fact Variable {0}\n context {1}\n matched text: {2}").format( 
                                                 fact.qname, fact.contextID, v[max(0,fr-20):to+20])
                                         modelXbrl.info("arelle.rssInfo",
                                                        msg,
                                                        modelXbrl=modelXbrl) # msg as code passes it through to the status
                                         if rssWatchOptions.get("alertMatchedFactText"):
                                             emailAlert = True
                                     
                         if (rssWatchOptions.get("formulaFileUri") and rssWatchOptions.get("validateFormulaAssertions") and
                             self.instValidator): 
                             # attach formulas
                             ModelDocument.load(modelXbrl, rssWatchOptions["formulaFileUri"])
                             ValidateFormula.validate(self.instValidator)
                             
                     rssItem.setResults(modelXbrl)
                     modelXbrl.close()
                     del modelXbrl  # completely dereference
                     self.rssModelXbrl.modelManager.viewModelObject(self.rssModelXbrl, rssItem.objectId())
                     if rssItem.assertionUnsuccessful and rssWatchOptions.get("alertAssertionUnsuccessful"):
                         emailAlert = True
                     
                     msg = _("Filing CIK {0}\n "
                              "company {1}\n "
                              "published {2}\n "
                              "form type {3}\n "
                              "filing date {4}\n "
                              "period {5}\n "
                              "year end {6}\n "
                              "results: {7}").format(
                              rssItem.cikNumber,
                              rssItem.companyName,
                              rssItem.pubDate,
                              rssItem.formType,
                              rssItem.filingDate,
                              rssItem.period,
                              rssItem.fiscalYearEnd,
                              rssItem.status)
                     self.rssModelXbrl.info("arelle:rssWatch", msg, modelXbrl=self.rssModelXbrl)
                     emailAddress = rssWatchOptions.get("emailAddress")
                     if emailAlert and emailAddress:
                         self.rssModelXbrl.modelManager.showStatus(_("sending e-mail alert"))
                         import smtplib
                         from email.mime.text import MIMEText
                         emailMsg = MIMEText(msg)
                         emailMsg["Subject"] = _("Arelle RSS Watch alert on {0}").format(rssItem.companyName)
                         emailMsg["From"] = emailAddress
                         emailMsg["To"] = emailAddress
                         smtp = smtplib.SMTP()
                         smtp.sendmail(emailAddress, [emailAddress], emailMsg.as_string())
                         smtp.quit()
                     self.rssModelXbrl.modelManager.showStatus(_("RSS item {0}, {1} completed, status {2}").format(rssItem.companyName, rssItem.formType, rssItem.status), 3500)
                     self.rssModelXbrl.modelManager.cntlr.rssWatchUpdateOption(rssItem.pubDate.strftime('%Y-%m-%dT%H:%M:%S'))
                 except Exception as err:
                     self.rssModelXbrl.error("arelle.rssError",
                                             _("RSS item %(company)s, %(form)s, %(date)s, exception: %(error)s"),
                                             modelXbrl=self.rssModelXbrl, company=rssItem.companyName, 
                                             form=rssItem.formType, date=rssItem.filingDate, error=err,
                                             exc_info=True)
                 if self.stopRequested: break
         if self.stopRequested: 
             self.cntlr.showStatus(_("RSS watch, stop requested"), 10000)
         else:
             import time
             time.sleep(600)
         
     self.thread = None  # close thread
     self.stopRequested = False
     
             
def init(modelXbrl):
    # setup modelXbrl for rendering evaluation

    # dimension defaults required in advance of validation
    from arelle import ValidateXbrlDimensions, ValidateFormula, ModelDocument
    ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
    
    hasXbrlTables = False
    
    # validate table linkbase dimensions
    for baseSetKey in modelXbrl.baseSets.keys():
        arcrole, ELR, linkqname, arcqname = baseSetKey
        if ELR and linkqname and arcqname and XbrlConst.isTableRenderingArcrole(arcrole):
            ValidateFormula.checkBaseSet(modelXbrl, arcrole, ELR, modelXbrl.relationshipSet(arcrole,ELR,linkqname,arcqname))
            if arcrole in (XbrlConst.tableBreakdown, XbrlConst.tableAxis2011):
                hasXbrlTables = True

    # provide context for view
    if modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE:
        instance = None # use instance of the entry pont
    else: # need dummy instance
        instance = ModelDocument.create(modelXbrl, ModelDocument.Type.INSTANCE, 
                                        "dummy.xml",  # fake URI and fake schemaRef 
                                        ("http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd",))
        
    if hasXbrlTables:
        # formula processor is needed for 2011 XBRL tables but not for 2010 Eurofiling tables
        modelXbrl.rendrCntx = XPathContext.create(modelXbrl, instance)
        
        modelXbrl.profileStat(None)
        
        # setup fresh parameters from formula optoins
        modelXbrl.parameters = modelXbrl.modelManager.formulaOptions.typedParameters()
        
        # validate parameters and custom function signatures
        ValidateFormula.validate(modelXbrl, xpathContext=modelXbrl.rendrCntx, parametersOnly=True, statusMsg=_("compiling rendering tables"))
        
        # check and extract message expressions into compilable programs
        for msgArcrole in (XbrlConst.tableDefinitionNodeMessage, XbrlConst.tableDefinitionNodeSelectionMessage,
                           XbrlConst.tableAxisMessage2011, XbrlConst.tableAxisSelectionMessage2011):
            for msgRel in modelXbrl.relationshipSet(msgArcrole).modelRelationships:
                ValidateFormula.checkMessageExpressions(modelXbrl, msgRel.toModelObject)
                
        # compile and validate tables
        for modelTable in modelXbrl.modelRenderingTables:
            modelTable.fromInstanceQnames = None # required if referred to by variables scope chaining
            modelTable.compile()

            # check aspectModel
            if modelTable.aspectModel not in ("non-dimensional", "dimensional"):
                modelXbrl.error("xbrlte:unknownAspectModel",
                    _("Table %(xlinkLabel)s, aspect model %(aspectModel)s not recognized"),
                    modelObject=modelTable, xlinkLabel=modelTable.xlinkLabel, aspectModel=modelTable.aspectModel)
            else:
                modelTable.priorAspectAxisDisposition = {}
                # check ordinate aspects against aspectModel
                oppositeAspectModel = (_DICT_SET({'dimensional','non-dimensional'}) - _DICT_SET({modelTable.aspectModel})).pop()
                uncoverableAspects = aspectModels[oppositeAspectModel] - aspectModels[modelTable.aspectModel]
                for tblAxisRel in modelXbrl.relationshipSet((XbrlConst.tableBreakdown,XbrlConst.tableAxis2011)).fromModelObject(modelTable):
                    checkDefinitionNodeAspectModel(modelXbrl, modelTable, tblAxisRel, uncoverableAspects)
                del modelTable.priorAspectAxisDisposition
    
        modelXbrl.profileStat(_("compileTables"))
Example #7
0
 def executeCallTest(self, modelXbrl, name, callTuple, testTuple):
     self.modelXbrl = modelXbrl
     ValidateFormula.executeCallTest(self, name, callTuple, testTuple)
Example #8
0
    def validate(self, modelXbrl, parameters=None):
        self.parameters = parameters
        self.precisionPattern = re.compile("^([0-9]+|INF)$")
        self.decimalsPattern = re.compile("^(-?[0-9]+|INF)$")
        self.isoCurrencyPattern = re.compile(r"^[A-Z]{3}$")
        self.modelXbrl = modelXbrl
        self.validateDisclosureSystem = modelXbrl.modelManager.validateDisclosureSystem
        self.disclosureSystem = modelXbrl.modelManager.disclosureSystem
        self.validateEFM = self.validateDisclosureSystem and self.disclosureSystem.EFM
        self.validateGFM = self.validateDisclosureSystem and self.disclosureSystem.GFM
        self.validateEFMorGFM = self.validateDisclosureSystem and self.disclosureSystem.EFMorGFM
        self.validateHMRC = self.validateDisclosureSystem and self.disclosureSystem.HMRC
        self.validateSBRNL = self.validateDisclosureSystem and self.disclosureSystem.SBRNL
        self.validateXmlLang = self.validateDisclosureSystem and self.disclosureSystem.xmlLangPattern
        self.validateCalcLB = modelXbrl.modelManager.validateCalcLB
        self.validateInferDecimals = modelXbrl.modelManager.validateInferDecimals
        
        # xlink validation
        modelXbrl.profileStat(None)
        modelXbrl.modelManager.showStatus(_("validating links"))
        modelLinks = set()
        self.remoteResourceLocElements = set()
        self.genericArcArcroles = set()
        for baseSetExtLinks in modelXbrl.baseSets.values():
            for baseSetExtLink in baseSetExtLinks:
                modelLinks.add(baseSetExtLink)    # ext links are unique (no dups)
        for modelLink in modelLinks:
            fromToArcs = {}
            locLabels = {}
            resourceLabels = {}
            resourceArcTos = []
            for arcElt in modelLink.iterchildren():
                if isinstance(arcElt,ModelObject):
                    xlinkType = arcElt.get("{http://www.w3.org/1999/xlink}type")
                    # locator must have an href
                    if xlinkType == "locator":
                        if arcElt.get("{http://www.w3.org/1999/xlink}href") is None:
                            modelXbrl.error("xlink:locatorHref",
                                _("Xlink locator %(xlinkLabel)s missing href in extended link %(linkrole)s"),
                                modelObject=arcElt,
                                linkrole=modelLink.role, 
                                xlinkLabel=arcElt.get("{http://www.w3.org/1999/xlink}label")) 
                        locLabels[arcElt.get("{http://www.w3.org/1999/xlink}label")] = arcElt
                    elif xlinkType == "resource":
                        resourceLabels[arcElt.get("{http://www.w3.org/1999/xlink}label")] = arcElt
                    # can be no duplicated arcs between same from and to
                    elif xlinkType == "arc":
                        fromLabel = arcElt.get("{http://www.w3.org/1999/xlink}from")
                        toLabel = arcElt.get("{http://www.w3.org/1999/xlink}to")
                        fromTo = (fromLabel,toLabel)
                        if fromTo in fromToArcs:
                            modelXbrl.error("xlink:dupArcs",
                                _("Duplicate xlink arcs  in extended link %(linkrole)s from %(xlinkLabelFrom)s to %(xlinkLabelTo)s"),
                                modelObject=arcElt,
                                linkrole=modelLink.role, 
                                xlinkLabelFrom=fromLabel, xlinkLabelTo=toLabel)
                        else:
                            fromToArcs[fromTo] = arcElt
                        if arcElt.namespaceURI == XbrlConst.link:
                            if arcElt.localName in arcNamesTo21Resource: #("labelArc","referenceArc"):
                                resourceArcTos.append((toLabel, arcElt.get("use"), arcElt))
                        elif self.isGenericArc(arcElt):
                            arcrole = arcElt.get("{http://www.w3.org/1999/xlink}arcrole")
                            self.genericArcArcroles.add(arcrole)
                            if arcrole in (XbrlConst.elementLabel, XbrlConst.elementReference):
                                resourceArcTos.append((toLabel, arcrole, arcElt))
                    # values of type (not needed for validating parsers)
                    if xlinkType not in xlinkTypeValues: # ("", "simple", "extended", "locator", "arc", "resource", "title", "none"):
                        modelXbrl.error("xlink:type",
                            _("Xlink type %(xlinkType)s invalid in extended link %(linkrole)s"),
                            modelObject=arcElt, linkrole=modelLink.role, xlinkType=xlinkType)
                    # values of actuate (not needed for validating parsers)
                    xlinkActuate = arcElt.get("{http://www.w3.org/1999/xlink}actuate")
                    if xlinkActuate not in xlinkActuateValues: # ("", "onLoad", "onRequest", "other", "none"):
                        modelXbrl.error("xlink:actuate",
                            _("Actuate %(xlinkActuate)s invalid in extended link %(linkrole)s"),
                            modelObject=arcElt, linkrole=modelLink.role, xlinkActuate=xlinkActuate)
                    # values of show (not needed for validating parsers)
                    xlinkShow = arcElt.get("{http://www.w3.org/1999/xlink}show")
                    if xlinkShow not in xlinkShowValues: # ("", "new", "replace", "embed", "other", "none"):
                        modelXbrl.error("xlink:show",
                            _("Show %(xlinkShow)s invalid in extended link %(linkrole)s"),
                            modelObject=arcElt, linkrole=modelLink.role, xlinkShow=xlinkShow)
            # check from, to of arcs have a resource or loc
            for fromTo, arcElt in fromToArcs.items():
                fromLabel, toLabel = fromTo
                for name, value, sect in (("from", fromLabel, "3.5.3.9.2"),("to",toLabel, "3.5.3.9.3")):
                    if value not in locLabels and value not in resourceLabels:
                        modelXbrl.error("xbrl.{0}:arcResource".format(sect),
                            _("Arc in extended link %(linkrole)s from %(xlinkLabelFrom)s to %(xlinkLabelTo)s attribute '%(attribute)s' has no matching loc or resource label"),
                            modelObject=arcElt, 
                            linkrole=modelLink.role, xlinkLabelFrom=fromLabel, xlinkLabelTo=toLabel, 
                            attribute=name)
                if arcElt.localName == "footnoteArc" and arcElt.namespaceURI == XbrlConst.link and \
                   arcElt.get("{http://www.w3.org/1999/xlink}arcrole") == XbrlConst.factFootnote:
                    if fromLabel not in locLabels:
                        modelXbrl.error("xbrl.4.11.1.3.1:factFootnoteArcFrom",
                            _("Footnote arc in extended link %(linkrole)s from %(xlinkLabelFrom)s to %(xlinkLabelTo)s \"from\" is not a loc"),
                            modelObject=arcElt, 
                            linkrole=modelLink.role, xlinkLabelFrom=fromLabel, xlinkLabelTo=toLabel)
                    if toLabel not in resourceLabels or qname(resourceLabels[toLabel]) != XbrlConst.qnLinkFootnote:
                        modelXbrl.error("xbrl.4.11.1.3.1:factFootnoteArcTo",
                            _("Footnote arc in extended link %(linkrole)s from %(xlinkLabelFrom)s to %(xlinkLabelTo)s \"to\" is not a footnote resource"),
                            modelObject=arcElt, 
                            linkrole=modelLink.role, xlinkLabelFrom=fromLabel, xlinkLabelTo=toLabel)
            # check unprohibited label arcs to remote locs
            for resourceArcTo in resourceArcTos:
                resourceArcToLabel, resourceArcUse, arcElt = resourceArcTo
                if resourceArcToLabel in locLabels:
                    toLabel = locLabels[resourceArcToLabel]
                    if resourceArcUse == "prohibited":
                        self.remoteResourceLocElements.add(toLabel)
                    else:
                        modelXbrl.error("xbrl.5.2.2.3:labelArcRemoteResource",
                            _("Unprohibited labelArc in extended link %(linkrole)s has illegal remote resource loc labeled %(xlinkLabel)s href %(xlinkHref)s"),
                            modelObject=arcElt, 
                            linkrole=modelLink.role, 
                            xlinkLabel=resourceArcToLabel,
                            xlinkHref=toLabel.get("{http://www.w3.org/1999/xlink}href"))
                elif resourceArcToLabel in resourceLabels:
                    toResource = resourceLabels[resourceArcToLabel]
                    if resourceArcUse == XbrlConst.elementLabel:
                        if not self.isGenericLabel(toResource):
                            modelXbrl.error("xbrlle.2.1.1:genericLabelTarget",
                                _("Generic label arc in extended link %(linkrole)s to %(xlinkLabel)s must target a generic label"),
                                modelObject=arcElt, 
                                linkrole=modelLink.role, 
                                xlinkLabel=resourceArcToLabel)
                    elif resourceArcUse == XbrlConst.elementReference:
                        if not self.isGenericReference(toResource):
                            modelXbrl.error("xbrlre.2.1.1:genericReferenceTarget",
                                _("Generic reference arc in extended link %(linkrole)s to %(xlinkLabel)s must target a generic reference"),
                                modelObject=arcElt, 
                                linkrole=modelLink.role, 
                                xlinkLabel=resourceArcToLabel)
            resourceArcTos = None # dereference arcs
        modelXbrl.profileStat(_("validateLinks"))

        modelXbrl.dimensionDefaultConcepts = {}
        modelXbrl.qnameDimensionDefaults = {}
        modelXbrl.qnameDimensionContextElement = {}
        # check base set cycles, dimensions
        modelXbrl.modelManager.showStatus(_("validating relationship sets"))
        for baseSetKey in modelXbrl.baseSets.keys():
            arcrole, ELR, linkqname, arcqname = baseSetKey
            if arcrole.startswith("XBRL-") or ELR is None or \
                linkqname is None or arcqname is None:
                continue
            elif arcrole in XbrlConst.standardArcroleCyclesAllowed:
                # TODO: table should be in this module, where it is used
                cyclesAllowed, specSect = XbrlConst.standardArcroleCyclesAllowed[arcrole]
            elif arcrole in self.modelXbrl.arcroleTypes and len(self.modelXbrl.arcroleTypes[arcrole]) > 0:
                cyclesAllowed = self.modelXbrl.arcroleTypes[arcrole][0].cyclesAllowed
                if arcrole in self.genericArcArcroles:
                    specSect = "xbrlgene:violatedCyclesConstraint"
                else:
                    specSect = "xbrl.5.1.4.3:cycles"
            else:
                cyclesAllowed = "any"
                specSect = None
            if cyclesAllowed != "any" or arcrole in (XbrlConst.summationItem,) \
                                      or arcrole in self.genericArcArcroles  \
                                      or arcrole.startswith(XbrlConst.formulaStartsWith):
                relsSet = modelXbrl.relationshipSet(arcrole,ELR,linkqname,arcqname)
            if cyclesAllowed != "any" and \
                   (XbrlConst.isStandardExtLinkQname(linkqname) and XbrlConst.isStandardArcQname(arcqname)) \
                   or arcrole in self.genericArcArcroles:
                noUndirected = cyclesAllowed == "none"
                fromRelationships = relsSet.fromModelObjects()
                for relFrom, rels in fromRelationships.items():
                    cycleFound = self.fwdCycle(relsSet, rels, noUndirected, {relFrom})
                    if cycleFound is not None:
                        path = str(relFrom.qname) + " " + " - ".join(
                            "{0}:{1} {2}".format(rel.modelDocument.basename, rel.sourceline, rel.toModelObject.qname)
                            for rel in reversed(cycleFound[1:]))
                        modelXbrl.error(specSect,
                            _("Relationships have a %(cycle)s cycle in arcrole %(arcrole)s \nlink role %(linkrole)s \nlink %(linkname)s, \narc %(arcname)s, \npath %(path)s"),
                            modelObject=cycleFound[1:], cycle=cycleFound[0], path=path,
                            arcrole=arcrole, linkrole=ELR, linkname=linkqname, arcname=arcqname), 
                        break
                
            # check calculation arcs for weight issues (note calc arc is an "any" cycles)
            if arcrole == XbrlConst.summationItem:
                for modelRel in relsSet.modelRelationships:
                    weight = modelRel.weight
                    fromConcept = modelRel.fromModelObject
                    toConcept = modelRel.toModelObject
                    if fromConcept is not None and toConcept is not None:
                        if weight == 0:
                            modelXbrl.error("xbrl.5.2.5.2.1:zeroWeight",
                                _("Calculation relationship has zero weight from %(source)s to %(target)s in link role %(linkrole)s"),
                                modelObject=modelRel,
                                source=fromConcept.qname, target=toConcept.qname, linkrole=ELR), 
                        fromBalance = fromConcept.balance
                        toBalance = toConcept.balance
                        if fromBalance and toBalance:
                            if (fromBalance == toBalance and weight < 0) or \
                               (fromBalance != toBalance and weight > 0):
                                modelXbrl.error("xbrl.5.1.1.2:balanceCalcWeightIllegal" +
                                                ("Negative" if weight < 0 else "Positive"),
                                    _("Calculation relationship has illegal weight %(weight)s from %(source)s, %(sourceBalance)s, to %(target)s, %(targetBalance)s, in link role %(linkrole)s (per 5.1.1.2 Table 6)"),
                                    modelObject=modelRel, weight=weight,
                                    source=fromConcept.qname, target=toConcept.qname, linkrole=ELR, 
                                    sourceBalance=fromBalance, targetBalance=toBalance)
                        if not fromConcept.isNumeric or not toConcept.isNumeric:
                            modelXbrl.error("xbrl.5.2.5.2:nonNumericCalc",
                                _("Calculation relationship has illegal concept from %(source)s%(sourceNumericDecorator)s to %(target)s%(targetNumericDecorator)s in link role %(linkrole)s"),
                                modelObject=modelRel,
                                source=fromConcept.qname, target=toConcept.qname, linkrole=ELR, 
                                sourceNumericDecorator="" if fromConcept.isNumeric else _(" (non-numeric)"), 
                                targetNumericDecorator="" if toConcept.isNumeric else _(" (non-numeric)"))
            # check presentation relationships for preferredLabel issues
            elif arcrole == XbrlConst.parentChild:
                for modelRel in relsSet.modelRelationships:
                    preferredLabel = modelRel.preferredLabel
                    toConcept = modelRel.toModelObject
                    if preferredLabel is not None and toConcept is not None and \
                       toConcept.label(preferredLabel=preferredLabel,fallbackToQname=False) is None:
                        modelXbrl.error("xbrl.5.2.4.2.1:preferredLabelMissing",
                            _("Presentation relationship from %(source)s to %(target)s in link role %(linkrole)s missing preferredLabel %(preferredLabel)s"),
                            modelObject=modelRel,
                            source=modelRel.fromModelObject.qname, target=toConcept.qname, linkrole=ELR, 
                            preferredLabel=preferredLabel)
            # check essence-alias relationships
            elif arcrole == XbrlConst.essenceAlias:
                for modelRel in relsSet.modelRelationships:
                    fromConcept = modelRel.fromModelObject
                    toConcept = modelRel.toModelObject
                    if fromConcept is not None and toConcept is not None:
                        if fromConcept.type != toConcept.type or fromConcept.periodType != toConcept.periodType:
                            modelXbrl.error("xbrl.5.2.6.2.2:essenceAliasTypes",
                                _("Essence-alias relationship from %(source)s to %(target)s in link role %(linkrole)s has different types or periodTypes"),
                                modelObject=modelRel,
                                source=fromConcept.qname, target=toConcept.qname, linkrole=ELR)
                        fromBalance = fromConcept.balance
                        toBalance = toConcept.balance
                        if fromBalance and toBalance:
                            if fromBalance and toBalance and fromBalance != toBalance:
                                modelXbrl.error("xbrl.5.2.6.2.2:essenceAliasBalance",
                                    _("Essence-alias relationship from %(source)s to %(target)s in link role %(linkrole)s has different balances")).format(
                                    modelObject=modelRel,
                                    source=fromConcept.qname, target=toConcept.qname, linkrole=ELR)
            elif modelXbrl.hasXDT and arcrole.startswith(XbrlConst.dimStartsWith):
                ValidateXbrlDimensions.checkBaseSet(self, arcrole, ELR, relsSet)             
            elif (modelXbrl.hasFormulae or modelXbrl.hasTableRendering) and arcrole.startswith(XbrlConst.formulaStartsWith):
                ValidateFormula.checkBaseSet(self, arcrole, ELR, relsSet)
        modelXbrl.isDimensionsValidated = True
        modelXbrl.profileStat(_("validateRelationships"))
                            
        # instance checks
        modelXbrl.modelManager.showStatus(_("validating instance"))
        self.footnoteRefs = set()
        if modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE or \
           modelXbrl.modelDocument.type == ModelDocument.Type.INLINEXBRL:
            self.factsWithDeprecatedIxNamespace = []
            self.checkFacts(modelXbrl.facts)
            
            if self.factsWithDeprecatedIxNamespace:
                self.modelXbrl.info("arelle:info",
                    _("%(count)s facts have deprecated transformation namespace %(namespace)s"),
                        modelObject=self.factsWithDeprecatedIxNamespace,
                        count=len(self.factsWithDeprecatedIxNamespace), 
                        namespace=FunctionIxt.deprecatedNamespaceURI)
            del self.factsWithDeprecatedIxNamespace

            
            #instance checks
            for cntx in modelXbrl.contexts.values():
                if cntx.isStartEndPeriod:
                    try:
                        if cntx.endDatetime <= cntx.startDatetime:
                            self.modelXbrl.error("xbrl.4.7.2:periodStartBeforeEnd",
                                _("Context %(contextID)s must have startDate less than endDate"),
                                modelObject=cntx, contextID=cntx.id)
                    except (TypeError, ValueError) as err:
                        self.modelXbrl.error("xbrl.4.7.2:contextDateError",
                            _("Context %(contextID) startDate or endDate: %(error)s"),
                            modelObject=cntx, contextID=cntx.id, error=err)
                elif cntx.isInstantPeriod:
                    try:
                        cntx.instantDatetime #parse field
                    except ValueError as err:
                        self.modelXbrl.error("xbrl.4.7.2:contextDateError",
                            _("Context %(contextID)s instant date: %(error)s"),
                            modelObject=cntx, contextID=cntx.id, error=err)
                self.segmentScenario(cntx.segment, cntx.id, "segment", "4.7.3.2")
                self.segmentScenario(cntx.scenario, cntx.id, "scenario", "4.7.4")
                
            for unit in modelXbrl.units.values():
                mulDivMeasures = unit.measures
                if mulDivMeasures:
                    for measures in mulDivMeasures:
                        for measure in measures:
                            if measure.namespaceURI == XbrlConst.xbrli and not \
                                measure in (XbrlConst.qnXbrliPure, XbrlConst.qnXbrliShares):
                                    self.modelXbrl.error("xbrl.4.8.2:measureElement",
                                        _("Unit %(unitID)s illegal measure: %(measure)s"),
                                        modelObject=unit, unitID=unit.id, measure=measure)
                    for numeratorMeasure in mulDivMeasures[0]:
                        if numeratorMeasure in mulDivMeasures[1]:
                            self.modelXbrl.error("xbrl.4.8.4:measureBothNumDenom",
                                _("Unit %(unitID)s numerator measure: %(measure)s also appears as denominator measure"),
                                modelObject=unit, unitID=unit.id, measure=numeratorMeasure)
            modelXbrl.profileStat(_("validateInstance"))

            if modelXbrl.hasXDT:            
                modelXbrl.modelManager.showStatus(_("validating dimensions"))
                ''' uncomment if using otherFacts in checkFact
                dimCheckableFacts = set(f 
                                        for f in modelXbrl.factsInInstance
                                        if f.concept.isItem and f.context is not None)
                while (dimCheckableFacts): # check one and all of its compatible family members
                    f = dimCheckableFacts.pop()
                    ValidateXbrlDimensions.checkFact(self, f, dimCheckableFacts)
                del dimCheckableFacts
                '''
                self.checkFactDimensions(modelXbrl.facts) # check fact dimensions in document order
                for cntx in modelXbrl.contexts.values():
                    ValidateXbrlDimensions.checkContext(self,cntx)
                modelXbrl.profileStat(_("validateDimensions"))
                    
        # dimensional validity
        #concepts checks
        modelXbrl.modelManager.showStatus(_("validating concepts"))
        for concept in modelXbrl.qnameConcepts.values():
            conceptType = concept.type
            if XbrlConst.isStandardNamespace(concept.qname.namespaceURI) or \
               not concept.modelDocument.inDTS:
                continue
            
            if concept.isTuple:
                # must be global
                if not concept.getparent().localName == "schema":
                    self.modelXbrl.error("xbrl.4.9:tupleGloballyDeclared",
                        _("Tuple %(concept)s must be declared globally"),
                        modelObject=concept, concept=concept.qname)
                if concept.periodType:
                    self.modelXbrl.error("xbrl.4.9:tuplePeriodType",
                        _("Tuple %(concept)s must not have periodType"),
                        modelObject=concept, concept=concept.qname)
                if concept.balance:
                    self.modelXbrl.error("xbrl.4.9:tupleBalance",
                        _("Tuple %(concept)s must not have balance"),
                        modelObject=concept, concept=concept.qname)
                if conceptType is not None:
                    # check attribute declarations
                    for attribute in conceptType.attributes.values():
                        if attribute.qname.namespaceURI in (XbrlConst.xbrli, XbrlConst.link, XbrlConst.xlink, XbrlConst.xl):
                            self.modelXbrl.error("xbrl.4.9:tupleAttribute",
                                _("Tuple %(concept)s must not have attribute in this namespace %(attribute)s"),
                                modelObject=concept, concept=concept.qname, attribute=attribute.qname)
                    # check for mixed="true" or simple content
                    if XmlUtil.descendantAttr(conceptType, XbrlConst.xsd, ("complexType", "complexContent"), "mixed") == "true":
                        self.modelXbrl.error("xbrl.4.9:tupleMixedContent",
                            _("Tuple %(concept)s must not have mixed content"),
                            modelObject=concept, concept=concept.qname)
                    if XmlUtil.descendant(conceptType, XbrlConst.xsd, "simpleContent"):
                        self.modelXbrl.error("xbrl.4.9:tupleSimpleContent",
                            _("Tuple %(concept)s must not have simple content"),
                            modelObject=concept, concept=concept.qname)
                    # child elements must be item or tuple
                    for elementQname in conceptType.elements:
                        childConcept = self.modelXbrl.qnameConcepts.get(elementQname)
                        if childConcept is None:
                            self.modelXbrl.error("xbrl.4.9:tupleElementUndefined",
                                _("Tuple %(concept)s element %(tupleElement)s not defined"),
                                modelObject=concept, concept=str(concept.qname), tupleElement=elementQname)
                        elif not (childConcept.isItem or childConcept.isTuple or # isItem/isTuple do not include item or tuple itself
                                  childConcept.qname == XbrlConst.qnXbrliItem or # subs group includes item as member
                                  childConcept.qname == XbrlConst.qnXbrliTuple):
                            self.modelXbrl.error("xbrl.4.9:tupleElementItemOrTuple",
                                _("Tuple %(concept)s must not have element %(tupleElement)s not an item or tuple"),
                                modelObject=concept, concept=concept.qname, tupleElement=elementQname)
            elif concept.isItem:
                if concept.periodType not in periodTypeValues: #("instant","duration"):
                    self.modelXbrl.error("xbrl.5.1.1.1:itemPeriodType",
                        _("Item %(concept)s must have a valid periodType"),
                        modelObject=concept, concept=concept.qname)
                if concept.isMonetary:
                    if concept.balance not in balanceValues: #(None, "credit","debit"):
                        self.modelXbrl.error("xbrl.5.1.1.2:itemBalance",
                            _("Item %(concept)s must have a valid balance %(balance)s"),
                            modelObject=concept, concept=concept.qname, balance=concept.balance)
                else:
                    if concept.balance:
                        self.modelXbrl.error("xbrl.5.1.1.2:itemBalance",
                            _("Item %(concept)s may not have a balance"),
                            modelObject=concept, concept=concept.qname)
                if concept.baseXbrliType not in baseXbrliTypes:
                    self.modelXbrl.error("xbrl.5.1.1.3:itemType",
                        _("Item %(concept)s type %(itemType)s invalid"),
                        modelObject=concept, concept=concept.qname, itemType=concept.baseXbrliType)
                if modelXbrl.hasXDT:
                    if concept.isHypercubeItem and not concept.abstract == "true":
                        self.modelXbrl.error("xbrldte:HypercubeElementIsNotAbstractError",
                            _("Hypercube item %(concept)s must be abstract"),
                            modelObject=concept, concept=concept.qname)
                    elif concept.isDimensionItem and not concept.abstract == "true":
                        self.modelXbrl.error("xbrldte:DimensionElementIsNotAbstractError",
                            _("Dimension item %(concept)s must be abstract"),
                            modelObject=concept, concept=concept.qname)
            if modelXbrl.hasXDT:
                ValidateXbrlDimensions.checkConcept(self, concept)
        modelXbrl.profileStat(_("validateConcepts"))
            
        modelXbrl.modelManager.showStatus(_("validating DTS"))
        self.DTSreferenceResourceIDs = {}
        ValidateXbrlDTS.checkDTS(self, modelXbrl.modelDocument, [])
        del self.DTSreferenceResourceIDs
        
        global validateUniqueParticleAttribution
        if validateUniqueParticleAttribution is None:
            from arelle.XmlValidateParticles import validateUniqueParticleAttribution
        for modelType in modelXbrl.qnameTypes.values():
            validateUniqueParticleAttribution(modelXbrl, modelType.particlesList, modelType)
        modelXbrl.profileStat(_("validateDTS"))
        
        if self.validateCalcLB:
            modelXbrl.modelManager.showStatus(_("Validating instance calculations"))
            ValidateXbrlCalcs.validate(modelXbrl, inferDecimals=self.validateInferDecimals)
            modelXbrl.profileStat(_("validateCalculations"))
            
        if (modelXbrl.modelManager.validateUtr or
            (self.parameters and self.parameters.get(qname("forceUtrValidation",noPrefixIsNoNamespace=True),(None,"false"))[1] == "true") or
             #(self.validateEFM and 
             #any((concept.namespaceURI in self.disclosureSystem.standardTaxonomiesDict) 
             #    for concept in self.modelXbrl.nameConcepts.get("UTR",())))):
            (self.validateEFM and any(modelDoc.definesUTR for modelDoc in self.modelXbrl.urlDocs.values()))):
            ValidateUtr.validate(modelXbrl)
            modelXbrl.profileStat(_("validateUTR"))
            
        if modelXbrl.hasFormulae or modelXbrl.modelRenderingTables:
            ValidateFormula.validate(self, 
                                     statusMsg=_("compiling formulae and rendering tables") if (modelXbrl.hasFormulae and modelXbrl.modelRenderingTables)
                                     else (_("compiling formulae") if modelXbrl.hasFormulae
                                           else _("compiling rendering tables")))
            
        modelXbrl.modelManager.showStatus(_("ready"), 2000)
Example #9
0
    def validateTestcase(self, testcase):
        self.modelXbrl.info("info", "Testcase", modelDocument=testcase)
        self.modelXbrl.viewModelObject(testcase.objectId())
        if testcase.type in (Type.TESTCASESINDEX, Type.REGISTRY):
            for doc in sorted(testcase.referencesDocument.keys(), key=lambda doc: doc.uri):
                self.validateTestcase(doc)  # testcases doc's are sorted by their uri (file names), e.g., for formula
        elif hasattr(testcase, "testcaseVariations"):
            for modelTestcaseVariation in testcase.testcaseVariations:
                # update ui thread via modelManager (running in background here)
                self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
                # is this a versioning report?
                resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport
                resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance
                resultIsTaxonomyPackage = modelTestcaseVariation.resultIsTaxonomyPackage
                formulaOutputInstance = None
                inputDTSes = defaultdict(list)
                baseForElement = testcase.baseForElement(modelTestcaseVariation)
                # try to load instance document
                self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s - %(description)s"),
                                    modelObject=modelTestcaseVariation, 
                                    id=modelTestcaseVariation.id, 
                                    name=modelTestcaseVariation.name, 
                                    expected=modelTestcaseVariation.expected, 
                                    description=modelTestcaseVariation.description)
                if self.modelXbrl.modelManager.formulaOptions.testcaseResultsCaptureWarnings:
                    errorCaptureLevel = logging._checkLevel("WARNING")
                else:
                    errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY
                parameters = modelTestcaseVariation.parameters.copy()
                for readMeFirstUri in modelTestcaseVariation.readMeFirstUris:
                    if isinstance(readMeFirstUri,tuple):
                        # dtsName is for formula instances, but is from/to dts if versioning
                        dtsName, readMeFirstUri = readMeFirstUri
                    elif resultIsVersioningReport:
                        if inputDTSes: dtsName = "to"
                        else: dtsName = "from"
                    else:
                        dtsName = None
                    if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document
                        if dtsName in inputDTSes:
                            dtsName = inputDTSes[dtsName]
                        else:
                            modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager, 
                                         Type.DTSENTRIES,
                                         self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement),
                                         isEntry=True,
                                         errorCaptureLevel=errorCaptureLevel)
                        DTSdoc = modelXbrl.modelDocument
                        DTSdoc.inDTS = True
                        doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement)
                        if doc is not None:
                            DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement)  #fake import
                            doc.inDTS = True
                    elif resultIsTaxonomyPackage:
                        from arelle import PackageManager, PrototypeInstanceObject
                        dtsName = readMeFirstUri
                        modelXbrl = PrototypeInstanceObject.XbrlPrototype(self.modelXbrl.modelManager, readMeFirstUri)
                        PackageManager.packageInfo(self.modelXbrl.modelManager.cntlr, readMeFirstUri, reload=True, errors=modelXbrl.errors)
                    else: # not a multi-schemaRef versioning report
                        if self.useFileSource.isArchive:
                            modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, 
                                                       readMeFirstUri,
                                                       _("validating"), 
                                                       base=baseForElement,
                                                       useFileSource=self.useFileSource,
                                                       errorCaptureLevel=errorCaptureLevel)
                        else: # need own file source, may need instance discovery
                            filesource = FileSource.FileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr)
                            if filesource and not filesource.selection and filesource.isArchive:
                                for _archiveFile in filesource.dir or (): # find instance document in archive
                                    filesource.select(_archiveFile)
                                    if ModelDocument.Type.identify(filesource, filesource.url) in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL):
                                        break # use this selection
                            modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, 
                                                       filesource,
                                                       _("validating"), 
                                                       base=baseForElement,
                                                       errorCaptureLevel=errorCaptureLevel)
                        modelXbrl.isTestcaseVariation = True
                    if modelXbrl.modelDocument is None:
                        modelXbrl.error("arelle:notLoaded",
                             _("Variation %(id)s %(name)s readMeFirst document not loaded: %(file)s"),
                             modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
                        self.determineNotLoadedTestStatus(modelTestcaseVariation, modelXbrl.errors)
                        modelXbrl.close()
                    elif resultIsVersioningReport or resultIsTaxonomyPackage:
                        inputDTSes[dtsName] = modelXbrl
                    elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT:
                        ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl)
                        self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
                        modelXbrl.close()
                    elif testcase.type == Type.REGISTRYTESTCASE:
                        self.instValidator.validate(modelXbrl)  # required to set up dimensions, etc
                        self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id, 
                                   modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest)
                        self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
                        self.instValidator.close()
                        modelXbrl.close()
                    else:
                        inputDTSes[dtsName].append(modelXbrl)
                        # validate except for formulas
                        _hasFormulae = modelXbrl.hasFormulae
                        modelXbrl.hasFormulae = False
                        try:
                            for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"):
                                pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation)
                            self.instValidator.validate(modelXbrl, parameters)
                            for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"):
                                pluginXbrlMethod(self.modelXbrl, modelXbrl)
                        except Exception as err:
                            modelXbrl.error("exception:" + type(err).__name__,
                                _("Testcase variation validation exception: %(error)s, instance: %(instance)s"),
                                modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
                        modelXbrl.hasFormulae = _hasFormulae
                if resultIsVersioningReport and modelXbrl.modelDocument:
                    versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
                        modelTestcaseVariation.versioningReportUri, baseForElement)
                    if os.path.exists(versReportFile): #validate existing
                        modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report"))
                        if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT:
                            ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport)
                            self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors)
                            modelVersReport.close()
                    elif len(inputDTSes) == 2:
                        ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes(
                              versReportFile, inputDTSes["from"], inputDTSes["to"])
                        modelTestcaseVariation.status = "generated"
                    else:
                        modelXbrl.error("arelle:notLoaded",
                             _("Variation %(id)s %(name)s input DTSes not loaded, unable to generate versioning report: %(file)s"),
                             modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri))
                        modelTestcaseVariation.status = "failed"
                    for inputDTS in inputDTSes.values():
                        inputDTS.close()
                    del inputDTSes # dereference
                elif resultIsTaxonomyPackage:
                    self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors)
                    modelXbrl.close()
                elif inputDTSes:
                    # validate schema, linkbase, or instance
                    modelXbrl = inputDTSes[None][0]
                    expectedDataFiles = set(modelXbrl.modelManager.cntlr.webCache.normalizeUrl(uri, baseForElement)
                                            for d in modelTestcaseVariation.dataUris.values() for uri in d
                                            if not UrlUtil.isAbsolute(uri))
                    foundDataFiles = set()
                    variationBase = os.path.dirname(baseForElement)
                    for dtsName, inputDTS in inputDTSes.items():  # input instances are also parameters
                        if dtsName: # named instance
                            parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes)
                        elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents
                            parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula
                        for _inputDTS in inputDTS:
                            for docUrl, doc in _inputDTS.urlDocs.items():
                                if docUrl.startswith(variationBase) and not doc.type == Type.INLINEXBRLDOCUMENTSET:
                                    if getattr(doc,"loadedFromXbrlFormula", False): # may have been sourced from xf file
                                        if docUrl.replace("-formula.xml", ".xf") in expectedDataFiles:
                                            docUrl = docUrl.replace("-formula.xml", ".xf")
                                    foundDataFiles.add(docUrl)
                    if expectedDataFiles - foundDataFiles:
                        modelXbrl.info("arelle:testcaseDataNotUsed",
                            _("Variation %(id)s %(name)s data files not used: %(missingDataFiles)s"),
                            modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id, 
                            missingDataFiles=", ".join(sorted(os.path.basename(f) for f in expectedDataFiles - foundDataFiles)))
                    if foundDataFiles - expectedDataFiles:
                        modelXbrl.info("arelle:testcaseDataUnexpected",
                            _("Variation %(id)s %(name)s files not in variation data: %(unexpectedDataFiles)s"),
                            modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id,
                            unexpectedDataFiles=", ".join(sorted(os.path.basename(f) for f in foundDataFiles - expectedDataFiles)))
                    if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable:
                        try:
                            RenderingEvaluator.init(modelXbrl)
                        except Exception as err:
                            modelXbrl.error("exception:" + type(err).__name__,
                                _("Testcase RenderingEvaluator.init exception: %(error)s, instance: %(instance)s"),
                                modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
                    modelXbrlHasFormulae = modelXbrl.hasFormulae
                    if modelXbrlHasFormulae:
                        try:
                            # validate only formulae
                            self.instValidator.parameters = parameters
                            ValidateFormula.validate(self.instValidator)
                        except Exception as err:
                            modelXbrl.error("exception:" + type(err).__name__,
                                _("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"),
                                modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
                    if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset:
                        for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"):
                            pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri)
                        infoset = ModelXbrl.load(self.modelXbrl.modelManager, 
                                                 modelTestcaseVariation.resultInfosetUri,
                                                   _("loading result infoset"), 
                                                   base=baseForElement,
                                                   useFileSource=self.useFileSource,
                                                   errorCaptureLevel=errorCaptureLevel)
                        if infoset.modelDocument is None:
                            modelXbrl.error("arelle:notLoaded",
                                _("Variation %(id)s %(name)s result infoset not loaded: %(file)s"),
                                modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, 
                                file=os.path.basename(modelTestcaseVariation.resultXbrlInstance))
                            modelTestcaseVariation.status = "result infoset not loadable"
                        else:   # check infoset
                            ValidateInfoset.validate(self.instValidator, modelXbrl, infoset)
                        infoset.close()
                    if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset:
                        # diff (or generate) table infoset
                        resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement)
                        if not any(alternativeValidation(modelXbrl, resultTableUri)
                                   for alternativeValidation in pluginClassMethods("Validate.TableInfoset")):
                            try:
                                ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True)  # false to save infoset files
                            except Exception as err:
                                modelXbrl.error("exception:" + type(err).__name__,
                                    _("Testcase table linkbase validation exception: %(error)s, instance: %(instance)s"),
                                    modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True)
                    self.instValidator.close()
                    extraErrors = []
                    for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"):
                        pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors)
                    self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status
                    if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual): 
                        # if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it
                        modelXbrl.formulaOutputInstance.hasFormulae = False #  block formulae on output instance (so assertion of input is not lost)
                        self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters)
                        self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors)
                        if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance
                            formulaOutputInstance = modelXbrl.formulaOutputInstance
                            modelXbrl.formulaOutputInstance = None # prevent it from being closed now
                        self.instValidator.close()
                    compareIxResultInstance = (modelXbrl.modelDocument.type in (Type.INLINEXBRL, Type.INLINEXBRLDOCUMENTSET) and 
                                               modelTestcaseVariation.resultXbrlInstanceUri is not None)
                    if compareIxResultInstance:
                        formulaOutputInstance = modelXbrl # compare modelXbrl to generated output instance
                        errMsgPrefix = "ix"
                    else: # delete input instances before formula output comparision
                        for inputDTSlist in inputDTSes.values():
                            for inputDTS in inputDTSlist:
                                inputDTS.close()
                        del inputDTSes # dereference
                        errMsgPrefix = "formula"
                    if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument:
                        _matchExpectedResultIDs = not modelXbrlHasFormulae # formula restuls have inconsistent IDs
                        expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager, 
                                                   modelTestcaseVariation.resultXbrlInstanceUri,
                                                   _("loading expected result XBRL instance"), 
                                                   base=baseForElement,
                                                   useFileSource=self.useFileSource,
                                                   errorCaptureLevel=errorCaptureLevel)
                        if expectedInstance.modelDocument is None:
                            self.modelXbrl.error("{}:expectedResultNotLoaded".format(errMsgPrefix),
                                _("Testcase \"%(name)s\" %(id)s expected result instance not loaded: %(file)s"),
                                modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, 
                                file=os.path.basename(modelTestcaseVariation.resultXbrlInstanceUri),
                                messageCodes=("formula:expectedResultNotLoaded","ix:expectedResultNotLoaded"))
                            modelTestcaseVariation.status = "result not loadable"
                        else:   # compare facts
                            if len(expectedInstance.facts) != len(formulaOutputInstance.facts):
                                formulaOutputInstance.error("{}:resultFactCounts".format(errMsgPrefix),
                                    _("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"),
                                    modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts),
                                    expectedFacts=len(expectedInstance.facts),
                                    messageCodes=("formula:resultFactCounts","ix:resultFactCounts"))
                            else:
                                formulaOutputFootnotesRelSet = ModelRelationshipSet(formulaOutputInstance, "XBRL-footnotes")
                                expectedFootnotesRelSet = ModelRelationshipSet(expectedInstance, "XBRL-footnotes")
                                def factFootnotes(fact, footnotesRelSet):
                                    footnotes = {}
                                    footnoteRels = footnotesRelSet.fromModelObject(fact)
                                    if footnoteRels:
                                        # most process rels in same order between two instances, use labels to sort
                                        for i, footnoteRel in enumerate(sorted(footnoteRels,
                                                                               key=lambda r: (r.fromLabel,r.toLabel))):
                                            modelObject = footnoteRel.toModelObject
                                            if isinstance(modelObject, ModelResource):
                                                xml = modelObject.viewText().strip()
                                                footnotes["Footnote {}".format(i+1)] = xml #re.sub(r'\s+', ' ', collapseWhitespace(modelObject.stringValue))
                                            elif isinstance(modelObject, ModelFact):
                                                footnotes["Footnoted fact {}".format(i+1)] = \
                                                    "{} context: {} value: {}".format(
                                                    modelObject.qname,
                                                    modelObject.contextID,
                                                    collapseWhitespace(modelObject.value))
                                    return footnotes
                                for expectedInstanceFact in expectedInstance.facts:
                                    unmatchedFactsStack = []
                                    formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=_matchExpectedResultIDs, matchLang=False)
                                    #formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=True, matchLang=True)
                                    if formulaOutputFact is None:
                                        if unmatchedFactsStack: # get missing nested tuple fact, if possible
                                            missingFact = unmatchedFactsStack[-1]
                                        else:
                                            missingFact = expectedInstanceFact
                                        formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix),
                                            _("Output missing expected fact %(fact)s"),
                                            modelXbrl=missingFact, fact=missingFact.qname,
                                            messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing"))
                                    else: # compare footnotes
                                        expectedInstanceFactFootnotes = factFootnotes(expectedInstanceFact, expectedFootnotesRelSet)
                                        formulaOutputFactFootnotes = factFootnotes(formulaOutputFact, formulaOutputFootnotesRelSet)
                                        if (len(expectedInstanceFactFootnotes) != len(formulaOutputFactFootnotes) or
                                            set(expectedInstanceFactFootnotes.values()) != set(formulaOutputFactFootnotes.values())):
                                            formulaOutputInstance.error("{}:expectedFactFootnoteDifference".format(errMsgPrefix),
                                                _("Output expected fact %(fact)s expected footnotes %(footnotes1)s produced footnotes %(footnotes2)s"),
                                                modelXbrl=(formulaOutputFact,expectedInstanceFact), fact=expectedInstanceFact.qname, footnotes1=sorted(expectedInstanceFactFootnotes.items()), footnotes2=sorted(formulaOutputFactFootnotes.items()),
                                                messageCodes=("formula:expectedFactFootnoteDifference","ix:expectedFactFootnoteDifference"))

                            # for debugging uncomment next line to save generated instance document
                            # formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml")
                        expectedInstance.close()
                        del expectedInstance # dereference
                        self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors)
                        formulaOutputInstance.close()
                        del formulaOutputInstance
                    if compareIxResultInstance:
                        for inputDTSlist in inputDTSes.values():
                            for inputDTS in inputDTSlist:
                                inputDTS.close()
                        del inputDTSes # dereference
                # update ui thread via modelManager (running in background here)
                self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId())
                    
            _statusCounts = OrderedDict((("pass",0),("fail",0)))
            for tv in getattr(testcase, "testcaseVariations", ()):
                _statusCounts[tv.status] = _statusCounts.get(tv.status, 0) + 1    
            self.modelXbrl.info("arelle:testCaseResults", ", ".join("{}={}".format(k,c) for k, c in _statusCounts.items() if k))
            
            self.modelXbrl.modelManager.showStatus(_("ready"), 2000)