def backgroundProfileFormula(cntlr, profileReportFile, maxRunTime, excludeCompileTime): from arelle import Locale, XPathParser, ValidateXbrlDimensions, ValidateFormula # build grammar before profiling (if this is the first pass, so it doesn't count in profile statistics) XPathParser.initializeParser(cntlr.modelManager) # load dimension defaults ValidateXbrlDimensions.loadDimensionDefaults(cntlr.modelManager) import cProfile, pstats, sys, time # a minimal validation class for formula validator parameters that are needed class Validate: def __init__(self, modelXbrl, maxRunTime): self.modelXbrl = modelXbrl self.parameters = None self.validateSBRNL = False self.maxFormulaRunTime = maxRunTime def close(self): self.__dict__.clear() val = Validate(cntlr.modelManager.modelXbrl, maxRunTime) formulaOptions = val.modelXbrl.modelManager.formulaOptions if excludeCompileTime: startedAt = time.time() cntlr.addToLog(_("pre-compiling formulas before profiling")) val.validateFormulaCompileOnly = True ValidateFormula.validate(val) del val.validateFormulaCompileOnly cntlr.addToLog(Locale.format_string(cntlr.modelManager.locale, _("formula pre-compiling completed in %.2f secs"), time.time() - startedAt)) cntlr.addToLog(_("executing formulas for profiling")) else: cntlr.addToLog(_("compiling and executing formulas for profiling")) startedAt = time.time() statsFile = profileReportFile + ".bin" cProfile.runctx("ValidateFormula.validate(val)", globals(), locals(), statsFile) cntlr.addToLog(Locale.format_string(cntlr.modelManager.locale, _("formula profiling completed in %.2f secs"), time.time() - startedAt)) # dereference val val.close() # specify a file for log priorStdOut = sys.stdout sys.stdout = open(profileReportFile, "w") statObj = pstats.Stats(statsFile) statObj.strip_dirs() statObj.sort_stats("time") statObj.print_stats() statObj.print_callees() statObj.print_callers() sys.stdout.flush() sys.stdout.close() del statObj sys.stdout = priorStdOut os.remove(statsFile)
def executeCallTest(val, name, callTuple, testTuple): if callTuple: XPathParser.initializeParser(val) try: val.modelXbrl.modelManager.showStatus(_("Executing call")) callExprStack = XPathParser.parse(val, callTuple[0], callTuple[1], name + " call", Trace.CALL) xpathContext = XPathContext.create(val.modelXbrl, sourceElement=callTuple[1]) result = xpathContext.evaluate(callExprStack) xpathContext.inScopeVars[qname('result',noPrefixIsNoNamespace=True)] = result val.modelXbrl.error( _("{0} result {1}").format( name, result), "info", "formula:trace") if testTuple: val.modelXbrl.modelManager.showStatus(_("Executing test")) testExprStack = XPathParser.parse(val, testTuple[0], testTuple[1], name + " test", Trace.CALL) testResult = xpathContext.effectiveBooleanValue( None, xpathContext.evaluate(testExprStack) ) val.modelXbrl.error( _("Test {0} result {1}").format( name, testResult), "info" if testResult else "err", "cfcn:testPass" if testResult else "cfcn:testFail") except XPathContext.XPathException as err: val.modelXbrl.error( _("{0} evaluation error: {1} \n{2}").format(name, err.message, err.sourceErrorIndication), "err", err.code) val.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def __init__(self, cntlr, isCmdLine=False): self.cntlr = cntlr # setup tester xml = "<rootElement/>" self.modelXbrl = ModelXbrl.create(cntlr.modelManager, ModelDocument.Type.UnknownNonXML, initialXml=xml, isEntry=True) self.validator = ValidateXbrl.ValidateXbrl(self.modelXbrl) self.validator.validate(self.modelXbrl) # required to set up cntlr.showStatus(_("Initializing Formula Grammar")) XPathParser.initializeParser(cntlr.modelManager) cntlr.showStatus(None) self.trRegs = sorted(ixtNamespaces.keys()) self.trPrefixNSs = dict((qn.prefix, qn.namespaceURI) for qn in self.modelXbrl.modelManager.customTransforms.keys()) self.trRegs.extend(sorted(self.trPrefixNSs.keys())) self.trPrefixNSs.update(ixtNamespaces)
def __init__(self, cntlr, xfFile): self.modelXbrl = cntlr.modelManager.modelXbrl self.xfFile = xfFile self.xfLines = [] self.xmlns = {} self.eltTypeCount = {} self.nextIdDupNbr = {} cntlr.showStatus(_("Initializing Formula Grammar")) XPathParser.initializeParser(cntlr.modelManager) cntlr.showStatus(None) for cfQnameArity in sorted( qnameArity for qnameArity in self.modelXbrl.modelCustomFunctionSignatures.keys() if isinstance(qnameArity, (tuple, list))): cfObject = self.modelXbrl.modelCustomFunctionSignatures[ cfQnameArity] self.doObject(cfObject, None, "", set()) rootObjects = rootFormulaObjects(self) # sets var sets up # put parameters at root regardless of whether linked to for qn, param in sorted(self.modelXbrl.qnameParameters.items(), key=lambda i: i[0]): self.doObject(param, None, "", set()) for rootObject in sorted(rootObjects, key=formulaObjSortKey): self.doObject(rootObject, None, "", set()) if self.xmlns: self.xfLines.insert(0, "") for prefix, ns in sorted(self.xmlns.items(), reverse=True): self.xfLines.insert( 0, "namespace {} = \"{}\";".format(prefix, ns)) self.xfLines.insert(0, "") self.xfLines.insert( 0, "(: Generated from {} by Arelle on {} :)".format( self.modelXbrl.modelDocument.basename, XmlUtil.dateunionValue(datetime.datetime.now()))) with open(xfFile, "w", encoding="utf-8") as fh: fh.write("\n".join(self.xfLines)) self.modelXbrl.info("info", "saved formula file %(file)s", file=xfFile)
def executeCallTest(val, name, callTuple, testTuple): if callTuple: XPathParser.initializeParser(val) try: val.modelXbrl.modelManager.showStatus(_("Executing call")) callExprStack = XPathParser.parse(val, callTuple[0], callTuple[1], name + " call", Trace.CALL) xpathContext = XPathContext.create(val.modelXbrl, sourceElement=callTuple[1]) result = xpathContext.evaluate(callExprStack) xpathContext.inScopeVars[qname( 'result', noPrefixIsNoNamespace=True)] = result val.modelXbrl.error( _("{0} result {1}").format(name, result), "info", "formula:trace") if testTuple: val.modelXbrl.modelManager.showStatus(_("Executing test")) testExprStack = XPathParser.parse(val, testTuple[0], testTuple[1], name + " test", Trace.CALL) testResult = xpathContext.effectiveBooleanValue( None, xpathContext.evaluate(testExprStack)) val.modelXbrl.error( _("Test {0} result {1}").format(name, testResult), "info" if testResult else "err", "cfcn:testPass" if testResult else "cfcn:testFail") except XPathContext.XPathException as err: val.modelXbrl.error( _("{0} evaluation error: {1} \n{2}").format( name, err.message, err.sourceErrorIndication), "err", err.code) val.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def backgroundFind(self, expr, logViewLines): exprType = self.options["exprType"] inConceptLabel = self.options["conceptLabel"] inConceptName = self.options["conceptName"] inConceptType = self.options["conceptType"] inConceptSubs = self.options["conceptSubs"] inConceptPer = self.options["conceptPer"] inConceptBal = self.options["conceptBal"] inFactLabel = self.options["factLabel"] inFactName = self.options["factName"] inFactValue = self.options["factValue"] inFactCntx = self.options["factCntx"] inFactUnit = self.options["factUnit"] inMessagesLog = self.options["messagesLog"] nextIsDown = self.options["direction"] == "down" objsFound = set() try: if exprType == "text": # escape regex metacharacters pattern = re.compile( ''.join([(('\\' + c) if c in reMetaChars else c) for c in expr]), re.IGNORECASE) isRE = True isXP = False elif exprType == "regex": pattern = re.compile(expr, re.IGNORECASE) isRE = True isXP = False elif exprType == "xpath": isRE = False isXP = True self.resultText.setValue(_("Compiling xpath expression...")) XPathParser.initializeParser(self.modelManager) self.modelManager.showStatus( _("Compiling xpath expression...")) xpProg = XPathParser.parse( self, expr, XPathParser.staticExpressionFunctionContext(), "find expression", Trace.CALL) xpCtx = XPathContext.create(self.modelXbrl, sourceElement=None) else: return # nothing to do if inMessagesLog: for lineNumber, line in enumerate(logViewLines): if pattern.search(line): objsFound.add(lineNumber) elif self.modelXbrl.modelDocument.type == ModelDocument.Type.RSSFEED: for rssItem in self.modelXbrl.modelDocument.items: if any( pattern.search(str(value)) for name, value in rssItem.propertyView): objsFound.add(rssItem) else: # DTS search if inConceptLabel or inConceptName or inConceptType or inConceptSubs or inConceptPer or inConceptBal: self.modelManager.cntlr.uiThreadQueue.put( (self.resultText.setValue, [_("Matching concepts...") ])) self.modelManager.showStatus(_("Matching concepts...")) for conceptName, concepts in self.modelXbrl.nameConcepts.items( ): for concept in concepts: if ((isXP and xpCtx.evaluateBooleanValue( xpProg, contextItem=concept.qname)) or (isRE and (inConceptLabel and pattern.search(concept.label())) or (inConceptName and pattern.search(conceptName)) or (inConceptType and pattern.search(str(concept.typeQname))) or (inConceptSubs and pattern.search( str(concept.substitutionGroupQname))) or (inConceptPer and concept.periodType and pattern.search(concept.periodType)) or (inConceptBal and concept.balance and pattern.search(concept.balance)))): objsFound.add(concept) if inFactLabel or inFactName or inFactValue or inFactCntx or inFactUnit: self.modelManager.cntlr.uiThreadQueue.put( (self.resultText.setValue, [_("Matching facts...")])) self.modelManager.showStatus(_("Matching facts...")) for fact in self.modelXbrl.facts: if ((isXP and xpCtx.evaluateBooleanValue( xpProg, contextItem=fact)) or (isRE and (inFactName and pattern.search(fact.concept.name) or (inFactLabel and pattern.search(fact.concept.label())) or (inFactValue and pattern.search(fact.value)) or (inFactCntx and pattern.search( XmlUtil.innerText(fact.context.element))) or (inFactUnit and pattern.search( XmlUtil.innerText(fact.unit.element)))))): objsFound.add(fact) except XPathContext.XPathException as err: err = _("Find expression error: {0} \n{1}").format( err.message, err.sourceErrorIndication) self.modelManager.addToLog(err) self.modelManager.cntlr.uiThreadQueue.put( (self.resultText.setValue, [err])) self.modelManager.showStatus(_("Completed with errors"), 5000) numConcepts = 0 numFacts = 0 numRssItems = 0 numMessages = 0 self.objsList = [] for obj in objsFound: if inMessagesLog: numMessages += 1 self.objsList.append(('m', "{0:06}".format(obj), obj)) elif isinstance(obj, ModelConcept): numConcepts += 1 self.objsList.append(('c', obj.localName, obj.objectId())) elif isinstance(obj, ModelFact): numFacts += 1 self.objsList.append(('f', obj.__hash__(), obj.objectId())) elif isinstance(obj, ModelRssItem): numRssItems += 1 self.objsList.append(('r', obj.__hash__(), obj.objectId())) self.objsList.sort() self.result = "Found " if numConcepts: self.result += "{0} concepts".format(numConcepts) if numFacts: self.result += ", " if numFacts: self.result += "{0} facts".format(numFacts) if numRssItems: self.result += "{0} RSS items".format(numRssItems) if numMessages: self.result += "{0} Messages".format(numMessages) if numConcepts + numFacts + numRssItems + numMessages == 0: self.result += "no matches" self.foundIndex = -1 self.modelManager.cntlr.uiThreadQueue.put( (self.resultText.setValue, [self.result])) else: self.foundIndex = 0 if nextIsDown else (len(self.objsList) - 1) self.modelManager.cntlr.uiThreadQueue.put((self.next, [])) self.modelManager.showStatus(_("Ready..."), 2000)
def backgroundFind(self, expr, logViewLines): exprType = self.options["exprType"] inConceptLabel = self.options["conceptLabel"] inConceptName = self.options["conceptName"] inConceptType = self.options["conceptType"] inConceptSubs = self.options["conceptSubs"] inConceptPer = self.options["conceptPer"] inConceptBal = self.options["conceptBal"] inFactLabel = self.options["factLabel"] inFactName = self.options["factName"] inFactValue = self.options["factValue"] inFactCntx = self.options["factCntx"] inFactUnit = self.options["factUnit"] inMessagesLog = self.options["messagesLog"] nextIsDown = self.options["direction"] == "down" objsFound = set() try: if exprType == "text": # escape regex metacharacters pattern = re.compile(''.join( [(('\\' + c) if c in reMetaChars else c) for c in expr]), re.IGNORECASE) isRE = True isXP = False elif exprType == "regex": pattern = re.compile(expr, re.IGNORECASE) isRE = True isXP = False elif exprType == "xpath": isRE = False isXP = True self.resultText.setValue(_("Compiling xpath expression...")) XPathParser.initializeParser(self) self.modelManager.showStatus(_("Compiling xpath expression...")) xpProg= XPathParser.parse(self, expr, XPathParser.staticExpressionFunctionContext(), "find expression", Trace.CALL) xpCtx = XPathContext.create(self.modelXbrl, sourceElement=None) else: return # nothing to do if inMessagesLog: for lineNumber, line in enumerate(logViewLines): if pattern.search(line): objsFound.add(lineNumber) elif self.modelXbrl.modelDocument.type == ModelDocument.Type.RSSFEED: for rssItem in self.modelXbrl.modelDocument.items: if any(pattern.search(str(value)) for name, value in rssItem.propertyView): objsFound.add(rssItem) else: # DTS search if inConceptLabel or inConceptName or inConceptType or inConceptSubs or inConceptPer or inConceptBal: self.modelManager.cntlr.uiThreadQueue.put((self.resultText.setValue, [_("Matching concepts...")])) self.modelManager.showStatus(_("Matching concepts...")) for conceptName, concepts in self.modelXbrl.nameConcepts.items(): for concept in concepts: if ((isXP and xpCtx.evaluateBooleanValue(xpProg, contextItem=concept.qname)) or (isRE and (inConceptLabel and pattern.search(concept.label())) or (inConceptName and pattern.search(conceptName)) or (inConceptType and pattern.search(str(concept.typeQname))) or (inConceptSubs and pattern.search(str(concept.substitutionGroupQname))) or (inConceptPer and concept.periodType and pattern.search(concept.periodType)) or (inConceptBal and concept.balance and pattern.search(concept.balance)) ) ): objsFound.add(concept) if inFactLabel or inFactName or inFactValue or inFactCntx or inFactUnit: self.modelManager.cntlr.uiThreadQueue.put((self.resultText.setValue, [_("Matching facts...")])) self.modelManager.showStatus(_("Matching facts...")) for fact in self.modelXbrl.facts: if ((isXP and xpCtx.evaluateBooleanValue(xpProg, contextItem=fact)) or (isRE and (inFactName and pattern.search(fact.concept.name) or (inFactLabel and pattern.search(fact.concept.label())) or (inFactValue and pattern.search(fact.value)) or (inFactCntx and pattern.search(XmlUtil.innerText(fact.context.element))) or (inFactUnit and pattern.search(XmlUtil.innerText(fact.unit.element)))) ) ): objsFound.add(fact) except XPathContext.XPathException as err: err = _("Find expression error: {0} \n{1}").format(err.message, err.sourceErrorIndication) self.modelManager.addToLog(err) self.modelManager.cntlr.uiThreadQueue.put((self.resultText.setValue, [err])) self.modelManager.showStatus(_("Completed with errors"), 5000) numConcepts = 0 numFacts = 0 numRssItems = 0 numMessages = 0 self.objsList = [] for obj in objsFound: if inMessagesLog: numMessages += 1 self.objsList.append( ('m', "{0:06}".format(obj), obj) ) elif isinstance(obj,ModelConcept): numConcepts += 1 self.objsList.append( ('c', obj.localName, obj.objectId()) ) elif isinstance(obj,ModelFact): numFacts += 1 self.objsList.append( ('f', obj.__hash__(), obj.objectId()) ) elif isinstance(obj,ModelRssItem): numRssItems += 1 self.objsList.append( ('r', obj.__hash__(), obj.objectId()) ) self.objsList.sort() self.result = "Found " if numConcepts: self.result += "{0} concepts".format(numConcepts) if numFacts: self.result += ", " if numFacts: self.result += "{0} facts".format(numFacts) if numRssItems: self.result += "{0} RSS items".format(numRssItems) if numMessages: self.result += "{0} Messages".format(numMessages) if numConcepts + numFacts + numRssItems + numMessages == 0: self.result += "no matches" self.foundIndex = -1 self.modelManager.cntlr.uiThreadQueue.put((self.resultText.setValue, [self.result])) else: self.foundIndex = 0 if nextIsDown else (len(self.objsList) - 1) self.modelManager.cntlr.uiThreadQueue.put((self.next, [])) self.modelManager.showStatus(_("Ready..."), 2000)
def validate(val): formulaOptions = val.modelXbrl.modelManager.formulaOptions XPathParser.initializeParser(val) val.modelXbrl.modelManager.showStatus(_("Compiling formulae")) initialErrorCount = val.modelXbrl.logCountErr # global parameter names parameterQnames = set() instanceQnames = set() parameterDependencies = {} instanceDependencies = defaultdict( set) # None-key entries are non-formula dependencies dependencyResolvedParameters = set() orderedParameters = [] orderedInstances = [] for paramQname, modelParameter in val.modelXbrl.qnameParameters.items(): if isinstance(modelParameter, ModelParameter): modelParameter.compile() parameterDependencies[paramQname] = modelParameter.variableRefs() parameterQnames.add(paramQname) if isinstance(modelParameter, ModelInstance): instanceQnames.add(paramQname) # duplicates checked on loading modelDocument #resolve dependencies resolvedAParameter = True while (resolvedAParameter): resolvedAParameter = False for paramQname in parameterQnames: if paramQname not in dependencyResolvedParameters and \ len(parameterDependencies[paramQname] - dependencyResolvedParameters) == 0: dependencyResolvedParameters.add(paramQname) orderedParameters.append(paramQname) resolvedAParameter = True # anything unresolved? for paramQname in parameterQnames: if paramQname not in dependencyResolvedParameters: circularOrUndefDependencies = parameterDependencies[ paramQname] - dependencyResolvedParameters undefinedVars = circularOrUndefDependencies - parameterQnames paramsCircularDep = circularOrUndefDependencies - undefinedVars if len(undefinedVars) > 0: val.modelXbrl.error( _("Undefined dependencies in parameter {0}, to names {1}" ).format(paramQname, ", ".join( (str(v) for v in undefinedVars))), "err", "xbrlve:unresolvedDependency") if len(paramsCircularDep) > 0: val.modelXbrl.error( _("Cyclic dependencies in parameter {0}, to names {1}" ).format(paramQname, ", ".join( (str(d) for d in paramsCircularDep))), "err", "xbrlve:parameterCyclicDependencies") for custFnSig in val.modelXbrl.modelCustomFunctionSignatures.values(): custFnQname = custFnSig.qname if custFnQname.namespaceURI == "XbrlConst.xfi": val.modelXbrl.error( _("Custom function {0} has namespace reserved for functions in the function registry {1}" ).format(str(custFnQname), custFnQname.namespaceURI), "err", "xbrlve:noProhibitedNamespaceForCustomFunction") # any custom function implementations? for modelRel in val.modelXbrl.relationshipSet( XbrlConst.functionImplementation).fromModelObject(custFnSig): custFnImpl = modelRel.toModelObject custFnSig.customFunctionImplementation = custFnImpl if len(custFnImpl.inputNames) != len(custFnSig.inputTypes): val.modelXbrl.error( _("Custom function {0} signature has {1} parameters but implementation has {2}, must be matching" ).format(str(custFnQname), len(custFnSig.inputTypes), len(custFnImpl.inputNames)), "err", "xbrlcfie:inputMismatch") for custFnImpl in val.modelXbrl.modelCustomFunctionImplementations: if not val.modelXbrl.relationshipSet( XbrlConst.functionImplementation).toModelObject(custFnImpl): val.modelXbrl.error( _("Custom function implementation {0} has no relationship from any custom function signature" ).format(custFnImpl.xlinkLabel), "err", "xbrlcfie:missingCFIRelationship") custFnImpl.compile() # xpathContext is needed for filter setup for expressions such as aspect cover filter # determine parameter values xpathContext = XPathContext.create(val.modelXbrl) for paramQname in orderedParameters: if not isinstance(modelParameter, ModelInstance): modelParameter = val.modelXbrl.qnameParameters[paramQname] asType = modelParameter.asType asLocalName = asType.localName if asType else "string" try: if val.parameters and paramQname in val.parameters: paramDataType, paramValue = val.parameters[paramQname] typeLocalName = paramDataType.localName if paramDataType else "string" value = FunctionXs.call(xpathContext, None, typeLocalName, [paramValue]) result = FunctionXs.call(xpathContext, None, asLocalName, [value]) if formulaOptions.traceParameterInputValue: val.modelXbrl.error( _("Parameter {0} input {1}").format( paramQname, result), "info", "formula:trace") else: result = modelParameter.evaluate(xpathContext, asType) if formulaOptions.traceParameterExpressionResult: val.modelXbrl.error( _("Parameter {0} result {1}").format( paramQname, result), "info", "formula:trace") xpathContext.inScopeVars[ paramQname] = result # make visible to subsequent parameter expression except XPathContext.XPathException as err: val.modelXbrl.error( _("Parameter \n{0} \nException: \n{1}").format( paramQname, err.message), "err", "xbrlve:parameterTypeMismatch" if err.code == "err:FORG0001" else err.code) produceOutputXbrlInstance = False instanceProducingVariableSets = defaultdict(list) for modelVariableSet in val.modelXbrl.modelVariableSets: varSetInstanceDependencies = set() if isinstance(modelVariableSet, ModelFormula): instanceQname = None for modelRel in val.modelXbrl.relationshipSet( XbrlConst.formulaInstance).fromModelObject( modelVariableSet): instance = modelRel.toModelObject if isinstance(instance, ModelInstance): if instanceQname is None: instanceQname = instance.qname else: val.modelXbrl.error( _("Multiple output instances for formula {0}, to names {1}, {2}" ).format(modelVariableSet.xlinkLabel, instanceQname, instance.qname), "info", "arelle:multipleOutputInstances") if instanceQname is None: instanceQname = XbrlConst.qnStandardOutputInstance instanceQnames.add(instanceQname) modelVariableSet.outputInstanceQname = instanceQname if val.validateSBRNL: val.modelXbrl.error( _("Formula linkbase {0} formula:formula {1} is not allowed" ).format( os.path.basename(modelVariableSet.modelDocument.uri), modelVariableSet.xlinkLabel), "err", "SBR.NL.2.3.9.03") else: instanceQname = None modelVariableSet.countSatisfied = 0 modelVariableSet.countNotSatisfied = 0 checkValidationMessages(val, modelVariableSet) instanceProducingVariableSets[instanceQname].append(modelVariableSet) modelVariableSet.outputInstanceQname = instanceQname if modelVariableSet.aspectModel not in ("non-dimensional", "dimensional"): val.modelXbrl.error( _("Variable set {0}, aspect model {1} not recognized").format( modelVariableSet.xlinkLabel, modelVariableSet.aspectModel), "err", "xbrlve:unknownAspectModel") modelVariableSet.compile() modelVariableSet.hasConsistencyAssertion = False #determine dependencies within variable sets nameVariables = {} qnameRels = {} definedNamesSet = set() for modelRel in val.modelXbrl.relationshipSet( XbrlConst.variableSet).fromModelObject(modelVariableSet): varqname = modelRel.variableQname if varqname: qnameRels[varqname] = modelRel toVariable = modelRel.toModelObject if varqname not in definedNamesSet: definedNamesSet.add(varqname) if varqname not in nameVariables: nameVariables[varqname] = toVariable elif nameVariables[varqname] != toVariable: val.modelXbrl.error( _("Multiple variables named {1} in variable set {0}" ).format(modelVariableSet.xlinkLabel, varqname), "err", "xbrlve:duplicateVariableNames") fromInstanceQnames = None for instRel in val.modelXbrl.relationshipSet( XbrlConst.instanceVariable).toModelObject(toVariable): fromInstance = instRel.fromModelObject if isinstance(fromInstance, ModelInstance): fromInstanceQname = fromInstance.qname varSetInstanceDependencies.add(fromInstanceQname) instanceDependencies[instanceQname].add( fromInstanceQname) if fromInstanceQnames is None: fromInstanceQnames = set() fromInstanceQnames.add(fromInstanceQname) if fromInstanceQnames is None: varSetInstanceDependencies.add( XbrlConst.qnStandardInputInstance) if instanceQname: instanceDependencies[instanceQname].add( XbrlConst.qnStandardInputInstance) toVariable.fromInstanceQnames = fromInstanceQnames else: val.modelXbrl.error( _("Variables name {1} cannot be determined on arc from {0}" ).format(modelVariableSet.xlinkLabel, modelRel.variablename), "err", "xbrlve:variableNameResolutionFailure") definedNamesSet |= parameterQnames variableDependencies = {} for modelRel in val.modelXbrl.relationshipSet( XbrlConst.variableSet).fromModelObject(modelVariableSet): variable = modelRel.toModelObject if isinstance( variable, (ModelParameter, ModelVariable)): # ignore anything not parameter or variable varqname = modelRel.variableQname depVars = variable.variableRefs() variableDependencies[varqname] = depVars if len(depVars ) > 0 and formulaOptions.traceVariablesDependencies: val.modelXbrl.error( _("Variable set {0}, variable {1}, dependences {2}"). format(modelVariableSet.xlinkLabel, varqname, depVars), "info", "formula:trace") definedNamesSet.add(varqname) # check for fallback value variable references if isinstance(variable, ModelFactVariable): for depVar in XPathParser.variableReferencesSet( variable.fallbackValueProg, variable.element): if depVar in qnameRels and isinstance( qnameRels[depVar].toModelObject, ModelVariable): val.modelXbrl.error( _("Variable set {0} fallbackValue '{1}' cannot refer to variable {2}" ).format(modelVariableSet.xlinkLabel, variable.fallbackValue, depVar), "err", "xbrlve:factVariableReferenceNotAllowed") # check for covering aspect not in variable set aspect model checkFilterAspectModel(val, modelVariableSet, variable.filterRelationships, xpathContext) orderedNameSet = set() orderedNameList = [] orderedAVariable = True while (orderedAVariable): orderedAVariable = False for varqname, depVars in variableDependencies.items(): if varqname not in orderedNameSet and len(depVars - parameterQnames - orderedNameSet) == 0: orderedNameList.append(varqname) orderedNameSet.add(varqname) orderedAVariable = True if varqname in instanceQnames: varSetInstanceDependencies.add(varqname) instanceDependencies[instanceQname].add(varqname) elif isinstance(nameVariables.get(varqname), ModelInstance): instqname = nameVariables[varqname].qname varSetInstanceDependencies.add(instqname) instanceDependencies[instanceQname].add(instqname) # anything unresolved? for varqname, depVars in variableDependencies.items(): if varqname not in orderedNameSet: circularOrUndefVars = depVars - parameterQnames - orderedNameSet undefinedVars = circularOrUndefVars - definedNamesSet varsCircularDep = circularOrUndefVars - undefinedVars if len(undefinedVars) > 0: val.modelXbrl.error( _("Undefined variable dependencies in variable st {0}, from variable {1} to {2}" ).format(modelVariableSet.xlinkLabel, varqname, undefinedVars), "err", "xbrlve:unresolvedDependency") if len(varsCircularDep) > 0: val.modelXbrl.error( _("Cyclic dependencies in variable set {0}, from variable {1} to {2}" ).format(modelVariableSet.xlinkLabel, varqname, varsCircularDep), "err", "xbrlve:cyclicDependencies") # check unresolved variable set dependencies for varSetDepVarQname in modelVariableSet.variableRefs(): if varSetDepVarQname not in orderedNameSet and varSetDepVarQname not in parameterQnames: val.modelXbrl.error( _("Undefined variable dependency in variable set {0}, {1}" ).format(modelVariableSet.xlinkLabel, varSetDepVarQname), "err", "xbrlve:unresolvedDependency") if varSetDepVarQname in instanceQnames: varSetInstanceDependencies.add(varSetDepVarQname) instanceDependencies[instanceQname].add(varSetDepVarQname) elif isinstance(nameVariables.get(varSetDepVarQname), ModelInstance): instqname = nameVariables[varSetDepVarQname].qname varSetInstanceDependencies.add(instqname) instanceDependencies[instanceQname].add(instqname) if formulaOptions.traceVariablesOrder: val.modelXbrl.error( _("Variable set {0}, variables order: {1}").format( modelVariableSet.xlinkLabel, orderedNameList), "info", "formula:trace") if (formulaOptions.traceVariablesDependencies and len(varSetInstanceDependencies) > 0 and varSetInstanceDependencies != {XbrlConst.qnStandardInputInstance}): val.modelXbrl.error( _("Variable set {0}, instance dependences {1}").format( modelVariableSet.xlinkLabel, varSetInstanceDependencies), "info", "formula:trace") modelVariableSet.orderedVariableRelationships = [] for varqname in orderedNameList: if varqname in qnameRels: modelVariableSet.orderedVariableRelationships.append( qnameRels[varqname]) # check existence assertion variable dependencies if isinstance(modelVariableSet, ModelExistenceAssertion): for depVar in modelVariableSet.variableRefs(): if depVar in qnameRels and isinstance( qnameRels[depVar].toModelObject, ModelVariable): val.modelXbrl.error( _("Existence Assertion {0}, cannot refer to variable {1}" ).format(modelVariableSet.xlinkLabel, depVar), "err", "xbrleae:variableReferenceNotAllowed") # check messages variable dependencies checkValidationMessageVariables(val, modelVariableSet, qnameRels) # check preconditions modelVariableSet.preconditions = [] for modelRel in val.modelXbrl.relationshipSet( XbrlConst.variableSetPrecondition).fromModelObject( modelVariableSet): precondition = modelRel.toModelObject if isinstance(precondition, ModelPrecondition): modelVariableSet.preconditions.append(precondition) # check for variable sets referencing fact or general variables for modelRel in val.modelXbrl.relationshipSet( XbrlConst.variableSetFilter).fromModelObject(modelVariableSet): varSetFilter = modelRel.toModelObject if modelRel.isCovered: val.modelXbrl.error( _("Variable set {0}, filter {1}, cannot be covered" ).format(modelVariableSet.xlinkLabel, varSetFilter.xlinkLabel), "wrn", "arelle:variableSetFilterCovered") modelRel._isCovered = False # block group filter from being able to covere for depVar in varSetFilter.variableRefs(): if depVar in qnameRels and isinstance( qnameRels[depVar].toModelObject, ModelVariable): val.modelXbrl.error( _("Variable set {0}, filter {1}, cannot refer to variable {2}" ).format(modelVariableSet.xlinkLabel, varSetFilter.xlinkLabel, depVar), "err", "xbrlve:factVariableReferenceNotAllowed") # check aspects of formula if isinstance(modelVariableSet, ModelFormula): checkFormulaRules(val, modelVariableSet, nameVariables) # determine instance dependency order orderedInstancesSet = set() stdInpInst = {XbrlConst.qnStandardInputInstance} orderedInstancesList = [] orderedAnInstance = True while (orderedAnInstance): orderedAnInstance = False for instqname, depInsts in instanceDependencies.items(): if instqname and instqname not in orderedInstancesSet and len( depInsts - stdInpInst - orderedInstancesSet) == 0: orderedInstancesList.append(instqname) orderedInstancesSet.add(instqname) orderedAnInstance = True orderedInstancesList.append( None) # assertions come after all formulas that produce outputs # anything unresolved? for instqname, depInsts in instanceDependencies.items(): if instqname not in orderedInstancesSet: # can also be satisfied from an input DTS missingDependentInstances = depInsts - stdInpInst if val.parameters: missingDependentInstances -= val.parameters.keys() if instqname: if missingDependentInstances: val.modelXbrl.error( _("Cyclic dependencies of instance {0} produced by a formula, with variables consuming instances {1}" ).format(instqname, missingDependentInstances), "err", "xbrlvarinste:instanceVariableRecursionCycle") elif instqname == XbrlConst.qnStandardOutputInstance: orderedInstancesSet.add(instqname) orderedInstancesList.append( instqname ) # standard output formula, all input dependencies in parameters ''' future check? if instance has no external input or producing formula else: val.modelXbrl.error( _("Unresolved dependencies of an assertion's variables on instances {0}").format( depInsts - stdInpInst ), "err", "xbrlvarinste:instanceVariableRecursionCycle") ''' if formulaOptions.traceVariablesOrder and len(orderedInstancesList) > 1: val.modelXbrl.error( _("Variable instances processing order: {0}").format( orderedInstancesList), "info", "formula:trace") # linked consistency assertions for modelRel in val.modelXbrl.relationshipSet( XbrlConst.consistencyAssertionFormula).modelRelationships: if modelRel.fromModelObject and modelRel.toModelObject and isinstance( modelRel.toModelObject, ModelFormula): consisAsser = modelRel.fromModelObject consisAsser.countSatisfied = 0 consisAsser.countNotSatisfied = 0 if consisAsser.hasProportionalAcceptanceRadius and consisAsser.hasAbsoluteAcceptanceRadius: val.modelXbrl.error( _("Consistency assertion {0} has both absolute and proportional acceptance radii" ).format(consisAsser.xlinkLabel), "err", "xbrlcae:acceptanceRadiusConflict") consisAsser.orderedVariableRelationships = [] for consisParamRel in val.modelXbrl.relationshipSet( XbrlConst.consistencyAssertionParameter).fromModelObject( consisAsser): if isinstance(consisParamRel.toModelObject, ModelVariable): val.modelXbrl.error( _("Consistency assertion {0} has relationship to a {1} {2}" ).format( consisAsser.xlinkLabel, consisParamRel.toModelObject.element.localName, consisParamRel.toModelObject.xlinkLabel), "err", "xbrlcae:variablesNotAllowed") else: consisAsser.orderedVariableRelationships.append( consisParamRel) consisAsser.compile() modelRel.toModelObject.hasConsistencyAssertion = True if initialErrorCount < val.modelXbrl.logCountErr: return # don't try to execute # formula output instances if instanceQnames: schemaRefs = [ val.modelXbrl.modelDocument.relativeUri(referencedDoc.uri) for referencedDoc in val.modelXbrl.modelDocument.referencesDocument.keys() if referencedDoc.type == ModelDocument.Type.SCHEMA ] outputXbrlInstance = None for instanceQname in instanceQnames: if instanceQname == XbrlConst.qnStandardInputInstance: continue # always present the standard way if val.parameters and instanceQname in val.parameters: namedInstance = val.parameters[instanceQname][1] else: # empty intermediate instance uri = val.modelXbrl.modelDocument.filepath[:-4] + "-output-XBRL-instance" if instanceQname != XbrlConst.qnStandardOutputInstance: uri = uri + "-" + instanceQname.localName uri = uri + ".xml" namedInstance = ModelXbrl.create( val.modelXbrl.modelManager, newDocumentType=ModelDocument.Type.INSTANCE, url=uri, schemaRefs=schemaRefs, isEntry=True) xpathContext.inScopeVars[instanceQname] = namedInstance if instanceQname == XbrlConst.qnStandardOutputInstance: outputXbrlInstance = namedInstance # evaluate consistency assertions # evaluate variable sets not in consistency assertions for instanceQname in orderedInstancesList: for modelVariableSet in instanceProducingVariableSets[instanceQname]: # produce variable evaluations from arelle.FormulaEvaluator import evaluate try: evaluate(xpathContext, modelVariableSet) except XPathContext.XPathException as err: val.modelXbrl.error( _("Variable set \n{0} \nException: \n{1}").format( modelVariableSet, err.message), "err", err.code) # log assertion result counts asserTests = {} for exisValAsser in val.modelXbrl.modelVariableSets: if isinstance(exisValAsser, ModelVariableSetAssertion): asserTests[exisValAsser.id] = (exisValAsser.countSatisfied, exisValAsser.countNotSatisfied) if formulaOptions.traceAssertionResultCounts: val.modelXbrl.error( _("{0} Assertion {1} evaluations : {2} satisfied, {3} not satisfied" ).format( "Existence" if isinstance(exisValAsser, ModelExistenceAssertion) else "Value", exisValAsser.id, exisValAsser.countSatisfied, exisValAsser.countNotSatisfied), "info", "formula:trace") for modelRel in val.modelXbrl.relationshipSet( XbrlConst.consistencyAssertionFormula).modelRelationships: if modelRel.fromModelObject and modelRel.toModelObject and isinstance( modelRel.toModelObject, ModelFormula): consisAsser = modelRel.fromModelObject asserTests[consisAsser.id] = (consisAsser.countSatisfied, consisAsser.countNotSatisfied) if formulaOptions.traceAssertionResultCounts: val.modelXbrl.error( _("Consistency Assertion {0} evaluations : {1} satisfied, {2} not satisfied" ).format(consisAsser.id, consisAsser.countSatisfied, consisAsser.countNotSatisfied), "info", "formula:trace") if asserTests: val.modelXbrl.error( _("Assertion results {0}").format(asserTests), "asrtNoLog", asserTests) # display output instance if outputXbrlInstance: if val.modelXbrl.formulaOutputInstance: # close prior instance, usually closed by caller to validate as it may affect UI on different thread val.modelXbrl.formulaOutputInstance.close() val.modelXbrl.formulaOutputInstance = outputXbrlInstance
def __init__(self, mainWin): parent = mainWin.parent super(DialogTransformTester, self).__init__(parent) self.mainWin = mainWin self.parent = parent parentGeometry = re.match("(\d+)x(\d+)[+]?([-]?\d+)[+]?([-]?\d+)", parent.geometry()) dialogX = int(parentGeometry.group(3)) dialogY = int(parentGeometry.group(4)) self.selectedGroup = None self.transient(self.parent) self.title(_("Transformation Tester")) frame = Frame(self) # setup tester xml = "<rootElement/>" self.modelXbrl = ModelXbrl.create(mainWin.modelManager, ModelDocument.Type.UnknownNonXML, initialXml=xml, isEntry=True) self.validator = ValidateXbrl.ValidateXbrl(self.modelXbrl) self.validator.validate(self.modelXbrl) # required to set up mainWin.showStatus(_("Initializing Formula Grammar")) XPathParser.initializeParser(mainWin.modelManager) mainWin.showStatus(None) self.trRegs = sorted(ixtNamespaces.keys()) self.trReg = self.trRegs[-1] # default is latest self.trPrefixNSs = dict( (qn.prefix, qn.namespaceURI) for qn in self.modelXbrl.modelManager.customTransforms.keys()) self.trRegs.extend(sorted(self.trPrefixNSs.keys())) self.trPrefixNSs.update(ixtNamespaces) self.trNames = self.getTrNames() # load grid trRegLabel = label(frame, 0, 0, _("Registry:")) self.trRegName = gridCombobox( frame, 1, 0, value=self.trReg, values=self.trRegs, comboboxselected=self.trRegComboBoxSelected) trRegToolTipMessage = _("Select Transformation Registry") ToolTip(self.trRegName, text=trRegToolTipMessage, wraplength=360) ToolTip(trRegLabel, text=trRegToolTipMessage, wraplength=360) trNameLabel = label(frame, 0, 1, _("Transform:")) self.trNameName = gridCombobox( frame, 1, 1, value="", values=self.trNames, comboboxselected=self.trNameComboBoxSelected) trRegToolTipMessage = _("Select or enter transform") ToolTip(self.trRegName, text=trRegToolTipMessage, wraplength=360) ToolTip(trRegLabel, text=trRegToolTipMessage, wraplength=360) sourceLabel = label(frame, 0, 2, _("Source text:")) ToolTip(sourceLabel, text=_("Enter the source text which is to be transformed. "), wraplength=240) self.sourceVar = StringVar() self.sourceVar.set("") sourceEntry = Entry(frame, textvariable=self.sourceVar, width=50) sourceLabel.grid(row=2, column=0, sticky=W) sourceEntry.grid(row=2, column=1, sticky=EW, pady=3, padx=3) resultLabel = label(frame, 1, 3, _("Result:")) ToolTip(sourceLabel, text=_("Transformation result. "), wraplength=240) self.resultVar = StringVar() self.resultVar.set("") resultEntry = Entry(frame, textvariable=self.resultVar, width=50) resultLabel.grid(row=3, column=0, sticky=W) resultEntry.grid(row=3, column=1, sticky=EW, pady=3, padx=3) mainWin.showStatus(None) btnPad = 2 if mainWin.isMSW else 0 # buttons too narrow on windows okButton = Button(frame, text=_("Transform"), width=8 + btnPad, command=self.ok) cancelButton = Button(frame, text=_("Done"), width=4 + btnPad, command=self.close) cancelButton.grid(row=4, column=0, sticky=E, columnspan=2, pady=3, padx=3) okButton.grid(row=4, column=0, sticky=E, columnspan=2, pady=3, padx=64) ToolTip(okButton, text=_("Transform the source entered. "), wraplength=240) ToolTip(cancelButton, text=_("Close this dialog. "), wraplength=240) frame.grid(row=0, column=0, sticky=(N, S, E, W)) frame.columnconfigure(1, weight=3) frame.columnconfigure(2, weight=1) window = self.winfo_toplevel() window.columnconfigure(0, weight=1) self.geometry("+{0}+{1}".format(dialogX + 150, dialogY + 100)) #self.bind("<Return>", self.ok) #self.bind("<Escape>", self.close) self.protocol("WM_DELETE_WINDOW", self.close) self.grab_set() self.wait_window(self)
def validate(val): formulaOptions = val.modelXbrl.modelManager.formulaOptions XPathParser.initializeParser(val) val.modelXbrl.modelManager.showStatus(_("Compiling formulae")) initialErrorCount = val.modelXbrl.logCountErr # global parameter names parameterQnames = set() instanceQnames = set() parameterDependencies = {} instanceDependencies = defaultdict(set) # None-key entries are non-formula dependencies dependencyResolvedParameters = set() orderedParameters = [] orderedInstances = [] for paramQname, modelParameter in val.modelXbrl.qnameParameters.items(): if isinstance(modelParameter, ModelParameter): modelParameter.compile() parameterDependencies[paramQname] = modelParameter.variableRefs() parameterQnames.add(paramQname) if isinstance(modelParameter, ModelInstance): instanceQnames.add(paramQname) # duplicates checked on loading modelDocument #resolve dependencies resolvedAParameter = True while (resolvedAParameter): resolvedAParameter = False for paramQname in parameterQnames: if paramQname not in dependencyResolvedParameters and \ len(parameterDependencies[paramQname] - dependencyResolvedParameters) == 0: dependencyResolvedParameters.add(paramQname) orderedParameters.append(paramQname) resolvedAParameter = True # anything unresolved? for paramQname in parameterQnames: if paramQname not in dependencyResolvedParameters: circularOrUndefDependencies = parameterDependencies[paramQname] - dependencyResolvedParameters undefinedVars = circularOrUndefDependencies - parameterQnames paramsCircularDep = circularOrUndefDependencies - undefinedVars if len(undefinedVars) > 0: val.modelXbrl.error( _("Undefined dependencies in parameter {0}, to names {1}").format( paramQname, ", ".join((str(v) for v in undefinedVars))), "err", "xbrlve:unresolvedDependency") if len(paramsCircularDep) > 0: val.modelXbrl.error( _("Cyclic dependencies in parameter {0}, to names {1}").format( paramQname, ", ".join((str(d) for d in paramsCircularDep)) ), "err", "xbrlve:parameterCyclicDependencies") for custFnSig in val.modelXbrl.modelCustomFunctionSignatures.values(): custFnQname = custFnSig.qname if custFnQname.namespaceURI == "XbrlConst.xfi": val.modelXbrl.error( _("Custom function {0} has namespace reserved for functions in the function registry {1}").format( str(custFnQname), custFnQname.namespaceURI ), "err", "xbrlve:noProhibitedNamespaceForCustomFunction") # any custom function implementations? for modelRel in val.modelXbrl.relationshipSet(XbrlConst.functionImplementation).fromModelObject(custFnSig): custFnImpl = modelRel.toModelObject custFnSig.customFunctionImplementation = custFnImpl if len(custFnImpl.inputNames) != len(custFnSig.inputTypes): val.modelXbrl.error( _("Custom function {0} signature has {1} parameters but implementation has {2}, must be matching").format( str(custFnQname), len(custFnSig.inputTypes), len(custFnImpl.inputNames) ), "err", "xbrlcfie:inputMismatch") for custFnImpl in val.modelXbrl.modelCustomFunctionImplementations: if not val.modelXbrl.relationshipSet(XbrlConst.functionImplementation).toModelObject(custFnImpl): val.modelXbrl.error( _("Custom function implementation {0} has no relationship from any custom function signature").format( custFnImpl.xlinkLabel), "err", "xbrlcfie:missingCFIRelationship") custFnImpl.compile() # xpathContext is needed for filter setup for expressions such as aspect cover filter # determine parameter values xpathContext = XPathContext.create(val.modelXbrl) for paramQname in orderedParameters: if not isinstance(modelParameter, ModelInstance): modelParameter = val.modelXbrl.qnameParameters[paramQname] asType = modelParameter.asType asLocalName = asType.localName if asType else "string" try: if val.parameters and paramQname in val.parameters: paramDataType, paramValue = val.parameters[paramQname] typeLocalName = paramDataType.localName if paramDataType else "string" value = FunctionXs.call(xpathContext, None, typeLocalName, [paramValue]) result = FunctionXs.call(xpathContext, None, asLocalName, [value]) if formulaOptions.traceParameterInputValue: val.modelXbrl.error( _("Parameter {0} input {1}").format( paramQname, result), "info", "formula:trace") else: result = modelParameter.evaluate(xpathContext, asType) if formulaOptions.traceParameterExpressionResult: val.modelXbrl.error( _("Parameter {0} result {1}").format( paramQname, result), "info", "formula:trace") xpathContext.inScopeVars[paramQname] = result # make visible to subsequent parameter expression except XPathContext.XPathException as err: val.modelXbrl.error( _("Parameter \n{0} \nException: \n{1}").format( paramQname, err.message), "err", "xbrlve:parameterTypeMismatch" if err.code == "err:FORG0001" else err.code) produceOutputXbrlInstance = False instanceProducingVariableSets = defaultdict(list) for modelVariableSet in val.modelXbrl.modelVariableSets: varSetInstanceDependencies = set() if isinstance(modelVariableSet, ModelFormula): instanceQname = None for modelRel in val.modelXbrl.relationshipSet(XbrlConst.formulaInstance).fromModelObject(modelVariableSet): instance = modelRel.toModelObject if isinstance(instance, ModelInstance): if instanceQname is None: instanceQname = instance.qname else: val.modelXbrl.error( _("Multiple output instances for formula {0}, to names {1}, {2}").format( modelVariableSet.xlinkLabel, instanceQname, instance.qname ), "info", "arelle:multipleOutputInstances") if instanceQname is None: instanceQname = XbrlConst.qnStandardOutputInstance instanceQnames.add(instanceQname) modelVariableSet.outputInstanceQname = instanceQname if val.validateSBRNL: val.modelXbrl.error( _("Formula linkbase {0} formula:formula {1} is not allowed").format( os.path.basename(modelVariableSet.modelDocument.uri), modelVariableSet.xlinkLabel), "err", "SBR.NL.2.3.9.03") else: instanceQname = None modelVariableSet.countSatisfied = 0 modelVariableSet.countNotSatisfied = 0 checkValidationMessages(val, modelVariableSet) instanceProducingVariableSets[instanceQname].append(modelVariableSet) modelVariableSet.outputInstanceQname = instanceQname if modelVariableSet.aspectModel not in ("non-dimensional", "dimensional"): val.modelXbrl.error( _("Variable set {0}, aspect model {1} not recognized").format( modelVariableSet.xlinkLabel, modelVariableSet.aspectModel), "err", "xbrlve:unknownAspectModel") modelVariableSet.compile() modelVariableSet.hasConsistencyAssertion = False #determine dependencies within variable sets nameVariables = {} qnameRels = {} definedNamesSet = set() for modelRel in val.modelXbrl.relationshipSet(XbrlConst.variableSet).fromModelObject(modelVariableSet): varqname = modelRel.variableQname if varqname: qnameRels[varqname] = modelRel toVariable = modelRel.toModelObject if varqname not in definedNamesSet: definedNamesSet.add(varqname) if varqname not in nameVariables: nameVariables[varqname] = toVariable elif nameVariables[varqname] != toVariable: val.modelXbrl.error( _("Multiple variables named {1} in variable set {0}").format( modelVariableSet.xlinkLabel, varqname ), "err", "xbrlve:duplicateVariableNames") fromInstanceQnames = None for instRel in val.modelXbrl.relationshipSet(XbrlConst.instanceVariable).toModelObject(toVariable): fromInstance = instRel.fromModelObject if isinstance(fromInstance, ModelInstance): fromInstanceQname = fromInstance.qname varSetInstanceDependencies.add(fromInstanceQname) instanceDependencies[instanceQname].add(fromInstanceQname) if fromInstanceQnames is None: fromInstanceQnames = set() fromInstanceQnames.add(fromInstanceQname) if fromInstanceQnames is None: varSetInstanceDependencies.add(XbrlConst.qnStandardInputInstance) if instanceQname: instanceDependencies[instanceQname].add(XbrlConst.qnStandardInputInstance) toVariable.fromInstanceQnames = fromInstanceQnames else: val.modelXbrl.error( _("Variables name {1} cannot be determined on arc from {0}").format( modelVariableSet.xlinkLabel, modelRel.variablename ), "err", "xbrlve:variableNameResolutionFailure") definedNamesSet |= parameterQnames variableDependencies = {} for modelRel in val.modelXbrl.relationshipSet(XbrlConst.variableSet).fromModelObject(modelVariableSet): variable = modelRel.toModelObject if isinstance(variable, (ModelParameter,ModelVariable)): # ignore anything not parameter or variable varqname = modelRel.variableQname depVars = variable.variableRefs() variableDependencies[varqname] = depVars if len(depVars) > 0 and formulaOptions.traceVariablesDependencies: val.modelXbrl.error(_("Variable set {0}, variable {1}, dependences {2}").format( modelVariableSet.xlinkLabel, varqname, depVars), "info", "formula:trace") definedNamesSet.add(varqname) # check for fallback value variable references if isinstance(variable, ModelFactVariable): for depVar in XPathParser.variableReferencesSet(variable.fallbackValueProg, variable.element): if depVar in qnameRels and isinstance(qnameRels[depVar].toModelObject,ModelVariable): val.modelXbrl.error(_("Variable set {0} fallbackValue '{1}' cannot refer to variable {2}").format( modelVariableSet.xlinkLabel, variable.fallbackValue, depVar), "err", "xbrlve:factVariableReferenceNotAllowed") # check for covering aspect not in variable set aspect model checkFilterAspectModel(val, modelVariableSet, variable.filterRelationships, xpathContext) orderedNameSet = set() orderedNameList = [] orderedAVariable = True while (orderedAVariable): orderedAVariable = False for varqname, depVars in variableDependencies.items(): if varqname not in orderedNameSet and len(depVars - parameterQnames - orderedNameSet) == 0: orderedNameList.append(varqname) orderedNameSet.add(varqname) orderedAVariable = True if varqname in instanceQnames: varSetInstanceDependencies.add(varqname) instanceDependencies[instanceQname].add(varqname) elif isinstance(nameVariables.get(varqname), ModelInstance): instqname = nameVariables[varqname].qname varSetInstanceDependencies.add(instqname) instanceDependencies[instanceQname].add(instqname) # anything unresolved? for varqname, depVars in variableDependencies.items(): if varqname not in orderedNameSet: circularOrUndefVars = depVars - parameterQnames - orderedNameSet undefinedVars = circularOrUndefVars - definedNamesSet varsCircularDep = circularOrUndefVars - undefinedVars if len(undefinedVars) > 0: val.modelXbrl.error( _("Undefined variable dependencies in variable st {0}, from variable {1} to {2}").format( modelVariableSet.xlinkLabel, varqname, undefinedVars), "err", "xbrlve:unresolvedDependency") if len(varsCircularDep) > 0: val.modelXbrl.error( _("Cyclic dependencies in variable set {0}, from variable {1} to {2}").format( modelVariableSet.xlinkLabel, varqname, varsCircularDep ), "err", "xbrlve:cyclicDependencies") # check unresolved variable set dependencies for varSetDepVarQname in modelVariableSet.variableRefs(): if varSetDepVarQname not in orderedNameSet and varSetDepVarQname not in parameterQnames: val.modelXbrl.error( _("Undefined variable dependency in variable set {0}, {1}").format( modelVariableSet.xlinkLabel, varSetDepVarQname), "err", "xbrlve:unresolvedDependency") if varSetDepVarQname in instanceQnames: varSetInstanceDependencies.add(varSetDepVarQname) instanceDependencies[instanceQname].add(varSetDepVarQname) elif isinstance(nameVariables.get(varSetDepVarQname), ModelInstance): instqname = nameVariables[varSetDepVarQname].qname varSetInstanceDependencies.add(instqname) instanceDependencies[instanceQname].add(instqname) if formulaOptions.traceVariablesOrder: val.modelXbrl.error(_("Variable set {0}, variables order: {1}").format( modelVariableSet.xlinkLabel, orderedNameList), "info", "formula:trace") if (formulaOptions.traceVariablesDependencies and len(varSetInstanceDependencies) > 0 and varSetInstanceDependencies != {XbrlConst.qnStandardInputInstance}): val.modelXbrl.error(_("Variable set {0}, instance dependences {1}").format( modelVariableSet.xlinkLabel, varSetInstanceDependencies), "info", "formula:trace") modelVariableSet.orderedVariableRelationships = [] for varqname in orderedNameList: if varqname in qnameRels: modelVariableSet.orderedVariableRelationships.append(qnameRels[varqname]) # check existence assertion variable dependencies if isinstance(modelVariableSet, ModelExistenceAssertion): for depVar in modelVariableSet.variableRefs(): if depVar in qnameRels and isinstance(qnameRels[depVar].toModelObject,ModelVariable): val.modelXbrl.error(_("Existence Assertion {0}, cannot refer to variable {1}").format( modelVariableSet.xlinkLabel, depVar), "err", "xbrleae:variableReferenceNotAllowed") # check messages variable dependencies checkValidationMessageVariables(val, modelVariableSet, qnameRels) # check preconditions modelVariableSet.preconditions = [] for modelRel in val.modelXbrl.relationshipSet(XbrlConst.variableSetPrecondition).fromModelObject(modelVariableSet): precondition = modelRel.toModelObject if isinstance(precondition, ModelPrecondition): modelVariableSet.preconditions.append(precondition) # check for variable sets referencing fact or general variables for modelRel in val.modelXbrl.relationshipSet(XbrlConst.variableSetFilter).fromModelObject(modelVariableSet): varSetFilter = modelRel.toModelObject if modelRel.isCovered: val.modelXbrl.error(_("Variable set {0}, filter {1}, cannot be covered").format( modelVariableSet.xlinkLabel, varSetFilter.xlinkLabel), "wrn", "arelle:variableSetFilterCovered") modelRel._isCovered = False # block group filter from being able to covere for depVar in varSetFilter.variableRefs(): if depVar in qnameRels and isinstance(qnameRels[depVar].toModelObject,ModelVariable): val.modelXbrl.error(_("Variable set {0}, filter {1}, cannot refer to variable {2}").format( modelVariableSet.xlinkLabel, varSetFilter.xlinkLabel, depVar), "err", "xbrlve:factVariableReferenceNotAllowed") # check aspects of formula if isinstance(modelVariableSet, ModelFormula): checkFormulaRules(val, modelVariableSet, nameVariables) # determine instance dependency order orderedInstancesSet = set() stdInpInst = {XbrlConst.qnStandardInputInstance} orderedInstancesList = [] orderedAnInstance = True while (orderedAnInstance): orderedAnInstance = False for instqname, depInsts in instanceDependencies.items(): if instqname and instqname not in orderedInstancesSet and len(depInsts - stdInpInst - orderedInstancesSet) == 0: orderedInstancesList.append(instqname) orderedInstancesSet.add(instqname) orderedAnInstance = True orderedInstancesList.append(None) # assertions come after all formulas that produce outputs # anything unresolved? for instqname, depInsts in instanceDependencies.items(): if instqname not in orderedInstancesSet: # can also be satisfied from an input DTS missingDependentInstances = depInsts - stdInpInst if val.parameters: missingDependentInstances -= val.parameters.keys() if instqname: if missingDependentInstances: val.modelXbrl.error( _("Cyclic dependencies of instance {0} produced by a formula, with variables consuming instances {1}").format( instqname, missingDependentInstances ), "err", "xbrlvarinste:instanceVariableRecursionCycle") elif instqname == XbrlConst.qnStandardOutputInstance: orderedInstancesSet.add(instqname) orderedInstancesList.append(instqname) # standard output formula, all input dependencies in parameters ''' future check? if instance has no external input or producing formula else: val.modelXbrl.error( _("Unresolved dependencies of an assertion's variables on instances {0}").format( depInsts - stdInpInst ), "err", "xbrlvarinste:instanceVariableRecursionCycle") ''' if formulaOptions.traceVariablesOrder and len(orderedInstancesList) > 1: val.modelXbrl.error(_("Variable instances processing order: {0}").format( orderedInstancesList), "info", "formula:trace") # linked consistency assertions for modelRel in val.modelXbrl.relationshipSet(XbrlConst.consistencyAssertionFormula).modelRelationships: if modelRel.fromModelObject and modelRel.toModelObject and isinstance(modelRel.toModelObject,ModelFormula): consisAsser = modelRel.fromModelObject consisAsser.countSatisfied = 0 consisAsser.countNotSatisfied = 0 if consisAsser.hasProportionalAcceptanceRadius and consisAsser.hasAbsoluteAcceptanceRadius: val.modelXbrl.error( _("Consistency assertion {0} has both absolute and proportional acceptance radii").format( consisAsser.xlinkLabel), "err", "xbrlcae:acceptanceRadiusConflict") consisAsser.orderedVariableRelationships = [] for consisParamRel in val.modelXbrl.relationshipSet(XbrlConst.consistencyAssertionParameter).fromModelObject(consisAsser): if isinstance(consisParamRel.toModelObject, ModelVariable): val.modelXbrl.error( _("Consistency assertion {0} has relationship to a {1} {2}").format( consisAsser.xlinkLabel, consisParamRel.toModelObject.element.localName, consisParamRel.toModelObject.xlinkLabel), "err", "xbrlcae:variablesNotAllowed") else: consisAsser.orderedVariableRelationships.append(consisParamRel) consisAsser.compile() modelRel.toModelObject.hasConsistencyAssertion = True if initialErrorCount < val.modelXbrl.logCountErr: return # don't try to execute # formula output instances if instanceQnames: schemaRefs = [val.modelXbrl.modelDocument.relativeUri(referencedDoc.uri) for referencedDoc in val.modelXbrl.modelDocument.referencesDocument.keys() if referencedDoc.type == ModelDocument.Type.SCHEMA] outputXbrlInstance = None for instanceQname in instanceQnames: if instanceQname == XbrlConst.qnStandardInputInstance: continue # always present the standard way if val.parameters and instanceQname in val.parameters: namedInstance = val.parameters[instanceQname][1] else: # empty intermediate instance uri = val.modelXbrl.modelDocument.filepath[:-4] + "-output-XBRL-instance" if instanceQname != XbrlConst.qnStandardOutputInstance: uri = uri + "-" + instanceQname.localName uri = uri + ".xml" namedInstance = ModelXbrl.create(val.modelXbrl.modelManager, newDocumentType=ModelDocument.Type.INSTANCE, url=uri, schemaRefs=schemaRefs, isEntry=True) xpathContext.inScopeVars[instanceQname] = namedInstance if instanceQname == XbrlConst.qnStandardOutputInstance: outputXbrlInstance = namedInstance # evaluate consistency assertions # evaluate variable sets not in consistency assertions for instanceQname in orderedInstancesList: for modelVariableSet in instanceProducingVariableSets[instanceQname]: # produce variable evaluations from arelle.FormulaEvaluator import evaluate try: evaluate(xpathContext, modelVariableSet) except XPathContext.XPathException as err: val.modelXbrl.error( _("Variable set \n{0} \nException: \n{1}").format( modelVariableSet, err.message), "err", err.code) # log assertion result counts asserTests = {} for exisValAsser in val.modelXbrl.modelVariableSets: if isinstance(exisValAsser, ModelVariableSetAssertion): asserTests[exisValAsser.id] = (exisValAsser.countSatisfied, exisValAsser.countNotSatisfied) if formulaOptions.traceAssertionResultCounts: val.modelXbrl.error( _("{0} Assertion {1} evaluations : {2} satisfied, {3} not satisfied").format( "Existence" if isinstance(exisValAsser, ModelExistenceAssertion) else "Value", exisValAsser.id, exisValAsser.countSatisfied, exisValAsser.countNotSatisfied), "info", "formula:trace") for modelRel in val.modelXbrl.relationshipSet(XbrlConst.consistencyAssertionFormula).modelRelationships: if modelRel.fromModelObject and modelRel.toModelObject and isinstance(modelRel.toModelObject,ModelFormula): consisAsser = modelRel.fromModelObject asserTests[consisAsser.id] = (consisAsser.countSatisfied, consisAsser.countNotSatisfied) if formulaOptions.traceAssertionResultCounts: val.modelXbrl.error( _("Consistency Assertion {0} evaluations : {1} satisfied, {2} not satisfied").format( consisAsser.id, consisAsser.countSatisfied, consisAsser.countNotSatisfied), "info", "formula:trace") if asserTests: val.modelXbrl.error( _("Assertion results {0}").format(asserTests), "asrtNoLog", asserTests) # display output instance if outputXbrlInstance: if val.modelXbrl.formulaOutputInstance: # close prior instance, usually closed by caller to validate as it may affect UI on different thread val.modelXbrl.formulaOutputInstance.close() val.modelXbrl.formulaOutputInstance = outputXbrlInstance