def executeCallTest(val, name, callTuple, testTuple): if callTuple: XPathParser.initializeParser(val) try: val.modelXbrl.modelManager.showStatus(_("Executing call")) callExprStack = XPathParser.parse(val, callTuple[0], callTuple[1], name + " call", Trace.CALL) xpathContext = XPathContext.create(val.modelXbrl, sourceElement=callTuple[1]) result = xpathContext.evaluate(callExprStack) xpathContext.inScopeVars[qname('result',noPrefixIsNoNamespace=True)] = result val.modelXbrl.error( _("{0} result {1}").format( name, result), "info", "formula:trace") if testTuple: val.modelXbrl.modelManager.showStatus(_("Executing test")) testExprStack = XPathParser.parse(val, testTuple[0], testTuple[1], name + " test", Trace.CALL) testResult = xpathContext.effectiveBooleanValue( None, xpathContext.evaluate(testExprStack) ) val.modelXbrl.error( _("Test {0} result {1}").format( name, testResult), "info" if testResult else "err", "cfcn:testPass" if testResult else "cfcn:testFail") except XPathContext.XPathException as err: val.modelXbrl.error( _("{0} evaluation error: {1} \n{2}").format(name, err.message, err.sourceErrorIndication), "err", err.code) val.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def ok(self, event=None): trName = self.trNameName.value sourceValue = self.sourceVar.get() try: self.modelXbrl.modelManager.showStatus(_("Executing call")) elt = self.modelXbrl.modelDocument.xmlRootElement callExprStack = XPathParser.parse( self.validator, '{}("{}")'.format(trName, sourceValue), elt, trName + " call", Trace.CALL) xpathContext = XPathContext.create(self.modelXbrl, sourceElement=elt) result = xpathContext.evaluate(callExprStack) while result and isinstance(result, (tuple, list, set)): result = next(iter(result)) # de-sequence result self.resultVar.set(str(result)) except XPathContext.XPathException as err: self.resultVar.set(str(err)) self.modelXbrl.error(err.code, err.message)
def transform(self, trReg, trName, sourceValue): try: trNS = self.trPrefixNSs[trReg] trPrefix = trReg.split()[0] # for ixt remove TRn part setXmlns(self.modelXbrl.modelDocument, trPrefix, trNS) self.modelXbrl.modelManager.showStatus(_("Executing call")) elt = self.modelXbrl.modelDocument.xmlRootElement if ':' in trName: prefixedFnName = trName else: prefixedFnName = "{}:{}".format(trPrefix, trName) callExprStack = XPathParser.parse(self.validator, '{}("{}")'.format(prefixedFnName, sourceValue), elt, trName + " call", Trace.CALL) xpathContext = XPathContext.create(self.modelXbrl, sourceElement=elt) result = xpathContext.evaluate(callExprStack) while result and isinstance(result, (tuple,list,set)): result = next(iter(result)) # de-sequence result return result except XPathContext.XPathException as err: self.modelXbrl.error(err.code, err.message) return err
def executeCallTest(val, name, callTuple, testTuple): if callTuple: XPathParser.initializeParser(val) try: val.modelXbrl.modelManager.showStatus(_("Executing call")) callExprStack = XPathParser.parse(val, callTuple[0], callTuple[1], name + " call", Trace.CALL) xpathContext = XPathContext.create(val.modelXbrl, sourceElement=callTuple[1]) result = xpathContext.evaluate(callExprStack) xpathContext.inScopeVars[qname( 'result', noPrefixIsNoNamespace=True)] = result val.modelXbrl.error( _("{0} result {1}").format(name, result), "info", "formula:trace") if testTuple: val.modelXbrl.modelManager.showStatus(_("Executing test")) testExprStack = XPathParser.parse(val, testTuple[0], testTuple[1], name + " test", Trace.CALL) testResult = xpathContext.effectiveBooleanValue( None, xpathContext.evaluate(testExprStack)) val.modelXbrl.error( _("Test {0} result {1}").format(name, testResult), "info" if testResult else "err", "cfcn:testPass" if testResult else "cfcn:testFail") except XPathContext.XPathException as err: val.modelXbrl.error( _("{0} evaluation error: {1} \n{2}").format( name, err.message, err.sourceErrorIndication), "err", err.code) val.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def init(modelXbrl): # setup modelXbrl for rendering evaluation # dimension defaults required in advance of validation from arelle import ValidateXbrlDimensions, ValidateFormula, ModelDocument ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) hasXbrlTables = False # validate table linkbase dimensions for baseSetKey in modelXbrl.baseSets.keys(): arcrole, ELR, linkqname, arcqname = baseSetKey if ELR and linkqname and arcqname and XbrlConst.isTableRenderingArcrole( arcrole): ValidateFormula.checkBaseSet( modelXbrl, arcrole, ELR, modelXbrl.relationshipSet(arcrole, ELR, linkqname, arcqname)) if arcrole in (XbrlConst.tableBreakdown, XbrlConst.tableBreakdownMMDD, XbrlConst.tableBreakdown201305, XbrlConst.tableBreakdown201301, XbrlConst.tableAxis2011): hasXbrlTables = True # provide context for view if modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE: instance = None # use instance of the entry pont else: # need dummy instance instance = ModelDocument.create( modelXbrl, ModelDocument.Type.INSTANCE, "dummy.xml", # fake URI and fake schemaRef ("http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd", )) if hasXbrlTables: # formula processor is needed for 2011 XBRL tables but not for 2010 Eurofiling tables modelXbrl.rendrCntx = XPathContext.create(modelXbrl, instance) modelXbrl.profileStat(None) # setup fresh parameters from formula options modelXbrl.parameters = modelXbrl.modelManager.formulaOptions.typedParameters( ) # validate parameters and custom function signatures ValidateFormula.validate(modelXbrl, xpathContext=modelXbrl.rendrCntx, parametersOnly=True, statusMsg=_("compiling rendering tables")) # deprecated as of 2013-05-17 # check and extract message expressions into compilable programs for msgArcrole in (XbrlConst.tableDefinitionNodeMessage201301, XbrlConst.tableDefinitionNodeSelectionMessage201301, XbrlConst.tableAxisMessage2011, XbrlConst.tableAxisSelectionMessage2011): for msgRel in modelXbrl.relationshipSet( msgArcrole).modelRelationships: ValidateFormula.checkMessageExpressions( modelXbrl, msgRel.toModelObject) # compile and validate tables for modelTable in modelXbrl.modelRenderingTables: modelTable.fromInstanceQnames = None # required if referred to by variables scope chaining modelTable.compile() hasNsWithAspectModel = modelTable.namespaceURI in ( XbrlConst.euRend, XbrlConst.table2011, XbrlConst.table201301, XbrlConst.table201305) # check aspectModel (attribute removed 2013-06, now always dimensional) if modelTable.aspectModel not in ( "non-dimensional", "dimensional") and hasNsWithAspectModel: modelXbrl.error( "xbrlte:unknownAspectModel", _("Table %(xlinkLabel)s, aspect model %(aspectModel)s not recognized" ), modelObject=modelTable, xlinkLabel=modelTable.xlinkLabel, aspectModel=modelTable.aspectModel) else: modelTable.priorAspectAxisDisposition = {} # check ordinate aspects against aspectModel oppositeAspectModel = ( _DICT_SET({'dimensional', 'non-dimensional'}) - _DICT_SET({modelTable.aspectModel})).pop() if hasNsWithAspectModel: uncoverableAspects = aspectModels[ oppositeAspectModel] - aspectModels[ modelTable.aspectModel] else: uncoverableAspects = () aspectsCovered = set() for tblAxisRel in modelXbrl.relationshipSet( (XbrlConst.tableBreakdown, XbrlConst.tableBreakdownMMDD, XbrlConst.tableBreakdown201305, XbrlConst.tableBreakdown201301, XbrlConst.tableAxis2011)).fromModelObject(modelTable): breakdownAspectsCovered = set() hasCoveredAspect = checkBreakdownDefinitionNode( modelXbrl, modelTable, tblAxisRel, tblAxisRel.axisDisposition, uncoverableAspects, breakdownAspectsCovered) ''' removed 2013-10 if not hasCoveredAspect: definitionNode = tblAxisRel.toModelObject modelXbrl.error("xbrlte:breakdownDefinesNoAspects", _("Breakdown %(xlinkLabel)s has no participating aspects"), modelObject=(modelTable,definitionNode), xlinkLabel=definitionNode.xlinkLabel, axis=definitionNode.localName) ''' aspectsCovered |= breakdownAspectsCovered checkBreakdownLeafNodeAspects(modelXbrl, modelTable, tblAxisRel, set(), breakdownAspectsCovered) if Aspect.CONCEPT not in aspectsCovered and not hasNsWithAspectModel: modelXbrl.error( "xbrlte:tableMissingConceptAspect", _("Table %(xlinkLabel)s does not include the concept aspect as one of its participating aspects" ), modelObject=modelTable, xlinkLabel=modelTable.xlinkLabel) del modelTable.priorAspectAxisDisposition # check for table-parameter name clash parameterNames = {} for tblParamRel in modelXbrl.relationshipSet( (XbrlConst.tableParameter, XbrlConst.tableParameterMMDD )).fromModelObject(modelTable): parameterName = tblParamRel.variableQname if parameterName in parameterNames: modelXbrl.error( "xbrlte:tableParameterNameClash ", _("Table %(xlinkLabel)s has parameter name clash for variable %(name)s" ), modelObject=(modelTable, tblParamRel, parameterNames[parameterName]), xlinkLabel=modelTable.xlinkLabel, name=parameterName) else: parameterNames[parameterName] = tblParamRel modelXbrl.profileStat(_("compileTables"))
def init(modelXbrl): # setup modelXbrl for rendering evaluation # dimension defaults required in advance of validation from arelle import ValidateXbrlDimensions, ValidateFormula, ModelDocument ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) hasXbrlTables = False # validate table linkbase dimensions for baseSetKey in modelXbrl.baseSets.keys(): arcrole, ELR, linkqname, arcqname = baseSetKey if ELR and linkqname and arcqname and XbrlConst.isTableRenderingArcrole(arcrole): ValidateFormula.checkBaseSet(modelXbrl, arcrole, ELR, modelXbrl.relationshipSet(arcrole,ELR,linkqname,arcqname)) if arcrole in (XbrlConst.tableBreakdown, XbrlConst.tableBreakdownMMDD, XbrlConst.tableBreakdown201305, XbrlConst.tableBreakdown201301, XbrlConst.tableAxis2011): hasXbrlTables = True # provide context for view if modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE: instance = None # use instance of the entry pont else: # need dummy instance instance = ModelDocument.create(modelXbrl, ModelDocument.Type.INSTANCE, "dummy.xml", # fake URI and fake schemaRef ("http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd",)) if hasXbrlTables: # formula processor is needed for 2011 XBRL tables but not for 2010 Eurofiling tables modelXbrl.rendrCntx = XPathContext.create(modelXbrl, instance) modelXbrl.profileStat(None) # setup fresh parameters from formula options modelXbrl.parameters = modelXbrl.modelManager.formulaOptions.typedParameters() # validate parameters and custom function signatures ValidateFormula.validate(modelXbrl, xpathContext=modelXbrl.rendrCntx, parametersOnly=True, statusMsg=_("compiling rendering tables")) # deprecated as of 2013-05-17 # check and extract message expressions into compilable programs for msgArcrole in (XbrlConst.tableDefinitionNodeMessage201301, XbrlConst.tableDefinitionNodeSelectionMessage201301, XbrlConst.tableAxisMessage2011, XbrlConst.tableAxisSelectionMessage2011): for msgRel in modelXbrl.relationshipSet(msgArcrole).modelRelationships: ValidateFormula.checkMessageExpressions(modelXbrl, msgRel.toModelObject) # compile and validate tables for modelTable in modelXbrl.modelRenderingTables: modelTable.fromInstanceQnames = None # required if referred to by variables scope chaining modelTable.compile() hasNsWithAspectModel = modelTable.namespaceURI in (XbrlConst.euRend, XbrlConst.table2011, XbrlConst.table201301, XbrlConst.table201305) # check aspectModel (attribute removed 2013-06, now always dimensional) if modelTable.aspectModel not in ("non-dimensional", "dimensional") and hasNsWithAspectModel: modelXbrl.error("xbrlte:unknownAspectModel", _("Table %(xlinkLabel)s, aspect model %(aspectModel)s not recognized"), modelObject=modelTable, xlinkLabel=modelTable.xlinkLabel, aspectModel=modelTable.aspectModel) else: modelTable.priorAspectAxisDisposition = {} # check ordinate aspects against aspectModel oppositeAspectModel = (_DICT_SET({'dimensional','non-dimensional'}) - _DICT_SET({modelTable.aspectModel})).pop() if hasNsWithAspectModel: uncoverableAspects = aspectModels[oppositeAspectModel] - aspectModels[modelTable.aspectModel] else: uncoverableAspects = () aspectsCovered = set() for tblAxisRel in modelXbrl.relationshipSet((XbrlConst.tableBreakdown, XbrlConst.tableBreakdownMMDD, XbrlConst.tableBreakdown201305, XbrlConst.tableBreakdown201301,XbrlConst.tableAxis2011)).fromModelObject(modelTable): breakdownAspectsCovered = set() hasCoveredAspect = checkBreakdownDefinitionNode(modelXbrl, modelTable, tblAxisRel, tblAxisRel.axisDisposition, uncoverableAspects, breakdownAspectsCovered) ''' removed 2013-10 if not hasCoveredAspect: definitionNode = tblAxisRel.toModelObject modelXbrl.error("xbrlte:breakdownDefinesNoAspects", _("Breakdown %(xlinkLabel)s has no participating aspects"), modelObject=(modelTable,definitionNode), xlinkLabel=definitionNode.xlinkLabel, axis=definitionNode.localName) ''' aspectsCovered |= breakdownAspectsCovered checkBreakdownLeafNodeAspects(modelXbrl, modelTable, tblAxisRel, set(), breakdownAspectsCovered) if Aspect.CONCEPT not in aspectsCovered and not hasNsWithAspectModel: modelXbrl.error("xbrlte:tableMissingConceptAspect", _("Table %(xlinkLabel)s does not include the concept aspect as one of its participating aspects"), modelObject=modelTable, xlinkLabel=modelTable.xlinkLabel) del modelTable.priorAspectAxisDisposition # check for table-parameter name clash parameterNames = {} for tblParamRel in modelXbrl.relationshipSet((XbrlConst.tableParameter, XbrlConst.tableParameterMMDD)).fromModelObject(modelTable): parameterName = tblParamRel.variableQname if parameterName in parameterNames: modelXbrl.error("xbrlte:tableParameterNameClash ", _("Table %(xlinkLabel)s has parameter name clash for variable %(name)s"), modelObject=(modelTable,tblParamRel,parameterNames[parameterName]), xlinkLabel=modelTable.xlinkLabel, name=parameterName) else: parameterNames[parameterName] = tblParamRel modelXbrl.profileStat(_("compileTables"))
def backgroundFind(self, expr, logViewLines): exprType = self.options["exprType"] inConceptLabel = self.options["conceptLabel"] inConceptName = self.options["conceptName"] inConceptType = self.options["conceptType"] inConceptSubs = self.options["conceptSubs"] inConceptPer = self.options["conceptPer"] inConceptBal = self.options["conceptBal"] inFactLabel = self.options["factLabel"] inFactName = self.options["factName"] inFactValue = self.options["factValue"] inFactCntx = self.options["factCntx"] inFactUnit = self.options["factUnit"] inMessagesLog = self.options["messagesLog"] nextIsDown = self.options["direction"] == "down" objsFound = set() try: if exprType == "text": # escape regex metacharacters pattern = re.compile( ''.join([(('\\' + c) if c in reMetaChars else c) for c in expr]), re.IGNORECASE) isRE = True isXP = False elif exprType == "regex": pattern = re.compile(expr, re.IGNORECASE) isRE = True isXP = False elif exprType == "xpath": isRE = False isXP = True self.resultText.setValue(_("Compiling xpath expression...")) XPathParser.initializeParser(self.modelManager) self.modelManager.showStatus( _("Compiling xpath expression...")) xpProg = XPathParser.parse( self, expr, XPathParser.staticExpressionFunctionContext(), "find expression", Trace.CALL) xpCtx = XPathContext.create(self.modelXbrl, sourceElement=None) else: return # nothing to do if inMessagesLog: for lineNumber, line in enumerate(logViewLines): if pattern.search(line): objsFound.add(lineNumber) elif self.modelXbrl.modelDocument.type == ModelDocument.Type.RSSFEED: for rssItem in self.modelXbrl.modelDocument.items: if any( pattern.search(str(value)) for name, value in rssItem.propertyView): objsFound.add(rssItem) else: # DTS search if inConceptLabel or inConceptName or inConceptType or inConceptSubs or inConceptPer or inConceptBal: self.modelManager.cntlr.uiThreadQueue.put( (self.resultText.setValue, [_("Matching concepts...") ])) self.modelManager.showStatus(_("Matching concepts...")) for conceptName, concepts in self.modelXbrl.nameConcepts.items( ): for concept in concepts: if ((isXP and xpCtx.evaluateBooleanValue( xpProg, contextItem=concept.qname)) or (isRE and (inConceptLabel and pattern.search(concept.label())) or (inConceptName and pattern.search(conceptName)) or (inConceptType and pattern.search(str(concept.typeQname))) or (inConceptSubs and pattern.search( str(concept.substitutionGroupQname))) or (inConceptPer and concept.periodType and pattern.search(concept.periodType)) or (inConceptBal and concept.balance and pattern.search(concept.balance)))): objsFound.add(concept) if inFactLabel or inFactName or inFactValue or inFactCntx or inFactUnit: self.modelManager.cntlr.uiThreadQueue.put( (self.resultText.setValue, [_("Matching facts...")])) self.modelManager.showStatus(_("Matching facts...")) for fact in self.modelXbrl.facts: if ((isXP and xpCtx.evaluateBooleanValue( xpProg, contextItem=fact)) or (isRE and (inFactName and pattern.search(fact.concept.name) or (inFactLabel and pattern.search(fact.concept.label())) or (inFactValue and pattern.search(fact.value)) or (inFactCntx and pattern.search( XmlUtil.innerText(fact.context.element))) or (inFactUnit and pattern.search( XmlUtil.innerText(fact.unit.element)))))): objsFound.add(fact) except XPathContext.XPathException as err: err = _("Find expression error: {0} \n{1}").format( err.message, err.sourceErrorIndication) self.modelManager.addToLog(err) self.modelManager.cntlr.uiThreadQueue.put( (self.resultText.setValue, [err])) self.modelManager.showStatus(_("Completed with errors"), 5000) numConcepts = 0 numFacts = 0 numRssItems = 0 numMessages = 0 self.objsList = [] for obj in objsFound: if inMessagesLog: numMessages += 1 self.objsList.append(('m', "{0:06}".format(obj), obj)) elif isinstance(obj, ModelConcept): numConcepts += 1 self.objsList.append(('c', obj.localName, obj.objectId())) elif isinstance(obj, ModelFact): numFacts += 1 self.objsList.append(('f', obj.__hash__(), obj.objectId())) elif isinstance(obj, ModelRssItem): numRssItems += 1 self.objsList.append(('r', obj.__hash__(), obj.objectId())) self.objsList.sort() self.result = "Found " if numConcepts: self.result += "{0} concepts".format(numConcepts) if numFacts: self.result += ", " if numFacts: self.result += "{0} facts".format(numFacts) if numRssItems: self.result += "{0} RSS items".format(numRssItems) if numMessages: self.result += "{0} Messages".format(numMessages) if numConcepts + numFacts + numRssItems + numMessages == 0: self.result += "no matches" self.foundIndex = -1 self.modelManager.cntlr.uiThreadQueue.put( (self.resultText.setValue, [self.result])) else: self.foundIndex = 0 if nextIsDown else (len(self.objsList) - 1) self.modelManager.cntlr.uiThreadQueue.put((self.next, [])) self.modelManager.showStatus(_("Ready..."), 2000)
def backgroundFind(self, expr, logViewLines): exprType = self.options["exprType"] inConceptLabel = self.options["conceptLabel"] inConceptName = self.options["conceptName"] inConceptType = self.options["conceptType"] inConceptSubs = self.options["conceptSubs"] inConceptPer = self.options["conceptPer"] inConceptBal = self.options["conceptBal"] inFactLabel = self.options["factLabel"] inFactName = self.options["factName"] inFactValue = self.options["factValue"] inFactCntx = self.options["factCntx"] inFactUnit = self.options["factUnit"] inMessagesLog = self.options["messagesLog"] nextIsDown = self.options["direction"] == "down" objsFound = set() try: if exprType == "text": # escape regex metacharacters pattern = re.compile(''.join( [(('\\' + c) if c in reMetaChars else c) for c in expr]), re.IGNORECASE) isRE = True isXP = False elif exprType == "regex": pattern = re.compile(expr, re.IGNORECASE) isRE = True isXP = False elif exprType == "xpath": isRE = False isXP = True self.resultText.setValue(_("Compiling xpath expression...")) XPathParser.initializeParser(self) self.modelManager.showStatus(_("Compiling xpath expression...")) xpProg= XPathParser.parse(self, expr, XPathParser.staticExpressionFunctionContext(), "find expression", Trace.CALL) xpCtx = XPathContext.create(self.modelXbrl, sourceElement=None) else: return # nothing to do if inMessagesLog: for lineNumber, line in enumerate(logViewLines): if pattern.search(line): objsFound.add(lineNumber) elif self.modelXbrl.modelDocument.type == ModelDocument.Type.RSSFEED: for rssItem in self.modelXbrl.modelDocument.items: if any(pattern.search(str(value)) for name, value in rssItem.propertyView): objsFound.add(rssItem) else: # DTS search if inConceptLabel or inConceptName or inConceptType or inConceptSubs or inConceptPer or inConceptBal: self.modelManager.cntlr.uiThreadQueue.put((self.resultText.setValue, [_("Matching concepts...")])) self.modelManager.showStatus(_("Matching concepts...")) for conceptName, concepts in self.modelXbrl.nameConcepts.items(): for concept in concepts: if ((isXP and xpCtx.evaluateBooleanValue(xpProg, contextItem=concept.qname)) or (isRE and (inConceptLabel and pattern.search(concept.label())) or (inConceptName and pattern.search(conceptName)) or (inConceptType and pattern.search(str(concept.typeQname))) or (inConceptSubs and pattern.search(str(concept.substitutionGroupQname))) or (inConceptPer and concept.periodType and pattern.search(concept.periodType)) or (inConceptBal and concept.balance and pattern.search(concept.balance)) ) ): objsFound.add(concept) if inFactLabel or inFactName or inFactValue or inFactCntx or inFactUnit: self.modelManager.cntlr.uiThreadQueue.put((self.resultText.setValue, [_("Matching facts...")])) self.modelManager.showStatus(_("Matching facts...")) for fact in self.modelXbrl.facts: if ((isXP and xpCtx.evaluateBooleanValue(xpProg, contextItem=fact)) or (isRE and (inFactName and pattern.search(fact.concept.name) or (inFactLabel and pattern.search(fact.concept.label())) or (inFactValue and pattern.search(fact.value)) or (inFactCntx and pattern.search(XmlUtil.innerText(fact.context.element))) or (inFactUnit and pattern.search(XmlUtil.innerText(fact.unit.element)))) ) ): objsFound.add(fact) except XPathContext.XPathException as err: err = _("Find expression error: {0} \n{1}").format(err.message, err.sourceErrorIndication) self.modelManager.addToLog(err) self.modelManager.cntlr.uiThreadQueue.put((self.resultText.setValue, [err])) self.modelManager.showStatus(_("Completed with errors"), 5000) numConcepts = 0 numFacts = 0 numRssItems = 0 numMessages = 0 self.objsList = [] for obj in objsFound: if inMessagesLog: numMessages += 1 self.objsList.append( ('m', "{0:06}".format(obj), obj) ) elif isinstance(obj,ModelConcept): numConcepts += 1 self.objsList.append( ('c', obj.localName, obj.objectId()) ) elif isinstance(obj,ModelFact): numFacts += 1 self.objsList.append( ('f', obj.__hash__(), obj.objectId()) ) elif isinstance(obj,ModelRssItem): numRssItems += 1 self.objsList.append( ('r', obj.__hash__(), obj.objectId()) ) self.objsList.sort() self.result = "Found " if numConcepts: self.result += "{0} concepts".format(numConcepts) if numFacts: self.result += ", " if numFacts: self.result += "{0} facts".format(numFacts) if numRssItems: self.result += "{0} RSS items".format(numRssItems) if numMessages: self.result += "{0} Messages".format(numMessages) if numConcepts + numFacts + numRssItems + numMessages == 0: self.result += "no matches" self.foundIndex = -1 self.modelManager.cntlr.uiThreadQueue.put((self.resultText.setValue, [self.result])) else: self.foundIndex = 0 if nextIsDown else (len(self.objsList) - 1) self.modelManager.cntlr.uiThreadQueue.put((self.next, [])) self.modelManager.showStatus(_("Ready..."), 2000)
def init(modelXbrl): # setup modelXbrl for rendering evaluation # dimension defaults required in advance of validation from arelle import ValidateXbrlDimensions, ValidateFormula, ModelDocument ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) hasXbrlTables = False # validate table linkbase dimensions for baseSetKey in modelXbrl.baseSets.keys(): arcrole, ELR, linkqname, arcqname = baseSetKey if ELR and linkqname and arcqname and XbrlConst.isTableRenderingArcrole(arcrole): ValidateFormula.checkBaseSet(modelXbrl, arcrole, ELR, modelXbrl.relationshipSet(arcrole,ELR,linkqname,arcqname)) if arcrole in (XbrlConst.tableBreakdown, XbrlConst.tableAxis2011): hasXbrlTables = True # provide context for view if modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE: instance = None # use instance of the entry pont else: # need dummy instance instance = ModelDocument.create(modelXbrl, ModelDocument.Type.INSTANCE, "dummy.xml", # fake URI and fake schemaRef ("http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd",)) if hasXbrlTables: # formula processor is needed for 2011 XBRL tables but not for 2010 Eurofiling tables modelXbrl.rendrCntx = XPathContext.create(modelXbrl, instance) modelXbrl.profileStat(None) # setup fresh parameters from formula optoins modelXbrl.parameters = modelXbrl.modelManager.formulaOptions.typedParameters() # validate parameters and custom function signatures ValidateFormula.validate(modelXbrl, xpathContext=modelXbrl.rendrCntx, parametersOnly=True, statusMsg=_("compiling rendering tables")) # check and extract message expressions into compilable programs for msgArcrole in (XbrlConst.tableDefinitionNodeMessage, XbrlConst.tableDefinitionNodeSelectionMessage, XbrlConst.tableAxisMessage2011, XbrlConst.tableAxisSelectionMessage2011): for msgRel in modelXbrl.relationshipSet(msgArcrole).modelRelationships: ValidateFormula.checkMessageExpressions(modelXbrl, msgRel.toModelObject) # compile and validate tables for modelTable in modelXbrl.modelRenderingTables: modelTable.fromInstanceQnames = None # required if referred to by variables scope chaining modelTable.compile() # check aspectModel if modelTable.aspectModel not in ("non-dimensional", "dimensional"): modelXbrl.error("xbrlte:unknownAspectModel", _("Table %(xlinkLabel)s, aspect model %(aspectModel)s not recognized"), modelObject=modelTable, xlinkLabel=modelTable.xlinkLabel, aspectModel=modelTable.aspectModel) else: modelTable.priorAspectAxisDisposition = {} # check ordinate aspects against aspectModel oppositeAspectModel = (_DICT_SET({'dimensional','non-dimensional'}) - _DICT_SET({modelTable.aspectModel})).pop() uncoverableAspects = aspectModels[oppositeAspectModel] - aspectModels[modelTable.aspectModel] for tblAxisRel in modelXbrl.relationshipSet((XbrlConst.tableBreakdown,XbrlConst.tableAxis2011)).fromModelObject(modelTable): checkDefinitionNodeAspectModel(modelXbrl, modelTable, tblAxisRel, uncoverableAspects) del modelTable.priorAspectAxisDisposition modelXbrl.profileStat(_("compileTables"))
def init(modelXbrl): # setup modelXbrl for rendering evaluation # dimension defaults required in advance of validation from arelle import ValidateXbrlDimensions, ValidateFormula, ModelDocument ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) hasXbrlTables = False # validate table linkbase dimensions for baseSetKey in modelXbrl.baseSets.keys(): arcrole, ELR, linkqname, arcqname = baseSetKey if ELR and linkqname and arcqname and XbrlConst.isTableRenderingArcrole( arcrole): ValidateFormula.checkBaseSet( modelXbrl, arcrole, ELR, modelXbrl.relationshipSet(arcrole, ELR, linkqname, arcqname)) if arcrole in (XbrlConst.tableBreakdown, XbrlConst.tableBreakdown201301, XbrlConst.tableAxis2011): hasXbrlTables = True # provide context for view if modelXbrl.modelDocument.type == ModelDocument.Type.INSTANCE: instance = None # use instance of the entry pont else: # need dummy instance instance = ModelDocument.create( modelXbrl, ModelDocument.Type.INSTANCE, "dummy.xml", # fake URI and fake schemaRef ("http://www.xbrl.org/2003/xbrl-instance-2003-12-31.xsd", )) if hasXbrlTables: # formula processor is needed for 2011 XBRL tables but not for 2010 Eurofiling tables modelXbrl.rendrCntx = XPathContext.create(modelXbrl, instance) modelXbrl.profileStat(None) # setup fresh parameters from formula optoins modelXbrl.parameters = modelXbrl.modelManager.formulaOptions.typedParameters( ) # validate parameters and custom function signatures ValidateFormula.validate(modelXbrl, xpathContext=modelXbrl.rendrCntx, parametersOnly=True, statusMsg=_("compiling rendering tables")) # deprecated as of 2013-05-17 # check and extract message expressions into compilable programs for msgArcrole in (XbrlConst.tableDefinitionNodeMessage201301, XbrlConst.tableDefinitionNodeSelectionMessage201301, XbrlConst.tableAxisMessage2011, XbrlConst.tableAxisSelectionMessage2011): for msgRel in modelXbrl.relationshipSet( msgArcrole).modelRelationships: ValidateFormula.checkMessageExpressions( modelXbrl, msgRel.toModelObject) # compile and validate tables for modelTable in modelXbrl.modelRenderingTables: modelTable.fromInstanceQnames = None # required if referred to by variables scope chaining modelTable.compile() # check aspectModel if modelTable.aspectModel not in ("non-dimensional", "dimensional"): modelXbrl.error( "xbrlte:unknownAspectModel", _("Table %(xlinkLabel)s, aspect model %(aspectModel)s not recognized" ), modelObject=modelTable, xlinkLabel=modelTable.xlinkLabel, aspectModel=modelTable.aspectModel) else: modelTable.priorAspectAxisDisposition = {} # check ordinate aspects against aspectModel oppositeAspectModel = ( _DICT_SET({'dimensional', 'non-dimensional'}) - _DICT_SET({modelTable.aspectModel})).pop() uncoverableAspects = aspectModels[ oppositeAspectModel] - aspectModels[modelTable.aspectModel] for tblAxisRel in modelXbrl.relationshipSet( (XbrlConst.tableBreakdown, XbrlConst.tableBreakdown201301, XbrlConst.tableAxis2011)).fromModelObject(modelTable): checkDefinitionNodeAspectModel(modelXbrl, modelTable, tblAxisRel, uncoverableAspects) del modelTable.priorAspectAxisDisposition modelXbrl.profileStat(_("compileTables"))
def validate(val): formulaOptions = val.modelXbrl.modelManager.formulaOptions XPathParser.initializeParser(val) val.modelXbrl.modelManager.showStatus(_("Compiling formulae")) initialErrorCount = val.modelXbrl.logCountErr # global parameter names parameterQnames = set() instanceQnames = set() parameterDependencies = {} instanceDependencies = defaultdict( set) # None-key entries are non-formula dependencies dependencyResolvedParameters = set() orderedParameters = [] orderedInstances = [] for paramQname, modelParameter in val.modelXbrl.qnameParameters.items(): if isinstance(modelParameter, ModelParameter): modelParameter.compile() parameterDependencies[paramQname] = modelParameter.variableRefs() parameterQnames.add(paramQname) if isinstance(modelParameter, ModelInstance): instanceQnames.add(paramQname) # duplicates checked on loading modelDocument #resolve dependencies resolvedAParameter = True while (resolvedAParameter): resolvedAParameter = False for paramQname in parameterQnames: if paramQname not in dependencyResolvedParameters and \ len(parameterDependencies[paramQname] - dependencyResolvedParameters) == 0: dependencyResolvedParameters.add(paramQname) orderedParameters.append(paramQname) resolvedAParameter = True # anything unresolved? for paramQname in parameterQnames: if paramQname not in dependencyResolvedParameters: circularOrUndefDependencies = parameterDependencies[ paramQname] - dependencyResolvedParameters undefinedVars = circularOrUndefDependencies - parameterQnames paramsCircularDep = circularOrUndefDependencies - undefinedVars if len(undefinedVars) > 0: val.modelXbrl.error( _("Undefined dependencies in parameter {0}, to names {1}" ).format(paramQname, ", ".join( (str(v) for v in undefinedVars))), "err", "xbrlve:unresolvedDependency") if len(paramsCircularDep) > 0: val.modelXbrl.error( _("Cyclic dependencies in parameter {0}, to names {1}" ).format(paramQname, ", ".join( (str(d) for d in paramsCircularDep))), "err", "xbrlve:parameterCyclicDependencies") for custFnSig in val.modelXbrl.modelCustomFunctionSignatures.values(): custFnQname = custFnSig.qname if custFnQname.namespaceURI == "XbrlConst.xfi": val.modelXbrl.error( _("Custom function {0} has namespace reserved for functions in the function registry {1}" ).format(str(custFnQname), custFnQname.namespaceURI), "err", "xbrlve:noProhibitedNamespaceForCustomFunction") # any custom function implementations? for modelRel in val.modelXbrl.relationshipSet( XbrlConst.functionImplementation).fromModelObject(custFnSig): custFnImpl = modelRel.toModelObject custFnSig.customFunctionImplementation = custFnImpl if len(custFnImpl.inputNames) != len(custFnSig.inputTypes): val.modelXbrl.error( _("Custom function {0} signature has {1} parameters but implementation has {2}, must be matching" ).format(str(custFnQname), len(custFnSig.inputTypes), len(custFnImpl.inputNames)), "err", "xbrlcfie:inputMismatch") for custFnImpl in val.modelXbrl.modelCustomFunctionImplementations: if not val.modelXbrl.relationshipSet( XbrlConst.functionImplementation).toModelObject(custFnImpl): val.modelXbrl.error( _("Custom function implementation {0} has no relationship from any custom function signature" ).format(custFnImpl.xlinkLabel), "err", "xbrlcfie:missingCFIRelationship") custFnImpl.compile() # xpathContext is needed for filter setup for expressions such as aspect cover filter # determine parameter values xpathContext = XPathContext.create(val.modelXbrl) for paramQname in orderedParameters: if not isinstance(modelParameter, ModelInstance): modelParameter = val.modelXbrl.qnameParameters[paramQname] asType = modelParameter.asType asLocalName = asType.localName if asType else "string" try: if val.parameters and paramQname in val.parameters: paramDataType, paramValue = val.parameters[paramQname] typeLocalName = paramDataType.localName if paramDataType else "string" value = FunctionXs.call(xpathContext, None, typeLocalName, [paramValue]) result = FunctionXs.call(xpathContext, None, asLocalName, [value]) if formulaOptions.traceParameterInputValue: val.modelXbrl.error( _("Parameter {0} input {1}").format( paramQname, result), "info", "formula:trace") else: result = modelParameter.evaluate(xpathContext, asType) if formulaOptions.traceParameterExpressionResult: val.modelXbrl.error( _("Parameter {0} result {1}").format( paramQname, result), "info", "formula:trace") xpathContext.inScopeVars[ paramQname] = result # make visible to subsequent parameter expression except XPathContext.XPathException as err: val.modelXbrl.error( _("Parameter \n{0} \nException: \n{1}").format( paramQname, err.message), "err", "xbrlve:parameterTypeMismatch" if err.code == "err:FORG0001" else err.code) produceOutputXbrlInstance = False instanceProducingVariableSets = defaultdict(list) for modelVariableSet in val.modelXbrl.modelVariableSets: varSetInstanceDependencies = set() if isinstance(modelVariableSet, ModelFormula): instanceQname = None for modelRel in val.modelXbrl.relationshipSet( XbrlConst.formulaInstance).fromModelObject( modelVariableSet): instance = modelRel.toModelObject if isinstance(instance, ModelInstance): if instanceQname is None: instanceQname = instance.qname else: val.modelXbrl.error( _("Multiple output instances for formula {0}, to names {1}, {2}" ).format(modelVariableSet.xlinkLabel, instanceQname, instance.qname), "info", "arelle:multipleOutputInstances") if instanceQname is None: instanceQname = XbrlConst.qnStandardOutputInstance instanceQnames.add(instanceQname) modelVariableSet.outputInstanceQname = instanceQname if val.validateSBRNL: val.modelXbrl.error( _("Formula linkbase {0} formula:formula {1} is not allowed" ).format( os.path.basename(modelVariableSet.modelDocument.uri), modelVariableSet.xlinkLabel), "err", "SBR.NL.2.3.9.03") else: instanceQname = None modelVariableSet.countSatisfied = 0 modelVariableSet.countNotSatisfied = 0 checkValidationMessages(val, modelVariableSet) instanceProducingVariableSets[instanceQname].append(modelVariableSet) modelVariableSet.outputInstanceQname = instanceQname if modelVariableSet.aspectModel not in ("non-dimensional", "dimensional"): val.modelXbrl.error( _("Variable set {0}, aspect model {1} not recognized").format( modelVariableSet.xlinkLabel, modelVariableSet.aspectModel), "err", "xbrlve:unknownAspectModel") modelVariableSet.compile() modelVariableSet.hasConsistencyAssertion = False #determine dependencies within variable sets nameVariables = {} qnameRels = {} definedNamesSet = set() for modelRel in val.modelXbrl.relationshipSet( XbrlConst.variableSet).fromModelObject(modelVariableSet): varqname = modelRel.variableQname if varqname: qnameRels[varqname] = modelRel toVariable = modelRel.toModelObject if varqname not in definedNamesSet: definedNamesSet.add(varqname) if varqname not in nameVariables: nameVariables[varqname] = toVariable elif nameVariables[varqname] != toVariable: val.modelXbrl.error( _("Multiple variables named {1} in variable set {0}" ).format(modelVariableSet.xlinkLabel, varqname), "err", "xbrlve:duplicateVariableNames") fromInstanceQnames = None for instRel in val.modelXbrl.relationshipSet( XbrlConst.instanceVariable).toModelObject(toVariable): fromInstance = instRel.fromModelObject if isinstance(fromInstance, ModelInstance): fromInstanceQname = fromInstance.qname varSetInstanceDependencies.add(fromInstanceQname) instanceDependencies[instanceQname].add( fromInstanceQname) if fromInstanceQnames is None: fromInstanceQnames = set() fromInstanceQnames.add(fromInstanceQname) if fromInstanceQnames is None: varSetInstanceDependencies.add( XbrlConst.qnStandardInputInstance) if instanceQname: instanceDependencies[instanceQname].add( XbrlConst.qnStandardInputInstance) toVariable.fromInstanceQnames = fromInstanceQnames else: val.modelXbrl.error( _("Variables name {1} cannot be determined on arc from {0}" ).format(modelVariableSet.xlinkLabel, modelRel.variablename), "err", "xbrlve:variableNameResolutionFailure") definedNamesSet |= parameterQnames variableDependencies = {} for modelRel in val.modelXbrl.relationshipSet( XbrlConst.variableSet).fromModelObject(modelVariableSet): variable = modelRel.toModelObject if isinstance( variable, (ModelParameter, ModelVariable)): # ignore anything not parameter or variable varqname = modelRel.variableQname depVars = variable.variableRefs() variableDependencies[varqname] = depVars if len(depVars ) > 0 and formulaOptions.traceVariablesDependencies: val.modelXbrl.error( _("Variable set {0}, variable {1}, dependences {2}"). format(modelVariableSet.xlinkLabel, varqname, depVars), "info", "formula:trace") definedNamesSet.add(varqname) # check for fallback value variable references if isinstance(variable, ModelFactVariable): for depVar in XPathParser.variableReferencesSet( variable.fallbackValueProg, variable.element): if depVar in qnameRels and isinstance( qnameRels[depVar].toModelObject, ModelVariable): val.modelXbrl.error( _("Variable set {0} fallbackValue '{1}' cannot refer to variable {2}" ).format(modelVariableSet.xlinkLabel, variable.fallbackValue, depVar), "err", "xbrlve:factVariableReferenceNotAllowed") # check for covering aspect not in variable set aspect model checkFilterAspectModel(val, modelVariableSet, variable.filterRelationships, xpathContext) orderedNameSet = set() orderedNameList = [] orderedAVariable = True while (orderedAVariable): orderedAVariable = False for varqname, depVars in variableDependencies.items(): if varqname not in orderedNameSet and len(depVars - parameterQnames - orderedNameSet) == 0: orderedNameList.append(varqname) orderedNameSet.add(varqname) orderedAVariable = True if varqname in instanceQnames: varSetInstanceDependencies.add(varqname) instanceDependencies[instanceQname].add(varqname) elif isinstance(nameVariables.get(varqname), ModelInstance): instqname = nameVariables[varqname].qname varSetInstanceDependencies.add(instqname) instanceDependencies[instanceQname].add(instqname) # anything unresolved? for varqname, depVars in variableDependencies.items(): if varqname not in orderedNameSet: circularOrUndefVars = depVars - parameterQnames - orderedNameSet undefinedVars = circularOrUndefVars - definedNamesSet varsCircularDep = circularOrUndefVars - undefinedVars if len(undefinedVars) > 0: val.modelXbrl.error( _("Undefined variable dependencies in variable st {0}, from variable {1} to {2}" ).format(modelVariableSet.xlinkLabel, varqname, undefinedVars), "err", "xbrlve:unresolvedDependency") if len(varsCircularDep) > 0: val.modelXbrl.error( _("Cyclic dependencies in variable set {0}, from variable {1} to {2}" ).format(modelVariableSet.xlinkLabel, varqname, varsCircularDep), "err", "xbrlve:cyclicDependencies") # check unresolved variable set dependencies for varSetDepVarQname in modelVariableSet.variableRefs(): if varSetDepVarQname not in orderedNameSet and varSetDepVarQname not in parameterQnames: val.modelXbrl.error( _("Undefined variable dependency in variable set {0}, {1}" ).format(modelVariableSet.xlinkLabel, varSetDepVarQname), "err", "xbrlve:unresolvedDependency") if varSetDepVarQname in instanceQnames: varSetInstanceDependencies.add(varSetDepVarQname) instanceDependencies[instanceQname].add(varSetDepVarQname) elif isinstance(nameVariables.get(varSetDepVarQname), ModelInstance): instqname = nameVariables[varSetDepVarQname].qname varSetInstanceDependencies.add(instqname) instanceDependencies[instanceQname].add(instqname) if formulaOptions.traceVariablesOrder: val.modelXbrl.error( _("Variable set {0}, variables order: {1}").format( modelVariableSet.xlinkLabel, orderedNameList), "info", "formula:trace") if (formulaOptions.traceVariablesDependencies and len(varSetInstanceDependencies) > 0 and varSetInstanceDependencies != {XbrlConst.qnStandardInputInstance}): val.modelXbrl.error( _("Variable set {0}, instance dependences {1}").format( modelVariableSet.xlinkLabel, varSetInstanceDependencies), "info", "formula:trace") modelVariableSet.orderedVariableRelationships = [] for varqname in orderedNameList: if varqname in qnameRels: modelVariableSet.orderedVariableRelationships.append( qnameRels[varqname]) # check existence assertion variable dependencies if isinstance(modelVariableSet, ModelExistenceAssertion): for depVar in modelVariableSet.variableRefs(): if depVar in qnameRels and isinstance( qnameRels[depVar].toModelObject, ModelVariable): val.modelXbrl.error( _("Existence Assertion {0}, cannot refer to variable {1}" ).format(modelVariableSet.xlinkLabel, depVar), "err", "xbrleae:variableReferenceNotAllowed") # check messages variable dependencies checkValidationMessageVariables(val, modelVariableSet, qnameRels) # check preconditions modelVariableSet.preconditions = [] for modelRel in val.modelXbrl.relationshipSet( XbrlConst.variableSetPrecondition).fromModelObject( modelVariableSet): precondition = modelRel.toModelObject if isinstance(precondition, ModelPrecondition): modelVariableSet.preconditions.append(precondition) # check for variable sets referencing fact or general variables for modelRel in val.modelXbrl.relationshipSet( XbrlConst.variableSetFilter).fromModelObject(modelVariableSet): varSetFilter = modelRel.toModelObject if modelRel.isCovered: val.modelXbrl.error( _("Variable set {0}, filter {1}, cannot be covered" ).format(modelVariableSet.xlinkLabel, varSetFilter.xlinkLabel), "wrn", "arelle:variableSetFilterCovered") modelRel._isCovered = False # block group filter from being able to covere for depVar in varSetFilter.variableRefs(): if depVar in qnameRels and isinstance( qnameRels[depVar].toModelObject, ModelVariable): val.modelXbrl.error( _("Variable set {0}, filter {1}, cannot refer to variable {2}" ).format(modelVariableSet.xlinkLabel, varSetFilter.xlinkLabel, depVar), "err", "xbrlve:factVariableReferenceNotAllowed") # check aspects of formula if isinstance(modelVariableSet, ModelFormula): checkFormulaRules(val, modelVariableSet, nameVariables) # determine instance dependency order orderedInstancesSet = set() stdInpInst = {XbrlConst.qnStandardInputInstance} orderedInstancesList = [] orderedAnInstance = True while (orderedAnInstance): orderedAnInstance = False for instqname, depInsts in instanceDependencies.items(): if instqname and instqname not in orderedInstancesSet and len( depInsts - stdInpInst - orderedInstancesSet) == 0: orderedInstancesList.append(instqname) orderedInstancesSet.add(instqname) orderedAnInstance = True orderedInstancesList.append( None) # assertions come after all formulas that produce outputs # anything unresolved? for instqname, depInsts in instanceDependencies.items(): if instqname not in orderedInstancesSet: # can also be satisfied from an input DTS missingDependentInstances = depInsts - stdInpInst if val.parameters: missingDependentInstances -= val.parameters.keys() if instqname: if missingDependentInstances: val.modelXbrl.error( _("Cyclic dependencies of instance {0} produced by a formula, with variables consuming instances {1}" ).format(instqname, missingDependentInstances), "err", "xbrlvarinste:instanceVariableRecursionCycle") elif instqname == XbrlConst.qnStandardOutputInstance: orderedInstancesSet.add(instqname) orderedInstancesList.append( instqname ) # standard output formula, all input dependencies in parameters ''' future check? if instance has no external input or producing formula else: val.modelXbrl.error( _("Unresolved dependencies of an assertion's variables on instances {0}").format( depInsts - stdInpInst ), "err", "xbrlvarinste:instanceVariableRecursionCycle") ''' if formulaOptions.traceVariablesOrder and len(orderedInstancesList) > 1: val.modelXbrl.error( _("Variable instances processing order: {0}").format( orderedInstancesList), "info", "formula:trace") # linked consistency assertions for modelRel in val.modelXbrl.relationshipSet( XbrlConst.consistencyAssertionFormula).modelRelationships: if modelRel.fromModelObject and modelRel.toModelObject and isinstance( modelRel.toModelObject, ModelFormula): consisAsser = modelRel.fromModelObject consisAsser.countSatisfied = 0 consisAsser.countNotSatisfied = 0 if consisAsser.hasProportionalAcceptanceRadius and consisAsser.hasAbsoluteAcceptanceRadius: val.modelXbrl.error( _("Consistency assertion {0} has both absolute and proportional acceptance radii" ).format(consisAsser.xlinkLabel), "err", "xbrlcae:acceptanceRadiusConflict") consisAsser.orderedVariableRelationships = [] for consisParamRel in val.modelXbrl.relationshipSet( XbrlConst.consistencyAssertionParameter).fromModelObject( consisAsser): if isinstance(consisParamRel.toModelObject, ModelVariable): val.modelXbrl.error( _("Consistency assertion {0} has relationship to a {1} {2}" ).format( consisAsser.xlinkLabel, consisParamRel.toModelObject.element.localName, consisParamRel.toModelObject.xlinkLabel), "err", "xbrlcae:variablesNotAllowed") else: consisAsser.orderedVariableRelationships.append( consisParamRel) consisAsser.compile() modelRel.toModelObject.hasConsistencyAssertion = True if initialErrorCount < val.modelXbrl.logCountErr: return # don't try to execute # formula output instances if instanceQnames: schemaRefs = [ val.modelXbrl.modelDocument.relativeUri(referencedDoc.uri) for referencedDoc in val.modelXbrl.modelDocument.referencesDocument.keys() if referencedDoc.type == ModelDocument.Type.SCHEMA ] outputXbrlInstance = None for instanceQname in instanceQnames: if instanceQname == XbrlConst.qnStandardInputInstance: continue # always present the standard way if val.parameters and instanceQname in val.parameters: namedInstance = val.parameters[instanceQname][1] else: # empty intermediate instance uri = val.modelXbrl.modelDocument.filepath[:-4] + "-output-XBRL-instance" if instanceQname != XbrlConst.qnStandardOutputInstance: uri = uri + "-" + instanceQname.localName uri = uri + ".xml" namedInstance = ModelXbrl.create( val.modelXbrl.modelManager, newDocumentType=ModelDocument.Type.INSTANCE, url=uri, schemaRefs=schemaRefs, isEntry=True) xpathContext.inScopeVars[instanceQname] = namedInstance if instanceQname == XbrlConst.qnStandardOutputInstance: outputXbrlInstance = namedInstance # evaluate consistency assertions # evaluate variable sets not in consistency assertions for instanceQname in orderedInstancesList: for modelVariableSet in instanceProducingVariableSets[instanceQname]: # produce variable evaluations from arelle.FormulaEvaluator import evaluate try: evaluate(xpathContext, modelVariableSet) except XPathContext.XPathException as err: val.modelXbrl.error( _("Variable set \n{0} \nException: \n{1}").format( modelVariableSet, err.message), "err", err.code) # log assertion result counts asserTests = {} for exisValAsser in val.modelXbrl.modelVariableSets: if isinstance(exisValAsser, ModelVariableSetAssertion): asserTests[exisValAsser.id] = (exisValAsser.countSatisfied, exisValAsser.countNotSatisfied) if formulaOptions.traceAssertionResultCounts: val.modelXbrl.error( _("{0} Assertion {1} evaluations : {2} satisfied, {3} not satisfied" ).format( "Existence" if isinstance(exisValAsser, ModelExistenceAssertion) else "Value", exisValAsser.id, exisValAsser.countSatisfied, exisValAsser.countNotSatisfied), "info", "formula:trace") for modelRel in val.modelXbrl.relationshipSet( XbrlConst.consistencyAssertionFormula).modelRelationships: if modelRel.fromModelObject and modelRel.toModelObject and isinstance( modelRel.toModelObject, ModelFormula): consisAsser = modelRel.fromModelObject asserTests[consisAsser.id] = (consisAsser.countSatisfied, consisAsser.countNotSatisfied) if formulaOptions.traceAssertionResultCounts: val.modelXbrl.error( _("Consistency Assertion {0} evaluations : {1} satisfied, {2} not satisfied" ).format(consisAsser.id, consisAsser.countSatisfied, consisAsser.countNotSatisfied), "info", "formula:trace") if asserTests: val.modelXbrl.error( _("Assertion results {0}").format(asserTests), "asrtNoLog", asserTests) # display output instance if outputXbrlInstance: if val.modelXbrl.formulaOutputInstance: # close prior instance, usually closed by caller to validate as it may affect UI on different thread val.modelXbrl.formulaOutputInstance.close() val.modelXbrl.formulaOutputInstance = outputXbrlInstance
def validate(val): formulaOptions = val.modelXbrl.modelManager.formulaOptions XPathParser.initializeParser(val) val.modelXbrl.modelManager.showStatus(_("Compiling formulae")) initialErrorCount = val.modelXbrl.logCountErr # global parameter names parameterQnames = set() instanceQnames = set() parameterDependencies = {} instanceDependencies = defaultdict(set) # None-key entries are non-formula dependencies dependencyResolvedParameters = set() orderedParameters = [] orderedInstances = [] for paramQname, modelParameter in val.modelXbrl.qnameParameters.items(): if isinstance(modelParameter, ModelParameter): modelParameter.compile() parameterDependencies[paramQname] = modelParameter.variableRefs() parameterQnames.add(paramQname) if isinstance(modelParameter, ModelInstance): instanceQnames.add(paramQname) # duplicates checked on loading modelDocument #resolve dependencies resolvedAParameter = True while (resolvedAParameter): resolvedAParameter = False for paramQname in parameterQnames: if paramQname not in dependencyResolvedParameters and \ len(parameterDependencies[paramQname] - dependencyResolvedParameters) == 0: dependencyResolvedParameters.add(paramQname) orderedParameters.append(paramQname) resolvedAParameter = True # anything unresolved? for paramQname in parameterQnames: if paramQname not in dependencyResolvedParameters: circularOrUndefDependencies = parameterDependencies[paramQname] - dependencyResolvedParameters undefinedVars = circularOrUndefDependencies - parameterQnames paramsCircularDep = circularOrUndefDependencies - undefinedVars if len(undefinedVars) > 0: val.modelXbrl.error( _("Undefined dependencies in parameter {0}, to names {1}").format( paramQname, ", ".join((str(v) for v in undefinedVars))), "err", "xbrlve:unresolvedDependency") if len(paramsCircularDep) > 0: val.modelXbrl.error( _("Cyclic dependencies in parameter {0}, to names {1}").format( paramQname, ", ".join((str(d) for d in paramsCircularDep)) ), "err", "xbrlve:parameterCyclicDependencies") for custFnSig in val.modelXbrl.modelCustomFunctionSignatures.values(): custFnQname = custFnSig.qname if custFnQname.namespaceURI == "XbrlConst.xfi": val.modelXbrl.error( _("Custom function {0} has namespace reserved for functions in the function registry {1}").format( str(custFnQname), custFnQname.namespaceURI ), "err", "xbrlve:noProhibitedNamespaceForCustomFunction") # any custom function implementations? for modelRel in val.modelXbrl.relationshipSet(XbrlConst.functionImplementation).fromModelObject(custFnSig): custFnImpl = modelRel.toModelObject custFnSig.customFunctionImplementation = custFnImpl if len(custFnImpl.inputNames) != len(custFnSig.inputTypes): val.modelXbrl.error( _("Custom function {0} signature has {1} parameters but implementation has {2}, must be matching").format( str(custFnQname), len(custFnSig.inputTypes), len(custFnImpl.inputNames) ), "err", "xbrlcfie:inputMismatch") for custFnImpl in val.modelXbrl.modelCustomFunctionImplementations: if not val.modelXbrl.relationshipSet(XbrlConst.functionImplementation).toModelObject(custFnImpl): val.modelXbrl.error( _("Custom function implementation {0} has no relationship from any custom function signature").format( custFnImpl.xlinkLabel), "err", "xbrlcfie:missingCFIRelationship") custFnImpl.compile() # xpathContext is needed for filter setup for expressions such as aspect cover filter # determine parameter values xpathContext = XPathContext.create(val.modelXbrl) for paramQname in orderedParameters: if not isinstance(modelParameter, ModelInstance): modelParameter = val.modelXbrl.qnameParameters[paramQname] asType = modelParameter.asType asLocalName = asType.localName if asType else "string" try: if val.parameters and paramQname in val.parameters: paramDataType, paramValue = val.parameters[paramQname] typeLocalName = paramDataType.localName if paramDataType else "string" value = FunctionXs.call(xpathContext, None, typeLocalName, [paramValue]) result = FunctionXs.call(xpathContext, None, asLocalName, [value]) if formulaOptions.traceParameterInputValue: val.modelXbrl.error( _("Parameter {0} input {1}").format( paramQname, result), "info", "formula:trace") else: result = modelParameter.evaluate(xpathContext, asType) if formulaOptions.traceParameterExpressionResult: val.modelXbrl.error( _("Parameter {0} result {1}").format( paramQname, result), "info", "formula:trace") xpathContext.inScopeVars[paramQname] = result # make visible to subsequent parameter expression except XPathContext.XPathException as err: val.modelXbrl.error( _("Parameter \n{0} \nException: \n{1}").format( paramQname, err.message), "err", "xbrlve:parameterTypeMismatch" if err.code == "err:FORG0001" else err.code) produceOutputXbrlInstance = False instanceProducingVariableSets = defaultdict(list) for modelVariableSet in val.modelXbrl.modelVariableSets: varSetInstanceDependencies = set() if isinstance(modelVariableSet, ModelFormula): instanceQname = None for modelRel in val.modelXbrl.relationshipSet(XbrlConst.formulaInstance).fromModelObject(modelVariableSet): instance = modelRel.toModelObject if isinstance(instance, ModelInstance): if instanceQname is None: instanceQname = instance.qname else: val.modelXbrl.error( _("Multiple output instances for formula {0}, to names {1}, {2}").format( modelVariableSet.xlinkLabel, instanceQname, instance.qname ), "info", "arelle:multipleOutputInstances") if instanceQname is None: instanceQname = XbrlConst.qnStandardOutputInstance instanceQnames.add(instanceQname) modelVariableSet.outputInstanceQname = instanceQname if val.validateSBRNL: val.modelXbrl.error( _("Formula linkbase {0} formula:formula {1} is not allowed").format( os.path.basename(modelVariableSet.modelDocument.uri), modelVariableSet.xlinkLabel), "err", "SBR.NL.2.3.9.03") else: instanceQname = None modelVariableSet.countSatisfied = 0 modelVariableSet.countNotSatisfied = 0 checkValidationMessages(val, modelVariableSet) instanceProducingVariableSets[instanceQname].append(modelVariableSet) modelVariableSet.outputInstanceQname = instanceQname if modelVariableSet.aspectModel not in ("non-dimensional", "dimensional"): val.modelXbrl.error( _("Variable set {0}, aspect model {1} not recognized").format( modelVariableSet.xlinkLabel, modelVariableSet.aspectModel), "err", "xbrlve:unknownAspectModel") modelVariableSet.compile() modelVariableSet.hasConsistencyAssertion = False #determine dependencies within variable sets nameVariables = {} qnameRels = {} definedNamesSet = set() for modelRel in val.modelXbrl.relationshipSet(XbrlConst.variableSet).fromModelObject(modelVariableSet): varqname = modelRel.variableQname if varqname: qnameRels[varqname] = modelRel toVariable = modelRel.toModelObject if varqname not in definedNamesSet: definedNamesSet.add(varqname) if varqname not in nameVariables: nameVariables[varqname] = toVariable elif nameVariables[varqname] != toVariable: val.modelXbrl.error( _("Multiple variables named {1} in variable set {0}").format( modelVariableSet.xlinkLabel, varqname ), "err", "xbrlve:duplicateVariableNames") fromInstanceQnames = None for instRel in val.modelXbrl.relationshipSet(XbrlConst.instanceVariable).toModelObject(toVariable): fromInstance = instRel.fromModelObject if isinstance(fromInstance, ModelInstance): fromInstanceQname = fromInstance.qname varSetInstanceDependencies.add(fromInstanceQname) instanceDependencies[instanceQname].add(fromInstanceQname) if fromInstanceQnames is None: fromInstanceQnames = set() fromInstanceQnames.add(fromInstanceQname) if fromInstanceQnames is None: varSetInstanceDependencies.add(XbrlConst.qnStandardInputInstance) if instanceQname: instanceDependencies[instanceQname].add(XbrlConst.qnStandardInputInstance) toVariable.fromInstanceQnames = fromInstanceQnames else: val.modelXbrl.error( _("Variables name {1} cannot be determined on arc from {0}").format( modelVariableSet.xlinkLabel, modelRel.variablename ), "err", "xbrlve:variableNameResolutionFailure") definedNamesSet |= parameterQnames variableDependencies = {} for modelRel in val.modelXbrl.relationshipSet(XbrlConst.variableSet).fromModelObject(modelVariableSet): variable = modelRel.toModelObject if isinstance(variable, (ModelParameter,ModelVariable)): # ignore anything not parameter or variable varqname = modelRel.variableQname depVars = variable.variableRefs() variableDependencies[varqname] = depVars if len(depVars) > 0 and formulaOptions.traceVariablesDependencies: val.modelXbrl.error(_("Variable set {0}, variable {1}, dependences {2}").format( modelVariableSet.xlinkLabel, varqname, depVars), "info", "formula:trace") definedNamesSet.add(varqname) # check for fallback value variable references if isinstance(variable, ModelFactVariable): for depVar in XPathParser.variableReferencesSet(variable.fallbackValueProg, variable.element): if depVar in qnameRels and isinstance(qnameRels[depVar].toModelObject,ModelVariable): val.modelXbrl.error(_("Variable set {0} fallbackValue '{1}' cannot refer to variable {2}").format( modelVariableSet.xlinkLabel, variable.fallbackValue, depVar), "err", "xbrlve:factVariableReferenceNotAllowed") # check for covering aspect not in variable set aspect model checkFilterAspectModel(val, modelVariableSet, variable.filterRelationships, xpathContext) orderedNameSet = set() orderedNameList = [] orderedAVariable = True while (orderedAVariable): orderedAVariable = False for varqname, depVars in variableDependencies.items(): if varqname not in orderedNameSet and len(depVars - parameterQnames - orderedNameSet) == 0: orderedNameList.append(varqname) orderedNameSet.add(varqname) orderedAVariable = True if varqname in instanceQnames: varSetInstanceDependencies.add(varqname) instanceDependencies[instanceQname].add(varqname) elif isinstance(nameVariables.get(varqname), ModelInstance): instqname = nameVariables[varqname].qname varSetInstanceDependencies.add(instqname) instanceDependencies[instanceQname].add(instqname) # anything unresolved? for varqname, depVars in variableDependencies.items(): if varqname not in orderedNameSet: circularOrUndefVars = depVars - parameterQnames - orderedNameSet undefinedVars = circularOrUndefVars - definedNamesSet varsCircularDep = circularOrUndefVars - undefinedVars if len(undefinedVars) > 0: val.modelXbrl.error( _("Undefined variable dependencies in variable st {0}, from variable {1} to {2}").format( modelVariableSet.xlinkLabel, varqname, undefinedVars), "err", "xbrlve:unresolvedDependency") if len(varsCircularDep) > 0: val.modelXbrl.error( _("Cyclic dependencies in variable set {0}, from variable {1} to {2}").format( modelVariableSet.xlinkLabel, varqname, varsCircularDep ), "err", "xbrlve:cyclicDependencies") # check unresolved variable set dependencies for varSetDepVarQname in modelVariableSet.variableRefs(): if varSetDepVarQname not in orderedNameSet and varSetDepVarQname not in parameterQnames: val.modelXbrl.error( _("Undefined variable dependency in variable set {0}, {1}").format( modelVariableSet.xlinkLabel, varSetDepVarQname), "err", "xbrlve:unresolvedDependency") if varSetDepVarQname in instanceQnames: varSetInstanceDependencies.add(varSetDepVarQname) instanceDependencies[instanceQname].add(varSetDepVarQname) elif isinstance(nameVariables.get(varSetDepVarQname), ModelInstance): instqname = nameVariables[varSetDepVarQname].qname varSetInstanceDependencies.add(instqname) instanceDependencies[instanceQname].add(instqname) if formulaOptions.traceVariablesOrder: val.modelXbrl.error(_("Variable set {0}, variables order: {1}").format( modelVariableSet.xlinkLabel, orderedNameList), "info", "formula:trace") if (formulaOptions.traceVariablesDependencies and len(varSetInstanceDependencies) > 0 and varSetInstanceDependencies != {XbrlConst.qnStandardInputInstance}): val.modelXbrl.error(_("Variable set {0}, instance dependences {1}").format( modelVariableSet.xlinkLabel, varSetInstanceDependencies), "info", "formula:trace") modelVariableSet.orderedVariableRelationships = [] for varqname in orderedNameList: if varqname in qnameRels: modelVariableSet.orderedVariableRelationships.append(qnameRels[varqname]) # check existence assertion variable dependencies if isinstance(modelVariableSet, ModelExistenceAssertion): for depVar in modelVariableSet.variableRefs(): if depVar in qnameRels and isinstance(qnameRels[depVar].toModelObject,ModelVariable): val.modelXbrl.error(_("Existence Assertion {0}, cannot refer to variable {1}").format( modelVariableSet.xlinkLabel, depVar), "err", "xbrleae:variableReferenceNotAllowed") # check messages variable dependencies checkValidationMessageVariables(val, modelVariableSet, qnameRels) # check preconditions modelVariableSet.preconditions = [] for modelRel in val.modelXbrl.relationshipSet(XbrlConst.variableSetPrecondition).fromModelObject(modelVariableSet): precondition = modelRel.toModelObject if isinstance(precondition, ModelPrecondition): modelVariableSet.preconditions.append(precondition) # check for variable sets referencing fact or general variables for modelRel in val.modelXbrl.relationshipSet(XbrlConst.variableSetFilter).fromModelObject(modelVariableSet): varSetFilter = modelRel.toModelObject if modelRel.isCovered: val.modelXbrl.error(_("Variable set {0}, filter {1}, cannot be covered").format( modelVariableSet.xlinkLabel, varSetFilter.xlinkLabel), "wrn", "arelle:variableSetFilterCovered") modelRel._isCovered = False # block group filter from being able to covere for depVar in varSetFilter.variableRefs(): if depVar in qnameRels and isinstance(qnameRels[depVar].toModelObject,ModelVariable): val.modelXbrl.error(_("Variable set {0}, filter {1}, cannot refer to variable {2}").format( modelVariableSet.xlinkLabel, varSetFilter.xlinkLabel, depVar), "err", "xbrlve:factVariableReferenceNotAllowed") # check aspects of formula if isinstance(modelVariableSet, ModelFormula): checkFormulaRules(val, modelVariableSet, nameVariables) # determine instance dependency order orderedInstancesSet = set() stdInpInst = {XbrlConst.qnStandardInputInstance} orderedInstancesList = [] orderedAnInstance = True while (orderedAnInstance): orderedAnInstance = False for instqname, depInsts in instanceDependencies.items(): if instqname and instqname not in orderedInstancesSet and len(depInsts - stdInpInst - orderedInstancesSet) == 0: orderedInstancesList.append(instqname) orderedInstancesSet.add(instqname) orderedAnInstance = True orderedInstancesList.append(None) # assertions come after all formulas that produce outputs # anything unresolved? for instqname, depInsts in instanceDependencies.items(): if instqname not in orderedInstancesSet: # can also be satisfied from an input DTS missingDependentInstances = depInsts - stdInpInst if val.parameters: missingDependentInstances -= val.parameters.keys() if instqname: if missingDependentInstances: val.modelXbrl.error( _("Cyclic dependencies of instance {0} produced by a formula, with variables consuming instances {1}").format( instqname, missingDependentInstances ), "err", "xbrlvarinste:instanceVariableRecursionCycle") elif instqname == XbrlConst.qnStandardOutputInstance: orderedInstancesSet.add(instqname) orderedInstancesList.append(instqname) # standard output formula, all input dependencies in parameters ''' future check? if instance has no external input or producing formula else: val.modelXbrl.error( _("Unresolved dependencies of an assertion's variables on instances {0}").format( depInsts - stdInpInst ), "err", "xbrlvarinste:instanceVariableRecursionCycle") ''' if formulaOptions.traceVariablesOrder and len(orderedInstancesList) > 1: val.modelXbrl.error(_("Variable instances processing order: {0}").format( orderedInstancesList), "info", "formula:trace") # linked consistency assertions for modelRel in val.modelXbrl.relationshipSet(XbrlConst.consistencyAssertionFormula).modelRelationships: if modelRel.fromModelObject and modelRel.toModelObject and isinstance(modelRel.toModelObject,ModelFormula): consisAsser = modelRel.fromModelObject consisAsser.countSatisfied = 0 consisAsser.countNotSatisfied = 0 if consisAsser.hasProportionalAcceptanceRadius and consisAsser.hasAbsoluteAcceptanceRadius: val.modelXbrl.error( _("Consistency assertion {0} has both absolute and proportional acceptance radii").format( consisAsser.xlinkLabel), "err", "xbrlcae:acceptanceRadiusConflict") consisAsser.orderedVariableRelationships = [] for consisParamRel in val.modelXbrl.relationshipSet(XbrlConst.consistencyAssertionParameter).fromModelObject(consisAsser): if isinstance(consisParamRel.toModelObject, ModelVariable): val.modelXbrl.error( _("Consistency assertion {0} has relationship to a {1} {2}").format( consisAsser.xlinkLabel, consisParamRel.toModelObject.element.localName, consisParamRel.toModelObject.xlinkLabel), "err", "xbrlcae:variablesNotAllowed") else: consisAsser.orderedVariableRelationships.append(consisParamRel) consisAsser.compile() modelRel.toModelObject.hasConsistencyAssertion = True if initialErrorCount < val.modelXbrl.logCountErr: return # don't try to execute # formula output instances if instanceQnames: schemaRefs = [val.modelXbrl.modelDocument.relativeUri(referencedDoc.uri) for referencedDoc in val.modelXbrl.modelDocument.referencesDocument.keys() if referencedDoc.type == ModelDocument.Type.SCHEMA] outputXbrlInstance = None for instanceQname in instanceQnames: if instanceQname == XbrlConst.qnStandardInputInstance: continue # always present the standard way if val.parameters and instanceQname in val.parameters: namedInstance = val.parameters[instanceQname][1] else: # empty intermediate instance uri = val.modelXbrl.modelDocument.filepath[:-4] + "-output-XBRL-instance" if instanceQname != XbrlConst.qnStandardOutputInstance: uri = uri + "-" + instanceQname.localName uri = uri + ".xml" namedInstance = ModelXbrl.create(val.modelXbrl.modelManager, newDocumentType=ModelDocument.Type.INSTANCE, url=uri, schemaRefs=schemaRefs, isEntry=True) xpathContext.inScopeVars[instanceQname] = namedInstance if instanceQname == XbrlConst.qnStandardOutputInstance: outputXbrlInstance = namedInstance # evaluate consistency assertions # evaluate variable sets not in consistency assertions for instanceQname in orderedInstancesList: for modelVariableSet in instanceProducingVariableSets[instanceQname]: # produce variable evaluations from arelle.FormulaEvaluator import evaluate try: evaluate(xpathContext, modelVariableSet) except XPathContext.XPathException as err: val.modelXbrl.error( _("Variable set \n{0} \nException: \n{1}").format( modelVariableSet, err.message), "err", err.code) # log assertion result counts asserTests = {} for exisValAsser in val.modelXbrl.modelVariableSets: if isinstance(exisValAsser, ModelVariableSetAssertion): asserTests[exisValAsser.id] = (exisValAsser.countSatisfied, exisValAsser.countNotSatisfied) if formulaOptions.traceAssertionResultCounts: val.modelXbrl.error( _("{0} Assertion {1} evaluations : {2} satisfied, {3} not satisfied").format( "Existence" if isinstance(exisValAsser, ModelExistenceAssertion) else "Value", exisValAsser.id, exisValAsser.countSatisfied, exisValAsser.countNotSatisfied), "info", "formula:trace") for modelRel in val.modelXbrl.relationshipSet(XbrlConst.consistencyAssertionFormula).modelRelationships: if modelRel.fromModelObject and modelRel.toModelObject and isinstance(modelRel.toModelObject,ModelFormula): consisAsser = modelRel.fromModelObject asserTests[consisAsser.id] = (consisAsser.countSatisfied, consisAsser.countNotSatisfied) if formulaOptions.traceAssertionResultCounts: val.modelXbrl.error( _("Consistency Assertion {0} evaluations : {1} satisfied, {2} not satisfied").format( consisAsser.id, consisAsser.countSatisfied, consisAsser.countNotSatisfied), "info", "formula:trace") if asserTests: val.modelXbrl.error( _("Assertion results {0}").format(asserTests), "asrtNoLog", asserTests) # display output instance if outputXbrlInstance: if val.modelXbrl.formulaOutputInstance: # close prior instance, usually closed by caller to validate as it may affect UI on different thread val.modelXbrl.formulaOutputInstance.close() val.modelXbrl.formulaOutputInstance = outputXbrlInstance