def create(self, newDocumentType=None, url=None, schemaRefs=None, createModelDocument=True): self.modelXbrl = ModelXbrl.create(self, newDocumentType, url, schemaRefs, createModelDocument) self.loadedModelXbrls.append(self.modelXbrl) return self.modelXbrl
def saveTargetDocument(self): targetUrl = self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl( self.targetDocumentPreferredFilename, self.filepath) targetUrlParts = targetUrl.rpartition(".") targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2] self.modelXbrl.modelManager.showStatus( _("Extracting instance ") + os.path.basename(targetUrl)) targetInstance = ModelXbrl.create( self.modelXbrl.modelManager, newDocumentType=Type.INSTANCE, url=targetUrl, schemaRefs=self.targetDocumentSchemaRefs, isEntry=True) ValidateXbrlDimensions.loadDimensionDefaults( targetInstance) # need dimension defaults for context in self.modelXbrl.contexts.values(): newCntx = targetInstance.createContext( context.entityIdentifier[0], context.entityIdentifier[1], 'instant' if context.isInstantPeriod else 'duration' if context.isStartEndPeriod else 'forever', context.startDatetime, context.endDatetime, None, context.qnameDims, [], [], id=context.id) for unit in self.modelXbrl.units.values(): measures = unit.measures newUnit = targetInstance.createUnit(measures[0], measures[1], id=unit.id) self.modelXbrl.modelManager.showStatus( _("Creating and validating facts")) for fact in self.modelXbrl.facts: if fact.isItem: attrs = [("contextRef", fact.contextID)] if fact.isNumeric: attrs.append(("unitRef", fact.unitID)) if fact.get("decimals"): attrs.append(("decimals", fact.get("decimals"))) if fact.get("precision"): attrs.append(("precision", fact.get("precision"))) if fact.isNil: attrs.append( ("{http://www.w3.org/2001/XMLSchema-instance}nil", "true")) text = None else: text = fact.xValue if fact.xValid else fact.elementText newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text) targetInstance.saveInstance(overrideFilepath=targetUrl) self.modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def create(self, newDocumentType=None, url=None, schemaRefs=None, createModelDocument=True, isEntry=False, errorCaptureLevel=None, initialXml=None, base=None): self.modelXbrl = ModelXbrl.create(self, newDocumentType, url, schemaRefs, createModelDocument, isEntry, errorCaptureLevel, initialXml, base) self.loadedModelXbrls.append(self.modelXbrl) return self.modelXbrl
def __init__(self, cntlr, isCmdLine=False): self.cntlr = cntlr # setup tester xml = "<rootElement/>" self.modelXbrl = ModelXbrl.create(cntlr.modelManager, ModelDocument.Type.UnknownNonXML, initialXml=xml, isEntry=True) self.validator = ValidateXbrl.ValidateXbrl(self.modelXbrl) self.validator.validate(self.modelXbrl) # required to set up cntlr.showStatus(_("Initializing Formula Grammar")) XPathParser.initializeParser(cntlr.modelManager) cntlr.showStatus(None) self.trRegs = sorted(ixtNamespaces.keys()) self.trPrefixNSs = dict((qn.prefix, qn.namespaceURI) for qn in self.modelXbrl.modelManager.customTransforms.keys()) self.trRegs.extend(sorted(self.trPrefixNSs.keys())) self.trPrefixNSs.update(ixtNamespaces)
def saveTargetDocument(self): targetUrl = self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(self.targetDocumentPreferredFilename, self.filepath) targetUrlParts = targetUrl.rpartition(".") targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2] self.modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl)) targetInstance = ModelXbrl.create(self.modelXbrl.modelManager, newDocumentType=Type.INSTANCE, url=targetUrl, schemaRefs=self.targetDocumentSchemaRefs, isEntry=True) ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults for context in self.modelXbrl.contexts.values(): newCntx = targetInstance.createContext(context.entityIdentifier[0], context.entityIdentifier[1], 'instant' if context.isInstantPeriod else 'duration' if context.isStartEndPeriod else 'forever', context.startDatetime, context.endDatetime, None, context.qnameDims, [], [], id=context.id) for unit in self.modelXbrl.units.values(): measures = unit.measures newUnit = targetInstance.createUnit(measures[0], measures[1], id=unit.id) self.modelXbrl.modelManager.showStatus(_("Creating and validating facts")) for fact in self.modelXbrl.facts: if fact.isItem: attrs = [("contextRef", fact.contextID)] if fact.isNumeric: attrs.append(("unitRef", fact.unitID)) if fact.get("decimals"): attrs.append(("decimals", fact.get("decimals"))) if fact.get("precision"): attrs.append(("precision", fact.get("precision"))) if fact.isNil: attrs.append(("{http://www.w3.org/2001/XMLSchema-instance}nil","true")) text = None else: text = fact.xValue if fact.xValid else fact.elementText newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text) targetInstance.saveInstance(overrideFilepath=targetUrl) self.modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs, outputZip=None, filingFiles=None, *args, **kwargs): targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl( targetDocumentFilename, modelXbrl.modelDocument.filepath) def addLocallyReferencedFile(elt, filingFiles): if elt.tag in ("a", "img"): for attrTag, attrValue in elt.items(): if attrTag in ("href", "src") and not isHttpUrl( attrValue) and not os.path.isabs(attrValue): attrValue = attrValue.partition('#')[0] # remove anchor if attrValue: # ignore anchor references to base document attrValue = os.path.normpath( attrValue ) # change url path separators to host separators file = os.path.join(sourceDir, attrValue) if modelXbrl.fileSource.isInArchive( file, checkExistence=True) or os.path.exists(file): filingFiles.add(file) targetUrlParts = targetUrl.rpartition(".") targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2] modelXbrl.modelManager.showStatus( _("Extracting instance ") + os.path.basename(targetUrl)) rootElt = modelXbrl.modelDocument.xmlRootElement # take baseXmlLang from <html> or <base> baseXmlLang = rootElt.get( "{http://www.w3.org/XML/1998/namespace}lang") or rootElt.get("lang") for ixElt in modelXbrl.modelDocument.xmlRootElement.iterdescendants( tag="{http://www.w3.org/1999/xhtml}body"): baseXmlLang = ixElt.get("{http://www.w3.org/XML/1998/namespace}lang" ) or rootElt.get("lang") or baseXmlLang targetInstance = ModelXbrl.create( modelXbrl.modelManager, newDocumentType=Type.INSTANCE, url=targetUrl, schemaRefs=targetDocumentSchemaRefs, isEntry=True, discover=False) # don't attempt to load DTS if baseXmlLang: targetInstance.modelDocument.xmlRootElement.set( "{http://www.w3.org/XML/1998/namespace}lang", baseXmlLang) ValidateXbrlDimensions.loadDimensionDefaults( targetInstance) # need dimension defaults # roleRef and arcroleRef (of each inline document) for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs): for roleRefElt in sourceRefs.values(): addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname, attributes=roleRefElt.items()) # contexts for context in sorted(modelXbrl.contexts.values(), key=lambda c: elementChildSequence(c)): ignore = targetInstance.createContext( context.entityIdentifier[0], context.entityIdentifier[1], 'instant' if context.isInstantPeriod else 'duration' if context.isStartEndPeriod else 'forever', context.startDatetime, context.endDatetime, None, context.qnameDims, [], [], id=context.id) for unit in modelXbrl.units.values(): measures = unit.measures ignore = targetInstance.createUnit(measures[0], measures[1], id=unit.id) modelXbrl.modelManager.showStatus(_("Creating and validating facts")) newFactForOldObjId = {} def createFacts(facts, parent): for fact in facts: if fact.isItem: # HF does not de-duplicate, which is currently-desired behavior attrs = {"contextRef": fact.contextID} if fact.id: attrs["id"] = fact.id if fact.isNumeric: attrs["unitRef"] = fact.unitID if fact.get("decimals"): attrs["decimals"] = fact.get("decimals") if fact.get("precision"): attrs["precision"] = fact.get("precision") if fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact.xValue if fact.xValid else fact.textValue if fact.concept is not None and fact.concept.baseXsdType in ( "string", "normalizedString"): # default xmlLang = fact.xmlLang if xmlLang is not None and xmlLang != baseXmlLang: attrs[ "{http://www.w3.org/XML/1998/namespace}lang"] = xmlLang newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent) # if fact.isFraction, create numerator and denominator newFactForOldObjId[fact.objectIndex] = newFact if filingFiles is not None and fact.concept is not None and fact.concept.isTextBlock: # check for img and other filing references so that referenced files are included in the zip. for xmltext in [text] + CDATApattern.findall(text): try: for elt in XML("<body>\n{0}\n</body>\n".format( xmltext)).iter(): addLocallyReferencedFile(elt, filingFiles) except (XMLSyntaxError, UnicodeDecodeError): pass # TODO: Why ignore UnicodeDecodeError? elif fact.isTuple: newTuple = targetInstance.createFact(fact.qname, parent=parent) newFactForOldObjId[fact.objectIndex] = newTuple createFacts(fact.modelTupleFacts, newTuple) createFacts(modelXbrl.facts, None) modelXbrl.modelManager.showStatus( _("Creating and validating footnotes and relationships")) HREF = "{http://www.w3.org/1999/xlink}href" footnoteLinks = defaultdict(list) footnoteIdCount = {} for linkKey, linkPrototypes in modelXbrl.baseSets.items(): arcrole, linkrole, linkqname, arcqname = linkKey if (linkrole and linkqname and arcqname and # fully specified roles arcrole != "XBRL-footnotes" and any( lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)): for linkPrototype in linkPrototypes: if linkPrototype not in footnoteLinks[linkrole]: footnoteLinks[linkrole].append(linkPrototype) for linkrole in sorted(footnoteLinks.keys()): for linkPrototype in footnoteLinks[linkrole]: newLink = addChild(targetInstance.modelDocument.xmlRootElement, linkPrototype.qname, attributes=linkPrototype.attributes) for linkChild in linkPrototype: attributes = linkChild.attributes if isinstance(linkChild, LocPrototype): if HREF not in linkChild.attributes: linkChild.attributes[HREF] = \ "#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex]) addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ArcPrototype): addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ModelInlineFootnote): idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1 if idUseCount > 1: # if footnote with id in other links bump the id number attributes = linkChild.attributes.copy() attributes["id"] = "{}_{}".format( attributes["id"], idUseCount) footnoteIdCount[linkChild.footnoteID] = idUseCount newChild = addChild(newLink, linkChild.qname, attributes=attributes) xmlLang = linkChild.xmlLang if xmlLang is not None and xmlLang != baseXmlLang: # default newChild.set( "{http://www.w3.org/XML/1998/namespace}lang", xmlLang) copyIxFootnoteHtml( linkChild, newChild, targetModelDocument=targetInstance.modelDocument, withText=True) if filingFiles and linkChild.textValue: footnoteHtml = XML("<body/>") copyIxFootnoteHtml(linkChild, footnoteHtml) for elt in footnoteHtml.iter(): addLocallyReferencedFile(elt, filingFiles) targetInstance.saveInstance(overrideFilepath=targetUrl, outputZip=outputZip) if getattr(modelXbrl, "isTestcaseVariation", False): modelXbrl.extractedInlineInstance = True # for validation comparison modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def runFromExcel(self, options): #testGenFileName = options.excelfilename testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation-index.xls" testGenDir = os.path.dirname(testGenFileName) schemaDir = os.path.dirname(testGenDir) + os.sep + "schema" timeNow = XmlUtil.dateunionValue(datetime.datetime.now()) if options.testfiledate: today = options.testfiledate else: today = XmlUtil.dateunionValue(datetime.date.today()) startedAt = time.time() LogHandler(self) # start logger self.logMessages = [] logMessagesFile = testGenDir + os.sep + 'log-generation-messages.txt' modelTestcases = ModelXbrl.create(self.modelManager, url=testGenFileName, isEntry=True) testcaseIndexBook = xlrd.open_workbook(testGenFileName) testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0) self.addToLog(_("[info] xls loaded in {0:.2} secs at {1}").format(time.time() - startedAt, timeNow)) # start index file indexFiles = [testGenDir + os.sep + 'creation-testcases-index.xml', testGenDir + os.sep + 'consumption-testcases-index.xml'] indexDocs = [] testcasesElements = [] for purpose in ("Creation","Consumption"): file = io.StringIO( #'<?xml version="1.0" encoding="UTF-8"?>' '<!-- XBRL Versioning 1.0 {0} Tests -->' '<!-- Copyright 2011 XBRL International. All Rights Reserved. -->' '<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>' '<testcases name="XBRL Versioning 1.0 {0} Tests" ' ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' ' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">' '</testcases>'.format(purpose, today) ) doc = etree.parse(file) file.close() indexDocs.append(doc) testcasesElements.append(doc.getroot()) priorTestcasesDir = None testcaseFiles = None testcaseDocs = None for iRow in range(1, testcaseIndexSheet.nrows): try: row = testcaseIndexSheet.row(iRow) if (row[0].ctype == xlrd.XL_CELL_EMPTY or # must have directory row[1].ctype == xlrd.XL_CELL_EMPTY or # from row[2].ctype == xlrd.XL_CELL_EMPTY): # to continue testDir = row[0].value uriFrom = row[1].value uriTo = row[2].value overrideReport = row[3].value description = row[4].value if description is None or len(description) == 0: continue # test not ready to run assignment = row[5].value expectedEvents = row[6].value # comma space separated if multiple note = row[7].value useCase = row[8].value base = os.path.join(os.path.dirname(testGenFileName),testDir) + os.sep self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom)) if uriFrom and uriTo and assignment.lower() not in ("n.a.", "error") and expectedEvents != "N.A.": modelDTSfrom = modelDTSto = None for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"), True), (uriTo, _("loading to DTS"), False)): if ',' not in URIs: modelDTS = ModelXbrl.load(self.modelManager, URIs, msg, base=base) else: modelDTS = ModelXbrl.create(self.modelManager, ModelDocument.Type.DTSENTRIES, self.webCache.normalizeUrl(URIs.replace(", ","_") + ".dts", base), isEntry=True) DTSdoc = modelDTS.modelDocument DTSdoc.inDTS = True for uri in URIs.split(','): doc = ModelDocument.load(modelDTS, uri.strip(), base=base) if doc is not None: DTSdoc.referencesDocument[doc] = "import" #fake import doc.inDTS = True if isFrom: modelDTSfrom = modelDTS else: modelDTSto = modelDTS if modelDTSfrom is not None and modelDTSto is not None: # generate differences report reportUri = uriFrom.partition(',')[0] # first file reportDir = os.path.dirname(reportUri) if reportDir: reportDir += os.sep reportName = os.path.basename(reportUri).replace("from.xsd","report.xml") reportFile = reportDir + "out" + os.sep + reportName #reportFile = reportDir + "report" + os.sep + reportName reportFullPath = self.webCache.normalizeUrl( reportFile, base) testcasesDir = os.path.dirname(os.path.dirname(reportFullPath)) if testcasesDir != priorTestcasesDir: # close prior report if priorTestcasesDir: for i,testcaseFile in enumerate(testcaseFiles): with open(testcaseFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8") testcaseName = os.path.basename(testcasesDir) testcaseFiles = [testcasesDir + os.sep + testcaseName + "-creation-testcase.xml", testcasesDir + os.sep + testcaseName + "-consumption-testcase.xml"] for i,testcaseFile in enumerate(testcaseFiles): etree.SubElement(testcasesElements[i], "testcase", attrib={"uri": testcaseFile[len(testGenDir)+1:].replace("\\","/")} ) # start testcase file testcaseDocs = [] testcaseElements = [] testcaseNumber = testcaseName[0:4] if testcaseNumber.isnumeric(): testcaseNumberElement = "<number>{0}</number>".format(testcaseNumber) testcaseName = testcaseName[5:] else: testcaseNumberElement = "" testDirSegments = testDir.split('/') if len(testDirSegments) >= 2 and '-' in testDirSegments[1]: testedModule = testDirSegments[1][testDirSegments[1].index('-') + 1:] else: testedModule = '' for purpose in ("Creation","Consumption"): file = io.StringIO( #'<?xml version="1.0" encoding="UTF-8"?>' '<!-- Copyright 2011 XBRL International. All Rights Reserved. -->' '<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>' '<testcase ' ' xmlns="http://xbrl.org/2008/conformance"' ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' ' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">' '<creator>' '<name>Roland Hommes</name>' '<email>[email protected]</email>' '</creator>' '{0}' '<name>{1}</name>' # '<description>{0}</description>' '<reference>' '{2}' '{3}' '</reference>' '</testcase>'.format(testcaseNumberElement, testcaseName, '<name>{0}</name>'.format(testedModule) if testedModule else '', '<id>{0}</id>'.format(useCase) if useCase else '') ) doc = etree.parse(file) file.close() testcaseDocs.append(doc) testcaseElements.append(doc.getroot()) priorTestcasesDir = testcasesDir variationSeq = 1 try: os.makedirs(os.path.dirname(reportFullPath)) except WindowsError: pass # dir already exists modelVersReport = ModelVersReport.ModelVersReport(modelTestcases) modelVersReport.diffDTSes(reportFullPath,modelDTSfrom, modelDTSto, assignment=assignment, schemaDir=schemaDir) # check for expected elements if expectedEvents: for expectedEvent in expectedEvents.split(","): if expectedEvent not in ("No change", "N.A."): prefix, sep, localName = expectedEvent.partition(':') if sep and len(modelVersReport.xmlDocument.findall( '//{{{0}}}{1}'.format( XbrlConst.verPrefixNS.get(prefix), localName))) == 0: modelTestcases.warning("warning", "Generated test case %(reportName)s missing expected event %(event)s", reportName=reportName, event=expectedEvent) modelVersReport.close() uriFromParts = uriFrom.split('_') if len(uriFromParts) >= 2: variationId = uriFromParts[1] else: variationId = "_{0:02n}".format(variationSeq) for i,testcaseElt in enumerate(testcaseElements): variationElement = etree.SubElement(testcaseElt, "{http://xbrl.org/2008/conformance}variation", attrib={"id": variationId}) nameElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}description") nameElement.text = description ''' (removed per RH 2011/10/04 if note: paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}description") paramElement.text = "Note: " + note if useCase: paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}reference") paramElement.set("specification", "versioning-requirements") paramElement.set("useCase", useCase) ''' dataElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}data") if i == 0: # result is report if expectedEvents: paramElement = etree.SubElement(dataElement, "{http://xbrl.org/2008/conformance}parameter", attrib={"name":"expectedEvent", "value":expectedEvents.replace(',',' ')}, nsmap={"conf":"http://xbrl.org/2008/conformance", None:""}) if assignment: paramElement = etree.SubElement(dataElement, "{http://xbrl.org/2008/conformance}parameter", attrib={"name":"assignment", "value":assignment}, nsmap={"conf":"http://xbrl.org/2008/conformance", None:""}) for schemaURIs, dtsAttr in ((uriFrom,"from"), (uriTo,"to")): for schemaURI in schemaURIs.split(","): schemaElement = etree.SubElement(dataElement, "{http://xbrl.org/2008/conformance}schema") schemaElement.set("dts",dtsAttr) if i == 0: schemaElement.set("readMeFirst","true") schemaElement.text=os.path.basename(schemaURI.strip()) resultElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}result") reportElement = etree.SubElement(resultElement if i == 0 else dataElement, "{http://xbrl.org/2008/conformance}versioningReport") if i == 1: reportElement.set("readMeFirst","true") reportElement.text = "report/" + reportName variationSeq += 1 except Exception as err: modelTestcases.error("exception", _("Exception: %(error)s, Excel row: %(excelRow)s"), error=err, excelRow=iRow, exc_info=True) # add tests-error-code index files to consumption for testcaseFile in self.testcaseFiles(testGenDir + os.sep + "tests-error-code"): etree.SubElement(testcasesElements[1], "testcase", attrib={"uri": testcaseFile[len(testGenDir)+1:].replace("\\","/")} ) with open(logMessagesFile, "w") as fh: fh.writelines(self.logMessages) if priorTestcasesDir: for i,testcaseFile in enumerate(testcaseFiles): with open(testcaseFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8") for i,indexFile in enumerate(indexFiles): with open(indexFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def createTargetInstance(modelXbrl, targetUrl, targetDocumentSchemaRefs, filingFiles, baseXmlLang=None, defaultXmlLang=None): targetInstance = ModelXbrl.create(modelXbrl.modelManager, newDocumentType=Type.INSTANCE, url=targetUrl, schemaRefs=targetDocumentSchemaRefs, isEntry=True, discover=False) # don't attempt to load DTS if baseXmlLang: targetInstance.modelDocument.xmlRootElement.set("{http://www.w3.org/XML/1998/namespace}lang", baseXmlLang) if defaultXmlLang is None: defaultXmlLang = baseXmlLang # allows facts/footnotes to override baseXmlLang ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults # roleRef and arcroleRef (of each inline document) for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs): for roleRefElt in sourceRefs.values(): addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname, attributes=roleRefElt.items()) # contexts for context in sorted(modelXbrl.contexts.values(), key=lambda c: c.objectIndex): # contexts may come from multiple IXDS files ignore = targetInstance.createContext(context.entityIdentifier[0], context.entityIdentifier[1], 'instant' if context.isInstantPeriod else 'duration' if context.isStartEndPeriod else 'forever', context.startDatetime, context.endDatetime, None, context.qnameDims, [], [], id=context.id) for unit in sorted(modelXbrl.units.values(), key=lambda u: u.objectIndex): # units may come from multiple IXDS files measures = unit.measures ignore = targetInstance.createUnit(measures[0], measures[1], id=unit.id) modelXbrl.modelManager.showStatus(_("Creating and validating facts")) newFactForOldObjId = {} def createFacts(facts, parent): for fact in facts: if fact.isItem: # HF does not de-duplicate, which is currently-desired behavior attrs = {"contextRef": fact.contextID} if fact.id: attrs["id"] = fact.id if fact.isNumeric: attrs["unitRef"] = fact.unitID if fact.get("decimals"): attrs["decimals"] = fact.get("decimals") if fact.get("precision"): attrs["precision"] = fact.get("precision") if fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact.xValue if fact.xValid else fact.textValue if fact.concept is not None and fact.concept.baseXsdType in ("string", "normalizedString"): # default xmlLang = fact.xmlLang if xmlLang is not None and xmlLang != defaultXmlLang: attrs["{http://www.w3.org/XML/1998/namespace}lang"] = xmlLang newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent) # if fact.isFraction, create numerator and denominator newFactForOldObjId[fact.objectIndex] = newFact if filingFiles is not None and fact.concept is not None and fact.concept.isTextBlock: # check for img and other filing references so that referenced files are included in the zip. for xmltext in [text] + CDATApattern.findall(text): try: for elt in XML("<body>\n{0}\n</body>\n".format(xmltext)).iter(): addLocallyReferencedFile(elt, filingFiles) except (XMLSyntaxError, UnicodeDecodeError): pass # TODO: Why ignore UnicodeDecodeError? elif fact.isTuple: newTuple = targetInstance.createFact(fact.qname, parent=parent) newFactForOldObjId[fact.objectIndex] = newTuple createFacts(fact.modelTupleFacts, newTuple) createFacts(modelXbrl.facts, None) modelXbrl.modelManager.showStatus(_("Creating and validating footnotes and relationships")) HREF = "{http://www.w3.org/1999/xlink}href" footnoteLinks = defaultdict(list) footnoteIdCount = {} for linkKey, linkPrototypes in modelXbrl.baseSets.items(): arcrole, linkrole, linkqname, arcqname = linkKey if (linkrole and linkqname and arcqname and # fully specified roles arcrole != "XBRL-footnotes" and any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)): for linkPrototype in linkPrototypes: if linkPrototype not in footnoteLinks[linkrole]: footnoteLinks[linkrole].append(linkPrototype) for linkrole in sorted(footnoteLinks.keys()): for linkPrototype in footnoteLinks[linkrole]: newLink = addChild(targetInstance.modelDocument.xmlRootElement, linkPrototype.qname, attributes=linkPrototype.attributes) for linkChild in linkPrototype: attributes = linkChild.attributes if isinstance(linkChild, LocPrototype): if HREF not in linkChild.attributes: linkChild.attributes[HREF] = \ "#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex]) addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ArcPrototype): addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ModelInlineFootnote): idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1 if idUseCount > 1: # if footnote with id in other links bump the id number attributes = linkChild.attributes.copy() attributes["id"] = "{}_{}".format(attributes["id"], idUseCount) footnoteIdCount[linkChild.footnoteID] = idUseCount newChild = addChild(newLink, linkChild.qname, attributes=attributes) xmlLang = linkChild.xmlLang if xmlLang is not None and xmlLang != defaultXmlLang: # default newChild.set("{http://www.w3.org/XML/1998/namespace}lang", xmlLang) copyIxFootnoteHtml(linkChild, newChild, targetModelDocument=targetInstance.modelDocument, withText=True) if filingFiles and linkChild.textValue: footnoteHtml = XML("<body/>") copyIxFootnoteHtml(linkChild, footnoteHtml) for elt in footnoteHtml.iter(): addLocallyReferencedFile(elt,filingFiles) return targetInstance
def validateTestcase(self, testcase): self.modelXbrl.info("info", "Testcase", modelDocument=testcase) self.modelXbrl.viewModelObject(testcase.objectId()) if hasattr(testcase, "testcaseVariations"): for modelTestcaseVariation in testcase.testcaseVariations: # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) # is this a versioning report? resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance resultIsTaxonomyPackage = modelTestcaseVariation.resultIsTaxonomyPackage formulaOutputInstance = None inputDTSes = defaultdict(list) baseForElement = testcase.baseForElement(modelTestcaseVariation) # try to load instance document self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s - %(description)s"), modelObject=modelTestcaseVariation, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, expected=modelTestcaseVariation.expected, description=modelTestcaseVariation.description) errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY parameters = modelTestcaseVariation.parameters.copy() for readMeFirstUri in modelTestcaseVariation.readMeFirstUris: if isinstance(readMeFirstUri,tuple): # dtsName is for formula instances, but is from/to dts if versioning dtsName, readMeFirstUri = readMeFirstUri elif resultIsVersioningReport: if inputDTSes: dtsName = "to" else: dtsName = "from" else: dtsName = None if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document if dtsName in inputDTSes: dtsName = inputDTSes[dtsName] else: modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager, Type.DTSENTRIES, self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement), isEntry=True, errorCaptureLevel=errorCaptureLevel) DTSdoc = modelXbrl.modelDocument DTSdoc.inDTS = True doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement) if doc is not None: DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement) #fake import doc.inDTS = True elif resultIsTaxonomyPackage: from arelle import PackageManager, PrototypeInstanceObject dtsName = readMeFirstUri modelXbrl = PrototypeInstanceObject.XbrlPrototype(self.modelXbrl.modelManager, readMeFirstUri) PackageManager.packageInfo(self.modelXbrl.modelManager.cntlr, readMeFirstUri, reload=True, errors=modelXbrl.errors) else: # not a multi-schemaRef versioning report if self.useFileSource.isArchive: modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, readMeFirstUri, _("validating"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) else: # need own file source, may need instance discovery filesource = FileSource.FileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr) if filesource and not filesource.selection and filesource.isArchive: for _archiveFile in filesource.dir: # find instance document in archive filesource.select(_archiveFile) if ModelDocument.Type.identify(filesource, filesource.url) in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL): break # use this selection modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, filesource, _("validating"), base=baseForElement, errorCaptureLevel=errorCaptureLevel) modelXbrl.isTestcaseVariation = True if modelXbrl.modelDocument is None: modelXbrl.error("arelle:notLoaded", _("Testcase %(id)s %(name)s document not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) self.determineNotLoadedTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif resultIsVersioningReport or resultIsTaxonomyPackage: inputDTSes[dtsName] = modelXbrl elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl) self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif testcase.type == Type.REGISTRYTESTCASE: self.instValidator.validate(modelXbrl) # required to set up dimensions, etc self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id, modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest) self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) self.instValidator.close() modelXbrl.close() else: inputDTSes[dtsName].append(modelXbrl) # validate except for formulas _hasFormulae = modelXbrl.hasFormulae modelXbrl.hasFormulae = False try: for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"): pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation) self.instValidator.validate(modelXbrl, parameters) for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"): pluginXbrlMethod(self.modelXbrl, modelXbrl) except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase variation validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) modelXbrl.hasFormulae = _hasFormulae if resultIsVersioningReport and modelXbrl.modelDocument: versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl( modelTestcaseVariation.versioningReportUri, baseForElement) if os.path.exists(versReportFile): #validate existing modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report")) if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport) self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors) modelVersReport.close() elif len(inputDTSes) == 2: ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes( versReportFile, inputDTSes["from"], inputDTSes["to"]) modelTestcaseVariation.status = "generated" else: modelXbrl.error("arelle:notLoaded", _("Testcase %(id)s %(name)s DTSes not loaded, unable to generate versioning report: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) modelTestcaseVariation.status = "failed" for inputDTS in inputDTSes.values(): inputDTS.close() del inputDTSes # dereference elif resultIsTaxonomyPackage: self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif inputDTSes: # validate schema, linkbase, or instance modelXbrl = inputDTSes[None][0] for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters if dtsName: # named instance parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes) elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable: RenderingEvaluator.init(modelXbrl) if modelXbrl.hasFormulae: try: # validate only formulae self.instValidator.parameters = parameters ValidateFormula.validate(self.instValidator) except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset: for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"): pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri) infoset = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultInfosetUri, _("loading result infoset"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) if infoset.modelDocument is None: modelXbrl.error("arelle:notLoaded", _("Testcase %(id)s %(name)s result infoset not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstance)) modelTestcaseVariation.status = "result infoset not loadable" else: # check infoset ValidateInfoset.validate(self.instValidator, modelXbrl, infoset) infoset.close() if modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset: # diff (or generate) table infoset resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement) if not any(alternativeValidation(modelXbrl, resultTableUri) for alternativeValidation in pluginClassMethods("Validate.TableInfoset")): ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files self.instValidator.close() extraErrors = [] for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"): pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors) self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual): # if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost) self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters) self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors) if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance formulaOutputInstance = modelXbrl.formulaOutputInstance modelXbrl.formulaOutputInstance = None # prevent it from being closed now self.instValidator.close() compareIxResultInstance = getattr(modelXbrl, "extractedInlineInstance", False) and modelTestcaseVariation.resultXbrlInstanceUri if compareIxResultInstance: formulaOutputInstance = modelXbrl # compare modelXbrl to generated output instance errMsgPrefix = "ix" else: # delete input instances before formula output comparision for inputDTSlist in inputDTSes.values(): for inputDTS in inputDTSlist: inputDTS.close() del inputDTSes # dereference errMsgPrefix = "formula" if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument: expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultXbrlInstanceUri, _("loading expected result XBRL instance"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) if expectedInstance.modelDocument is None: self.modelXbrl.error("{}:expectedResultNotLoaded".format(errMsgPrefix), _("Testcase %(id)s %(name)s expected result instance not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstanceUri), messageCodes=("formula:expectedResultNotLoaded","ix:expectedResultNotLoaded")) modelTestcaseVariation.status = "result not loadable" else: # compare facts if len(expectedInstance.facts) != len(formulaOutputInstance.facts): formulaOutputInstance.error("{}:resultFactCounts".format(errMsgPrefix), _("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"), modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts), expectedFacts=len(expectedInstance.facts), messageCodes=("formula:resultFactCounts","ix:resultFactCounts")) else: formulaOutputFootnotesRelSet = ModelRelationshipSet(formulaOutputInstance, "XBRL-footnotes") expectedFootnotesRelSet = ModelRelationshipSet(expectedInstance, "XBRL-footnotes") def factFootnotes(fact, footnotesRelSet): footnotes = [] footnoteRels = footnotesRelSet.fromModelObject(fact) if footnoteRels: # most process rels in same order between two instances, use labels to sort for i, footnoteRel in enumerate(sorted(footnoteRels, key=lambda r: (r.fromLabel,r.toLabel))): modelObject = footnoteRel.toModelObject if isinstance(modelObject, ModelResource): xml = modelObject.viewText().strip() footnotes.append("Footnote {}: {}".format( i+1, # compare footnote with HTML serialized xml, #re.sub(r'\s+', ' ', collapseWhitespace(modelObject.stringValue)) )) elif isinstance(modelObject, ModelFact): footnotes.append("Footnoted fact {}: {} context: {} value: {}".format( i+1, modelObject.qname, modelObject.contextID, collapseWhitespace(modelObject.value))) return footnotes for expectedInstanceFact in expectedInstance.facts: unmatchedFactsStack = [] formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True) if formulaOutputFact is None: if unmatchedFactsStack: # get missing nested tuple fact, if possible missingFact = unmatchedFactsStack[-1] else: missingFact = expectedInstanceFact formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix), _("Output missing expected fact %(fact)s"), modelXbrl=missingFact, fact=missingFact.qname, messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing")) else: # compare footnotes expectedInstanceFactFootnotes = factFootnotes(expectedInstanceFact, expectedFootnotesRelSet) formulaOutputFactFootnotes = factFootnotes(formulaOutputFact, formulaOutputFootnotesRelSet) if expectedInstanceFactFootnotes != formulaOutputFactFootnotes: formulaOutputInstance.error("{}:expectedFactFootnoteDifference".format(errMsgPrefix), _("Output expected fact %(fact)s expected footnotes %(footnotes1)s produced footnotes %(footnotes2)s"), modelXbrl=(formulaOutputFact,expectedInstanceFact), fact=expectedInstanceFact.qname, footnotes1=expectedInstanceFactFootnotes, footnotes2=formulaOutputFactFootnotes, messageCodes=("formula:expectedFactFootnoteDifference","ix:expectedFactFootnoteDifference")) # for debugging uncomment next line to save generated instance document # formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml") expectedInstance.close() del expectedInstance # dereference self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors) formulaOutputInstance.close() del formulaOutputInstance if compareIxResultInstance: for inputDTSlist in inputDTSes.values(): for inputDTS in inputDTSlist: inputDTS.close() del inputDTSes # dereference # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def __init__(self, mainWin): parent = mainWin.parent super(DialogTransformTester, self).__init__(parent) self.mainWin = mainWin self.parent = parent parentGeometry = re.match("(\d+)x(\d+)[+]?([-]?\d+)[+]?([-]?\d+)", parent.geometry()) dialogX = int(parentGeometry.group(3)) dialogY = int(parentGeometry.group(4)) self.selectedGroup = None self.transient(self.parent) self.title(_("Transformation Tester")) frame = Frame(self) # setup tester xml = "<rootElement/>" self.modelXbrl = ModelXbrl.create(mainWin.modelManager, ModelDocument.Type.UnknownNonXML, initialXml=xml, isEntry=True) self.validator = ValidateXbrl.ValidateXbrl(self.modelXbrl) self.validator.validate(self.modelXbrl) # required to set up mainWin.showStatus(_("Initializing Formula Grammar")) XPathParser.initializeParser(mainWin.modelManager) mainWin.showStatus(None) self.trRegs = sorted(ixtNamespaces.keys()) self.trReg = self.trRegs[-1] # default is latest self.trPrefixNSs = dict( (qn.prefix, qn.namespaceURI) for qn in self.modelXbrl.modelManager.customTransforms.keys()) self.trRegs.extend(sorted(self.trPrefixNSs.keys())) self.trPrefixNSs.update(ixtNamespaces) self.trNames = self.getTrNames() # load grid trRegLabel = label(frame, 0, 0, _("Registry:")) self.trRegName = gridCombobox( frame, 1, 0, value=self.trReg, values=self.trRegs, comboboxselected=self.trRegComboBoxSelected) trRegToolTipMessage = _("Select Transformation Registry") ToolTip(self.trRegName, text=trRegToolTipMessage, wraplength=360) ToolTip(trRegLabel, text=trRegToolTipMessage, wraplength=360) trNameLabel = label(frame, 0, 1, _("Transform:")) self.trNameName = gridCombobox( frame, 1, 1, value="", values=self.trNames, comboboxselected=self.trNameComboBoxSelected) trRegToolTipMessage = _("Select or enter transform") ToolTip(self.trRegName, text=trRegToolTipMessage, wraplength=360) ToolTip(trRegLabel, text=trRegToolTipMessage, wraplength=360) sourceLabel = label(frame, 0, 2, _("Source text:")) ToolTip(sourceLabel, text=_("Enter the source text which is to be transformed. "), wraplength=240) self.sourceVar = StringVar() self.sourceVar.set("") sourceEntry = Entry(frame, textvariable=self.sourceVar, width=50) sourceLabel.grid(row=2, column=0, sticky=W) sourceEntry.grid(row=2, column=1, sticky=EW, pady=3, padx=3) resultLabel = label(frame, 1, 3, _("Result:")) ToolTip(sourceLabel, text=_("Transformation result. "), wraplength=240) self.resultVar = StringVar() self.resultVar.set("") resultEntry = Entry(frame, textvariable=self.resultVar, width=50) resultLabel.grid(row=3, column=0, sticky=W) resultEntry.grid(row=3, column=1, sticky=EW, pady=3, padx=3) mainWin.showStatus(None) btnPad = 2 if mainWin.isMSW else 0 # buttons too narrow on windows okButton = Button(frame, text=_("Transform"), width=8 + btnPad, command=self.ok) cancelButton = Button(frame, text=_("Done"), width=4 + btnPad, command=self.close) cancelButton.grid(row=4, column=0, sticky=E, columnspan=2, pady=3, padx=3) okButton.grid(row=4, column=0, sticky=E, columnspan=2, pady=3, padx=64) ToolTip(okButton, text=_("Transform the source entered. "), wraplength=240) ToolTip(cancelButton, text=_("Close this dialog. "), wraplength=240) frame.grid(row=0, column=0, sticky=(N, S, E, W)) frame.columnconfigure(1, weight=3) frame.columnconfigure(2, weight=1) window = self.winfo_toplevel() window.columnconfigure(0, weight=1) self.geometry("+{0}+{1}".format(dialogX + 150, dialogY + 100)) #self.bind("<Return>", self.ok) #self.bind("<Escape>", self.close) self.protocol("WM_DELETE_WINDOW", self.close) self.grab_set() self.wait_window(self)
def runFromExcel(self, options): #testGenFileName = options.excelfilename testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation-index.xls" testGenDir = os.path.dirname(testGenFileName) schemaDir = os.path.dirname(testGenDir) + os.sep + "schema" timeNow = XmlUtil.dateunionValue(datetime.datetime.now()) if options.testfiledate: today = options.testfiledate else: today = XmlUtil.dateunionValue(datetime.date.today()) startedAt = time.time() LogHandler(self) # start logger self.logMessages = [] logMessagesFile = testGenDir + os.sep + 'log-generation-messages.txt' modelTestcases = ModelXbrl.create(self.modelManager, url=testGenFileName, isEntry=True) testcaseIndexBook = xlrd.open_workbook(testGenFileName) testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0) self.addToLog( _("[info] xls loaded in {0:.2} secs at {1}").format( time.time() - startedAt, timeNow)) # start index file indexFiles = [ testGenDir + os.sep + 'creation-testcases-index.xml', testGenDir + os.sep + 'consumption-testcases-index.xml' ] indexDocs = [] testcasesElements = [] for purpose in ("Creation", "Consumption"): file = io.StringIO( #'<?xml version="1.0" encoding="UTF-8"?>' '<!-- XBRL Versioning 1.0 {0} Tests -->' '<!-- Copyright 2011 XBRL International. All Rights Reserved. -->' '<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>' '<testcases name="XBRL Versioning 1.0 {0} Tests" ' ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' ' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">' '</testcases>'.format(purpose, today)) doc = etree.parse(file) file.close() indexDocs.append(doc) testcasesElements.append(doc.getroot()) priorTestcasesDir = None testcaseFiles = None testcaseDocs = None for iRow in range(1, testcaseIndexSheet.nrows): try: row = testcaseIndexSheet.row(iRow) if (row[0].ctype == xlrd.XL_CELL_EMPTY or # must have directory row[1].ctype == xlrd.XL_CELL_EMPTY or # from row[2].ctype == xlrd.XL_CELL_EMPTY): # to continue testDir = row[0].value uriFrom = row[1].value uriTo = row[2].value overrideReport = row[3].value description = row[4].value if description is None or len(description) == 0: continue # test not ready to run assignment = row[5].value expectedEvents = row[ 6].value # comma space separated if multiple note = row[7].value useCase = row[8].value base = os.path.join(os.path.dirname(testGenFileName), testDir) + os.sep self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom)) if uriFrom and uriTo and assignment.lower() not in ( "n.a.", "error") and expectedEvents != "N.A.": modelDTSfrom = modelDTSto = None for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"), True), (uriTo, _("loading to DTS"), False)): if ',' not in URIs: modelDTS = ModelXbrl.load(self.modelManager, URIs, msg, base=base) else: modelDTS = ModelXbrl.create( self.modelManager, ModelDocument.Type.DTSENTRIES, self.webCache.normalizeUrl( URIs.replace(", ", "_") + ".dts", base), isEntry=True) DTSdoc = modelDTS.modelDocument DTSdoc.inDTS = True for uri in URIs.split(','): doc = ModelDocument.load(modelDTS, uri.strip(), base=base) if doc is not None: DTSdoc.referencesDocument[ doc] = "import" #fake import doc.inDTS = True if isFrom: modelDTSfrom = modelDTS else: modelDTSto = modelDTS if modelDTSfrom is not None and modelDTSto is not None: # generate differences report reportUri = uriFrom.partition(',')[0] # first file reportDir = os.path.dirname(reportUri) if reportDir: reportDir += os.sep reportName = os.path.basename(reportUri).replace( "from.xsd", "report.xml") reportFile = reportDir + "out" + os.sep + reportName #reportFile = reportDir + "report" + os.sep + reportName reportFullPath = self.webCache.normalizeUrl( reportFile, base) testcasesDir = os.path.dirname( os.path.dirname(reportFullPath)) if testcasesDir != priorTestcasesDir: # close prior report if priorTestcasesDir: for i, testcaseFile in enumerate( testcaseFiles): with open(testcaseFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8") testcaseName = os.path.basename(testcasesDir) testcaseFiles = [ testcasesDir + os.sep + testcaseName + "-creation-testcase.xml", testcasesDir + os.sep + testcaseName + "-consumption-testcase.xml" ] for i, testcaseFile in enumerate(testcaseFiles): etree.SubElement( testcasesElements[i], "testcase", attrib={ "uri": testcaseFile[len(testGenDir) + 1:].replace("\\", "/") }) # start testcase file testcaseDocs = [] testcaseElements = [] testcaseNumber = testcaseName[0:4] if testcaseNumber.isnumeric(): testcaseNumberElement = "<number>{0}</number>".format( testcaseNumber) testcaseName = testcaseName[5:] else: testcaseNumberElement = "" testDirSegments = testDir.split('/') if len(testDirSegments ) >= 2 and '-' in testDirSegments[1]: testedModule = testDirSegments[1][ testDirSegments[1].index('-') + 1:] else: testedModule = '' for purpose in ("Creation", "Consumption"): file = io.StringIO( #'<?xml version="1.0" encoding="UTF-8"?>' '<!-- Copyright 2011 XBRL International. All Rights Reserved. -->' '<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>' '<testcase ' ' xmlns="http://xbrl.org/2008/conformance"' ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' ' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">' '<creator>' '<name>Roland Hommes</name>' '<email>[email protected]</email>' '</creator>' '{0}' '<name>{1}</name>' # '<description>{0}</description>' '<reference>' '{2}' '{3}' '</reference>' '</testcase>'.format( testcaseNumberElement, testcaseName, '<name>{0}</name>'.format(testedModule) if testedModule else '', '<id>{0}</id>'.format(useCase) if useCase else '')) doc = etree.parse(file) file.close() testcaseDocs.append(doc) testcaseElements.append(doc.getroot()) priorTestcasesDir = testcasesDir variationSeq = 1 try: os.makedirs(os.path.dirname(reportFullPath)) except WindowsError: pass # dir already exists modelVersReport = ModelVersReport.ModelVersReport( modelTestcases) modelVersReport.diffDTSes(reportFullPath, modelDTSfrom, modelDTSto, assignment=assignment, schemaDir=schemaDir) # check for expected elements if expectedEvents: for expectedEvent in expectedEvents.split(","): if expectedEvent not in ("No change", "N.A."): prefix, sep, localName = expectedEvent.partition( ':') if sep and len( modelVersReport.xmlDocument. findall('//{{{0}}}{1}'.format( XbrlConst.verPrefixNS.get( prefix), localName))) == 0: modelTestcases.warning( "warning", "Generated test case %(reportName)s missing expected event %(event)s", reportName=reportName, event=expectedEvent) modelVersReport.close() uriFromParts = uriFrom.split('_') if len(uriFromParts) >= 2: variationId = uriFromParts[1] else: variationId = "_{0:02n}".format(variationSeq) for i, testcaseElt in enumerate(testcaseElements): variationElement = etree.SubElement( testcaseElt, "{http://xbrl.org/2008/conformance}variation", attrib={"id": variationId}) nameElement = etree.SubElement( variationElement, "{http://xbrl.org/2008/conformance}description" ) nameElement.text = description ''' (removed per RH 2011/10/04 if note: paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}description") paramElement.text = "Note: " + note if useCase: paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}reference") paramElement.set("specification", "versioning-requirements") paramElement.set("useCase", useCase) ''' dataElement = etree.SubElement( variationElement, "{http://xbrl.org/2008/conformance}data") if i == 0: # result is report if expectedEvents: paramElement = etree.SubElement( dataElement, "{http://xbrl.org/2008/conformance}parameter", attrib={ "name": "expectedEvent", "value": expectedEvents.replace(',', ' ') }, nsmap={ "conf": "http://xbrl.org/2008/conformance", None: "" }) if assignment: paramElement = etree.SubElement( dataElement, "{http://xbrl.org/2008/conformance}parameter", attrib={ "name": "assignment", "value": assignment }, nsmap={ "conf": "http://xbrl.org/2008/conformance", None: "" }) for schemaURIs, dtsAttr in ((uriFrom, "from"), (uriTo, "to")): for schemaURI in schemaURIs.split(","): schemaElement = etree.SubElement( dataElement, "{http://xbrl.org/2008/conformance}schema" ) schemaElement.set("dts", dtsAttr) if i == 0: schemaElement.set( "readMeFirst", "true") schemaElement.text = os.path.basename( schemaURI.strip()) resultElement = etree.SubElement( variationElement, "{http://xbrl.org/2008/conformance}result") reportElement = etree.SubElement( resultElement if i == 0 else dataElement, "{http://xbrl.org/2008/conformance}versioningReport" ) if i == 1: reportElement.set("readMeFirst", "true") reportElement.text = "report/" + reportName variationSeq += 1 except Exception as err: modelTestcases.error( "exception", _("Exception: %(error)s, Excel row: %(excelRow)s"), error=err, excelRow=iRow, exc_info=True) # add tests-error-code index files to consumption for testcaseFile in self.testcaseFiles(testGenDir + os.sep + "tests-error-code"): etree.SubElement(testcasesElements[1], "testcase", attrib={ "uri": testcaseFile[len(testGenDir) + 1:].replace( "\\", "/") }) with open(logMessagesFile, "w") as fh: fh.writelines(self.logMessages) if priorTestcasesDir: for i, testcaseFile in enumerate(testcaseFiles): with open(testcaseFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8") for i, indexFile in enumerate(indexFiles): with open(indexFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def validateTestcase(self, testcase): self.modelXbrl.info("info", "Testcase", modelDocument=testcase) self.modelXbrl.viewModelObject(testcase.objectId()) if testcase.type in (Type.TESTCASESINDEX, Type.REGISTRY): for doc in sorted(testcase.referencesDocument.keys(), key=lambda doc: doc.uri): self.validateTestcase(doc) # testcases doc's are sorted by their uri (file names), e.g., for formula elif hasattr(testcase, "testcaseVariations"): for modelTestcaseVariation in testcase.testcaseVariations: # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) # is this a versioning report? resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance resultIsTaxonomyPackage = modelTestcaseVariation.resultIsTaxonomyPackage formulaOutputInstance = None inputDTSes = defaultdict(list) baseForElement = testcase.baseForElement(modelTestcaseVariation) # try to load instance document self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s - %(description)s"), modelObject=modelTestcaseVariation, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, expected=modelTestcaseVariation.expected, description=modelTestcaseVariation.description) if self.modelXbrl.modelManager.formulaOptions.testcaseResultsCaptureWarnings: errorCaptureLevel = logging._checkLevel("WARNING") else: errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY parameters = modelTestcaseVariation.parameters.copy() for readMeFirstUri in modelTestcaseVariation.readMeFirstUris: if isinstance(readMeFirstUri,tuple): # dtsName is for formula instances, but is from/to dts if versioning dtsName, readMeFirstUri = readMeFirstUri elif resultIsVersioningReport: if inputDTSes: dtsName = "to" else: dtsName = "from" else: dtsName = None if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document if dtsName in inputDTSes: dtsName = inputDTSes[dtsName] else: modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager, Type.DTSENTRIES, self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement), isEntry=True, errorCaptureLevel=errorCaptureLevel) DTSdoc = modelXbrl.modelDocument DTSdoc.inDTS = True doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement) if doc is not None: DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement) #fake import doc.inDTS = True elif resultIsTaxonomyPackage: from arelle import PackageManager, PrototypeInstanceObject dtsName = readMeFirstUri modelXbrl = PrototypeInstanceObject.XbrlPrototype(self.modelXbrl.modelManager, readMeFirstUri) PackageManager.packageInfo(self.modelXbrl.modelManager.cntlr, readMeFirstUri, reload=True, errors=modelXbrl.errors) else: # not a multi-schemaRef versioning report if self.useFileSource.isArchive: modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, readMeFirstUri, _("validating"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) else: # need own file source, may need instance discovery filesource = FileSource.FileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr) if filesource and not filesource.selection and filesource.isArchive: for _archiveFile in filesource.dir or (): # find instance document in archive filesource.select(_archiveFile) if ModelDocument.Type.identify(filesource, filesource.url) in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL): break # use this selection modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, filesource, _("validating"), base=baseForElement, errorCaptureLevel=errorCaptureLevel) modelXbrl.isTestcaseVariation = True if modelXbrl.modelDocument is None: modelXbrl.error("arelle:notLoaded", _("Variation %(id)s %(name)s readMeFirst document not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) self.determineNotLoadedTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif resultIsVersioningReport or resultIsTaxonomyPackage: inputDTSes[dtsName] = modelXbrl elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl) self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif testcase.type == Type.REGISTRYTESTCASE: self.instValidator.validate(modelXbrl) # required to set up dimensions, etc self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id, modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest) self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) self.instValidator.close() modelXbrl.close() else: inputDTSes[dtsName].append(modelXbrl) # validate except for formulas _hasFormulae = modelXbrl.hasFormulae modelXbrl.hasFormulae = False try: for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"): pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation) self.instValidator.validate(modelXbrl, parameters) for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"): pluginXbrlMethod(self.modelXbrl, modelXbrl) except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase variation validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) modelXbrl.hasFormulae = _hasFormulae if resultIsVersioningReport and modelXbrl.modelDocument: versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl( modelTestcaseVariation.versioningReportUri, baseForElement) if os.path.exists(versReportFile): #validate existing modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report")) if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport) self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors) modelVersReport.close() elif len(inputDTSes) == 2: ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes( versReportFile, inputDTSes["from"], inputDTSes["to"]) modelTestcaseVariation.status = "generated" else: modelXbrl.error("arelle:notLoaded", _("Variation %(id)s %(name)s input DTSes not loaded, unable to generate versioning report: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) modelTestcaseVariation.status = "failed" for inputDTS in inputDTSes.values(): inputDTS.close() del inputDTSes # dereference elif resultIsTaxonomyPackage: self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif inputDTSes: # validate schema, linkbase, or instance modelXbrl = inputDTSes[None][0] expectedDataFiles = set(modelXbrl.modelManager.cntlr.webCache.normalizeUrl(uri, baseForElement) for d in modelTestcaseVariation.dataUris.values() for uri in d if not UrlUtil.isAbsolute(uri)) foundDataFiles = set() variationBase = os.path.dirname(baseForElement) for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters if dtsName: # named instance parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes) elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula for _inputDTS in inputDTS: for docUrl, doc in _inputDTS.urlDocs.items(): if docUrl.startswith(variationBase) and not doc.type == Type.INLINEXBRLDOCUMENTSET: if getattr(doc,"loadedFromXbrlFormula", False): # may have been sourced from xf file if docUrl.replace("-formula.xml", ".xf") in expectedDataFiles: docUrl = docUrl.replace("-formula.xml", ".xf") foundDataFiles.add(docUrl) if expectedDataFiles - foundDataFiles: modelXbrl.info("arelle:testcaseDataNotUsed", _("Variation %(id)s %(name)s data files not used: %(missingDataFiles)s"), modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id, missingDataFiles=", ".join(sorted(os.path.basename(f) for f in expectedDataFiles - foundDataFiles))) if foundDataFiles - expectedDataFiles: modelXbrl.info("arelle:testcaseDataUnexpected", _("Variation %(id)s %(name)s files not in variation data: %(unexpectedDataFiles)s"), modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id, unexpectedDataFiles=", ".join(sorted(os.path.basename(f) for f in foundDataFiles - expectedDataFiles))) if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable: try: RenderingEvaluator.init(modelXbrl) except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase RenderingEvaluator.init exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) modelXbrlHasFormulae = modelXbrl.hasFormulae if modelXbrlHasFormulae: try: # validate only formulae self.instValidator.parameters = parameters ValidateFormula.validate(self.instValidator) except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset: for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"): pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri) infoset = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultInfosetUri, _("loading result infoset"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) if infoset.modelDocument is None: modelXbrl.error("arelle:notLoaded", _("Variation %(id)s %(name)s result infoset not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstance)) modelTestcaseVariation.status = "result infoset not loadable" else: # check infoset ValidateInfoset.validate(self.instValidator, modelXbrl, infoset) infoset.close() if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset: # diff (or generate) table infoset resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement) if not any(alternativeValidation(modelXbrl, resultTableUri) for alternativeValidation in pluginClassMethods("Validate.TableInfoset")): try: ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase table linkbase validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) self.instValidator.close() extraErrors = [] for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"): pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors) self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual): # if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost) self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters) self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors) if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance formulaOutputInstance = modelXbrl.formulaOutputInstance modelXbrl.formulaOutputInstance = None # prevent it from being closed now self.instValidator.close() compareIxResultInstance = (modelXbrl.modelDocument.type in (Type.INLINEXBRL, Type.INLINEXBRLDOCUMENTSET) and modelTestcaseVariation.resultXbrlInstanceUri is not None) if compareIxResultInstance: formulaOutputInstance = modelXbrl # compare modelXbrl to generated output instance errMsgPrefix = "ix" else: # delete input instances before formula output comparision for inputDTSlist in inputDTSes.values(): for inputDTS in inputDTSlist: inputDTS.close() del inputDTSes # dereference errMsgPrefix = "formula" if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument: _matchExpectedResultIDs = not modelXbrlHasFormulae # formula restuls have inconsistent IDs expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultXbrlInstanceUri, _("loading expected result XBRL instance"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) if expectedInstance.modelDocument is None: self.modelXbrl.error("{}:expectedResultNotLoaded".format(errMsgPrefix), _("Testcase \"%(name)s\" %(id)s expected result instance not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstanceUri), messageCodes=("formula:expectedResultNotLoaded","ix:expectedResultNotLoaded")) modelTestcaseVariation.status = "result not loadable" else: # compare facts if len(expectedInstance.facts) != len(formulaOutputInstance.facts): formulaOutputInstance.error("{}:resultFactCounts".format(errMsgPrefix), _("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"), modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts), expectedFacts=len(expectedInstance.facts), messageCodes=("formula:resultFactCounts","ix:resultFactCounts")) else: formulaOutputFootnotesRelSet = ModelRelationshipSet(formulaOutputInstance, "XBRL-footnotes") expectedFootnotesRelSet = ModelRelationshipSet(expectedInstance, "XBRL-footnotes") def factFootnotes(fact, footnotesRelSet): footnotes = {} footnoteRels = footnotesRelSet.fromModelObject(fact) if footnoteRels: # most process rels in same order between two instances, use labels to sort for i, footnoteRel in enumerate(sorted(footnoteRels, key=lambda r: (r.fromLabel,r.toLabel))): modelObject = footnoteRel.toModelObject if isinstance(modelObject, ModelResource): xml = modelObject.viewText().strip() footnotes["Footnote {}".format(i+1)] = xml #re.sub(r'\s+', ' ', collapseWhitespace(modelObject.stringValue)) elif isinstance(modelObject, ModelFact): footnotes["Footnoted fact {}".format(i+1)] = \ "{} context: {} value: {}".format( modelObject.qname, modelObject.contextID, collapseWhitespace(modelObject.value)) return footnotes for expectedInstanceFact in expectedInstance.facts: unmatchedFactsStack = [] formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=_matchExpectedResultIDs, matchLang=False) #formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=True, matchLang=True) if formulaOutputFact is None: if unmatchedFactsStack: # get missing nested tuple fact, if possible missingFact = unmatchedFactsStack[-1] else: missingFact = expectedInstanceFact formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix), _("Output missing expected fact %(fact)s"), modelXbrl=missingFact, fact=missingFact.qname, messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing")) else: # compare footnotes expectedInstanceFactFootnotes = factFootnotes(expectedInstanceFact, expectedFootnotesRelSet) formulaOutputFactFootnotes = factFootnotes(formulaOutputFact, formulaOutputFootnotesRelSet) if (len(expectedInstanceFactFootnotes) != len(formulaOutputFactFootnotes) or set(expectedInstanceFactFootnotes.values()) != set(formulaOutputFactFootnotes.values())): formulaOutputInstance.error("{}:expectedFactFootnoteDifference".format(errMsgPrefix), _("Output expected fact %(fact)s expected footnotes %(footnotes1)s produced footnotes %(footnotes2)s"), modelXbrl=(formulaOutputFact,expectedInstanceFact), fact=expectedInstanceFact.qname, footnotes1=sorted(expectedInstanceFactFootnotes.items()), footnotes2=sorted(formulaOutputFactFootnotes.items()), messageCodes=("formula:expectedFactFootnoteDifference","ix:expectedFactFootnoteDifference")) # for debugging uncomment next line to save generated instance document # formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml") expectedInstance.close() del expectedInstance # dereference self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors) formulaOutputInstance.close() del formulaOutputInstance if compareIxResultInstance: for inputDTSlist in inputDTSes.values(): for inputDTS in inputDTSlist: inputDTS.close() del inputDTSes # dereference # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) _statusCounts = OrderedDict((("pass",0),("fail",0))) for tv in getattr(testcase, "testcaseVariations", ()): _statusCounts[tv.status] = _statusCounts.get(tv.status, 0) + 1 self.modelXbrl.info("arelle:testCaseResults", ", ".join("{}={}".format(k,c) for k, c in _statusCounts.items() if k)) self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def validateTestcase(self, testcase): self.modelXbrl.error(_("testcase {0}").format(os.path.basename(testcase.uri))) self.modelXbrl.viewModelObject(testcase.objectId()) if hasattr(testcase, "testcaseVariations"): for modelTestcaseVariation in testcase.testcaseVariations: # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) # is this a versioning report? resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance formulaOutputInstance = None inputDTSes = defaultdict(list) baseForElement = testcase.baseForElement(modelTestcaseVariation.element) # try to load instance document self.modelXbrl.error(_("variation {0} {1}: {2}").format(modelTestcaseVariation.id, modelTestcaseVariation.name, modelTestcaseVariation.expected)) for readMeFirstUri in modelTestcaseVariation.readMeFirstUris: if isinstance(readMeFirstUri,tuple): # dtsName is for formula instances, but is from/to dts if versioning dtsName, readMeFirstUri = readMeFirstUri elif resultIsVersioningReport: if inputDTSes: dtsName = "to" else: dtsName = "from" else: dtsName = None if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document if dtsName in inputDTSes: dtsName = inputDTSes[dtsName] else: modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager, ModelDocument.Type.DTSENTRIES, self.modelXbrl.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement), isEntry=True) DTSdoc = modelXbrl.modelDocument DTSdoc.inDTS = True doc = ModelDocument.load(modelXbrl, readMeFirstUri, base=baseForElement) DTSdoc.referencesDocument[doc] = "import" #fake import doc.inDTS = True else: # not a multi-schemaRef versioning report modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, readMeFirstUri, _("validating"), base=baseForElement) if modelXbrl.modelDocument is None: self.modelXbrl.error(_("Testcase {0} {1} document not loaded: {2}").format( modelTestcaseVariation.id, modelTestcaseVariation.name, os.path.basename(readMeFirstUri))) modelTestcaseVariation.status = "not loadable" modelXbrl.close() elif resultIsVersioningReport: inputDTSes[dtsName] = modelXbrl elif modelXbrl.modelDocument.type == ModelDocument.Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl) \ .validate(modelXbrl) self.determineTestStatus(modelTestcaseVariation, modelXbrl) modelXbrl.close() elif testcase.type == ModelDocument.Type.REGISTRYTESTCASE: self.instValidator.validate(modelXbrl) # required to set up dimensions, etc self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id, modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest) self.determineTestStatus(modelTestcaseVariation, modelXbrl) modelXbrl.close() else: inputDTSes[dtsName].append(modelXbrl) if resultIsVersioningReport and modelXbrl.modelDocument: versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl( modelTestcaseVariation.versioningReportUri, baseForElement) if os.path.exists(versReportFile): #validate existing modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report")) if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == ModelDocument.Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport) self.determineTestStatus(modelTestcaseVariation, modelVersReport) modelVersReport.close() elif len(inputDTSes) == 2: ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes( versReportFile, inputDTSes["from"], inputDTSes["to"]) modelTestcaseVariation.status = "generated" else: self.modelXbrl.error(_("Testcase {0} {1} DTSes not loaded, unable to generate versioning report").format( modelTestcaseVariation.id, modelTestcaseVariation.name, os.path.basename(readMeFirstUri))) modelTestcaseVariation.status = "failed" for inputDTS in inputDTSes: inputDTS.close() elif inputDTSes: # validate schema, linkbase, or instance modelXbrl = inputDTSes[None][0] parameters = modelTestcaseVariation.parameters.copy() for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters if dtsName: parameters[dtsName] = (None, inputDTS) self.instValidator.validate(modelXbrl, parameters) self.determineTestStatus(modelTestcaseVariation, modelXbrl) if modelXbrl.formulaOutputInstance and len(modelTestcaseVariation.actual) == 0: # if an output instance is created, validate it self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters) self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance) if len(modelTestcaseVariation.actual) == 0: # if still 'clean' pass it forward for comparison to expected result instance formulaOutputInstance = modelXbrl.formulaOutputInstance modelXbrl.formulaOutputInstance = None # prevent it from being closed now for inputDTSlist in inputDTSes.values(): for inputDTS in inputDTSlist: inputDTS.close() if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument: expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultXbrlInstanceUri, _("loading expected result XBRL instance"), base=baseForElement) if expectedInstance.modelDocument is None: self.modelXbrl.error(_("Testcase {0} {1} expected result instance not loaded: {2}").format( modelTestcaseVariation.id, modelTestcaseVariation.name, os.path.basename(modelTestcaseVariation.resultXbrlInstance))) modelTestcaseVariation.status = "result not loadable" expectedInstance.close() else: # compare facts if len(expectedInstance.facts) != len(formulaOutputInstance.facts): formulaOutputInstance.error( _("Formula output {0} facts, expected {1} facts").format( len(formulaOutputInstance.facts), len(expectedInstance.facts)), "err", "formula:resultFactCounts") else: for fact in expectedInstance.facts: if not formulaOutputInstance.matchFact(fact): formulaOutputInstance.error( _("Formula output missing expected fact {0}").format( fact), "err", "formula:expectedFactMissing") self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance) formulaOutputInstance.close() formulaOutputInstance = None # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def runFromExcel(self, options): testGenFileName = options.excelfilename #testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation\1000-2000-index.xls" testGenDir = os.path.dirname(testGenFileName) timeNow = XmlUtil.dateunionValue(datetime.datetime.now()) if options.testfiledate: today = options.testfiledate else: today = XmlUtil.dateunionValue(datetime.date.today()) startedAt = time.time() self.logMessages = [] logMessagesFile = testGenDir + os.sep + 'logGenerationMessages.txt' modelTestcases = ModelXbrl.create(self.modelManager) testcaseIndexBook = xlrd.open_workbook(testGenFileName) testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0) self.addToLog( _("[info] xls loaded in {0:.2} secs at {1}").format( time.time() - startedAt, timeNow)) # start index file indexFiles = [ testGenDir + os.sep + 'creationTestcasesIndex.xml', testGenDir + os.sep + 'consumptionTestcasesIndex.xml' ] indexDocs = [ xml.dom.minidom.parseString( '<?xml version="1.0" encoding="UTF-8"?>' '<!-- XBRL Versioning 1.0 {0} Tests -->' '<!-- Copyright 2011 XBRL International. All Rights Reserved. -->' '<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>' '<testcases name="XBRL Versioning 1.0 Consumption Tests" date="{1}" ' ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' ' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">' '</testcases>'.format(purpose, today)) for purpose in ("Creation", "Consumption") ] testcasesElements = [ XmlUtil.child(indexDoc, None, "testcases") for indexDoc in indexDocs ] priorTestcasesDir = None testcaseFiles = None testcaseDocs = None for iRow in range(1, testcaseIndexSheet.nrows): row = testcaseIndexSheet.row(iRow) if row[0].ctype == xlrd.XL_CELL_EMPTY or row[ 1].ctype == xlrd.XL_CELL_EMPTY or row[ 2].ctype == xlrd.XL_CELL_EMPTY: continue testDir = row[0].value uriFrom = row[1].value uriTo = row[2].value intention = row[3].value if intention is None or len(intention) == 0: continue # test not ready to run reason = row[4].value expectedEvent = row[5].value base = os.path.join(os.path.dirname(testGenFileName), testDir) + os.sep self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom)) if uriFrom and uriTo and reason.lower() not in ( "n.a.", "error") and expectedEvent != "N.A.": for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"), True), (uriTo, _("loading to DTS"), False)): if ',' not in URIs: modelDTS = ModelXbrl.load(self.modelManager, URIs, msg, base=base) else: modelDTS = ModelXbrl.create( self.modelManager, ModelDocument.Type.DTSENTRIES, self.webCache.normalizeUrl( URIs.replace(", ", "_") + ".dts", base), isEntry=True) DTSdoc = modelDTS.modelDocument DTSdoc.inDTS = True for uri in URIs.split(','): doc = ModelDocument.load(modelDTS, uri.strip(), base=base) DTSdoc.referencesDocument[ doc] = "import" #fake import doc.inDTS = True if isFrom: modelDTSfrom = modelDTS else: modelDTSto = modelDTS if modelDTSfrom and modelDTSto: # generate differences report reportUri = uriFrom.partition(',')[0] # first file reportDir = os.path.dirname(reportUri) if reportDir: reportDir += os.sep reportName = os.path.basename(reportUri).replace( "from.xsd", "report.xml") reportFile = reportDir + "report" + os.sep + reportName reportFullPath = self.webCache.normalizeUrl( reportFile, base) testcasesDir = os.path.dirname( os.path.dirname(reportFullPath)) if testcasesDir != priorTestcasesDir: # close prior report if priorTestcasesDir: for i, testcaseFile in enumerate(testcaseFiles): with open(testcaseFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8") testcaseName = os.path.basename(testcasesDir) testcaseFiles = [ testcasesDir + os.sep + testcaseName + "-creation-testcase.xml", testcasesDir + os.sep + testcaseName + "-consumption-testcase.xml" ] for i, testcaseFile in enumerate(testcaseFiles): XmlUtil.addChild( testcasesElements[i], None, "testcase", ("uri", testcaseFile[len(testGenDir) + 1:].replace("\\", "/"))) # start testcase file testcaseDocs = [ xml.dom.minidom.parseString( '<?xml version="1.0" encoding="UTF-8"?>' '<!-- Copyright 2011 XBRL International. All Rights Reserved. -->' '<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>' '<testcase name="XBRL Versioning 1.0 {1} Tests" date="{2}" ' ' xmlns="http://xbrl.org/2008/conformance"' ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' ' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">' '<creator>' '<name>Roland Hommes</name>' '<email>[email protected]</email>' '</creator>' '<name>{0}</name>' '<description>{0}</description>' '</testcase>'.format(testcaseName, purpose, today)) for purpose in ("Creation", "Consumption") ] testcaseElements = [ XmlUtil.child(testcaseDoc, conformanceNS, "testcase") for testcaseDoc in testcaseDocs ] priorTestcasesDir = testcasesDir variationID = 1 try: os.makedirs(os.path.dirname(reportFullPath)) except WindowsError: pass # dir already exists modelVersReport = ModelVersReport.ModelVersReport( modelTestcases) modelVersReport.diffDTSes(reportFullPath, modelDTSfrom, modelDTSto) # check for expected elements if expectedEvent and expectedEvent not in ("No change", "N.A."): if len( modelVersReport.xmlDocument. getElementsByTagNameNS('*', expectedEvent)) == 0: modelTestcases.error( "Generated test case {0} missing expected event {1}" .format(reportName, expectedEvent), "wrn", "missingEvent") modelVersReport.close([]) for i, testcaseElt in enumerate(testcaseElements): variationElement = XmlUtil.addChild( testcaseElt, conformanceNS, "variation", attributes=("id", "_{0:02n}".format(variationID))) XmlUtil.addChild(variationElement, conformanceNS, "name", text=intention) dataElement = XmlUtil.addChild(variationElement, conformanceNS, "data") for schemaURIs, dtsAttr in ((uriFrom, "from"), (uriTo, "to")): for schemaURI in schemaURIs.split(","): XmlUtil.addChild( dataElement, conformanceNS, "schema", attributes=((("dts", dtsAttr), ) + ((("readMeFirst", "true"), ) if i == 0 else ())), text=os.path.basename(schemaURI.strip())) resultElement = XmlUtil.addChild( variationElement, conformanceNS, "result") XmlUtil.addChild( resultElement if i == 0 else dataElement, conformanceNS, "versioningReport", attributes=(("readMeFirst", "true") if i == 1 else ()), text="report/" + reportName) variationID += 1 with open(logMessagesFile, "w") as fh: fh.writelines(self.logMessages) if priorTestcasesDir: for i, testcaseFile in enumerate(testcaseFiles): with open(testcaseFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8") for i, indexFile in enumerate(indexFiles): with open(indexFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def runFromExcel(self, options): testGenFileName = options.excelfilename #testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation\1000-2000-index.xls" testGenDir = os.path.dirname(testGenFileName) timeNow = XmlUtil.dateunionValue(datetime.datetime.now()) if options.testfiledate: today = options.testfiledate else: today = XmlUtil.dateunionValue(datetime.date.today()) startedAt = time.time() self.logMessages = [] logMessagesFile = testGenDir + os.sep + 'logGenerationMessages.txt' modelTestcases = ModelXbrl.create(self.modelManager) testcaseIndexBook = xlrd.open_workbook(testGenFileName) testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0) self.addToLog(_("[info] xls loaded in {0:.2} secs at {1}").format(time.time() - startedAt, timeNow)) # start index file indexFiles = [testGenDir + os.sep + 'creationTestcasesIndex.xml', testGenDir + os.sep + 'consumptionTestcasesIndex.xml'] indexDocs = [xml.dom.minidom.parseString( '<?xml version="1.0" encoding="UTF-8"?>' '<!-- XBRL Versioning 1.0 {0} Tests -->' '<!-- Copyright 2011 XBRL International. All Rights Reserved. -->' '<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>' '<testcases name="XBRL Versioning 1.0 Consumption Tests" date="{1}" ' ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' ' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">' '</testcases>'.format(purpose, today) ) for purpose in ("Creation","Consumption")] testcasesElements = [XmlUtil.child(indexDoc, None, "testcases") for indexDoc in indexDocs] priorTestcasesDir = None testcaseFiles = None testcaseDocs = None for iRow in range(1, testcaseIndexSheet.nrows): row = testcaseIndexSheet.row(iRow) if row[0].ctype == xlrd.XL_CELL_EMPTY or row[1].ctype == xlrd.XL_CELL_EMPTY or row[2].ctype == xlrd.XL_CELL_EMPTY: continue testDir = row[0].value uriFrom = row[1].value uriTo = row[2].value intention = row[3].value if intention is None or len(intention) == 0: continue # test not ready to run reason = row[4].value expectedEvent = row[5].value base = os.path.join(os.path.dirname(testGenFileName),testDir) + os.sep self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom)) if uriFrom and uriTo and reason.lower() not in ("n.a.", "error") and expectedEvent != "N.A.": for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"), True), (uriTo, _("loading to DTS"), False)): if ',' not in URIs: modelDTS = ModelXbrl.load(self.modelManager, URIs, msg, base=base) else: modelDTS = ModelXbrl.create(self.modelManager, ModelDocument.Type.DTSENTRIES, self.webCache.normalizeUrl(URIs.replace(", ","_") + ".dts", base), isEntry=True) DTSdoc = modelDTS.modelDocument DTSdoc.inDTS = True for uri in URIs.split(','): doc = ModelDocument.load(modelDTS, uri.strip(), base=base) DTSdoc.referencesDocument[doc] = "import" #fake import doc.inDTS = True if isFrom: modelDTSfrom = modelDTS else: modelDTSto = modelDTS if modelDTSfrom and modelDTSto: # generate differences report reportUri = uriFrom.partition(',')[0] # first file reportDir = os.path.dirname(reportUri) if reportDir: reportDir += os.sep reportName = os.path.basename(reportUri).replace("from.xsd","report.xml") reportFile = reportDir + "report" + os.sep + reportName reportFullPath = self.webCache.normalizeUrl( reportFile, base) testcasesDir = os.path.dirname(os.path.dirname(reportFullPath)) if testcasesDir != priorTestcasesDir: # close prior report if priorTestcasesDir: for i,testcaseFile in enumerate(testcaseFiles): with open(testcaseFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8") testcaseName = os.path.basename(testcasesDir) testcaseFiles = [testcasesDir + os.sep + testcaseName + "-creation-testcase.xml", testcasesDir + os.sep + testcaseName + "-consumption-testcase.xml"] for i,testcaseFile in enumerate(testcaseFiles): XmlUtil.addChild(testcasesElements[i], None, "testcase", ("uri", testcaseFile[len(testGenDir)+1:].replace("\\","/")) ) # start testcase file testcaseDocs = [xml.dom.minidom.parseString( '<?xml version="1.0" encoding="UTF-8"?>' '<!-- Copyright 2011 XBRL International. All Rights Reserved. -->' '<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>' '<testcase name="XBRL Versioning 1.0 {1} Tests" date="{2}" ' ' xmlns="http://xbrl.org/2008/conformance"' ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' ' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">' '<creator>' '<name>Roland Hommes</name>' '<email>[email protected]</email>' '</creator>' '<name>{0}</name>' '<description>{0}</description>' '</testcase>'.format(testcaseName,purpose,today) ) for purpose in ("Creation","Consumption")] testcaseElements = [XmlUtil.child(testcaseDoc, conformanceNS, "testcase") for testcaseDoc in testcaseDocs] priorTestcasesDir = testcasesDir variationID = 1 try: os.makedirs(os.path.dirname(reportFullPath)) except WindowsError: pass # dir already exists modelVersReport = ModelVersReport.ModelVersReport(modelTestcases) modelVersReport.diffDTSes(reportFullPath,modelDTSfrom, modelDTSto) # check for expected elements if expectedEvent and expectedEvent not in ( "No change", "N.A."): if len(modelVersReport.xmlDocument.getElementsByTagNameNS('*',expectedEvent)) == 0: modelTestcases.error( "Generated test case {0} missing expected event {1}".format( reportName, expectedEvent), "wrn", "missingEvent") modelVersReport.close([]) for i,testcaseElt in enumerate(testcaseElements): variationElement = XmlUtil.addChild(testcaseElt, conformanceNS, "variation", attributes=("id", "_{0:02n}".format(variationID))) XmlUtil.addChild(variationElement, conformanceNS, "name", text=intention) dataElement = XmlUtil.addChild(variationElement, conformanceNS, "data") for schemaURIs, dtsAttr in ((uriFrom,"from"), (uriTo,"to")): for schemaURI in schemaURIs.split(","): XmlUtil.addChild(dataElement, conformanceNS, "schema", attributes=((("dts",dtsAttr),) + ((("readMeFirst","true"),) if i == 0 else ())), text=os.path.basename(schemaURI.strip())) resultElement = XmlUtil.addChild(variationElement, conformanceNS, "result") XmlUtil.addChild(resultElement if i == 0 else dataElement, conformanceNS, "versioningReport", attributes=(("readMeFirst","true") if i == 1 else ()), text="report/" + reportName) variationID += 1 with open(logMessagesFile, "w") as fh: fh.writelines(self.logMessages) if priorTestcasesDir: for i,testcaseFile in enumerate(testcaseFiles): with open(testcaseFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8") for i,indexFile in enumerate(indexFiles): with open(indexFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def validate(val): formulaOptions = val.modelXbrl.modelManager.formulaOptions XPathParser.initializeParser(val) val.modelXbrl.modelManager.showStatus(_("Compiling formulae")) initialErrorCount = val.modelXbrl.logCountErr # global parameter names parameterQnames = set() instanceQnames = set() parameterDependencies = {} instanceDependencies = defaultdict( set) # None-key entries are non-formula dependencies dependencyResolvedParameters = set() orderedParameters = [] orderedInstances = [] for paramQname, modelParameter in val.modelXbrl.qnameParameters.items(): if isinstance(modelParameter, ModelParameter): modelParameter.compile() parameterDependencies[paramQname] = modelParameter.variableRefs() parameterQnames.add(paramQname) if isinstance(modelParameter, ModelInstance): instanceQnames.add(paramQname) # duplicates checked on loading modelDocument #resolve dependencies resolvedAParameter = True while (resolvedAParameter): resolvedAParameter = False for paramQname in parameterQnames: if paramQname not in dependencyResolvedParameters and \ len(parameterDependencies[paramQname] - dependencyResolvedParameters) == 0: dependencyResolvedParameters.add(paramQname) orderedParameters.append(paramQname) resolvedAParameter = True # anything unresolved? for paramQname in parameterQnames: if paramQname not in dependencyResolvedParameters: circularOrUndefDependencies = parameterDependencies[ paramQname] - dependencyResolvedParameters undefinedVars = circularOrUndefDependencies - parameterQnames paramsCircularDep = circularOrUndefDependencies - undefinedVars if len(undefinedVars) > 0: val.modelXbrl.error( _("Undefined dependencies in parameter {0}, to names {1}" ).format(paramQname, ", ".join( (str(v) for v in undefinedVars))), "err", "xbrlve:unresolvedDependency") if len(paramsCircularDep) > 0: val.modelXbrl.error( _("Cyclic dependencies in parameter {0}, to names {1}" ).format(paramQname, ", ".join( (str(d) for d in paramsCircularDep))), "err", "xbrlve:parameterCyclicDependencies") for custFnSig in val.modelXbrl.modelCustomFunctionSignatures.values(): custFnQname = custFnSig.qname if custFnQname.namespaceURI == "XbrlConst.xfi": val.modelXbrl.error( _("Custom function {0} has namespace reserved for functions in the function registry {1}" ).format(str(custFnQname), custFnQname.namespaceURI), "err", "xbrlve:noProhibitedNamespaceForCustomFunction") # any custom function implementations? for modelRel in val.modelXbrl.relationshipSet( XbrlConst.functionImplementation).fromModelObject(custFnSig): custFnImpl = modelRel.toModelObject custFnSig.customFunctionImplementation = custFnImpl if len(custFnImpl.inputNames) != len(custFnSig.inputTypes): val.modelXbrl.error( _("Custom function {0} signature has {1} parameters but implementation has {2}, must be matching" ).format(str(custFnQname), len(custFnSig.inputTypes), len(custFnImpl.inputNames)), "err", "xbrlcfie:inputMismatch") for custFnImpl in val.modelXbrl.modelCustomFunctionImplementations: if not val.modelXbrl.relationshipSet( XbrlConst.functionImplementation).toModelObject(custFnImpl): val.modelXbrl.error( _("Custom function implementation {0} has no relationship from any custom function signature" ).format(custFnImpl.xlinkLabel), "err", "xbrlcfie:missingCFIRelationship") custFnImpl.compile() # xpathContext is needed for filter setup for expressions such as aspect cover filter # determine parameter values xpathContext = XPathContext.create(val.modelXbrl) for paramQname in orderedParameters: if not isinstance(modelParameter, ModelInstance): modelParameter = val.modelXbrl.qnameParameters[paramQname] asType = modelParameter.asType asLocalName = asType.localName if asType else "string" try: if val.parameters and paramQname in val.parameters: paramDataType, paramValue = val.parameters[paramQname] typeLocalName = paramDataType.localName if paramDataType else "string" value = FunctionXs.call(xpathContext, None, typeLocalName, [paramValue]) result = FunctionXs.call(xpathContext, None, asLocalName, [value]) if formulaOptions.traceParameterInputValue: val.modelXbrl.error( _("Parameter {0} input {1}").format( paramQname, result), "info", "formula:trace") else: result = modelParameter.evaluate(xpathContext, asType) if formulaOptions.traceParameterExpressionResult: val.modelXbrl.error( _("Parameter {0} result {1}").format( paramQname, result), "info", "formula:trace") xpathContext.inScopeVars[ paramQname] = result # make visible to subsequent parameter expression except XPathContext.XPathException as err: val.modelXbrl.error( _("Parameter \n{0} \nException: \n{1}").format( paramQname, err.message), "err", "xbrlve:parameterTypeMismatch" if err.code == "err:FORG0001" else err.code) produceOutputXbrlInstance = False instanceProducingVariableSets = defaultdict(list) for modelVariableSet in val.modelXbrl.modelVariableSets: varSetInstanceDependencies = set() if isinstance(modelVariableSet, ModelFormula): instanceQname = None for modelRel in val.modelXbrl.relationshipSet( XbrlConst.formulaInstance).fromModelObject( modelVariableSet): instance = modelRel.toModelObject if isinstance(instance, ModelInstance): if instanceQname is None: instanceQname = instance.qname else: val.modelXbrl.error( _("Multiple output instances for formula {0}, to names {1}, {2}" ).format(modelVariableSet.xlinkLabel, instanceQname, instance.qname), "info", "arelle:multipleOutputInstances") if instanceQname is None: instanceQname = XbrlConst.qnStandardOutputInstance instanceQnames.add(instanceQname) modelVariableSet.outputInstanceQname = instanceQname if val.validateSBRNL: val.modelXbrl.error( _("Formula linkbase {0} formula:formula {1} is not allowed" ).format( os.path.basename(modelVariableSet.modelDocument.uri), modelVariableSet.xlinkLabel), "err", "SBR.NL.2.3.9.03") else: instanceQname = None modelVariableSet.countSatisfied = 0 modelVariableSet.countNotSatisfied = 0 checkValidationMessages(val, modelVariableSet) instanceProducingVariableSets[instanceQname].append(modelVariableSet) modelVariableSet.outputInstanceQname = instanceQname if modelVariableSet.aspectModel not in ("non-dimensional", "dimensional"): val.modelXbrl.error( _("Variable set {0}, aspect model {1} not recognized").format( modelVariableSet.xlinkLabel, modelVariableSet.aspectModel), "err", "xbrlve:unknownAspectModel") modelVariableSet.compile() modelVariableSet.hasConsistencyAssertion = False #determine dependencies within variable sets nameVariables = {} qnameRels = {} definedNamesSet = set() for modelRel in val.modelXbrl.relationshipSet( XbrlConst.variableSet).fromModelObject(modelVariableSet): varqname = modelRel.variableQname if varqname: qnameRels[varqname] = modelRel toVariable = modelRel.toModelObject if varqname not in definedNamesSet: definedNamesSet.add(varqname) if varqname not in nameVariables: nameVariables[varqname] = toVariable elif nameVariables[varqname] != toVariable: val.modelXbrl.error( _("Multiple variables named {1} in variable set {0}" ).format(modelVariableSet.xlinkLabel, varqname), "err", "xbrlve:duplicateVariableNames") fromInstanceQnames = None for instRel in val.modelXbrl.relationshipSet( XbrlConst.instanceVariable).toModelObject(toVariable): fromInstance = instRel.fromModelObject if isinstance(fromInstance, ModelInstance): fromInstanceQname = fromInstance.qname varSetInstanceDependencies.add(fromInstanceQname) instanceDependencies[instanceQname].add( fromInstanceQname) if fromInstanceQnames is None: fromInstanceQnames = set() fromInstanceQnames.add(fromInstanceQname) if fromInstanceQnames is None: varSetInstanceDependencies.add( XbrlConst.qnStandardInputInstance) if instanceQname: instanceDependencies[instanceQname].add( XbrlConst.qnStandardInputInstance) toVariable.fromInstanceQnames = fromInstanceQnames else: val.modelXbrl.error( _("Variables name {1} cannot be determined on arc from {0}" ).format(modelVariableSet.xlinkLabel, modelRel.variablename), "err", "xbrlve:variableNameResolutionFailure") definedNamesSet |= parameterQnames variableDependencies = {} for modelRel in val.modelXbrl.relationshipSet( XbrlConst.variableSet).fromModelObject(modelVariableSet): variable = modelRel.toModelObject if isinstance( variable, (ModelParameter, ModelVariable)): # ignore anything not parameter or variable varqname = modelRel.variableQname depVars = variable.variableRefs() variableDependencies[varqname] = depVars if len(depVars ) > 0 and formulaOptions.traceVariablesDependencies: val.modelXbrl.error( _("Variable set {0}, variable {1}, dependences {2}"). format(modelVariableSet.xlinkLabel, varqname, depVars), "info", "formula:trace") definedNamesSet.add(varqname) # check for fallback value variable references if isinstance(variable, ModelFactVariable): for depVar in XPathParser.variableReferencesSet( variable.fallbackValueProg, variable.element): if depVar in qnameRels and isinstance( qnameRels[depVar].toModelObject, ModelVariable): val.modelXbrl.error( _("Variable set {0} fallbackValue '{1}' cannot refer to variable {2}" ).format(modelVariableSet.xlinkLabel, variable.fallbackValue, depVar), "err", "xbrlve:factVariableReferenceNotAllowed") # check for covering aspect not in variable set aspect model checkFilterAspectModel(val, modelVariableSet, variable.filterRelationships, xpathContext) orderedNameSet = set() orderedNameList = [] orderedAVariable = True while (orderedAVariable): orderedAVariable = False for varqname, depVars in variableDependencies.items(): if varqname not in orderedNameSet and len(depVars - parameterQnames - orderedNameSet) == 0: orderedNameList.append(varqname) orderedNameSet.add(varqname) orderedAVariable = True if varqname in instanceQnames: varSetInstanceDependencies.add(varqname) instanceDependencies[instanceQname].add(varqname) elif isinstance(nameVariables.get(varqname), ModelInstance): instqname = nameVariables[varqname].qname varSetInstanceDependencies.add(instqname) instanceDependencies[instanceQname].add(instqname) # anything unresolved? for varqname, depVars in variableDependencies.items(): if varqname not in orderedNameSet: circularOrUndefVars = depVars - parameterQnames - orderedNameSet undefinedVars = circularOrUndefVars - definedNamesSet varsCircularDep = circularOrUndefVars - undefinedVars if len(undefinedVars) > 0: val.modelXbrl.error( _("Undefined variable dependencies in variable st {0}, from variable {1} to {2}" ).format(modelVariableSet.xlinkLabel, varqname, undefinedVars), "err", "xbrlve:unresolvedDependency") if len(varsCircularDep) > 0: val.modelXbrl.error( _("Cyclic dependencies in variable set {0}, from variable {1} to {2}" ).format(modelVariableSet.xlinkLabel, varqname, varsCircularDep), "err", "xbrlve:cyclicDependencies") # check unresolved variable set dependencies for varSetDepVarQname in modelVariableSet.variableRefs(): if varSetDepVarQname not in orderedNameSet and varSetDepVarQname not in parameterQnames: val.modelXbrl.error( _("Undefined variable dependency in variable set {0}, {1}" ).format(modelVariableSet.xlinkLabel, varSetDepVarQname), "err", "xbrlve:unresolvedDependency") if varSetDepVarQname in instanceQnames: varSetInstanceDependencies.add(varSetDepVarQname) instanceDependencies[instanceQname].add(varSetDepVarQname) elif isinstance(nameVariables.get(varSetDepVarQname), ModelInstance): instqname = nameVariables[varSetDepVarQname].qname varSetInstanceDependencies.add(instqname) instanceDependencies[instanceQname].add(instqname) if formulaOptions.traceVariablesOrder: val.modelXbrl.error( _("Variable set {0}, variables order: {1}").format( modelVariableSet.xlinkLabel, orderedNameList), "info", "formula:trace") if (formulaOptions.traceVariablesDependencies and len(varSetInstanceDependencies) > 0 and varSetInstanceDependencies != {XbrlConst.qnStandardInputInstance}): val.modelXbrl.error( _("Variable set {0}, instance dependences {1}").format( modelVariableSet.xlinkLabel, varSetInstanceDependencies), "info", "formula:trace") modelVariableSet.orderedVariableRelationships = [] for varqname in orderedNameList: if varqname in qnameRels: modelVariableSet.orderedVariableRelationships.append( qnameRels[varqname]) # check existence assertion variable dependencies if isinstance(modelVariableSet, ModelExistenceAssertion): for depVar in modelVariableSet.variableRefs(): if depVar in qnameRels and isinstance( qnameRels[depVar].toModelObject, ModelVariable): val.modelXbrl.error( _("Existence Assertion {0}, cannot refer to variable {1}" ).format(modelVariableSet.xlinkLabel, depVar), "err", "xbrleae:variableReferenceNotAllowed") # check messages variable dependencies checkValidationMessageVariables(val, modelVariableSet, qnameRels) # check preconditions modelVariableSet.preconditions = [] for modelRel in val.modelXbrl.relationshipSet( XbrlConst.variableSetPrecondition).fromModelObject( modelVariableSet): precondition = modelRel.toModelObject if isinstance(precondition, ModelPrecondition): modelVariableSet.preconditions.append(precondition) # check for variable sets referencing fact or general variables for modelRel in val.modelXbrl.relationshipSet( XbrlConst.variableSetFilter).fromModelObject(modelVariableSet): varSetFilter = modelRel.toModelObject if modelRel.isCovered: val.modelXbrl.error( _("Variable set {0}, filter {1}, cannot be covered" ).format(modelVariableSet.xlinkLabel, varSetFilter.xlinkLabel), "wrn", "arelle:variableSetFilterCovered") modelRel._isCovered = False # block group filter from being able to covere for depVar in varSetFilter.variableRefs(): if depVar in qnameRels and isinstance( qnameRels[depVar].toModelObject, ModelVariable): val.modelXbrl.error( _("Variable set {0}, filter {1}, cannot refer to variable {2}" ).format(modelVariableSet.xlinkLabel, varSetFilter.xlinkLabel, depVar), "err", "xbrlve:factVariableReferenceNotAllowed") # check aspects of formula if isinstance(modelVariableSet, ModelFormula): checkFormulaRules(val, modelVariableSet, nameVariables) # determine instance dependency order orderedInstancesSet = set() stdInpInst = {XbrlConst.qnStandardInputInstance} orderedInstancesList = [] orderedAnInstance = True while (orderedAnInstance): orderedAnInstance = False for instqname, depInsts in instanceDependencies.items(): if instqname and instqname not in orderedInstancesSet and len( depInsts - stdInpInst - orderedInstancesSet) == 0: orderedInstancesList.append(instqname) orderedInstancesSet.add(instqname) orderedAnInstance = True orderedInstancesList.append( None) # assertions come after all formulas that produce outputs # anything unresolved? for instqname, depInsts in instanceDependencies.items(): if instqname not in orderedInstancesSet: # can also be satisfied from an input DTS missingDependentInstances = depInsts - stdInpInst if val.parameters: missingDependentInstances -= val.parameters.keys() if instqname: if missingDependentInstances: val.modelXbrl.error( _("Cyclic dependencies of instance {0} produced by a formula, with variables consuming instances {1}" ).format(instqname, missingDependentInstances), "err", "xbrlvarinste:instanceVariableRecursionCycle") elif instqname == XbrlConst.qnStandardOutputInstance: orderedInstancesSet.add(instqname) orderedInstancesList.append( instqname ) # standard output formula, all input dependencies in parameters ''' future check? if instance has no external input or producing formula else: val.modelXbrl.error( _("Unresolved dependencies of an assertion's variables on instances {0}").format( depInsts - stdInpInst ), "err", "xbrlvarinste:instanceVariableRecursionCycle") ''' if formulaOptions.traceVariablesOrder and len(orderedInstancesList) > 1: val.modelXbrl.error( _("Variable instances processing order: {0}").format( orderedInstancesList), "info", "formula:trace") # linked consistency assertions for modelRel in val.modelXbrl.relationshipSet( XbrlConst.consistencyAssertionFormula).modelRelationships: if modelRel.fromModelObject and modelRel.toModelObject and isinstance( modelRel.toModelObject, ModelFormula): consisAsser = modelRel.fromModelObject consisAsser.countSatisfied = 0 consisAsser.countNotSatisfied = 0 if consisAsser.hasProportionalAcceptanceRadius and consisAsser.hasAbsoluteAcceptanceRadius: val.modelXbrl.error( _("Consistency assertion {0} has both absolute and proportional acceptance radii" ).format(consisAsser.xlinkLabel), "err", "xbrlcae:acceptanceRadiusConflict") consisAsser.orderedVariableRelationships = [] for consisParamRel in val.modelXbrl.relationshipSet( XbrlConst.consistencyAssertionParameter).fromModelObject( consisAsser): if isinstance(consisParamRel.toModelObject, ModelVariable): val.modelXbrl.error( _("Consistency assertion {0} has relationship to a {1} {2}" ).format( consisAsser.xlinkLabel, consisParamRel.toModelObject.element.localName, consisParamRel.toModelObject.xlinkLabel), "err", "xbrlcae:variablesNotAllowed") else: consisAsser.orderedVariableRelationships.append( consisParamRel) consisAsser.compile() modelRel.toModelObject.hasConsistencyAssertion = True if initialErrorCount < val.modelXbrl.logCountErr: return # don't try to execute # formula output instances if instanceQnames: schemaRefs = [ val.modelXbrl.modelDocument.relativeUri(referencedDoc.uri) for referencedDoc in val.modelXbrl.modelDocument.referencesDocument.keys() if referencedDoc.type == ModelDocument.Type.SCHEMA ] outputXbrlInstance = None for instanceQname in instanceQnames: if instanceQname == XbrlConst.qnStandardInputInstance: continue # always present the standard way if val.parameters and instanceQname in val.parameters: namedInstance = val.parameters[instanceQname][1] else: # empty intermediate instance uri = val.modelXbrl.modelDocument.filepath[:-4] + "-output-XBRL-instance" if instanceQname != XbrlConst.qnStandardOutputInstance: uri = uri + "-" + instanceQname.localName uri = uri + ".xml" namedInstance = ModelXbrl.create( val.modelXbrl.modelManager, newDocumentType=ModelDocument.Type.INSTANCE, url=uri, schemaRefs=schemaRefs, isEntry=True) xpathContext.inScopeVars[instanceQname] = namedInstance if instanceQname == XbrlConst.qnStandardOutputInstance: outputXbrlInstance = namedInstance # evaluate consistency assertions # evaluate variable sets not in consistency assertions for instanceQname in orderedInstancesList: for modelVariableSet in instanceProducingVariableSets[instanceQname]: # produce variable evaluations from arelle.FormulaEvaluator import evaluate try: evaluate(xpathContext, modelVariableSet) except XPathContext.XPathException as err: val.modelXbrl.error( _("Variable set \n{0} \nException: \n{1}").format( modelVariableSet, err.message), "err", err.code) # log assertion result counts asserTests = {} for exisValAsser in val.modelXbrl.modelVariableSets: if isinstance(exisValAsser, ModelVariableSetAssertion): asserTests[exisValAsser.id] = (exisValAsser.countSatisfied, exisValAsser.countNotSatisfied) if formulaOptions.traceAssertionResultCounts: val.modelXbrl.error( _("{0} Assertion {1} evaluations : {2} satisfied, {3} not satisfied" ).format( "Existence" if isinstance(exisValAsser, ModelExistenceAssertion) else "Value", exisValAsser.id, exisValAsser.countSatisfied, exisValAsser.countNotSatisfied), "info", "formula:trace") for modelRel in val.modelXbrl.relationshipSet( XbrlConst.consistencyAssertionFormula).modelRelationships: if modelRel.fromModelObject and modelRel.toModelObject and isinstance( modelRel.toModelObject, ModelFormula): consisAsser = modelRel.fromModelObject asserTests[consisAsser.id] = (consisAsser.countSatisfied, consisAsser.countNotSatisfied) if formulaOptions.traceAssertionResultCounts: val.modelXbrl.error( _("Consistency Assertion {0} evaluations : {1} satisfied, {2} not satisfied" ).format(consisAsser.id, consisAsser.countSatisfied, consisAsser.countNotSatisfied), "info", "formula:trace") if asserTests: val.modelXbrl.error( _("Assertion results {0}").format(asserTests), "asrtNoLog", asserTests) # display output instance if outputXbrlInstance: if val.modelXbrl.formulaOutputInstance: # close prior instance, usually closed by caller to validate as it may affect UI on different thread val.modelXbrl.formulaOutputInstance.close() val.modelXbrl.formulaOutputInstance = outputXbrlInstance
def validateTestcase(self, testcase): self.modelXbrl.info("info", "Testcase", modelDocument=testcase) self.modelXbrl.viewModelObject(testcase.objectId()) if hasattr(testcase, "testcaseVariations"): for modelTestcaseVariation in testcase.testcaseVariations: # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject( self.modelXbrl, modelTestcaseVariation.objectId()) # is this a versioning report? resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance formulaOutputInstance = None inputDTSes = defaultdict(list) baseForElement = testcase.baseForElement( modelTestcaseVariation) # try to load instance document self.modelXbrl.info( "info", _("Variation %(id)s %(name)s: %(expected)s - %(description)s" ), modelObject=modelTestcaseVariation, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, expected=modelTestcaseVariation.expected, description=modelTestcaseVariation.description) errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY for readMeFirstUri in modelTestcaseVariation.readMeFirstUris: if isinstance(readMeFirstUri, tuple): # dtsName is for formula instances, but is from/to dts if versioning dtsName, readMeFirstUri = readMeFirstUri elif resultIsVersioningReport: if inputDTSes: dtsName = "to" else: dtsName = "from" else: dtsName = None if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document if dtsName in inputDTSes: dtsName = inputDTSes[dtsName] else: modelXbrl = ModelXbrl.create( self.modelXbrl.modelManager, Type.DTSENTRIES, self.modelXbrl.modelManager.cntlr.webCache. normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement), isEntry=True, errorCaptureLevel=errorCaptureLevel) DTSdoc = modelXbrl.modelDocument DTSdoc.inDTS = True doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement) if doc is not None: DTSdoc.referencesDocument[ doc] = ModelDocumentReference( "import", DTSdoc.xmlRootElement) #fake import doc.inDTS = True else: # not a multi-schemaRef versioning report modelXbrl = ModelXbrl.load( self.modelXbrl.modelManager, readMeFirstUri, _("validating"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) if modelXbrl.modelDocument is None: self.modelXbrl.error( "arelle:notLoaded", _("Testcase %(id)s %(name)s document not loaded: %(file)s" ), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) modelXbrl.close() self.determineNotLoadedTestStatus( modelTestcaseVariation) elif resultIsVersioningReport: inputDTSes[dtsName] = modelXbrl elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport( self.modelXbrl).validate(modelXbrl) self.determineTestStatus(modelTestcaseVariation, modelXbrl) modelXbrl.close() elif testcase.type == Type.REGISTRYTESTCASE: self.instValidator.validate( modelXbrl) # required to set up dimensions, etc self.instValidator.executeCallTest( modelXbrl, modelTestcaseVariation.id, modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest) self.determineTestStatus(modelTestcaseVariation, modelXbrl) self.instValidator.close() modelXbrl.close() else: inputDTSes[dtsName].append(modelXbrl) if resultIsVersioningReport and modelXbrl.modelDocument: versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl( modelTestcaseVariation.versioningReportUri, baseForElement) if os.path.exists(versReportFile): #validate existing modelVersReport = ModelXbrl.load( self.modelXbrl.modelManager, versReportFile, _("validating existing version report")) if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport( self.modelXbrl).validate(modelVersReport) self.determineTestStatus(modelTestcaseVariation, modelVersReport) modelVersReport.close() elif len(inputDTSes) == 2: ModelVersReport.ModelVersReport( self.modelXbrl).diffDTSes(versReportFile, inputDTSes["from"], inputDTSes["to"]) modelTestcaseVariation.status = "generated" else: self.modelXbrl.error( "arelle:notLoaded", _("Testcase %(id)s %(name)s DTSes not loaded, unable to generate versioning report: %(file)s" ), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) modelTestcaseVariation.status = "failed" for inputDTS in inputDTSes.values(): inputDTS.close() del inputDTSes # dereference elif inputDTSes: # validate schema, linkbase, or instance modelXbrl = inputDTSes[None][0] for pluginXbrlMethod in pluginClassMethods( "TestcaseVariation.Xbrl.Loaded"): pluginXbrlMethod(self.modelXbrl, modelXbrl) parameters = modelTestcaseVariation.parameters.copy() for dtsName, inputDTS in inputDTSes.items( ): # input instances are also parameters if dtsName: # named instance parameters[dtsName] = ( None, inputDTS ) #inputDTS is a list of modelXbrl's (instance DTSes) elif len( inputDTS ) > 1: # standard-input-instance with multiple instance documents parameters[XbrlConst.qnStandardInputInstance] = ( None, inputDTS ) # allow error detection in validateFormula if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable: RenderingEvaluator.init(modelXbrl) try: self.instValidator.validate(modelXbrl, parameters) except Exception as err: self.modelXbrl.error( "exception", _("Testcase variation validation exception: %(error)s, instance: %(instance)s" ), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset: for pluginXbrlMethod in pluginClassMethods( "Validate.Infoset"): pluginXbrlMethod( modelXbrl, modelTestcaseVariation.resultInfosetUri) infoset = ModelXbrl.load( self.modelXbrl.modelManager, modelTestcaseVariation.resultInfosetUri, _("loading result infoset"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) if infoset.modelDocument is None: self.modelXbrl.error( "arelle:notLoaded", _("Testcase %(id)s %(name)s result infoset not loaded: %(file)s" ), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename( modelTestcaseVariation.resultXbrlInstance)) modelTestcaseVariation.status = "result infoset not loadable" else: # check infoset ValidateInfoset.validate(self.instValidator, modelXbrl, infoset) infoset.close() if modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset: # diff (or generate) table infoset resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl( modelTestcaseVariation.resultTableUri, baseForElement) if not any( alternativeValidation(modelXbrl, resultTableUri) for alternativeValidation in pluginClassMethods("Validate.TableInfoset")): ViewFileRenderedGrid.viewRenderedGrid( modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files self.determineTestStatus( modelTestcaseVariation, modelXbrl) # include infoset errors in status self.instValidator.close() if modelXbrl.formulaOutputInstance and self.noErrorCodes( modelTestcaseVariation.actual): # if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost) self.instValidator.validate( modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters) self.determineTestStatus( modelTestcaseVariation, modelXbrl.formulaOutputInstance) if self.noErrorCodes( modelTestcaseVariation.actual ): # if still 'clean' pass it forward for comparison to expected result instance formulaOutputInstance = modelXbrl.formulaOutputInstance modelXbrl.formulaOutputInstance = None # prevent it from being closed now self.instValidator.close() for inputDTSlist in inputDTSes.values(): for inputDTS in inputDTSlist: inputDTS.close() del inputDTSes # dereference if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument: expectedInstance = ModelXbrl.load( self.modelXbrl.modelManager, modelTestcaseVariation.resultXbrlInstanceUri, _("loading expected result XBRL instance"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) if expectedInstance.modelDocument is None: self.modelXbrl.error( "arelle:notLoaded", _("Testcase %(id)s %(name)s expected result instance not loaded: %(file)s" ), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename( modelTestcaseVariation.resultXbrlInstance)) modelTestcaseVariation.status = "result not loadable" else: # compare facts if len(expectedInstance.facts) != len( formulaOutputInstance.facts): formulaOutputInstance.error( "formula:resultFactCounts", _("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts" ), modelXbrl=modelXbrl, countFacts=len( formulaOutputInstance.facts), expectedFacts=len(expectedInstance.facts)) else: for fact in expectedInstance.facts: unmatchedFactsStack = [] if formulaOutputInstance.matchFact( fact, unmatchedFactsStack, deemP0inf=True) is None: if unmatchedFactsStack: # get missing nested tuple fact, if possible missingFact = unmatchedFactsStack[ -1] else: missingFact = fact formulaOutputInstance.error( "formula:expectedFactMissing", _("Formula output missing expected fact %(fact)s" ), modelXbrl=missingFact, fact=missingFact.qname) # for debugging uncomment next line to save generated instance document # formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml") expectedInstance.close() del expectedInstance # dereference self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance) formulaOutputInstance.close() del formulaOutputInstance # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject( self.modelXbrl, modelTestcaseVariation.objectId()) self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs, outputZip=None, filingFiles=None, *args, **kwargs): targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl( targetDocumentFilename, modelXbrl.modelDocument.filepath) targetUrlParts = targetUrl.rpartition(".") targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2] modelXbrl.modelManager.showStatus( _("Extracting instance ") + os.path.basename(targetUrl)) targetInstance = ModelXbrl.create(modelXbrl.modelManager, newDocumentType=Type.INSTANCE, url=targetUrl, schemaRefs=targetDocumentSchemaRefs, isEntry=True) ValidateXbrlDimensions.loadDimensionDefaults( targetInstance) # need dimension defaults # roleRef and arcroleRef (of each inline document) for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs): for roleRefElt in sourceRefs.values(): addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname, attributes=roleRefElt.items()) # contexts for context in modelXbrl.contexts.values(): newCntx = targetInstance.createContext( context.entityIdentifier[0], context.entityIdentifier[1], 'instant' if context.isInstantPeriod else 'duration' if context.isStartEndPeriod else 'forever', context.startDatetime, context.endDatetime, None, context.qnameDims, [], [], id=context.id) for unit in modelXbrl.units.values(): measures = unit.measures newUnit = targetInstance.createUnit(measures[0], measures[1], id=unit.id) modelXbrl.modelManager.showStatus(_("Creating and validating facts")) newFactForOldObjId = {} def createFacts(facts, parent): for fact in facts: if fact.isItem: attrs = {"contextRef": fact.contextID} if fact.id: attrs["id"] = fact.id if fact.isNumeric: attrs["unitRef"] = fact.unitID if fact.get("decimals"): attrs["decimals"] = fact.get("decimals") if fact.get("precision"): attrs["precision"] = fact.get("precision") if fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact.xValue if fact.xValid else fact.textValue newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent) newFactForOldObjId[fact.objectIndex] = newFact if filingFiles and fact.concept is not None and fact.concept.isTextBlock: # check for img and other filing references for xmltext in [text] + CDATApattern.findall(text): try: for elt in XML( "<body>\n{0}\n</body>\n".format(xmltext)): if elt.tag in ("a", "img") and not isHttpUrl( attrValue) and not os.path.isabs( attrvalue): for attrTag, attrValue in elt.items(): if attrTag in ("href", "src"): filingFiles.add(attrValue) except (XMLSyntaxError, UnicodeDecodeError): pass elif fact.isTuple: newTuple = targetInstance.createFact(fact.qname, parent=parent) newFactForOldObjId[fact.objectIndex] = newTuple createFacts(fact.modelTupleFacts, newTuple) createFacts(modelXbrl.facts, None) # footnote links footnoteIdCount = {} modelXbrl.modelManager.showStatus( _("Creating and validating footnotes & relationships")) HREF = "{http://www.w3.org/1999/xlink}href" footnoteLinks = defaultdict(list) for linkKey, linkPrototypes in modelXbrl.baseSets.items(): arcrole, linkrole, linkqname, arcqname = linkKey if (linkrole and linkqname and arcqname and # fully specified roles arcrole != "XBRL-footnotes" and any( lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)): for linkPrototype in linkPrototypes: if linkPrototype not in footnoteLinks[linkrole]: footnoteLinks[linkrole].append(linkPrototype) for linkrole in sorted(footnoteLinks.keys()): for linkPrototype in footnoteLinks[linkrole]: newLink = addChild(targetInstance.modelDocument.xmlRootElement, linkPrototype.qname, attributes=linkPrototype.attributes) for linkChild in linkPrototype: attributes = linkChild.attributes if isinstance(linkChild, LocPrototype): if HREF not in linkChild.attributes: linkChild.attributes[HREF] = \ "#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex]) addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ArcPrototype): addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ModelInlineFootnote): idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1 if idUseCount > 1: # if footnote with id in other links bump the id number attributes = linkChild.attributes.copy() attributes["id"] = "{}_{}".format( attributes["id"], idUseCount) footnoteIdCount[linkChild.footnoteID] = idUseCount newChild = addChild(newLink, linkChild.qname, attributes=attributes) copyIxFootnoteHtml( linkChild, newChild, targetModelDocument=targetInstance.modelDocument, withText=True) if filingFiles and linkChild.textValue: footnoteHtml = XML("<body/>") copyIxFootnoteHtml(linkChild, footnoteHtml) for elt in footnoteHtml.iter(): if elt.tag in ("a", "img"): for attrTag, attrValue in elt.items(): if attrTag in ( "href", "src") and not isHttpUrl( attrValue ) and not os.path.isabs(attrvalue): filingFiles.add(attrValue) targetInstance.saveInstance(overrideFilepath=targetUrl, outputZip=outputZip) if getattr(modelXbrl, "isTestcaseVariation", False): modelXbrl.extractedInlineInstance = True # for validation comparison modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs, outputZip=None, filingFiles=None, *args, **kwargs): targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(targetDocumentFilename, modelXbrl.modelDocument.filepath) def addLocallyReferencedFile(elt,filingFiles): if elt.tag in ("a", "img"): for attrTag, attrValue in elt.items(): if attrTag in ("href", "src") and not isHttpUrl(attrValue) and not os.path.isabs(attrValue): attrValue = attrValue.partition('#')[0] # remove anchor if attrValue: # ignore anchor references to base document attrValue = os.path.normpath(attrValue) # change url path separators to host separators file = os.path.join(sourceDir,attrValue) if modelXbrl.fileSource.isInArchive(file, checkExistence=True) or os.path.exists(file): filingFiles.add(file) targetUrlParts = targetUrl.rpartition(".") targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2] modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl)) rootElt = modelXbrl.modelDocument.xmlRootElement # take baseXmlLang from <html> or <base> baseXmlLang = rootElt.get("{http://www.w3.org/XML/1998/namespace}lang") or rootElt.get("lang") for ixElt in modelXbrl.modelDocument.xmlRootElement.iterdescendants(tag="{http://www.w3.org/1999/xhtml}body"): baseXmlLang = ixElt.get("{http://www.w3.org/XML/1998/namespace}lang") or rootElt.get("lang") or baseXmlLang targetInstance = ModelXbrl.create(modelXbrl.modelManager, newDocumentType=Type.INSTANCE, url=targetUrl, schemaRefs=targetDocumentSchemaRefs, isEntry=True, discover=False) # don't attempt to load DTS if baseXmlLang: targetInstance.modelDocument.xmlRootElement.set("{http://www.w3.org/XML/1998/namespace}lang", baseXmlLang) ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults # roleRef and arcroleRef (of each inline document) for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs): for roleRefElt in sourceRefs.values(): addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname, attributes=roleRefElt.items()) # contexts for context in sorted(modelXbrl.contexts.values(), key=lambda c: elementChildSequence(c)): ignore = targetInstance.createContext(context.entityIdentifier[0], context.entityIdentifier[1], 'instant' if context.isInstantPeriod else 'duration' if context.isStartEndPeriod else 'forever', context.startDatetime, context.endDatetime, None, context.qnameDims, [], [], id=context.id) for unit in modelXbrl.units.values(): measures = unit.measures ignore = targetInstance.createUnit(measures[0], measures[1], id=unit.id) modelXbrl.modelManager.showStatus(_("Creating and validating facts")) newFactForOldObjId = {} def createFacts(facts, parent): for fact in facts: if fact.isItem: # HF does not de-duplicate, which is currently-desired behavior attrs = {"contextRef": fact.contextID} if fact.id: attrs["id"] = fact.id if fact.isNumeric: attrs["unitRef"] = fact.unitID if fact.get("decimals"): attrs["decimals"] = fact.get("decimals") if fact.get("precision"): attrs["precision"] = fact.get("precision") if fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact.xValue if fact.xValid else fact.textValue if fact.concept is not None and fact.concept.baseXsdType in ("string", "normalizedString"): # default xmlLang = fact.xmlLang if xmlLang is not None and xmlLang != baseXmlLang: attrs["{http://www.w3.org/XML/1998/namespace}lang"] = xmlLang newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent) # if fact.isFraction, create numerator and denominator newFactForOldObjId[fact.objectIndex] = newFact if filingFiles is not None and fact.concept is not None and fact.concept.isTextBlock: # check for img and other filing references so that referenced files are included in the zip. for xmltext in [text] + CDATApattern.findall(text): try: for elt in XML("<body>\n{0}\n</body>\n".format(xmltext)).iter(): addLocallyReferencedFile(elt, filingFiles) except (XMLSyntaxError, UnicodeDecodeError): pass # TODO: Why ignore UnicodeDecodeError? elif fact.isTuple: newTuple = targetInstance.createFact(fact.qname, parent=parent) newFactForOldObjId[fact.objectIndex] = newTuple createFacts(fact.modelTupleFacts, newTuple) createFacts(modelXbrl.facts, None) modelXbrl.modelManager.showStatus(_("Creating and validating footnotes and relationships")) HREF = "{http://www.w3.org/1999/xlink}href" footnoteLinks = defaultdict(list) footnoteIdCount = {} for linkKey, linkPrototypes in modelXbrl.baseSets.items(): arcrole, linkrole, linkqname, arcqname = linkKey if (linkrole and linkqname and arcqname and # fully specified roles arcrole != "XBRL-footnotes" and any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)): for linkPrototype in linkPrototypes: if linkPrototype not in footnoteLinks[linkrole]: footnoteLinks[linkrole].append(linkPrototype) for linkrole in sorted(footnoteLinks.keys()): for linkPrototype in footnoteLinks[linkrole]: newLink = addChild(targetInstance.modelDocument.xmlRootElement, linkPrototype.qname, attributes=linkPrototype.attributes) for linkChild in linkPrototype: attributes = linkChild.attributes if isinstance(linkChild, LocPrototype): if HREF not in linkChild.attributes: linkChild.attributes[HREF] = \ "#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex]) addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ArcPrototype): addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ModelInlineFootnote): idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1 if idUseCount > 1: # if footnote with id in other links bump the id number attributes = linkChild.attributes.copy() attributes["id"] = "{}_{}".format(attributes["id"], idUseCount) footnoteIdCount[linkChild.footnoteID] = idUseCount newChild = addChild(newLink, linkChild.qname, attributes=attributes) xmlLang = linkChild.xmlLang if xmlLang is not None and xmlLang != baseXmlLang: # default newChild.set("{http://www.w3.org/XML/1998/namespace}lang", xmlLang) copyIxFootnoteHtml(linkChild, newChild, targetModelDocument=targetInstance.modelDocument, withText=True) if filingFiles and linkChild.textValue: footnoteHtml = XML("<body/>") copyIxFootnoteHtml(linkChild, footnoteHtml) for elt in footnoteHtml.iter(): addLocallyReferencedFile(elt,filingFiles) targetInstance.saveInstance(overrideFilepath=targetUrl, outputZip=outputZip) if getattr(modelXbrl, "isTestcaseVariation", False): modelXbrl.extractedInlineInstance = True # for validation comparison modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def createTargetInstance(modelXbrl, targetUrl, targetDocumentSchemaRefs, filingFiles, baseXmlLang=None, defaultXmlLang=None): def addLocallyReferencedFile(elt, filingFiles): if elt.tag in ("a", "img"): for attrTag, attrValue in elt.items(): if attrTag in ("href", "src") and not isHttpUrl( attrValue) and not os.path.isabs(attrValue): attrValue = attrValue.partition('#')[0] # remove anchor if attrValue: # ignore anchor references to base document attrValue = os.path.normpath( attrValue ) # change url path separators to host separators file = os.path.join(sourceDir, attrValue) if modelXbrl.fileSource.isInArchive( file, checkExistence=True) or os.path.exists(file): filingFiles.add(file) targetInstance = ModelXbrl.create( modelXbrl.modelManager, newDocumentType=Type.INSTANCE, url=targetUrl, schemaRefs=targetDocumentSchemaRefs, isEntry=True, discover=False) # don't attempt to load DTS ixTargetRootElt = modelXbrl.ixTargetRootElements[getattr( modelXbrl, "ixdsTarget", None)] langIsSet = False # copy ix resources target root attributes for attrName, attrValue in ixTargetRootElt.items(): if attrName != "target": # ix:references target is not mapped to xbrli:xbrl targetInstance.modelDocument.xmlRootElement.set( attrName, attrValue) if attrName == "{http://www.w3.org/XML/1998/namespace}lang": langIsSet = True defaultXmlLang = attrValue if attrName.startswith("{"): ns, _sep, ln = attrName[1:].rpartition("}") if ns: prefix = xmlnsprefix(ixTargetRootElt, ns) if prefix not in (None, "xml"): setXmlns(targetInstance.modelDocument, prefix, ns) if not langIsSet and baseXmlLang: targetInstance.modelDocument.xmlRootElement.set( "{http://www.w3.org/XML/1998/namespace}lang", baseXmlLang) if defaultXmlLang is None: defaultXmlLang = baseXmlLang # allows facts/footnotes to override baseXmlLang ValidateXbrlDimensions.loadDimensionDefaults( targetInstance) # need dimension defaults # roleRef and arcroleRef (of each inline document) for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs): for roleRefElt in sourceRefs.values(): addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname, attributes=roleRefElt.items()) # contexts for context in sorted(modelXbrl.contexts.values(), key=lambda c: c.objectIndex ): # contexts may come from multiple IXDS files ignore = targetInstance.createContext( context.entityIdentifier[0], context.entityIdentifier[1], 'instant' if context.isInstantPeriod else 'duration' if context.isStartEndPeriod else 'forever', context.startDatetime, context.endDatetime, None, context.qnameDims, [], [], id=context.id) for unit in sorted(modelXbrl.units.values(), key=lambda u: u.objectIndex ): # units may come from multiple IXDS files measures = unit.measures ignore = targetInstance.createUnit(measures[0], measures[1], id=unit.id) modelXbrl.modelManager.showStatus(_("Creating and validating facts")) newFactForOldObjId = {} def createFacts(facts, parent): for fact in facts: if fact.isItem: # HF does not de-duplicate, which is currently-desired behavior attrs = {"contextRef": fact.contextID} if fact.id: attrs["id"] = fact.id if fact.isNumeric: attrs["unitRef"] = fact.unitID if fact.get("decimals"): attrs["decimals"] = fact.get("decimals") if fact.get("precision"): attrs["precision"] = fact.get("precision") if fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact.xValue if fact.xValid else fact.textValue for attrName, attrValue in fact.items(): if attrName.startswith("{"): attrs[qname( attrName, fact.nsmap )] = attrValue # using qname allows setting prefix in extracted instance newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent) # if fact.isFraction, create numerator and denominator newFactForOldObjId[fact.objectIndex] = newFact if filingFiles is not None and fact.concept is not None and fact.concept.isTextBlock: # check for img and other filing references so that referenced files are included in the zip. for xmltext in [text] + CDATApattern.findall(text): try: for elt in XML("<body>\n{0}\n</body>\n".format( xmltext)).iter(): addLocallyReferencedFile(elt, filingFiles) except (XMLSyntaxError, UnicodeDecodeError): pass # TODO: Why ignore UnicodeDecodeError? elif fact.isTuple: attrs = {} if fact.id: attrs["id"] = fact.id if fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" for attrName, attrValue in fact.items(): if attrName.startswith("{"): attrs[qname(attrName, fact.nsmap)] = attrValue newTuple = targetInstance.createFact(fact.qname, attributes=attrs, parent=parent) newFactForOldObjId[fact.objectIndex] = newTuple createFacts(fact.modelTupleFacts, newTuple) createFacts(modelXbrl.facts, None) modelXbrl.modelManager.showStatus( _("Creating and validating footnotes and relationships")) HREF = "{http://www.w3.org/1999/xlink}href" footnoteLinks = defaultdict(list) footnoteIdCount = {} for linkKey, linkPrototypes in modelXbrl.baseSets.items(): arcrole, linkrole, linkqname, arcqname = linkKey if (linkrole and linkqname and arcqname and # fully specified roles arcrole != "XBRL-footnotes" and any( lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)): for linkPrototype in linkPrototypes: if linkPrototype not in footnoteLinks[linkrole]: footnoteLinks[linkrole].append(linkPrototype) for linkrole in sorted(footnoteLinks.keys()): for linkPrototype in footnoteLinks[linkrole]: newLink = addChild(targetInstance.modelDocument.xmlRootElement, linkPrototype.qname, attributes=linkPrototype.attributes) for linkChild in linkPrototype: attributes = linkChild.attributes if isinstance(linkChild, LocPrototype): if HREF not in linkChild.attributes: linkChild.attributes[HREF] = \ "#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex]) addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ArcPrototype): addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ModelInlineFootnote): idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1 if idUseCount > 1: # if footnote with id in other links bump the id number attributes = linkChild.attributes.copy() attributes["id"] = "{}_{}".format( attributes["id"], idUseCount) footnoteIdCount[linkChild.footnoteID] = idUseCount newChild = addChild(newLink, linkChild.qname, attributes=attributes) xmlLang = linkChild.xmlLang if xmlLang is not None and xmlLang != defaultXmlLang: # default newChild.set( "{http://www.w3.org/XML/1998/namespace}lang", xmlLang) copyIxFootnoteHtml( linkChild, newChild, targetModelDocument=targetInstance.modelDocument, withText=True) if filingFiles and linkChild.textValue: footnoteHtml = XML("<body/>") copyIxFootnoteHtml(linkChild, footnoteHtml) for elt in footnoteHtml.iter(): addLocallyReferencedFile(elt, filingFiles) return targetInstance
def validateTestcase(self, testcase): self.modelXbrl.info("info", "Testcase", modelDocument=testcase) self.modelXbrl.viewModelObject(testcase.objectId()) if hasattr(testcase, "testcaseVariations"): for modelTestcaseVariation in testcase.testcaseVariations: # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) # is this a versioning report? resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance formulaOutputInstance = None inputDTSes = defaultdict(list) baseForElement = testcase.baseForElement(modelTestcaseVariation) # try to load instance document self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s"), modelObject=modelTestcaseVariation, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, expected=modelTestcaseVariation.expected) for readMeFirstUri in modelTestcaseVariation.readMeFirstUris: if isinstance(readMeFirstUri,tuple): # dtsName is for formula instances, but is from/to dts if versioning dtsName, readMeFirstUri = readMeFirstUri elif resultIsVersioningReport: if inputDTSes: dtsName = "to" else: dtsName = "from" else: dtsName = None if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document if dtsName in inputDTSes: dtsName = inputDTSes[dtsName] else: modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager, ModelDocument.Type.DTSENTRIES, self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement), isEntry=True) DTSdoc = modelXbrl.modelDocument DTSdoc.inDTS = True doc = ModelDocument.load(modelXbrl, readMeFirstUri, base=baseForElement) if doc is not None: DTSdoc.referencesDocument[doc] = "import" #fake import doc.inDTS = True else: # not a multi-schemaRef versioning report modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, readMeFirstUri, _("validating"), base=baseForElement, useFileSource=self.useFileSource) if modelXbrl.modelDocument is None: self.modelXbrl.error("arelle:notLoaded", _("Testcase %(id)s %(name)s document not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) modelXbrl.close() self.determineNotLoadedTestStatus(modelTestcaseVariation) elif resultIsVersioningReport: inputDTSes[dtsName] = modelXbrl elif modelXbrl.modelDocument.type == ModelDocument.Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl) self.determineTestStatus(modelTestcaseVariation, modelXbrl) modelXbrl.close() elif testcase.type == ModelDocument.Type.REGISTRYTESTCASE: self.instValidator.validate(modelXbrl) # required to set up dimensions, etc self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id, modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest) self.determineTestStatus(modelTestcaseVariation, modelXbrl) self.instValidator.close() modelXbrl.close() else: inputDTSes[dtsName].append(modelXbrl) if resultIsVersioningReport and modelXbrl.modelDocument: versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl( modelTestcaseVariation.versioningReportUri, baseForElement) if os.path.exists(versReportFile): #validate existing modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report")) if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == ModelDocument.Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport) self.determineTestStatus(modelTestcaseVariation, modelVersReport) modelVersReport.close() elif len(inputDTSes) == 2: ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes( versReportFile, inputDTSes["from"], inputDTSes["to"]) modelTestcaseVariation.status = "generated" else: self.modelXbrl.error("arelle:notLoaded", _("Testcase %(id)s %(name)s DTSes not loaded, unable to generate versioning report: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) modelTestcaseVariation.status = "failed" for inputDTS in inputDTSes.values(): inputDTS.close() del inputDTSes # dereference elif inputDTSes: # validate schema, linkbase, or instance modelXbrl = inputDTSes[None][0] parameters = modelTestcaseVariation.parameters.copy() for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters if dtsName: parameters[dtsName] = (None, inputDTS) self.instValidator.validate(modelXbrl, parameters) if modelTestcaseVariation.resultIsInfoset: infoset = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultInfosetUri, _("loading result infoset"), base=baseForElement, useFileSource=self.useFileSource) if infoset.modelDocument is None: self.modelXbrl.error("arelle:notLoaded", _("Testcase %(id)s %(name)s result infoset not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstance)) modelTestcaseVariation.status = "result infoset not loadable" else: # check infoset ValidateInfoset.validate(self.instValidator, modelXbrl, infoset) infoset.close() self.determineTestStatus(modelTestcaseVariation, modelXbrl) # include infoset errors in status self.instValidator.close() if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual): # if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost) self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters) self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance) if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance formulaOutputInstance = modelXbrl.formulaOutputInstance modelXbrl.formulaOutputInstance = None # prevent it from being closed now self.instValidator.close() for inputDTSlist in inputDTSes.values(): for inputDTS in inputDTSlist: inputDTS.close() if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument: expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultXbrlInstanceUri, _("loading expected result XBRL instance"), base=baseForElement, useFileSource=self.useFileSource) if expectedInstance.modelDocument is None: self.modelXbrl.error("arelle:notLoaded", _("Testcase %(id)s %(name)s expected result instance not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstance)) modelTestcaseVariation.status = "result not loadable" expectedInstance.close() else: # compare facts if len(expectedInstance.facts) != len(formulaOutputInstance.facts): formulaOutputInstance.error("formula:resultFactCounts", _("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"), modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts), expectedFacts=len(expectedInstance.facts)) else: for fact in expectedInstance.facts: if formulaOutputInstance.matchFact(fact) is None: formulaOutputInstance.error("formula:expectedFactMissing", _("Formula output missing expected fact %(fact)s"), modelXbrl=fact, fact=fact.qname) self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance) formulaOutputInstance.close() formulaOutputInstance = None # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def validateTestcase(self, testcase): self.modelXbrl.info("info", "Testcase", modelDocument=testcase) self.modelXbrl.viewModelObject(testcase.objectId()) if testcase.type in (Type.TESTCASESINDEX, Type.REGISTRY): for doc in sorted(testcase.referencesDocument.keys(), key=lambda doc: doc.uri): self.validateTestcase(doc) # testcases doc's are sorted by their uri (file names), e.g., for formula elif hasattr(testcase, "testcaseVariations"): for modelTestcaseVariation in testcaseVariationsByTarget(testcase.testcaseVariations): # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) # is this a versioning report? resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance resultIsTaxonomyPackage = modelTestcaseVariation.resultIsTaxonomyPackage formulaOutputInstance = None inputDTSes = defaultdict(list) baseForElement = testcase.baseForElement(modelTestcaseVariation) # try to load instance document self.modelXbrl.info("info", _("Variation %(id)s%(name)s%(target)s: %(expected)s - %(description)s"), modelObject=modelTestcaseVariation, id=modelTestcaseVariation.id, name=(" {}".format(modelTestcaseVariation.name) if modelTestcaseVariation.name else ""), target=(" target {}".format(modelTestcaseVariation.ixdsTarget) if modelTestcaseVariation.ixdsTarget else ""), expected=modelTestcaseVariation.expected, description=modelTestcaseVariation.description) if self.modelXbrl.modelManager.formulaOptions.testcaseResultsCaptureWarnings: errorCaptureLevel = logging._checkLevel("WARNING") else: errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY parameters = modelTestcaseVariation.parameters.copy() for readMeFirstUri in modelTestcaseVariation.readMeFirstUris: if isinstance(readMeFirstUri,tuple): # dtsName is for formula instances, but is from/to dts if versioning dtsName, readMeFirstUri = readMeFirstUri elif resultIsVersioningReport: if inputDTSes: dtsName = "to" else: dtsName = "from" else: dtsName = None if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document if dtsName in inputDTSes: dtsName = inputDTSes[dtsName] else: modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager, Type.DTSENTRIES, self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement), isEntry=True, errorCaptureLevel=errorCaptureLevel) DTSdoc = modelXbrl.modelDocument DTSdoc.inDTS = True doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement) if doc is not None: DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement) #fake import doc.inDTS = True elif resultIsTaxonomyPackage: from arelle import PackageManager, PrototypeInstanceObject dtsName = readMeFirstUri modelXbrl = PrototypeInstanceObject.XbrlPrototype(self.modelXbrl.modelManager, readMeFirstUri) PackageManager.packageInfo(self.modelXbrl.modelManager.cntlr, readMeFirstUri, reload=True, errors=modelXbrl.errors) else: # not a multi-schemaRef versioning report if self.useFileSource.isArchive: modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, readMeFirstUri, _("validating"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel, ixdsTarget=modelTestcaseVariation.ixdsTarget) else: # need own file source, may need instance discovery filesource = FileSource.openFileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr, base=baseForElement) if filesource and not filesource.selection and filesource.isArchive: try: if filesource.isTaxonomyPackage: _rptPkgIxdsOptions = {} for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ReportPackageIxdsOptions"): pluginXbrlMethod(self, _rptPkgIxdsOptions) filesource.loadTaxonomyPackageMappings() for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ReportPackageIxds"): filesource.select(pluginXbrlMethod(filesource, **_rptPkgIxdsOptions)) else: from arelle.CntlrCmdLine import filesourceEntrypointFiles entrypoints = filesourceEntrypointFiles(filesource) if entrypoints: # resolve an IXDS in entrypoints for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ArchiveIxds"): pluginXbrlMethod(self, filesource,entrypoints) filesource.select(entrypoints[0].get("file", None) ) except Exception as err: self.modelXbrl.error("exception:" + type(err).__name__, _("Testcase variation validation exception: %(error)s, entry URL: %(instance)s"), modelXbrl=self.modelXbrl, instance=readMeFirstUri, error=err) continue # don't try to load this entry URL modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, filesource, _("validating"), base=baseForElement, errorCaptureLevel=errorCaptureLevel, ixdsTarget=modelTestcaseVariation.ixdsTarget) modelXbrl.isTestcaseVariation = True if modelXbrl.modelDocument is None: modelXbrl.info("arelle:notLoaded", _("Variation %(id)s %(name)s readMeFirst document not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) self.determineNotLoadedTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif resultIsVersioningReport or resultIsTaxonomyPackage: inputDTSes[dtsName] = modelXbrl elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl) self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif testcase.type == Type.REGISTRYTESTCASE: self.instValidator.validate(modelXbrl) # required to set up dimensions, etc self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id, modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest) self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) self.instValidator.close() modelXbrl.close() else: inputDTSes[dtsName].append(modelXbrl) # validate except for formulas _hasFormulae = modelXbrl.hasFormulae modelXbrl.hasFormulae = False try: for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"): pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation) self.instValidator.validate(modelXbrl, parameters) for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"): pluginXbrlMethod(self.modelXbrl, modelXbrl) except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase variation validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=(type(err) is not AssertionError)) modelXbrl.hasFormulae = _hasFormulae if resultIsVersioningReport and modelXbrl.modelDocument: versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl( modelTestcaseVariation.versioningReportUri, baseForElement) if os.path.exists(versReportFile): #validate existing modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report")) if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport) self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors) modelVersReport.close() elif len(inputDTSes) == 2: ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes( versReportFile, inputDTSes["from"], inputDTSes["to"]) modelTestcaseVariation.status = "generated" else: modelXbrl.error("arelle:notLoaded", _("Variation %(id)s %(name)s input DTSes not loaded, unable to generate versioning report: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) modelTestcaseVariation.status = "failed" for inputDTS in inputDTSes.values(): inputDTS.close() del inputDTSes # dereference elif resultIsTaxonomyPackage: self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif inputDTSes: # validate schema, linkbase, or instance modelXbrl = inputDTSes[None][0] expectedDataFiles = set(modelXbrl.modelManager.cntlr.webCache.normalizeUrl(uri, baseForElement) for d in modelTestcaseVariation.dataUris.values() for uri in d if not UrlUtil.isAbsolute(uri)) foundDataFiles = set() variationBase = os.path.dirname(baseForElement) for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters if dtsName: # named instance parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes) elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula for _inputDTS in inputDTS: for docUrl, doc in _inputDTS.urlDocs.items(): if docUrl.startswith(variationBase) and not doc.type == Type.INLINEXBRLDOCUMENTSET: if getattr(doc,"loadedFromXbrlFormula", False): # may have been sourced from xf file if docUrl.replace("-formula.xml", ".xf") in expectedDataFiles: docUrl = docUrl.replace("-formula.xml", ".xf") foundDataFiles.add(docUrl) if expectedDataFiles - foundDataFiles: modelXbrl.info("arelle:testcaseDataNotUsed", _("Variation %(id)s %(name)s data files not used: %(missingDataFiles)s"), modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id, missingDataFiles=", ".join(sorted(os.path.basename(f) for f in expectedDataFiles - foundDataFiles))) if foundDataFiles - expectedDataFiles: modelXbrl.info("arelle:testcaseDataUnexpected", _("Variation %(id)s %(name)s files not in variation data: %(unexpectedDataFiles)s"), modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id, unexpectedDataFiles=", ".join(sorted(os.path.basename(f) for f in foundDataFiles - expectedDataFiles))) if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable: try: RenderingEvaluator.init(modelXbrl) except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase RenderingEvaluator.init exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) modelXbrlHasFormulae = modelXbrl.hasFormulae if modelXbrlHasFormulae and self.modelXbrl.modelManager.formulaOptions.formulaAction != "none": try: # validate only formulae self.instValidator.parameters = parameters ValidateFormula.validate(self.instValidator) except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=(type(err) is not AssertionError)) if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset: for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"): pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri) infoset = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultInfosetUri, _("loading result infoset"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) if infoset.modelDocument is None: modelXbrl.error("arelle:notLoaded", _("Variation %(id)s %(name)s result infoset not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstance)) modelTestcaseVariation.status = "result infoset not loadable" else: # check infoset ValidateInfoset.validate(self.instValidator, modelXbrl, infoset) infoset.close() if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset: # diff (or generate) table infoset resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement) if not any(alternativeValidation(modelXbrl, resultTableUri) for alternativeValidation in pluginClassMethods("Validate.TableInfoset")): try: ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase table linkbase validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) self.instValidator.close() extraErrors = [] for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"): pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors) self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual): # if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost) self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters) self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors) if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance formulaOutputInstance = modelXbrl.formulaOutputInstance modelXbrl.formulaOutputInstance = None # prevent it from being closed now self.instValidator.close() compareIxResultInstance = (modelXbrl.modelDocument.type in (Type.INLINEXBRL, Type.INLINEXBRLDOCUMENTSET) and modelTestcaseVariation.resultXbrlInstanceUri is not None) if compareIxResultInstance: formulaOutputInstance = modelXbrl # compare modelXbrl to generated output instance errMsgPrefix = "ix" else: # delete input instances before formula output comparision for inputDTSlist in inputDTSes.values(): for inputDTS in inputDTSlist: inputDTS.close() del inputDTSes # dereference errMsgPrefix = "formula" if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument: _matchExpectedResultIDs = not modelXbrlHasFormulae # formula restuls have inconsistent IDs expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultXbrlInstanceUri, _("loading expected result XBRL instance"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) if expectedInstance.modelDocument is None: self.modelXbrl.error("{}:expectedResultNotLoaded".format(errMsgPrefix), _("Testcase \"%(name)s\" %(id)s expected result instance not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstanceUri), messageCodes=("formula:expectedResultNotLoaded","ix:expectedResultNotLoaded")) modelTestcaseVariation.status = "result not loadable" else: # compare facts for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.ExpectedInstance.Loaded"): pluginXbrlMethod(expectedInstance, formulaOutputInstance) if len(expectedInstance.facts) != len(formulaOutputInstance.facts): formulaOutputInstance.error("{}:resultFactCounts".format(errMsgPrefix), _("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"), modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts), expectedFacts=len(expectedInstance.facts), messageCodes=("formula:resultFactCounts","ix:resultFactCounts")) else: formulaOutputFootnotesRelSet = ModelRelationshipSet(formulaOutputInstance, "XBRL-footnotes") expectedFootnotesRelSet = ModelRelationshipSet(expectedInstance, "XBRL-footnotes") def factFootnotes(fact, footnotesRelSet): footnotes = {} footnoteRels = footnotesRelSet.fromModelObject(fact) if footnoteRels: # most process rels in same order between two instances, use labels to sort for i, footnoteRel in enumerate(sorted(footnoteRels, key=lambda r: (r.fromLabel,r.toLabel))): modelObject = footnoteRel.toModelObject if isinstance(modelObject, ModelResource): xml = collapseWhitespace(modelObject.viewText().strip()) footnotes["Footnote {}".format(i+1)] = xml #re.sub(r'\s+', ' ', collapseWhitespace(modelObject.stringValue)) elif isinstance(modelObject, ModelFact): footnotes["Footnoted fact {}".format(i+1)] = \ "{} context: {} value: {}".format( modelObject.qname, modelObject.contextID, collapseWhitespace(modelObject.value)) return footnotes for expectedInstanceFact in expectedInstance.facts: unmatchedFactsStack = [] formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=_matchExpectedResultIDs, matchLang=False) #formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=True, matchLang=True) if formulaOutputFact is None: if unmatchedFactsStack: # get missing nested tuple fact, if possible missingFact = unmatchedFactsStack[-1] else: missingFact = expectedInstanceFact # is it possible to show value mismatches? expectedFacts = formulaOutputInstance.factsByQname.get(missingFact.qname) if len(expectedFacts) == 1: formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix), _("Output missing expected fact %(fact)s, extracted value \"%(value1)s\", expected value \"%(value2)s\""), modelXbrl=missingFact, fact=missingFact.qname, value1=missingFact.xValue, value2=next(iter(expectedFacts)).xValue, messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing")) else: formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix), _("Output missing expected fact %(fact)s"), modelXbrl=missingFact, fact=missingFact.qname, messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing")) else: # compare footnotes expectedInstanceFactFootnotes = factFootnotes(expectedInstanceFact, expectedFootnotesRelSet) formulaOutputFactFootnotes = factFootnotes(formulaOutputFact, formulaOutputFootnotesRelSet) if (len(expectedInstanceFactFootnotes) != len(formulaOutputFactFootnotes) or set(expectedInstanceFactFootnotes.values()) != set(formulaOutputFactFootnotes.values())): formulaOutputInstance.error("{}:expectedFactFootnoteDifference".format(errMsgPrefix), _("Output expected fact %(fact)s expected footnotes %(footnotes1)s produced footnotes %(footnotes2)s"), modelXbrl=(formulaOutputFact,expectedInstanceFact), fact=expectedInstanceFact.qname, footnotes1=sorted(expectedInstanceFactFootnotes.items()), footnotes2=sorted(formulaOutputFactFootnotes.items()), messageCodes=("formula:expectedFactFootnoteDifference","ix:expectedFactFootnoteDifference")) # for debugging uncomment next line to save generated instance document # formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml") expectedInstance.close() del expectedInstance # dereference self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors) formulaOutputInstance.close() del formulaOutputInstance if compareIxResultInstance: for inputDTSlist in inputDTSes.values(): for inputDTS in inputDTSlist: inputDTS.close() del inputDTSes # dereference # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) _statusCounts = OrderedDict((("pass",0),("fail",0))) for tv in getattr(testcase, "testcaseVariations", ()): _statusCounts[tv.status] = _statusCounts.get(tv.status, 0) + 1 self.modelXbrl.info("arelle:testCaseResults", ", ".join("{}={}".format(k,c) for k, c in _statusCounts.items() if k)) self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs, outputZip=None, filingFiles=None): targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(targetDocumentFilename, modelXbrl.modelDocument.filepath) targetUrlParts = targetUrl.rpartition(".") targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2] modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl)) targetInstance = ModelXbrl.create(modelXbrl.modelManager, newDocumentType=Type.INSTANCE, url=targetUrl, schemaRefs=targetDocumentSchemaRefs, isEntry=True) ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults # roleRef and arcroleRef (of each inline document) for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs): for roleRefElt in sourceRefs.values(): addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname, attributes=roleRefElt.items()) # contexts for context in modelXbrl.contexts.values(): newCntx = targetInstance.createContext(context.entityIdentifier[0], context.entityIdentifier[1], 'instant' if context.isInstantPeriod else 'duration' if context.isStartEndPeriod else 'forever', context.startDatetime, context.endDatetime, None, context.qnameDims, [], [], id=context.id) for unit in modelXbrl.units.values(): measures = unit.measures newUnit = targetInstance.createUnit(measures[0], measures[1], id=unit.id) modelXbrl.modelManager.showStatus(_("Creating and validating facts")) newFactForOldObjId = {} def createFacts(facts, parent): for fact in facts: if fact.isItem: attrs = {"contextRef": fact.contextID} if fact.id: attrs["id"] = fact.id if fact.isNumeric: attrs["unitRef"] = fact.unitID if fact.get("decimals"): attrs["decimals"] = fact.get("decimals") if fact.get("precision"): attrs["precision"] = fact.get("precision") if fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact.xValue if fact.xValid else fact.textValue newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent) newFactForOldObjId[fact.objectIndex] = newFact if filingFiles and fact.concept is not None and fact.concept.isTextBlock: # check for img and other filing references for xmltext in [text] + CDATApattern.findall(text): try: for elt in XML("<body>\n{0}\n</body>\n".format(xmltext)): if elt.tag in ("a", "img") and not isHttpUrl(attrValue) and not os.path.isabs(attrvalue): for attrTag, attrValue in elt.items(): if attrTag in ("href", "src"): filingFiles.add(attrValue) except (XMLSyntaxError, UnicodeDecodeError): pass elif fact.isTuple: newTuple = targetInstance.createFact(fact.qname, parent=parent) newFactForOldObjId[fact.objectIndex] = newTuple createFacts(fact.modelTupleFacts, newTuple) createFacts(modelXbrl.facts, None) # footnote links footnoteIdCount = {} modelXbrl.modelManager.showStatus(_("Creating and validating footnotes & relationships")) HREF = "{http://www.w3.org/1999/xlink}href" footnoteLinks = defaultdict(list) for linkKey, linkPrototypes in modelXbrl.baseSets.items(): arcrole, linkrole, linkqname, arcqname = linkKey if (linkrole and linkqname and arcqname and # fully specified roles arcrole != "XBRL-footnotes" and any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)): for linkPrototype in linkPrototypes: if linkPrototype not in footnoteLinks[linkrole]: footnoteLinks[linkrole].append(linkPrototype) for linkrole in sorted(footnoteLinks.keys()): for linkPrototype in footnoteLinks[linkrole]: newLink = addChild(targetInstance.modelDocument.xmlRootElement, linkPrototype.qname, attributes=linkPrototype.attributes) for linkChild in linkPrototype: attributes = linkChild.attributes if isinstance(linkChild, LocPrototype): if HREF not in linkChild.attributes: linkChild.attributes[HREF] = \ "#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex]) addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ArcPrototype): addChild(newLink, linkChild.qname, attributes=attributes) elif isinstance(linkChild, ModelInlineFootnote): idUseCount = footnoteIdCount.get(linkChild.footnoteID, 0) + 1 if idUseCount > 1: # if footnote with id in other links bump the id number attributes = linkChild.attributes.copy() attributes["id"] = "{}_{}".format(attributes["id"], idUseCount) footnoteIdCount[linkChild.footnoteID] = idUseCount newChild = addChild(newLink, linkChild.qname, attributes=attributes) copyIxFootnoteHtml(linkChild, newChild, withText=True) if filingFiles and linkChild.textValue: footnoteHtml = XML("<body/>") copyIxFootnoteHtml(linkChild, footnoteHtml) for elt in footnoteHtml.iter(): if elt.tag in ("a", "img"): for attrTag, attrValue in elt.items(): if attrTag in ("href", "src") and not isHttpUrl(attrValue) and not os.path.isabs(attrvalue): filingFiles.add(attrValue) targetInstance.saveInstance(overrideFilepath=targetUrl, outputZip=outputZip) modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def validateTestcase(self, testcase): self.modelXbrl.info("info", "Testcase", modelDocument=testcase) self.modelXbrl.viewModelObject(testcase.objectId()) if hasattr(testcase, "testcaseVariations"): for modelTestcaseVariation in testcase.testcaseVariations: # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) # is this a versioning report? resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance formulaOutputInstance = None inputDTSes = defaultdict(list) baseForElement = testcase.baseForElement(modelTestcaseVariation) # try to load instance document self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s - %(description)s"), modelObject=modelTestcaseVariation, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, expected=modelTestcaseVariation.expected, description=modelTestcaseVariation.description) errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY parameters = modelTestcaseVariation.parameters.copy() for readMeFirstUri in modelTestcaseVariation.readMeFirstUris: if isinstance(readMeFirstUri,tuple): # dtsName is for formula instances, but is from/to dts if versioning dtsName, readMeFirstUri = readMeFirstUri elif resultIsVersioningReport: if inputDTSes: dtsName = "to" else: dtsName = "from" else: dtsName = None if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document if dtsName in inputDTSes: dtsName = inputDTSes[dtsName] else: modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager, Type.DTSENTRIES, self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement), isEntry=True, errorCaptureLevel=errorCaptureLevel) DTSdoc = modelXbrl.modelDocument DTSdoc.inDTS = True doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement) if doc is not None: DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement) #fake import doc.inDTS = True else: # not a multi-schemaRef versioning report if self.useFileSource.isArchive: modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, readMeFirstUri, _("validating"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) else: # need own file source, may need instance discovery filesource = FileSource.FileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr) if filesource and not filesource.selection and filesource.isArchive: for _archiveFile in filesource.dir: # find instance document in archive filesource.select(_archiveFile) if ModelDocument.Type.identify(filesource, filesource.url) in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL): break # use this selection modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, filesource, _("validating"), base=baseForElement, errorCaptureLevel=errorCaptureLevel) if modelXbrl.modelDocument is None: self.modelXbrl.error("arelle:notLoaded", _("Testcase %(id)s %(name)s document not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) modelXbrl.close() self.determineNotLoadedTestStatus(modelTestcaseVariation) elif resultIsVersioningReport: inputDTSes[dtsName] = modelXbrl elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl) self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif testcase.type == Type.REGISTRYTESTCASE: self.instValidator.validate(modelXbrl) # required to set up dimensions, etc self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id, modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest) self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) self.instValidator.close() modelXbrl.close() else: inputDTSes[dtsName].append(modelXbrl) # validate except for formulas _hasFormulae = modelXbrl.hasFormulae modelXbrl.hasFormulae = False try: for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"): pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation) self.instValidator.validate(modelXbrl, parameters) for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"): pluginXbrlMethod(self.modelXbrl, modelXbrl) except Exception as err: self.modelXbrl.error("exception", _("Testcase variation validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) modelXbrl.hasFormulae = _hasFormulae if resultIsVersioningReport and modelXbrl.modelDocument: versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl( modelTestcaseVariation.versioningReportUri, baseForElement) if os.path.exists(versReportFile): #validate existing modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report")) if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport) self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors) modelVersReport.close() elif len(inputDTSes) == 2: ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes( versReportFile, inputDTSes["from"], inputDTSes["to"]) modelTestcaseVariation.status = "generated" else: self.modelXbrl.error("arelle:notLoaded", _("Testcase %(id)s %(name)s DTSes not loaded, unable to generate versioning report: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) modelTestcaseVariation.status = "failed" for inputDTS in inputDTSes.values(): inputDTS.close() del inputDTSes # dereference elif inputDTSes: # validate schema, linkbase, or instance modelXbrl = inputDTSes[None][0] for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters if dtsName: # named instance parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes) elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable: RenderingEvaluator.init(modelXbrl) if modelXbrl.hasFormulae: try: # validate only formulae self.instValidator.parameters = parameters ValidateFormula.validate(self.instValidator) except Exception as err: self.modelXbrl.error("exception", _("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset: for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"): pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri) infoset = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultInfosetUri, _("loading result infoset"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) if infoset.modelDocument is None: self.modelXbrl.error("arelle:notLoaded", _("Testcase %(id)s %(name)s result infoset not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstance)) modelTestcaseVariation.status = "result infoset not loadable" else: # check infoset ValidateInfoset.validate(self.instValidator, modelXbrl, infoset) infoset.close() if modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset: # diff (or generate) table infoset resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement) if not any(alternativeValidation(modelXbrl, resultTableUri) for alternativeValidation in pluginClassMethods("Validate.TableInfoset")): ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files self.instValidator.close() extraErrors = [] for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"): pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors) self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual): # if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost) self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters) self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors) if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance formulaOutputInstance = modelXbrl.formulaOutputInstance modelXbrl.formulaOutputInstance = None # prevent it from being closed now self.instValidator.close() for inputDTSlist in inputDTSes.values(): for inputDTS in inputDTSlist: inputDTS.close() del inputDTSes # dereference if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument: expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultXbrlInstanceUri, _("loading expected result XBRL instance"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) if expectedInstance.modelDocument is None: self.modelXbrl.error("arelle:notLoaded", _("Testcase %(id)s %(name)s expected result instance not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstance)) modelTestcaseVariation.status = "result not loadable" else: # compare facts if len(expectedInstance.facts) != len(formulaOutputInstance.facts): formulaOutputInstance.error("formula:resultFactCounts", _("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"), modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts), expectedFacts=len(expectedInstance.facts)) else: for fact in expectedInstance.facts: unmatchedFactsStack = [] if formulaOutputInstance.matchFact(fact, unmatchedFactsStack, deemP0inf=True) is None: if unmatchedFactsStack: # get missing nested tuple fact, if possible missingFact = unmatchedFactsStack[-1] else: missingFact = fact formulaOutputInstance.error("formula:expectedFactMissing", _("Formula output missing expected fact %(fact)s"), modelXbrl=missingFact, fact=missingFact.qname) # for debugging uncomment next line to save generated instance document # formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml") expectedInstance.close() del expectedInstance # dereference self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors) formulaOutputInstance.close() del formulaOutputInstance # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs): targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl( targetDocumentFilename, modelXbrl.modelDocument.filepath) targetUrlParts = targetUrl.rpartition(".") targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2] modelXbrl.modelManager.showStatus( _("Extracting instance ") + os.path.basename(targetUrl)) targetInstance = ModelXbrl.create(modelXbrl.modelManager, newDocumentType=Type.INSTANCE, url=targetUrl, schemaRefs=targetDocumentSchemaRefs, isEntry=True) ValidateXbrlDimensions.loadDimensionDefaults( targetInstance) # need dimension defaults # roleRef and arcroleRef (of each inline document) for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs): for roleRefElt in sourceRefs.values(): XmlUtil.addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname, attributes=roleRefElt.items()) # contexts for context in modelXbrl.contexts.values(): newCntx = targetInstance.createContext( context.entityIdentifier[0], context.entityIdentifier[1], 'instant' if context.isInstantPeriod else 'duration' if context.isStartEndPeriod else 'forever', context.startDatetime, context.endDatetime, None, context.qnameDims, [], [], id=context.id) for unit in modelXbrl.units.values(): measures = unit.measures newUnit = targetInstance.createUnit(measures[0], measures[1], id=unit.id) modelXbrl.modelManager.showStatus(_("Creating and validating facts")) newFactForOldObjId = {} def createFacts(facts, parent): for fact in modelXbrl.facts: if fact.isItem: attrs = {"contextRef": fact.contextID} if fact.isNumeric: attrs["unitRef"] = fact.unitID if fact.get("decimals"): attrs["decimals"] = fact.get("decimals") if fact.get("precision"): attrs["precision"] = fact.get("precision") if fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact.xValue if fact.xValid else fact.textValue newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent) newFactForOldObjId[fact.objectIndex] = newFact elif fact.isTuple: newTuple = targetInstance.createFact(fact.qname, parent=parent) newFactForOldObjId[fact.objectIndex] = newTuple createFacts(fact.modelTupleFacts, newTuple) createFacts(modelXbrl.facts, None) # footnote links modelXbrl.modelManager.showStatus( _("Creating and validating footnotes & relationships")) for linkKey, linkPrototypes in modelXbrl.baseSets.items(): arcrole, linkrole, linkqname, arcqname = linkKey if (linkrole and linkqname and arcqname and # fully specified roles any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)): for linkPrototype in linkPrototypes: newLink = XmlUtil.addChild( targetInstance.modelDocument.xmlRootElement, linkqname, attributes=linkPrototype.attributes) for linkChild in linkPrototype: if isinstance( linkChild, LocPrototype ) and "{http://www.w3.org/1999/xlink}href" not in linkChild.attributes: linkChild.attributes["{http://www.w3.org/1999/xlink}href"] = \ "#" + XmlUtil.elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex]) XmlUtil.addChild(newLink, linkChild.qname, attributes=linkChild.attributes, text=linkChild.textValue) targetInstance.saveInstance(overrideFilepath=targetUrl) modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def saveTargetDocument(modelXbrl, targetDocumentFilename, targetDocumentSchemaRefs): targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(targetDocumentFilename, modelXbrl.modelDocument.filepath) targetUrlParts = targetUrl.rpartition(".") targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2] modelXbrl.modelManager.showStatus(_("Extracting instance ") + os.path.basename(targetUrl)) targetInstance = ModelXbrl.create(modelXbrl.modelManager, newDocumentType=Type.INSTANCE, url=targetUrl, schemaRefs=targetDocumentSchemaRefs, isEntry=True) ValidateXbrlDimensions.loadDimensionDefaults(targetInstance) # need dimension defaults # roleRef and arcroleRef (of each inline document) for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs): for roleRefElt in sourceRefs.values(): XmlUtil.addChild(targetInstance.modelDocument.xmlRootElement, roleRefElt.qname, attributes=roleRefElt.items()) # contexts for context in modelXbrl.contexts.values(): newCntx = targetInstance.createContext(context.entityIdentifier[0], context.entityIdentifier[1], 'instant' if context.isInstantPeriod else 'duration' if context.isStartEndPeriod else 'forever', context.startDatetime, context.endDatetime, None, context.qnameDims, [], [], id=context.id) for unit in modelXbrl.units.values(): measures = unit.measures newUnit = targetInstance.createUnit(measures[0], measures[1], id=unit.id) modelXbrl.modelManager.showStatus(_("Creating and validating facts")) newFactForOldObjId = {} def createFacts(facts, parent): for fact in modelXbrl.facts: if fact.isItem: attrs = {"contextRef": fact.contextID} if fact.isNumeric: attrs["unitRef"] = fact.unitID if fact.get("decimals"): attrs["decimals"] = fact.get("decimals") if fact.get("precision"): attrs["precision"] = fact.get("precision") if fact.isNil: attrs[XbrlConst.qnXsiNil] = "true" text = None else: text = fact.xValue if fact.xValid else fact.textValue newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent) newFactForOldObjId[fact.objectIndex] = newFact elif fact.isTuple: newTuple = targetInstance.createFact(fact.qname, parent=parent) newFactForOldObjId[fact.objectIndex] = newTuple createFacts(fact.modelTupleFacts, newTuple) createFacts(modelXbrl.facts, None) # footnote links modelXbrl.modelManager.showStatus(_("Creating and validating footnotes & relationships")) for linkKey, linkPrototypes in modelXbrl.baseSets.items(): arcrole, linkrole, linkqname, arcqname = linkKey if (linkrole and linkqname and arcqname and # fully specified roles any(lP.modelDocument.type == Type.INLINEXBRL for lP in linkPrototypes)): for linkPrototype in linkPrototypes: newLink = XmlUtil.addChild(targetInstance.modelDocument.xmlRootElement, linkqname, attributes=linkPrototype.attributes) for linkChild in linkPrototype: if isinstance(linkChild, LocPrototype) and "{http://www.w3.org/1999/xlink}href" not in linkChild.attributes: linkChild.attributes["{http://www.w3.org/1999/xlink}href"] = \ "#" + XmlUtil.elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex]) XmlUtil.addChild(newLink, linkChild.qname, attributes=linkChild.attributes, text=linkChild.textValue) targetInstance.saveInstance(overrideFilepath=targetUrl) modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)
def validate(val): formulaOptions = val.modelXbrl.modelManager.formulaOptions XPathParser.initializeParser(val) val.modelXbrl.modelManager.showStatus(_("Compiling formulae")) initialErrorCount = val.modelXbrl.logCountErr # global parameter names parameterQnames = set() instanceQnames = set() parameterDependencies = {} instanceDependencies = defaultdict(set) # None-key entries are non-formula dependencies dependencyResolvedParameters = set() orderedParameters = [] orderedInstances = [] for paramQname, modelParameter in val.modelXbrl.qnameParameters.items(): if isinstance(modelParameter, ModelParameter): modelParameter.compile() parameterDependencies[paramQname] = modelParameter.variableRefs() parameterQnames.add(paramQname) if isinstance(modelParameter, ModelInstance): instanceQnames.add(paramQname) # duplicates checked on loading modelDocument #resolve dependencies resolvedAParameter = True while (resolvedAParameter): resolvedAParameter = False for paramQname in parameterQnames: if paramQname not in dependencyResolvedParameters and \ len(parameterDependencies[paramQname] - dependencyResolvedParameters) == 0: dependencyResolvedParameters.add(paramQname) orderedParameters.append(paramQname) resolvedAParameter = True # anything unresolved? for paramQname in parameterQnames: if paramQname not in dependencyResolvedParameters: circularOrUndefDependencies = parameterDependencies[paramQname] - dependencyResolvedParameters undefinedVars = circularOrUndefDependencies - parameterQnames paramsCircularDep = circularOrUndefDependencies - undefinedVars if len(undefinedVars) > 0: val.modelXbrl.error( _("Undefined dependencies in parameter {0}, to names {1}").format( paramQname, ", ".join((str(v) for v in undefinedVars))), "err", "xbrlve:unresolvedDependency") if len(paramsCircularDep) > 0: val.modelXbrl.error( _("Cyclic dependencies in parameter {0}, to names {1}").format( paramQname, ", ".join((str(d) for d in paramsCircularDep)) ), "err", "xbrlve:parameterCyclicDependencies") for custFnSig in val.modelXbrl.modelCustomFunctionSignatures.values(): custFnQname = custFnSig.qname if custFnQname.namespaceURI == "XbrlConst.xfi": val.modelXbrl.error( _("Custom function {0} has namespace reserved for functions in the function registry {1}").format( str(custFnQname), custFnQname.namespaceURI ), "err", "xbrlve:noProhibitedNamespaceForCustomFunction") # any custom function implementations? for modelRel in val.modelXbrl.relationshipSet(XbrlConst.functionImplementation).fromModelObject(custFnSig): custFnImpl = modelRel.toModelObject custFnSig.customFunctionImplementation = custFnImpl if len(custFnImpl.inputNames) != len(custFnSig.inputTypes): val.modelXbrl.error( _("Custom function {0} signature has {1} parameters but implementation has {2}, must be matching").format( str(custFnQname), len(custFnSig.inputTypes), len(custFnImpl.inputNames) ), "err", "xbrlcfie:inputMismatch") for custFnImpl in val.modelXbrl.modelCustomFunctionImplementations: if not val.modelXbrl.relationshipSet(XbrlConst.functionImplementation).toModelObject(custFnImpl): val.modelXbrl.error( _("Custom function implementation {0} has no relationship from any custom function signature").format( custFnImpl.xlinkLabel), "err", "xbrlcfie:missingCFIRelationship") custFnImpl.compile() # xpathContext is needed for filter setup for expressions such as aspect cover filter # determine parameter values xpathContext = XPathContext.create(val.modelXbrl) for paramQname in orderedParameters: if not isinstance(modelParameter, ModelInstance): modelParameter = val.modelXbrl.qnameParameters[paramQname] asType = modelParameter.asType asLocalName = asType.localName if asType else "string" try: if val.parameters and paramQname in val.parameters: paramDataType, paramValue = val.parameters[paramQname] typeLocalName = paramDataType.localName if paramDataType else "string" value = FunctionXs.call(xpathContext, None, typeLocalName, [paramValue]) result = FunctionXs.call(xpathContext, None, asLocalName, [value]) if formulaOptions.traceParameterInputValue: val.modelXbrl.error( _("Parameter {0} input {1}").format( paramQname, result), "info", "formula:trace") else: result = modelParameter.evaluate(xpathContext, asType) if formulaOptions.traceParameterExpressionResult: val.modelXbrl.error( _("Parameter {0} result {1}").format( paramQname, result), "info", "formula:trace") xpathContext.inScopeVars[paramQname] = result # make visible to subsequent parameter expression except XPathContext.XPathException as err: val.modelXbrl.error( _("Parameter \n{0} \nException: \n{1}").format( paramQname, err.message), "err", "xbrlve:parameterTypeMismatch" if err.code == "err:FORG0001" else err.code) produceOutputXbrlInstance = False instanceProducingVariableSets = defaultdict(list) for modelVariableSet in val.modelXbrl.modelVariableSets: varSetInstanceDependencies = set() if isinstance(modelVariableSet, ModelFormula): instanceQname = None for modelRel in val.modelXbrl.relationshipSet(XbrlConst.formulaInstance).fromModelObject(modelVariableSet): instance = modelRel.toModelObject if isinstance(instance, ModelInstance): if instanceQname is None: instanceQname = instance.qname else: val.modelXbrl.error( _("Multiple output instances for formula {0}, to names {1}, {2}").format( modelVariableSet.xlinkLabel, instanceQname, instance.qname ), "info", "arelle:multipleOutputInstances") if instanceQname is None: instanceQname = XbrlConst.qnStandardOutputInstance instanceQnames.add(instanceQname) modelVariableSet.outputInstanceQname = instanceQname if val.validateSBRNL: val.modelXbrl.error( _("Formula linkbase {0} formula:formula {1} is not allowed").format( os.path.basename(modelVariableSet.modelDocument.uri), modelVariableSet.xlinkLabel), "err", "SBR.NL.2.3.9.03") else: instanceQname = None modelVariableSet.countSatisfied = 0 modelVariableSet.countNotSatisfied = 0 checkValidationMessages(val, modelVariableSet) instanceProducingVariableSets[instanceQname].append(modelVariableSet) modelVariableSet.outputInstanceQname = instanceQname if modelVariableSet.aspectModel not in ("non-dimensional", "dimensional"): val.modelXbrl.error( _("Variable set {0}, aspect model {1} not recognized").format( modelVariableSet.xlinkLabel, modelVariableSet.aspectModel), "err", "xbrlve:unknownAspectModel") modelVariableSet.compile() modelVariableSet.hasConsistencyAssertion = False #determine dependencies within variable sets nameVariables = {} qnameRels = {} definedNamesSet = set() for modelRel in val.modelXbrl.relationshipSet(XbrlConst.variableSet).fromModelObject(modelVariableSet): varqname = modelRel.variableQname if varqname: qnameRels[varqname] = modelRel toVariable = modelRel.toModelObject if varqname not in definedNamesSet: definedNamesSet.add(varqname) if varqname not in nameVariables: nameVariables[varqname] = toVariable elif nameVariables[varqname] != toVariable: val.modelXbrl.error( _("Multiple variables named {1} in variable set {0}").format( modelVariableSet.xlinkLabel, varqname ), "err", "xbrlve:duplicateVariableNames") fromInstanceQnames = None for instRel in val.modelXbrl.relationshipSet(XbrlConst.instanceVariable).toModelObject(toVariable): fromInstance = instRel.fromModelObject if isinstance(fromInstance, ModelInstance): fromInstanceQname = fromInstance.qname varSetInstanceDependencies.add(fromInstanceQname) instanceDependencies[instanceQname].add(fromInstanceQname) if fromInstanceQnames is None: fromInstanceQnames = set() fromInstanceQnames.add(fromInstanceQname) if fromInstanceQnames is None: varSetInstanceDependencies.add(XbrlConst.qnStandardInputInstance) if instanceQname: instanceDependencies[instanceQname].add(XbrlConst.qnStandardInputInstance) toVariable.fromInstanceQnames = fromInstanceQnames else: val.modelXbrl.error( _("Variables name {1} cannot be determined on arc from {0}").format( modelVariableSet.xlinkLabel, modelRel.variablename ), "err", "xbrlve:variableNameResolutionFailure") definedNamesSet |= parameterQnames variableDependencies = {} for modelRel in val.modelXbrl.relationshipSet(XbrlConst.variableSet).fromModelObject(modelVariableSet): variable = modelRel.toModelObject if isinstance(variable, (ModelParameter,ModelVariable)): # ignore anything not parameter or variable varqname = modelRel.variableQname depVars = variable.variableRefs() variableDependencies[varqname] = depVars if len(depVars) > 0 and formulaOptions.traceVariablesDependencies: val.modelXbrl.error(_("Variable set {0}, variable {1}, dependences {2}").format( modelVariableSet.xlinkLabel, varqname, depVars), "info", "formula:trace") definedNamesSet.add(varqname) # check for fallback value variable references if isinstance(variable, ModelFactVariable): for depVar in XPathParser.variableReferencesSet(variable.fallbackValueProg, variable.element): if depVar in qnameRels and isinstance(qnameRels[depVar].toModelObject,ModelVariable): val.modelXbrl.error(_("Variable set {0} fallbackValue '{1}' cannot refer to variable {2}").format( modelVariableSet.xlinkLabel, variable.fallbackValue, depVar), "err", "xbrlve:factVariableReferenceNotAllowed") # check for covering aspect not in variable set aspect model checkFilterAspectModel(val, modelVariableSet, variable.filterRelationships, xpathContext) orderedNameSet = set() orderedNameList = [] orderedAVariable = True while (orderedAVariable): orderedAVariable = False for varqname, depVars in variableDependencies.items(): if varqname not in orderedNameSet and len(depVars - parameterQnames - orderedNameSet) == 0: orderedNameList.append(varqname) orderedNameSet.add(varqname) orderedAVariable = True if varqname in instanceQnames: varSetInstanceDependencies.add(varqname) instanceDependencies[instanceQname].add(varqname) elif isinstance(nameVariables.get(varqname), ModelInstance): instqname = nameVariables[varqname].qname varSetInstanceDependencies.add(instqname) instanceDependencies[instanceQname].add(instqname) # anything unresolved? for varqname, depVars in variableDependencies.items(): if varqname not in orderedNameSet: circularOrUndefVars = depVars - parameterQnames - orderedNameSet undefinedVars = circularOrUndefVars - definedNamesSet varsCircularDep = circularOrUndefVars - undefinedVars if len(undefinedVars) > 0: val.modelXbrl.error( _("Undefined variable dependencies in variable st {0}, from variable {1} to {2}").format( modelVariableSet.xlinkLabel, varqname, undefinedVars), "err", "xbrlve:unresolvedDependency") if len(varsCircularDep) > 0: val.modelXbrl.error( _("Cyclic dependencies in variable set {0}, from variable {1} to {2}").format( modelVariableSet.xlinkLabel, varqname, varsCircularDep ), "err", "xbrlve:cyclicDependencies") # check unresolved variable set dependencies for varSetDepVarQname in modelVariableSet.variableRefs(): if varSetDepVarQname not in orderedNameSet and varSetDepVarQname not in parameterQnames: val.modelXbrl.error( _("Undefined variable dependency in variable set {0}, {1}").format( modelVariableSet.xlinkLabel, varSetDepVarQname), "err", "xbrlve:unresolvedDependency") if varSetDepVarQname in instanceQnames: varSetInstanceDependencies.add(varSetDepVarQname) instanceDependencies[instanceQname].add(varSetDepVarQname) elif isinstance(nameVariables.get(varSetDepVarQname), ModelInstance): instqname = nameVariables[varSetDepVarQname].qname varSetInstanceDependencies.add(instqname) instanceDependencies[instanceQname].add(instqname) if formulaOptions.traceVariablesOrder: val.modelXbrl.error(_("Variable set {0}, variables order: {1}").format( modelVariableSet.xlinkLabel, orderedNameList), "info", "formula:trace") if (formulaOptions.traceVariablesDependencies and len(varSetInstanceDependencies) > 0 and varSetInstanceDependencies != {XbrlConst.qnStandardInputInstance}): val.modelXbrl.error(_("Variable set {0}, instance dependences {1}").format( modelVariableSet.xlinkLabel, varSetInstanceDependencies), "info", "formula:trace") modelVariableSet.orderedVariableRelationships = [] for varqname in orderedNameList: if varqname in qnameRels: modelVariableSet.orderedVariableRelationships.append(qnameRels[varqname]) # check existence assertion variable dependencies if isinstance(modelVariableSet, ModelExistenceAssertion): for depVar in modelVariableSet.variableRefs(): if depVar in qnameRels and isinstance(qnameRels[depVar].toModelObject,ModelVariable): val.modelXbrl.error(_("Existence Assertion {0}, cannot refer to variable {1}").format( modelVariableSet.xlinkLabel, depVar), "err", "xbrleae:variableReferenceNotAllowed") # check messages variable dependencies checkValidationMessageVariables(val, modelVariableSet, qnameRels) # check preconditions modelVariableSet.preconditions = [] for modelRel in val.modelXbrl.relationshipSet(XbrlConst.variableSetPrecondition).fromModelObject(modelVariableSet): precondition = modelRel.toModelObject if isinstance(precondition, ModelPrecondition): modelVariableSet.preconditions.append(precondition) # check for variable sets referencing fact or general variables for modelRel in val.modelXbrl.relationshipSet(XbrlConst.variableSetFilter).fromModelObject(modelVariableSet): varSetFilter = modelRel.toModelObject if modelRel.isCovered: val.modelXbrl.error(_("Variable set {0}, filter {1}, cannot be covered").format( modelVariableSet.xlinkLabel, varSetFilter.xlinkLabel), "wrn", "arelle:variableSetFilterCovered") modelRel._isCovered = False # block group filter from being able to covere for depVar in varSetFilter.variableRefs(): if depVar in qnameRels and isinstance(qnameRels[depVar].toModelObject,ModelVariable): val.modelXbrl.error(_("Variable set {0}, filter {1}, cannot refer to variable {2}").format( modelVariableSet.xlinkLabel, varSetFilter.xlinkLabel, depVar), "err", "xbrlve:factVariableReferenceNotAllowed") # check aspects of formula if isinstance(modelVariableSet, ModelFormula): checkFormulaRules(val, modelVariableSet, nameVariables) # determine instance dependency order orderedInstancesSet = set() stdInpInst = {XbrlConst.qnStandardInputInstance} orderedInstancesList = [] orderedAnInstance = True while (orderedAnInstance): orderedAnInstance = False for instqname, depInsts in instanceDependencies.items(): if instqname and instqname not in orderedInstancesSet and len(depInsts - stdInpInst - orderedInstancesSet) == 0: orderedInstancesList.append(instqname) orderedInstancesSet.add(instqname) orderedAnInstance = True orderedInstancesList.append(None) # assertions come after all formulas that produce outputs # anything unresolved? for instqname, depInsts in instanceDependencies.items(): if instqname not in orderedInstancesSet: # can also be satisfied from an input DTS missingDependentInstances = depInsts - stdInpInst if val.parameters: missingDependentInstances -= val.parameters.keys() if instqname: if missingDependentInstances: val.modelXbrl.error( _("Cyclic dependencies of instance {0} produced by a formula, with variables consuming instances {1}").format( instqname, missingDependentInstances ), "err", "xbrlvarinste:instanceVariableRecursionCycle") elif instqname == XbrlConst.qnStandardOutputInstance: orderedInstancesSet.add(instqname) orderedInstancesList.append(instqname) # standard output formula, all input dependencies in parameters ''' future check? if instance has no external input or producing formula else: val.modelXbrl.error( _("Unresolved dependencies of an assertion's variables on instances {0}").format( depInsts - stdInpInst ), "err", "xbrlvarinste:instanceVariableRecursionCycle") ''' if formulaOptions.traceVariablesOrder and len(orderedInstancesList) > 1: val.modelXbrl.error(_("Variable instances processing order: {0}").format( orderedInstancesList), "info", "formula:trace") # linked consistency assertions for modelRel in val.modelXbrl.relationshipSet(XbrlConst.consistencyAssertionFormula).modelRelationships: if modelRel.fromModelObject and modelRel.toModelObject and isinstance(modelRel.toModelObject,ModelFormula): consisAsser = modelRel.fromModelObject consisAsser.countSatisfied = 0 consisAsser.countNotSatisfied = 0 if consisAsser.hasProportionalAcceptanceRadius and consisAsser.hasAbsoluteAcceptanceRadius: val.modelXbrl.error( _("Consistency assertion {0} has both absolute and proportional acceptance radii").format( consisAsser.xlinkLabel), "err", "xbrlcae:acceptanceRadiusConflict") consisAsser.orderedVariableRelationships = [] for consisParamRel in val.modelXbrl.relationshipSet(XbrlConst.consistencyAssertionParameter).fromModelObject(consisAsser): if isinstance(consisParamRel.toModelObject, ModelVariable): val.modelXbrl.error( _("Consistency assertion {0} has relationship to a {1} {2}").format( consisAsser.xlinkLabel, consisParamRel.toModelObject.element.localName, consisParamRel.toModelObject.xlinkLabel), "err", "xbrlcae:variablesNotAllowed") else: consisAsser.orderedVariableRelationships.append(consisParamRel) consisAsser.compile() modelRel.toModelObject.hasConsistencyAssertion = True if initialErrorCount < val.modelXbrl.logCountErr: return # don't try to execute # formula output instances if instanceQnames: schemaRefs = [val.modelXbrl.modelDocument.relativeUri(referencedDoc.uri) for referencedDoc in val.modelXbrl.modelDocument.referencesDocument.keys() if referencedDoc.type == ModelDocument.Type.SCHEMA] outputXbrlInstance = None for instanceQname in instanceQnames: if instanceQname == XbrlConst.qnStandardInputInstance: continue # always present the standard way if val.parameters and instanceQname in val.parameters: namedInstance = val.parameters[instanceQname][1] else: # empty intermediate instance uri = val.modelXbrl.modelDocument.filepath[:-4] + "-output-XBRL-instance" if instanceQname != XbrlConst.qnStandardOutputInstance: uri = uri + "-" + instanceQname.localName uri = uri + ".xml" namedInstance = ModelXbrl.create(val.modelXbrl.modelManager, newDocumentType=ModelDocument.Type.INSTANCE, url=uri, schemaRefs=schemaRefs, isEntry=True) xpathContext.inScopeVars[instanceQname] = namedInstance if instanceQname == XbrlConst.qnStandardOutputInstance: outputXbrlInstance = namedInstance # evaluate consistency assertions # evaluate variable sets not in consistency assertions for instanceQname in orderedInstancesList: for modelVariableSet in instanceProducingVariableSets[instanceQname]: # produce variable evaluations from arelle.FormulaEvaluator import evaluate try: evaluate(xpathContext, modelVariableSet) except XPathContext.XPathException as err: val.modelXbrl.error( _("Variable set \n{0} \nException: \n{1}").format( modelVariableSet, err.message), "err", err.code) # log assertion result counts asserTests = {} for exisValAsser in val.modelXbrl.modelVariableSets: if isinstance(exisValAsser, ModelVariableSetAssertion): asserTests[exisValAsser.id] = (exisValAsser.countSatisfied, exisValAsser.countNotSatisfied) if formulaOptions.traceAssertionResultCounts: val.modelXbrl.error( _("{0} Assertion {1} evaluations : {2} satisfied, {3} not satisfied").format( "Existence" if isinstance(exisValAsser, ModelExistenceAssertion) else "Value", exisValAsser.id, exisValAsser.countSatisfied, exisValAsser.countNotSatisfied), "info", "formula:trace") for modelRel in val.modelXbrl.relationshipSet(XbrlConst.consistencyAssertionFormula).modelRelationships: if modelRel.fromModelObject and modelRel.toModelObject and isinstance(modelRel.toModelObject,ModelFormula): consisAsser = modelRel.fromModelObject asserTests[consisAsser.id] = (consisAsser.countSatisfied, consisAsser.countNotSatisfied) if formulaOptions.traceAssertionResultCounts: val.modelXbrl.error( _("Consistency Assertion {0} evaluations : {1} satisfied, {2} not satisfied").format( consisAsser.id, consisAsser.countSatisfied, consisAsser.countNotSatisfied), "info", "formula:trace") if asserTests: val.modelXbrl.error( _("Assertion results {0}").format(asserTests), "asrtNoLog", asserTests) # display output instance if outputXbrlInstance: if val.modelXbrl.formulaOutputInstance: # close prior instance, usually closed by caller to validate as it may affect UI on different thread val.modelXbrl.formulaOutputInstance.close() val.modelXbrl.formulaOutputInstance = outputXbrlInstance