def backgroundLoadXbrl(self, filesource, importToDTS): startedAt = time.time() try: if importToDTS: action = _("imported") modelXbrl = self.modelManager.modelXbrl if modelXbrl: ModelDocument.load(modelXbrl, filesource.url) else: action = _("loaded") modelXbrl = self.modelManager.load(filesource, _("views loading")) except Exception as err: msg = _("Exception loading {0}: {1}, at {2}").format( filesource.url, err, traceback.format_tb(sys.exc_info()[2])) # not sure if message box can be shown from background thread # tkinter.messagebox.showwarning(_("Exception loading"),msg, parent=self.parent) self.addToLog(msg); return if modelXbrl and modelXbrl.modelDocument: self.addToLog(format_string(self.modelManager.locale, _("%s in %.2f secs"), (action, time.time() - startedAt))) self.showStatus(_("{0}, preparing views").format(action)) self.uiThreadQueue.put((self.showLoadedXbrl, [modelXbrl, importToDTS])) else: self.addToLog(format_string(self.modelManager.locale, _("not successfully %s in %.2f secs"), (action, time.time() - startedAt)))
def load(modelManager, url, nextaction, base=None): from arelle import (ModelDocument, FileSource) modelXbrl = create(modelManager) if isinstance(url,FileSource.FileSource): modelXbrl.fileSource = url url = modelXbrl.fileSource.url else: modelXbrl.fileSource = FileSource.FileSource(url) modelXbrl.modelDocument = ModelDocument.load(modelXbrl, url, base, isEntry=True) # at this point DTS is fully discovered but schemaLocated xsd's are not yet loaded modelDocumentsSchemaLocated = set() while True: # need this logic because each new pass may add new urlDocs modelDocuments = set(modelXbrl.urlDocs.values()) - modelDocumentsSchemaLocated if not modelDocuments: break modelDocument = modelDocuments.pop() modelDocumentsSchemaLocated.add(modelDocument) modelDocument.loadSchemalocatedSchemas() #from arelle import XmlValidate #uncomment for trial use of lxml xml schema validation of entry document #XmlValidate.xmlValidate(modelXbrl.modelDocument) modelManager.cntlr.webCache.saveUrlCheckTimes() modelManager.showStatus(_("xbrl loading finished, {0}...").format(nextaction)) return modelXbrl
def load(modelManager, url, nextaction=None, base=None, useFileSource=None): if nextaction is None: nextaction = _("loading") from arelle import (ModelDocument, FileSource) modelXbrl = create(modelManager) if useFileSource is not None: modelXbrl.fileSource = useFileSource modelXbrl.closeFileSource = False url = url elif isinstance(url,FileSource.FileSource): modelXbrl.fileSource = url modelXbrl.closeFileSource= True url = modelXbrl.fileSource.url else: modelXbrl.fileSource = FileSource.FileSource(url) modelXbrl.closeFileSource= True modelXbrl.modelDocument = ModelDocument.load(modelXbrl, url, base, isEntry=True) del modelXbrl.entryLoadingUrl if modelXbrl.modelDocument is not None and modelXbrl.modelDocument.type < ModelDocument.Type.DTSENTRIES: # at this point DTS is fully discovered but schemaLocated xsd's are not yet loaded modelDocumentsSchemaLocated = set() while True: # need this logic because each new pass may add new urlDocs modelDocuments = set(modelXbrl.urlDocs.values()) - modelDocumentsSchemaLocated if not modelDocuments: break modelDocument = modelDocuments.pop() modelDocumentsSchemaLocated.add(modelDocument) modelDocument.loadSchemalocatedSchemas() #from arelle import XmlValidate #uncomment for trial use of lxml xml schema validation of entry document #XmlValidate.xmlValidate(modelXbrl.modelDocument) modelManager.cntlr.webCache.saveUrlCheckTimes() modelManager.showStatus(_("xbrl loading finished, {0}...").format(nextaction)) return modelXbrl
def doc(xc, p, contextItem, args): if len(args) != 1: raise XPathContext.FunctionNumArgs() uri = stringArg(xc, args, 0, "xs:string", emptyFallback=None) if uri is None: return () if xc.progHeader is None or xc.progHeader.element is None: raise XPathContext.XPathException( p, 'err:FODC0005', _('Function xf:doc no formula resource element for {0}').format( uri)) if not UrlUtil.isValid(uri): raise XPathContext.XPathException( p, 'err:FODC0005', _('Function xf:doc $uri is not valid {0}').format(uri)) normalizedUri = xc.modelXbrl.modelManager.cntlr.webCache.normalizeUrl( uri, xc.progHeader.element.modelDocument.baseForElement( xc.progHeader.element)) if normalizedUri in xc.modelXbrl.urlDocs: return xc.modelXbrl.urlDocs[normalizedUri].xmlDocument modelDocument = ModelDocument.load(xc.modelXbrl, normalizedUri) if modelDocument is None: raise XPathContext.XPathException( p, 'err:FODC0005', _('Function xf:doc $uri not successfully loaded {0}').format(uri)) # assure that document is validated XmlValidate.validate(xc.modelXbrl, modelDocument.xmlRootElement) return modelDocument.xmlDocument
def reload(self, nextaction, reloadCache=False): from arelle import ModelDocument self.init(keepViews=True) self.modelDocument = ModelDocument.load(self, self.fileSource.url, isEntry=True, reloadCache=reloadCache) self.modelManager.showStatus( _("xbrl loading finished, {0}...").format(nextaction), 5000) self.modelManager.reloadViews(self)
def reload(self,nextaction,reloadCache=False): """Reloads all model objects from their original entry point URL, preserving any open views (which are reloaded). :param nextAction: status line text string, if any, to show upon completion :type nextAction: str :param reloadCache: True to force clearing and reloading of web cache, if working online. :param reloadCache: bool """ from arelle import ModelDocument self.init(keepViews=True) self.modelDocument = ModelDocument.load(self, self.fileSource.url, isEntry=True, reloadCache=reloadCache) self.modelManager.showStatus(_("xbrl loading finished, {0}...").format(nextaction),5000) self.modelManager.reloadViews(self)
def load(modelManager, url, nextaction=None, base=None, useFileSource=None, errorCaptureLevel=None): """Each loaded instance, DTS, testcase, testsuite, versioning report, or RSS feed, is represented by an instance of a ModelXbrl object. The ModelXbrl object has a collection of ModelDocument objects, each representing an XML document (for now, with SQL whenever its time comes). One of the modelDocuments of the ModelXbrl is the entry point (of discovery or of the test suite). :param url: may be a filename or FileSource object :type url: str or FileSource :param nextaction: text to use as status line prompt on conclusion of loading and discovery :type nextaction: str :param base: the base URL if any (such as a versioning report's URL when loading to/from DTS modelXbrl). :type base: str :param useFileSource: for internal use (when an entry point is in a FileSource archive and discovered files expected to also be in the entry point's archive. :type useFileSource: bool :returns: ModelXbrl -- a new modelXbrl, performing DTS discovery for instance, inline XBRL, schema, linkbase, and versioning report entry urls """ if nextaction is None: nextaction = _("loading") from arelle import (ModelDocument, FileSource) modelXbrl = create(modelManager, errorCaptureLevel=errorCaptureLevel) if useFileSource is not None: modelXbrl.fileSource = useFileSource modelXbrl.closeFileSource = False url = url elif isinstance(url,FileSource.FileSource): modelXbrl.fileSource = url modelXbrl.closeFileSource= True url = modelXbrl.fileSource.url else: modelXbrl.fileSource = FileSource.FileSource(url) modelXbrl.closeFileSource= True modelXbrl.modelDocument = ModelDocument.load(modelXbrl, url, base, isEntry=True) del modelXbrl.entryLoadingUrl if modelXbrl.modelDocument is not None and modelXbrl.modelDocument.type < ModelDocument.Type.DTSENTRIES: # at this point DTS is fully discovered but schemaLocated xsd's are not yet loaded modelDocumentsSchemaLocated = set() while True: # need this logic because each new pass may add new urlDocs modelDocuments = set(modelXbrl.urlDocs.values()) - modelDocumentsSchemaLocated if not modelDocuments: break modelDocument = modelDocuments.pop() modelDocumentsSchemaLocated.add(modelDocument) modelDocument.loadSchemalocatedSchemas() #from arelle import XmlValidate #uncomment for trial use of lxml xml schema validation of entry document #XmlValidate.xmlValidate(modelXbrl.modelDocument) modelManager.cntlr.webCache.saveUrlCheckTimes() modelManager.showStatus(_("xbrl loading finished, {0}...").format(nextaction)) return modelXbrl
def doc(xc, p, contextItem, args): if len(args) != 1: raise XPathContext.FunctionNumArgs() uri = stringArg(xc, args, 0, "xs:string", emptyFallback=None) if uri is None: return () if xc.progHeader is None or xc.progHeader.element is None: raise XPathContext.XPathException(p, 'err:FODC0005', _('Function xf:doc no formula resource element for {0}').format(uri)) if not UrlUtil.isValid(uri): raise XPathContext.XPathException(p, 'err:FODC0005', _('Function xf:doc $uri is not valid {0}').format(uri)) normalizedUri = xc.modelXbrl.modelManager.cntlr.webCache.normalizeUrl( uri, xc.progHeader.element.modelDocument.baseForElement(xc.progHeader.element)) if normalizedUri in xc.modelXbrl.urlDocs: return xc.modelXbrl.urlDocs[normalizedUri].xmlDocument modelDocument = ModelDocument.load(xc.modelXbrl, normalizedUri) if modelDocument is None: raise XPathContext.XPathException(p, 'err:FODC0005', _('Function xf:doc $uri not successfully loaded {0}').format(uri)) # assure that document is validated XmlValidate.validate(xc.modelXbrl, modelDocument.xmlRootElement) return modelDocument.xmlDocument
def runFromExcel(self, options): #testGenFileName = options.excelfilename testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation-index.xls" testGenDir = os.path.dirname(testGenFileName) schemaDir = os.path.dirname(testGenDir) + os.sep + "schema" timeNow = XmlUtil.dateunionValue(datetime.datetime.now()) if options.testfiledate: today = options.testfiledate else: today = XmlUtil.dateunionValue(datetime.date.today()) startedAt = time.time() LogHandler(self) # start logger self.logMessages = [] logMessagesFile = testGenDir + os.sep + 'log-generation-messages.txt' modelTestcases = ModelXbrl.create(self.modelManager, url=testGenFileName, isEntry=True) testcaseIndexBook = xlrd.open_workbook(testGenFileName) testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0) self.addToLog(_("[info] xls loaded in {0:.2} secs at {1}").format(time.time() - startedAt, timeNow)) # start index file indexFiles = [testGenDir + os.sep + 'creation-testcases-index.xml', testGenDir + os.sep + 'consumption-testcases-index.xml'] indexDocs = [] testcasesElements = [] for purpose in ("Creation","Consumption"): file = io.StringIO( #'<?xml version="1.0" encoding="UTF-8"?>' '<!-- XBRL Versioning 1.0 {0} Tests -->' '<!-- Copyright 2011 XBRL International. All Rights Reserved. -->' '<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>' '<testcases name="XBRL Versioning 1.0 {0} Tests" ' ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' ' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">' '</testcases>'.format(purpose, today) ) doc = etree.parse(file) file.close() indexDocs.append(doc) testcasesElements.append(doc.getroot()) priorTestcasesDir = None testcaseFiles = None testcaseDocs = None for iRow in range(1, testcaseIndexSheet.nrows): try: row = testcaseIndexSheet.row(iRow) if (row[0].ctype == xlrd.XL_CELL_EMPTY or # must have directory row[1].ctype == xlrd.XL_CELL_EMPTY or # from row[2].ctype == xlrd.XL_CELL_EMPTY): # to continue testDir = row[0].value uriFrom = row[1].value uriTo = row[2].value overrideReport = row[3].value description = row[4].value if description is None or len(description) == 0: continue # test not ready to run assignment = row[5].value expectedEvents = row[6].value # comma space separated if multiple note = row[7].value useCase = row[8].value base = os.path.join(os.path.dirname(testGenFileName),testDir) + os.sep self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom)) if uriFrom and uriTo and assignment.lower() not in ("n.a.", "error") and expectedEvents != "N.A.": modelDTSfrom = modelDTSto = None for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"), True), (uriTo, _("loading to DTS"), False)): if ',' not in URIs: modelDTS = ModelXbrl.load(self.modelManager, URIs, msg, base=base) else: modelDTS = ModelXbrl.create(self.modelManager, ModelDocument.Type.DTSENTRIES, self.webCache.normalizeUrl(URIs.replace(", ","_") + ".dts", base), isEntry=True) DTSdoc = modelDTS.modelDocument DTSdoc.inDTS = True for uri in URIs.split(','): doc = ModelDocument.load(modelDTS, uri.strip(), base=base) if doc is not None: DTSdoc.referencesDocument[doc] = "import" #fake import doc.inDTS = True if isFrom: modelDTSfrom = modelDTS else: modelDTSto = modelDTS if modelDTSfrom is not None and modelDTSto is not None: # generate differences report reportUri = uriFrom.partition(',')[0] # first file reportDir = os.path.dirname(reportUri) if reportDir: reportDir += os.sep reportName = os.path.basename(reportUri).replace("from.xsd","report.xml") reportFile = reportDir + "out" + os.sep + reportName #reportFile = reportDir + "report" + os.sep + reportName reportFullPath = self.webCache.normalizeUrl( reportFile, base) testcasesDir = os.path.dirname(os.path.dirname(reportFullPath)) if testcasesDir != priorTestcasesDir: # close prior report if priorTestcasesDir: for i,testcaseFile in enumerate(testcaseFiles): with open(testcaseFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8") testcaseName = os.path.basename(testcasesDir) testcaseFiles = [testcasesDir + os.sep + testcaseName + "-creation-testcase.xml", testcasesDir + os.sep + testcaseName + "-consumption-testcase.xml"] for i,testcaseFile in enumerate(testcaseFiles): etree.SubElement(testcasesElements[i], "testcase", attrib={"uri": testcaseFile[len(testGenDir)+1:].replace("\\","/")} ) # start testcase file testcaseDocs = [] testcaseElements = [] testcaseNumber = testcaseName[0:4] if testcaseNumber.isnumeric(): testcaseNumberElement = "<number>{0}</number>".format(testcaseNumber) testcaseName = testcaseName[5:] else: testcaseNumberElement = "" testDirSegments = testDir.split('/') if len(testDirSegments) >= 2 and '-' in testDirSegments[1]: testedModule = testDirSegments[1][testDirSegments[1].index('-') + 1:] else: testedModule = '' for purpose in ("Creation","Consumption"): file = io.StringIO( #'<?xml version="1.0" encoding="UTF-8"?>' '<!-- Copyright 2011 XBRL International. All Rights Reserved. -->' '<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>' '<testcase ' ' xmlns="http://xbrl.org/2008/conformance"' ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' ' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">' '<creator>' '<name>Roland Hommes</name>' '<email>[email protected]</email>' '</creator>' '{0}' '<name>{1}</name>' # '<description>{0}</description>' '<reference>' '{2}' '{3}' '</reference>' '</testcase>'.format(testcaseNumberElement, testcaseName, '<name>{0}</name>'.format(testedModule) if testedModule else '', '<id>{0}</id>'.format(useCase) if useCase else '') ) doc = etree.parse(file) file.close() testcaseDocs.append(doc) testcaseElements.append(doc.getroot()) priorTestcasesDir = testcasesDir variationSeq = 1 try: os.makedirs(os.path.dirname(reportFullPath)) except WindowsError: pass # dir already exists modelVersReport = ModelVersReport.ModelVersReport(modelTestcases) modelVersReport.diffDTSes(reportFullPath,modelDTSfrom, modelDTSto, assignment=assignment, schemaDir=schemaDir) # check for expected elements if expectedEvents: for expectedEvent in expectedEvents.split(","): if expectedEvent not in ("No change", "N.A."): prefix, sep, localName = expectedEvent.partition(':') if sep and len(modelVersReport.xmlDocument.findall( '//{{{0}}}{1}'.format( XbrlConst.verPrefixNS.get(prefix), localName))) == 0: modelTestcases.warning("warning", "Generated test case %(reportName)s missing expected event %(event)s", reportName=reportName, event=expectedEvent) modelVersReport.close() uriFromParts = uriFrom.split('_') if len(uriFromParts) >= 2: variationId = uriFromParts[1] else: variationId = "_{0:02n}".format(variationSeq) for i,testcaseElt in enumerate(testcaseElements): variationElement = etree.SubElement(testcaseElt, "{http://xbrl.org/2008/conformance}variation", attrib={"id": variationId}) nameElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}description") nameElement.text = description ''' (removed per RH 2011/10/04 if note: paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}description") paramElement.text = "Note: " + note if useCase: paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}reference") paramElement.set("specification", "versioning-requirements") paramElement.set("useCase", useCase) ''' dataElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}data") if i == 0: # result is report if expectedEvents: paramElement = etree.SubElement(dataElement, "{http://xbrl.org/2008/conformance}parameter", attrib={"name":"expectedEvent", "value":expectedEvents.replace(',',' ')}, nsmap={"conf":"http://xbrl.org/2008/conformance", None:""}) if assignment: paramElement = etree.SubElement(dataElement, "{http://xbrl.org/2008/conformance}parameter", attrib={"name":"assignment", "value":assignment}, nsmap={"conf":"http://xbrl.org/2008/conformance", None:""}) for schemaURIs, dtsAttr in ((uriFrom,"from"), (uriTo,"to")): for schemaURI in schemaURIs.split(","): schemaElement = etree.SubElement(dataElement, "{http://xbrl.org/2008/conformance}schema") schemaElement.set("dts",dtsAttr) if i == 0: schemaElement.set("readMeFirst","true") schemaElement.text=os.path.basename(schemaURI.strip()) resultElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}result") reportElement = etree.SubElement(resultElement if i == 0 else dataElement, "{http://xbrl.org/2008/conformance}versioningReport") if i == 1: reportElement.set("readMeFirst","true") reportElement.text = "report/" + reportName variationSeq += 1 except Exception as err: modelTestcases.error("exception", _("Exception: %(error)s, Excel row: %(excelRow)s"), error=err, excelRow=iRow, exc_info=True) # add tests-error-code index files to consumption for testcaseFile in self.testcaseFiles(testGenDir + os.sep + "tests-error-code"): etree.SubElement(testcasesElements[1], "testcase", attrib={"uri": testcaseFile[len(testGenDir)+1:].replace("\\","/")} ) with open(logMessagesFile, "w") as fh: fh.writelines(self.logMessages) if priorTestcasesDir: for i,testcaseFile in enumerate(testcaseFiles): with open(testcaseFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8") for i,indexFile in enumerate(indexFiles): with open(indexFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def loadPrimaryDocumentFacts(dts, rssItem, entityInformation): # identify tables disclosureSystem = dts.modelManager.disclosureSystem if disclosureSystem.validationType != "EFM": return if rssItem is not None: formType = rssItem.formType fileUrl = rssItem.primaryDocumentURL reloadCache = getattr(rssItem.modelXbrl, "reloadCache", False) else: formType = entityInformation.get("form-type") fileUrl = entityInformation.get("primary-document-url") reloadCache = False if fileUrl and formType and (formType.startswith('10-K') or formType.startswith('10-Q')): if fileUrl.endswith(".txt") or fileUrl.endswith(".htm"): if formType.startswith('10-K'): parts = SEC10Kparts elif formType.startswith('10-Q'): parts = SEC10Qparts # try to load and use it normalizedUrl = dts.modelManager.cntlr.webCache.normalizeUrl( fileUrl) text = '' try: filePath = dts.modelManager.cntlr.webCache.getfilename( normalizedUrl, reload=reloadCache) if filePath: if filePath.endswith('.txt'): with open(filePath, encoding='utf-8') as fh: text = fh.read() elif filePath.endswith('.htm'): doc = html.parse(filePath) textParts = [] def iterTextParts(parent): for node in parent.iterchildren(): if isinstance(node, etree._Element): if node.tag in ('p', 'P', 'br', 'BR', 'div', 'DIV'): textParts.append('\n') textParts.append(node.text or '') iterTextParts(node) if node.tail: # use tail whether element, comment, or processing instruction textParts.append(node.tail) iterTextParts(doc.getroot()) text = ' '.join(textParts) except (IOError, EnvironmentError, AttributeError ) as err: # attribute err if html has no root element dts.info( "xpDB:primaryDocumentLoadingError", _("Loading XBRL DB: primary document loading error: %(error)s" ), modelObject=dts, error=err) #with open("/Users/hermf/temp/test.txt", "w", encoding='utf-8') as fh: # fh.write(text) class Span: def __init__(self, start): self.start = start self.end = -1 # find the parts partSpan = {} partPrev = None missing2ndPart1 = False for partMatch in partPattern.finditer(text): part = partMatch.group(1).upper() if partPrev is not None: if part != 'I' and part == partPrev: # two of these parts without second part 1, use signature or first item for 2nd part 1 missing2ndPart1 = True partSpan[partPrev].end = partMatch.start(0) partSpan[part] = Span(partMatch.end(1)) partPrev = part if partPrev is not None: partSpan[partPrev].end = len(text) if missing2ndPart1: # signatures signaturesStarts = [] for signaturesMatch in signaturesPattern.finditer(text): signaturesStarts.append(signaturesMatch.start(0)) #check if PART I missing then use first signatures if 'I' in partSpan and 'II' in partSpan: if len(signaturesStarts) == 2 and signaturesStarts[ 0] > partSpan['I'].start: partSpan['I'].start = signaturesStarts[0] partSpan['I'].end = partSpan['II'].start else: # use ASSETS as start of part 1 for assetsMatch in assetsPattern.finditer(text): partSpan['I'].start = assetsMatch.end(0) partSpan['I'].end = partSpan['II'].start break # find the items itemSpan = {} for part, span in partSpan.items(): item = None for itemMatch in itemPattern.finditer(text, span.start, span.end): if item is not None: itemSpan[(part, item)].end = itemMatch.start(0) item = itemMatch.group(1) itemSpan[(part, item)] = Span(itemMatch.end(1)) if item is not None: itemSpan[(part, item)].end = span.end if any(itemKey in parts for itemKey in itemSpan.keys()): # find default context for cntx in dts.contexts.values(): if cntx.isStartEndPeriod: if not cntx.hasSegment: # use c as default context # load extra datapoints taxonomy but not as discovered in DTS ModelDocument.load( dts, "http://arelle.org/2014/doc-2014-01-31.xsd", ) # add the facts for itemKey, itemSpan in itemSpan.items(): if itemKey in parts: dts.createFact(qname( "{http://arelle.org/doc/2014-01-31}doc:" + parts[itemKey]), attributes=[("contextRef", cntx.id)], text=text[itemSpan.start: itemSpan.end]) break
def loadPrimaryDocumentFacts(dts, rssItem, entityInformation): # identify tables disclosureSystem = dts.modelManager.disclosureSystem if not disclosureSystem.EFM: return if rssItem is not None: formType = rssItem.formType fileUrl = rssItem.primaryDocumentURL else: formType = entityInformation.get("form-type") fileUrl = entityInformation.get("primary-document-url") if fileUrl and formType and (formType.startswith("10-K") or formType.startswith("10-Q")): if fileUrl.endswith(".txt") or fileUrl.endswith(".htm"): if formType.startswith("10-K"): parts = SEC10Kparts elif formType.startswith("10-Q"): parts = SEC10Qparts # try to load and use it normalizedUrl = dts.modelManager.cntlr.webCache.normalizeUrl(fileUrl) try: filePath = dts.modelManager.cntlr.webCache.getfilename(normalizedUrl) if filePath: if filePath.endswith(".txt"): with open(filePath, encoding="utf-8") as fh: text = fh.read() elif filePath.endswith(".htm"): doc = html.parse(filePath) textParts = [] def iterTextParts(parent): for node in parent.iterchildren(): if isinstance(node, etree._Element): if node.tag in ("p", "P", "br", "BR", "div", "DIV"): textParts.append("\n") textParts.append(node.text or "") iterTextParts(node) if node.tail: # use tail whether element, comment, or processing instruction textParts.append(node.tail) iterTextParts(doc.getroot()) text = " ".join(textParts) except (IOError, EnvironmentError) as err: dts.info( "xpDB:primaryDocumentLoadingError", _("Loading XBRL DB: primary document loading error: %(error)s"), modelObject=dts, error=err, ) text = "" # with open("/Users/hermf/temp/test.txt", "w", encoding='utf-8') as fh: # fh.write(text) class Span: def __init__(self, start): self.start = start self.end = -1 # find the parts partSpan = {} partPrev = None missing2ndPart1 = False for partMatch in partPattern.finditer(text): part = partMatch.group(1).upper() if partPrev is not None: if part != "I" and part == partPrev: # two of these parts without second part 1, use signature or first item for 2nd part 1 missing2ndPart1 = True partSpan[partPrev].end = partMatch.start(0) partSpan[part] = Span(partMatch.end(1)) partPrev = part if partPrev is not None: partSpan[partPrev].end = len(text) if missing2ndPart1: # signatures signaturesStarts = [] for signaturesMatch in signaturesPattern.finditer(text): signaturesStarts.append(signaturesMatch.start(0)) # check if PART I missing then use first signatures if "I" in partSpan and "II" in partSpan: if len(signaturesStarts) == 2 and signaturesStarts[0] > partSpan["I"].start: partSpan["I"].start = signaturesStarts[0] partSpan["I"].end = partSpan["II"].start else: # use ASSETS as start of part 1 for assetsMatch in assetsPattern.finditer(text): partSpan["I"].start = assetsMatch.end(0) partSpan["I"].end = partSpan["II"].start break # find the items itemSpan = {} for part, span in partSpan.items(): item = None for itemMatch in itemPattern.finditer(text, span.start, span.end): if item is not None: itemSpan[(part, item)].end = itemMatch.start(0) item = itemMatch.group(1) itemSpan[(part, item)] = Span(itemMatch.end(1)) if item is not None: itemSpan[(part, item)].end = span.end if any(itemKey in parts for itemKey in itemSpan.keys()): # find default context for cntx in dts.contexts.values(): if cntx.isStartEndPeriod: if not cntx.hasSegment: # use c as default context # load extra datapoints taxonomy but not as discovered in DTS ModelDocument.load(dts, "http://arelle.org/2014/doc-2014-01-31.xsd") # add the facts for itemKey, itemSpan in itemSpan.items(): if itemKey in parts: dts.createFact( qname("{http://arelle.org/doc/2014-01-31}doc:" + parts[itemKey]), attributes=[("contextRef", cntx.id)], text=text[itemSpan.start : itemSpan.end], ) break
def watchCycle(self): while not self.stopRequested: rssWatchOptions = self.rssModelXbrl.modelManager.rssWatchOptions # check rss expiration rssHeaders = self.cntlr.webCache.getheaders(self.rssModelXbrl.modelManager.rssWatchOptions.get("feedSourceUri")) expires = parseRfcDatetime(rssHeaders.get("expires")) reloadNow = True # texpires and expires > datetime.datetime.now() # reload rss feed self.rssModelXbrl.reload('checking RSS items', reloadCache=reloadNow) if self.stopRequested: break # setup validator postLoadActions = [] if rssWatchOptions.get("validateDisclosureSystemRules"): self.instValidator = ValidateFiling.ValidateFiling(self.rssModelXbrl) postLoadActions.append(_("validating")) elif rssWatchOptions.get("validateXbrlRules") or rssWatchOptions.get("validateFormulaAssertions"): self.instValidator = ValidateXbrl.ValidateXbrl(self.rssModelXbrl) postLoadActions.append(_("validating")) if (rssWatchOptions.get("validateFormulaAssertions")): postLoadActions.append(_("running formulas")) else: self.instValidator = None matchTextExpr = rssWatchOptions.get("matchTextExpr") if matchTextExpr: matchPattern = re.compile(matchTextExpr) postLoadActions.append(_("matching text")) else: matchPattern= None postLoadAction = ', '.join(postLoadActions) # anything to check new filings for if (rssWatchOptions.get("validateDisclosureSystemRules") or rssWatchOptions.get("validateXbrlRules") or rssWatchOptions.get("validateCalcLinkbase") or rssWatchOptions.get("validateFormulaAssertions") or rssWatchOptions.get("alertMatchedFactText") or any(pluginXbrlMethod(rssWatchOptions) for pluginXbrlMethod in pluginClassMethods("RssWatch.HasWatchAction")) ): # form keys in ascending order of pubdate pubDateRssItems = [] for rssItem in self.rssModelXbrl.modelDocument.rssItems: pubDateRssItems.append((rssItem.pubDate,rssItem.objectId())) for pubDate, rssItemObjectId in sorted(pubDateRssItems): rssItem = self.rssModelXbrl.modelObject(rssItemObjectId) # update ui thread via modelManager (running in background here) self.rssModelXbrl.modelManager.viewModelObject(self.rssModelXbrl, rssItem.objectId()) if self.stopRequested: break latestPubDate = XmlUtil.datetimeValue(rssWatchOptions.get("latestPubDate")) if (latestPubDate and rssItem.pubDate < latestPubDate): continue try: # try zipped URL if possible, else expanded instance document modelXbrl = ModelXbrl.load(self.rssModelXbrl.modelManager, openFileSource(rssItem.zippedUrl, self.cntlr), postLoadAction) if self.stopRequested: modelXbrl.close() break emailAlert = False if modelXbrl.modelDocument is None: modelXbrl.error("arelle.rssWatch", _("RSS item %(company)s %(form)s document not loaded: %(date)s"), modelXbrl=modelXbrl, company=rssItem.companyName, form=rssItem.formType, date=rssItem.filingDate) rssItem.status = "not loadable" else: # validate schema, linkbase, or instance if self.stopRequested: modelXbrl.close() break if self.instValidator: self.instValidator.validate(modelXbrl) if modelXbrl.errors and rssWatchOptions.get("alertValiditionError"): emailAlert = True for pluginXbrlMethod in pluginClassMethods("RssWatch.DoWatchAction"): pluginXbrlMethod(modelXbrl, rssWatchOptions, rssItem) # check match expression if matchPattern: for fact in modelXbrl.factsInInstance: v = fact.value if v is not None: m = matchPattern.search(v) if m: fr, to = m.span() msg = _("Fact Variable {0}\n context {1}\n matched text: {2}").format( fact.qname, fact.contextID, v[max(0,fr-20):to+20]) modelXbrl.info("arelle.rssInfo", msg, modelXbrl=modelXbrl) # msg as code passes it through to the status if rssWatchOptions.get("alertMatchedFactText"): emailAlert = True if (rssWatchOptions.get("formulaFileUri") and rssWatchOptions.get("validateFormulaAssertions") and self.instValidator): # attach formulas ModelDocument.load(modelXbrl, rssWatchOptions["formulaFileUri"]) ValidateFormula.validate(self.instValidator) rssItem.setResults(modelXbrl) modelXbrl.close() del modelXbrl # completely dereference self.rssModelXbrl.modelManager.viewModelObject(self.rssModelXbrl, rssItem.objectId()) if rssItem.assertionUnsuccessful and rssWatchOptions.get("alertAssertionUnsuccessful"): emailAlert = True msg = _("Filing CIK {0}\n " "company {1}\n " "published {2}\n " "form type {3}\n " "filing date {4}\n " "period {5}\n " "year end {6}\n " "results: {7}").format( rssItem.cikNumber, rssItem.companyName, rssItem.pubDate, rssItem.formType, rssItem.filingDate, rssItem.period, rssItem.fiscalYearEnd, rssItem.status) self.rssModelXbrl.info("arelle:rssWatch", msg, modelXbrl=self.rssModelXbrl) emailAddress = rssWatchOptions.get("emailAddress") if emailAlert and emailAddress: self.rssModelXbrl.modelManager.showStatus(_("sending e-mail alert")) import smtplib from email.mime.text import MIMEText emailMsg = MIMEText(msg) emailMsg["Subject"] = _("Arelle RSS Watch alert on {0}").format(rssItem.companyName) emailMsg["From"] = emailAddress emailMsg["To"] = emailAddress smtp = smtplib.SMTP() smtp.sendmail(emailAddress, [emailAddress], emailMsg.as_string()) smtp.quit() self.rssModelXbrl.modelManager.showStatus(_("RSS item {0}, {1} completed, status {2}").format(rssItem.companyName, rssItem.formType, rssItem.status), 3500) self.rssModelXbrl.modelManager.cntlr.rssWatchUpdateOption(rssItem.pubDate.strftime('%Y-%m-%dT%H:%M:%S')) except Exception as err: self.rssModelXbrl.error("arelle.rssError", _("RSS item %(company)s, %(form)s, %(date)s, exception: %(error)s"), modelXbrl=self.rssModelXbrl, company=rssItem.companyName, form=rssItem.formType, date=rssItem.filingDate, error=err, exc_info=True) if self.stopRequested: break if self.stopRequested: self.cntlr.showStatus(_("RSS watch, stop requested"), 10000) else: import time time.sleep(600) self.thread = None # close thread self.stopRequested = False
def runFromExcel(self, options): testGenFileName = options.excelfilename #testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation\1000-2000-index.xls" testGenDir = os.path.dirname(testGenFileName) timeNow = XmlUtil.dateunionValue(datetime.datetime.now()) if options.testfiledate: today = options.testfiledate else: today = XmlUtil.dateunionValue(datetime.date.today()) startedAt = time.time() self.logMessages = [] logMessagesFile = testGenDir + os.sep + 'logGenerationMessages.txt' modelTestcases = ModelXbrl.create(self.modelManager) testcaseIndexBook = xlrd.open_workbook(testGenFileName) testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0) self.addToLog(_("[info] xls loaded in {0:.2} secs at {1}").format(time.time() - startedAt, timeNow)) # start index file indexFiles = [testGenDir + os.sep + 'creationTestcasesIndex.xml', testGenDir + os.sep + 'consumptionTestcasesIndex.xml'] indexDocs = [xml.dom.minidom.parseString( '<?xml version="1.0" encoding="UTF-8"?>' '<!-- XBRL Versioning 1.0 {0} Tests -->' '<!-- Copyright 2011 XBRL International. All Rights Reserved. -->' '<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>' '<testcases name="XBRL Versioning 1.0 Consumption Tests" date="{1}" ' ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' ' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">' '</testcases>'.format(purpose, today) ) for purpose in ("Creation","Consumption")] testcasesElements = [XmlUtil.child(indexDoc, None, "testcases") for indexDoc in indexDocs] priorTestcasesDir = None testcaseFiles = None testcaseDocs = None for iRow in range(1, testcaseIndexSheet.nrows): row = testcaseIndexSheet.row(iRow) if row[0].ctype == xlrd.XL_CELL_EMPTY or row[1].ctype == xlrd.XL_CELL_EMPTY or row[2].ctype == xlrd.XL_CELL_EMPTY: continue testDir = row[0].value uriFrom = row[1].value uriTo = row[2].value intention = row[3].value if intention is None or len(intention) == 0: continue # test not ready to run reason = row[4].value expectedEvent = row[5].value base = os.path.join(os.path.dirname(testGenFileName),testDir) + os.sep self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom)) if uriFrom and uriTo and reason.lower() not in ("n.a.", "error") and expectedEvent != "N.A.": for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"), True), (uriTo, _("loading to DTS"), False)): if ',' not in URIs: modelDTS = ModelXbrl.load(self.modelManager, URIs, msg, base=base) else: modelDTS = ModelXbrl.create(self.modelManager, ModelDocument.Type.DTSENTRIES, self.webCache.normalizeUrl(URIs.replace(", ","_") + ".dts", base), isEntry=True) DTSdoc = modelDTS.modelDocument DTSdoc.inDTS = True for uri in URIs.split(','): doc = ModelDocument.load(modelDTS, uri.strip(), base=base) DTSdoc.referencesDocument[doc] = "import" #fake import doc.inDTS = True if isFrom: modelDTSfrom = modelDTS else: modelDTSto = modelDTS if modelDTSfrom and modelDTSto: # generate differences report reportUri = uriFrom.partition(',')[0] # first file reportDir = os.path.dirname(reportUri) if reportDir: reportDir += os.sep reportName = os.path.basename(reportUri).replace("from.xsd","report.xml") reportFile = reportDir + "report" + os.sep + reportName reportFullPath = self.webCache.normalizeUrl( reportFile, base) testcasesDir = os.path.dirname(os.path.dirname(reportFullPath)) if testcasesDir != priorTestcasesDir: # close prior report if priorTestcasesDir: for i,testcaseFile in enumerate(testcaseFiles): with open(testcaseFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8") testcaseName = os.path.basename(testcasesDir) testcaseFiles = [testcasesDir + os.sep + testcaseName + "-creation-testcase.xml", testcasesDir + os.sep + testcaseName + "-consumption-testcase.xml"] for i,testcaseFile in enumerate(testcaseFiles): XmlUtil.addChild(testcasesElements[i], None, "testcase", ("uri", testcaseFile[len(testGenDir)+1:].replace("\\","/")) ) # start testcase file testcaseDocs = [xml.dom.minidom.parseString( '<?xml version="1.0" encoding="UTF-8"?>' '<!-- Copyright 2011 XBRL International. All Rights Reserved. -->' '<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>' '<testcase name="XBRL Versioning 1.0 {1} Tests" date="{2}" ' ' xmlns="http://xbrl.org/2008/conformance"' ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' ' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">' '<creator>' '<name>Roland Hommes</name>' '<email>[email protected]</email>' '</creator>' '<name>{0}</name>' '<description>{0}</description>' '</testcase>'.format(testcaseName,purpose,today) ) for purpose in ("Creation","Consumption")] testcaseElements = [XmlUtil.child(testcaseDoc, conformanceNS, "testcase") for testcaseDoc in testcaseDocs] priorTestcasesDir = testcasesDir variationID = 1 try: os.makedirs(os.path.dirname(reportFullPath)) except WindowsError: pass # dir already exists modelVersReport = ModelVersReport.ModelVersReport(modelTestcases) modelVersReport.diffDTSes(reportFullPath,modelDTSfrom, modelDTSto) # check for expected elements if expectedEvent and expectedEvent not in ( "No change", "N.A."): if len(modelVersReport.xmlDocument.getElementsByTagNameNS('*',expectedEvent)) == 0: modelTestcases.error( "Generated test case {0} missing expected event {1}".format( reportName, expectedEvent), "wrn", "missingEvent") modelVersReport.close([]) for i,testcaseElt in enumerate(testcaseElements): variationElement = XmlUtil.addChild(testcaseElt, conformanceNS, "variation", attributes=("id", "_{0:02n}".format(variationID))) XmlUtil.addChild(variationElement, conformanceNS, "name", text=intention) dataElement = XmlUtil.addChild(variationElement, conformanceNS, "data") for schemaURIs, dtsAttr in ((uriFrom,"from"), (uriTo,"to")): for schemaURI in schemaURIs.split(","): XmlUtil.addChild(dataElement, conformanceNS, "schema", attributes=((("dts",dtsAttr),) + ((("readMeFirst","true"),) if i == 0 else ())), text=os.path.basename(schemaURI.strip())) resultElement = XmlUtil.addChild(variationElement, conformanceNS, "result") XmlUtil.addChild(resultElement if i == 0 else dataElement, conformanceNS, "versioningReport", attributes=(("readMeFirst","true") if i == 1 else ()), text="report/" + reportName) variationID += 1 with open(logMessagesFile, "w") as fh: fh.writelines(self.logMessages) if priorTestcasesDir: for i,testcaseFile in enumerate(testcaseFiles): with open(testcaseFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8") for i,indexFile in enumerate(indexFiles): with open(indexFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def run(self, options, sourceZipStream=None): self.entrypointFile = options.entrypointFile filesource = FileSource.openFileSource(self.entrypointFile, self, sourceZipStream) if options.validateEFM: if options.gfmName: self.addToLog(_("both --efm and --gfm validation are requested, proceeding with --efm only"), messageCode="info", file=self.entrypointFile) self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select("efm") elif options.gfmName: self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select(options.gfmName) elif options.validateHMRC: self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select("hmrc") else: self.modelManager.disclosureSystem.select(None) # just load ordinary mappings if options.calcDecimals: if options.calcPrecision: self.addToLog(_("both --calcDecimals and --calcPrecision validation are requested, proceeding with --calcDecimals only"), messageCode="info", file=self.entrypointFile) self.modelManager.validateInferDecimals = True self.modelManager.validateCalcLB = True elif options.calcPrecision: self.modelManager.validateInferDecimals = False self.modelManager.validateCalcLB = True if options.utrValidate: self.modelManager.validateUtr = True fo = FormulaOptions() if options.parameters: fo.parameterValues = dict(((qname(key, noPrefixIsNoNamespace=True),(None,value)) for param in options.parameters.split(',') for key,sep,value in (param.partition('='),) ) ) if options.formulaParamExprResult: fo.traceParameterExpressionResult = True if options.formulaParamInputValue: fo.traceParameterInputValue = True if options.formulaCallExprSource: fo.traceCallExpressionSource = True if options.formulaCallExprCode: fo.traceCallExpressionCode = True if options.formulaCallExprEval: fo.traceCallExpressionEvaluation = True if options.formulaCallExprResult: fo.traceCallExpressionResult = True if options.formulaVarSetExprEval: fo.traceVariableSetExpressionEvaluation = True if options.formulaVarSetExprResult: fo.traceVariableSetExpressionResult = True if options.formulaAsserResultCounts: fo.traceAssertionResultCounts = True if options.formulaFormulaRules: fo.traceFormulaRules = True if options.formulaVarsOrder: fo.traceVariablesOrder = True if options.formulaVarExpressionSource: fo.traceVariableExpressionSource = True if options.formulaVarExpressionCode: fo.traceVariableExpressionCode = True if options.formulaVarExpressionEvaluation: fo.traceVariableExpressionEvaluation = True if options.formulaVarExpressionResult: fo.traceVariableExpressionResult = True if options.formulaVarFilterWinnowing: fo.traceVariableFilterWinnowing = True if options.formulaVarFiltersResult: fo.traceVariableFiltersResult = True self.modelManager.formulaOptions = fo timeNow = XmlUtil.dateunionValue(datetime.datetime.now()) startedAt = time.time() modelDiffReport = None success = True modelXbrl = None try: modelXbrl = self.modelManager.load(filesource, _("views loading")) except Exception as err: self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format( err, traceback.format_tb(sys.exc_info()[2]))) success = False # loading errors, don't attempt to utilize loaded DTS if modelXbrl and modelXbrl.modelDocument: self.addToLog(format_string(self.modelManager.locale, _("loaded in %.2f secs at %s"), (time.time() - startedAt, timeNow)), messageCode="info", file=self.entrypointFile) if options.importFiles: for importFile in options.importFiles.split("|"): ModelDocument.load(modelXbrl, importFile.strip()) self.addToLog(format_string(self.modelManager.locale, _("imported in %.2f secs at %s"), (time.time() - startedAt, timeNow)), messageCode="info", file=importFile) if modelXbrl.errors: success = False # loading errors, don't attempt to utilize loaded DTS else: success = False if success and options.diffFile and options.versReportFile: diffFilesource = FileSource.FileSource(options.diffFile,self) startedAt = time.time() modelXbrl2 = self.modelManager.load(diffFilesource, _("views loading")) if modelXbrl2.errors: if not options.keepOpen: modelXbrl2.close() success = False else: self.addToLog(format_string(self.modelManager.locale, _("diff comparison DTS loaded in %.2f secs"), time.time() - startedAt), messageCode="info", file=self.entrypointFile) startedAt = time.time() modelDiffReport = self.modelManager.compareDTSes(options.versReportFile) self.addToLog(format_string(self.modelManager.locale, _("compared in %.2f secs"), time.time() - startedAt), messageCode="info", file=self.entrypointFile) if success: try: if options.validate: startedAt = time.time() self.modelManager.validate() self.addToLog(format_string(self.modelManager.locale, _("validated in %.2f secs"), time.time() - startedAt), messageCode="info", file=self.entrypointFile) if (options.testReport and self.modelManager.modelXbrl.modelDocument.type in (ModelDocument.Type.TESTCASESINDEX, ModelDocument.Type.TESTCASE, ModelDocument.Type.REGISTRY)): ViewFileTests.viewTests(self.modelManager.modelXbrl, options.testReport) if options.DTSFile: ViewFileDTS.viewDTS(modelXbrl, options.DTSFile) if options.factsFile: ViewFileFactList.viewFacts(modelXbrl, options.factsFile, labelrole=options.labelRole, lang=options.labelLang, cols=options.factListCols) if options.factTableFile: ViewFileFactTable.viewFacts(modelXbrl, options.factTableFile, labelrole=options.labelRole, lang=options.labelLang) if options.conceptsFile: ViewFileConcepts.viewConcepts(modelXbrl, options.conceptsFile, labelrole=options.labelRole, lang=options.labelLang) if options.preFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.preFile, "Presentation Linkbase", "http://www.xbrl.org/2003/arcrole/parent-child", labelrole=options.labelRole, lang=options.labelLang) if options.calFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.calFile, "Calculation Linkbase", "http://www.xbrl.org/2003/arcrole/summation-item", labelrole=options.labelRole, lang=options.labelLang) if options.dimFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.dimFile, "Dimensions", "XBRL-dimensions", labelrole=options.labelRole, lang=options.labelLang) if options.formulaeFile: ViewFileFormulae.viewFormulae(modelXbrl, options.formulaeFile, "Formulae", lang=options.labelLang) for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Run"): pluginXbrlMethod(self, options, modelXbrl) except (IOError, EnvironmentError) as err: self.addToLog(_("[IOError] Failed to save output:\n {0}").format(err)) success = False except Exception as err: self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format( err, traceback.format_tb(sys.exc_info()[2]))) success = False if not options.keepOpen: if modelDiffReport: modelDiffReport.close() elif modelXbrl: modelXbrl.close() return success
def validateTestcase(self, testcase): self.modelXbrl.info("info", "Testcase", modelDocument=testcase) self.modelXbrl.viewModelObject(testcase.objectId()) if hasattr(testcase, "testcaseVariations"): for modelTestcaseVariation in testcase.testcaseVariations: # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) # is this a versioning report? resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance formulaOutputInstance = None inputDTSes = defaultdict(list) baseForElement = testcase.baseForElement(modelTestcaseVariation) # try to load instance document self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s"), modelObject=modelTestcaseVariation, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, expected=modelTestcaseVariation.expected) for readMeFirstUri in modelTestcaseVariation.readMeFirstUris: if isinstance(readMeFirstUri,tuple): # dtsName is for formula instances, but is from/to dts if versioning dtsName, readMeFirstUri = readMeFirstUri elif resultIsVersioningReport: if inputDTSes: dtsName = "to" else: dtsName = "from" else: dtsName = None if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document if dtsName in inputDTSes: dtsName = inputDTSes[dtsName] else: modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager, ModelDocument.Type.DTSENTRIES, self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement), isEntry=True) DTSdoc = modelXbrl.modelDocument DTSdoc.inDTS = True doc = ModelDocument.load(modelXbrl, readMeFirstUri, base=baseForElement) if doc is not None: DTSdoc.referencesDocument[doc] = "import" #fake import doc.inDTS = True else: # not a multi-schemaRef versioning report modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, readMeFirstUri, _("validating"), base=baseForElement, useFileSource=self.useFileSource) if modelXbrl.modelDocument is None: self.modelXbrl.error("arelle:notLoaded", _("Testcase %(id)s %(name)s document not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) modelXbrl.close() self.determineNotLoadedTestStatus(modelTestcaseVariation) elif resultIsVersioningReport: inputDTSes[dtsName] = modelXbrl elif modelXbrl.modelDocument.type == ModelDocument.Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl) self.determineTestStatus(modelTestcaseVariation, modelXbrl) modelXbrl.close() elif testcase.type == ModelDocument.Type.REGISTRYTESTCASE: self.instValidator.validate(modelXbrl) # required to set up dimensions, etc self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id, modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest) self.determineTestStatus(modelTestcaseVariation, modelXbrl) self.instValidator.close() modelXbrl.close() else: inputDTSes[dtsName].append(modelXbrl) if resultIsVersioningReport and modelXbrl.modelDocument: versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl( modelTestcaseVariation.versioningReportUri, baseForElement) if os.path.exists(versReportFile): #validate existing modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report")) if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == ModelDocument.Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport) self.determineTestStatus(modelTestcaseVariation, modelVersReport) modelVersReport.close() elif len(inputDTSes) == 2: ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes( versReportFile, inputDTSes["from"], inputDTSes["to"]) modelTestcaseVariation.status = "generated" else: self.modelXbrl.error("arelle:notLoaded", _("Testcase %(id)s %(name)s DTSes not loaded, unable to generate versioning report: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) modelTestcaseVariation.status = "failed" for inputDTS in inputDTSes.values(): inputDTS.close() del inputDTSes # dereference elif inputDTSes: # validate schema, linkbase, or instance modelXbrl = inputDTSes[None][0] parameters = modelTestcaseVariation.parameters.copy() for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters if dtsName: # named instance parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes) elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula if modelTestcaseVariation.resultIsTable: RenderingEvaluator.init(modelXbrl) self.instValidator.validate(modelXbrl, parameters) if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset: for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"): pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri) infoset = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultInfosetUri, _("loading result infoset"), base=baseForElement, useFileSource=self.useFileSource) if infoset.modelDocument is None: self.modelXbrl.error("arelle:notLoaded", _("Testcase %(id)s %(name)s result infoset not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstance)) modelTestcaseVariation.status = "result infoset not loadable" else: # check infoset ValidateInfoset.validate(self.instValidator, modelXbrl, infoset) infoset.close() if modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset: # diff (or generate) table infoset ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement), diffToFile=True) # false to save infoset files self.determineTestStatus(modelTestcaseVariation, modelXbrl) # include infoset errors in status self.instValidator.close() if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual): # if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost) self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters) self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance) if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance formulaOutputInstance = modelXbrl.formulaOutputInstance modelXbrl.formulaOutputInstance = None # prevent it from being closed now self.instValidator.close() for inputDTSlist in inputDTSes.values(): for inputDTS in inputDTSlist: inputDTS.close() del inputDTSes # dereference if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument: expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultXbrlInstanceUri, _("loading expected result XBRL instance"), base=baseForElement, useFileSource=self.useFileSource) if expectedInstance.modelDocument is None: self.modelXbrl.error("arelle:notLoaded", _("Testcase %(id)s %(name)s expected result instance not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstance)) modelTestcaseVariation.status = "result not loadable" else: # compare facts if len(expectedInstance.facts) != len(formulaOutputInstance.facts): formulaOutputInstance.error("formula:resultFactCounts", _("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"), modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts), expectedFacts=len(expectedInstance.facts)) else: for fact in expectedInstance.facts: unmatchedFactsStack = [] if formulaOutputInstance.matchFact(fact, unmatchedFactsStack) is None: if unmatchedFactsStack: # get missing nested tuple fact, if possible missingFact = unmatchedFactsStack[-1] else: missingFact = fact formulaOutputInstance.error("formula:expectedFactMissing", _("Formula output missing expected fact %(fact)s"), modelXbrl=missingFact, fact=missingFact.qname) # for debugging uncomment next line to save generated instance document # formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml") expectedInstance.close() del expectedInstance # dereference self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance) formulaOutputInstance.close() del formulaOutputInstance # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def reload(self,nextaction,reloadCache=False): from arelle import ModelDocument self.init(keepViews=True) self.modelDocument = ModelDocument.load(self, self.fileSource.url, isEntry=True, reloadCache=reloadCache) self.modelManager.showStatus(_("xbrl loading finished, {0}...").format(nextaction),5000) self.modelManager.reloadViews(self)
def validateTestcase(self, testcase): self.modelXbrl.error(_("testcase {0}").format(os.path.basename(testcase.uri))) self.modelXbrl.viewModelObject(testcase.objectId()) if hasattr(testcase, "testcaseVariations"): for modelTestcaseVariation in testcase.testcaseVariations: # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) # is this a versioning report? resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance formulaOutputInstance = None inputDTSes = defaultdict(list) baseForElement = testcase.baseForElement(modelTestcaseVariation.element) # try to load instance document self.modelXbrl.error(_("variation {0} {1}: {2}").format(modelTestcaseVariation.id, modelTestcaseVariation.name, modelTestcaseVariation.expected)) for readMeFirstUri in modelTestcaseVariation.readMeFirstUris: if isinstance(readMeFirstUri,tuple): # dtsName is for formula instances, but is from/to dts if versioning dtsName, readMeFirstUri = readMeFirstUri elif resultIsVersioningReport: if inputDTSes: dtsName = "to" else: dtsName = "from" else: dtsName = None if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document if dtsName in inputDTSes: dtsName = inputDTSes[dtsName] else: modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager, ModelDocument.Type.DTSENTRIES, self.modelXbrl.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement), isEntry=True) DTSdoc = modelXbrl.modelDocument DTSdoc.inDTS = True doc = ModelDocument.load(modelXbrl, readMeFirstUri, base=baseForElement) DTSdoc.referencesDocument[doc] = "import" #fake import doc.inDTS = True else: # not a multi-schemaRef versioning report modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, readMeFirstUri, _("validating"), base=baseForElement) if modelXbrl.modelDocument is None: self.modelXbrl.error(_("Testcase {0} {1} document not loaded: {2}").format( modelTestcaseVariation.id, modelTestcaseVariation.name, os.path.basename(readMeFirstUri))) modelTestcaseVariation.status = "not loadable" modelXbrl.close() elif resultIsVersioningReport: inputDTSes[dtsName] = modelXbrl elif modelXbrl.modelDocument.type == ModelDocument.Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl) \ .validate(modelXbrl) self.determineTestStatus(modelTestcaseVariation, modelXbrl) modelXbrl.close() elif testcase.type == ModelDocument.Type.REGISTRYTESTCASE: self.instValidator.validate(modelXbrl) # required to set up dimensions, etc self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id, modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest) self.determineTestStatus(modelTestcaseVariation, modelXbrl) modelXbrl.close() else: inputDTSes[dtsName].append(modelXbrl) if resultIsVersioningReport and modelXbrl.modelDocument: versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl( modelTestcaseVariation.versioningReportUri, baseForElement) if os.path.exists(versReportFile): #validate existing modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report")) if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == ModelDocument.Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport) self.determineTestStatus(modelTestcaseVariation, modelVersReport) modelVersReport.close() elif len(inputDTSes) == 2: ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes( versReportFile, inputDTSes["from"], inputDTSes["to"]) modelTestcaseVariation.status = "generated" else: self.modelXbrl.error(_("Testcase {0} {1} DTSes not loaded, unable to generate versioning report").format( modelTestcaseVariation.id, modelTestcaseVariation.name, os.path.basename(readMeFirstUri))) modelTestcaseVariation.status = "failed" for inputDTS in inputDTSes: inputDTS.close() elif inputDTSes: # validate schema, linkbase, or instance modelXbrl = inputDTSes[None][0] parameters = modelTestcaseVariation.parameters.copy() for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters if dtsName: parameters[dtsName] = (None, inputDTS) self.instValidator.validate(modelXbrl, parameters) self.determineTestStatus(modelTestcaseVariation, modelXbrl) if modelXbrl.formulaOutputInstance and len(modelTestcaseVariation.actual) == 0: # if an output instance is created, validate it self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters) self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance) if len(modelTestcaseVariation.actual) == 0: # if still 'clean' pass it forward for comparison to expected result instance formulaOutputInstance = modelXbrl.formulaOutputInstance modelXbrl.formulaOutputInstance = None # prevent it from being closed now for inputDTSlist in inputDTSes.values(): for inputDTS in inputDTSlist: inputDTS.close() if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument: expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultXbrlInstanceUri, _("loading expected result XBRL instance"), base=baseForElement) if expectedInstance.modelDocument is None: self.modelXbrl.error(_("Testcase {0} {1} expected result instance not loaded: {2}").format( modelTestcaseVariation.id, modelTestcaseVariation.name, os.path.basename(modelTestcaseVariation.resultXbrlInstance))) modelTestcaseVariation.status = "result not loadable" expectedInstance.close() else: # compare facts if len(expectedInstance.facts) != len(formulaOutputInstance.facts): formulaOutputInstance.error( _("Formula output {0} facts, expected {1} facts").format( len(formulaOutputInstance.facts), len(expectedInstance.facts)), "err", "formula:resultFactCounts") else: for fact in expectedInstance.facts: if not formulaOutputInstance.matchFact(fact): formulaOutputInstance.error( _("Formula output missing expected fact {0}").format( fact), "err", "formula:expectedFactMissing") self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance) formulaOutputInstance.close() formulaOutputInstance = None # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def runFromExcel(self, options): testGenFileName = options.excelfilename #testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation\1000-2000-index.xls" testGenDir = os.path.dirname(testGenFileName) timeNow = XmlUtil.dateunionValue(datetime.datetime.now()) if options.testfiledate: today = options.testfiledate else: today = XmlUtil.dateunionValue(datetime.date.today()) startedAt = time.time() self.logMessages = [] logMessagesFile = testGenDir + os.sep + 'logGenerationMessages.txt' modelTestcases = ModelXbrl.create(self.modelManager) testcaseIndexBook = xlrd.open_workbook(testGenFileName) testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0) self.addToLog( _("[info] xls loaded in {0:.2} secs at {1}").format( time.time() - startedAt, timeNow)) # start index file indexFiles = [ testGenDir + os.sep + 'creationTestcasesIndex.xml', testGenDir + os.sep + 'consumptionTestcasesIndex.xml' ] indexDocs = [ xml.dom.minidom.parseString( '<?xml version="1.0" encoding="UTF-8"?>' '<!-- XBRL Versioning 1.0 {0} Tests -->' '<!-- Copyright 2011 XBRL International. All Rights Reserved. -->' '<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>' '<testcases name="XBRL Versioning 1.0 Consumption Tests" date="{1}" ' ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' ' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">' '</testcases>'.format(purpose, today)) for purpose in ("Creation", "Consumption") ] testcasesElements = [ XmlUtil.child(indexDoc, None, "testcases") for indexDoc in indexDocs ] priorTestcasesDir = None testcaseFiles = None testcaseDocs = None for iRow in range(1, testcaseIndexSheet.nrows): row = testcaseIndexSheet.row(iRow) if row[0].ctype == xlrd.XL_CELL_EMPTY or row[ 1].ctype == xlrd.XL_CELL_EMPTY or row[ 2].ctype == xlrd.XL_CELL_EMPTY: continue testDir = row[0].value uriFrom = row[1].value uriTo = row[2].value intention = row[3].value if intention is None or len(intention) == 0: continue # test not ready to run reason = row[4].value expectedEvent = row[5].value base = os.path.join(os.path.dirname(testGenFileName), testDir) + os.sep self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom)) if uriFrom and uriTo and reason.lower() not in ( "n.a.", "error") and expectedEvent != "N.A.": for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"), True), (uriTo, _("loading to DTS"), False)): if ',' not in URIs: modelDTS = ModelXbrl.load(self.modelManager, URIs, msg, base=base) else: modelDTS = ModelXbrl.create( self.modelManager, ModelDocument.Type.DTSENTRIES, self.webCache.normalizeUrl( URIs.replace(", ", "_") + ".dts", base), isEntry=True) DTSdoc = modelDTS.modelDocument DTSdoc.inDTS = True for uri in URIs.split(','): doc = ModelDocument.load(modelDTS, uri.strip(), base=base) DTSdoc.referencesDocument[ doc] = "import" #fake import doc.inDTS = True if isFrom: modelDTSfrom = modelDTS else: modelDTSto = modelDTS if modelDTSfrom and modelDTSto: # generate differences report reportUri = uriFrom.partition(',')[0] # first file reportDir = os.path.dirname(reportUri) if reportDir: reportDir += os.sep reportName = os.path.basename(reportUri).replace( "from.xsd", "report.xml") reportFile = reportDir + "report" + os.sep + reportName reportFullPath = self.webCache.normalizeUrl( reportFile, base) testcasesDir = os.path.dirname( os.path.dirname(reportFullPath)) if testcasesDir != priorTestcasesDir: # close prior report if priorTestcasesDir: for i, testcaseFile in enumerate(testcaseFiles): with open(testcaseFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8") testcaseName = os.path.basename(testcasesDir) testcaseFiles = [ testcasesDir + os.sep + testcaseName + "-creation-testcase.xml", testcasesDir + os.sep + testcaseName + "-consumption-testcase.xml" ] for i, testcaseFile in enumerate(testcaseFiles): XmlUtil.addChild( testcasesElements[i], None, "testcase", ("uri", testcaseFile[len(testGenDir) + 1:].replace("\\", "/"))) # start testcase file testcaseDocs = [ xml.dom.minidom.parseString( '<?xml version="1.0" encoding="UTF-8"?>' '<!-- Copyright 2011 XBRL International. All Rights Reserved. -->' '<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>' '<testcase name="XBRL Versioning 1.0 {1} Tests" date="{2}" ' ' xmlns="http://xbrl.org/2008/conformance"' ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' ' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">' '<creator>' '<name>Roland Hommes</name>' '<email>[email protected]</email>' '</creator>' '<name>{0}</name>' '<description>{0}</description>' '</testcase>'.format(testcaseName, purpose, today)) for purpose in ("Creation", "Consumption") ] testcaseElements = [ XmlUtil.child(testcaseDoc, conformanceNS, "testcase") for testcaseDoc in testcaseDocs ] priorTestcasesDir = testcasesDir variationID = 1 try: os.makedirs(os.path.dirname(reportFullPath)) except WindowsError: pass # dir already exists modelVersReport = ModelVersReport.ModelVersReport( modelTestcases) modelVersReport.diffDTSes(reportFullPath, modelDTSfrom, modelDTSto) # check for expected elements if expectedEvent and expectedEvent not in ("No change", "N.A."): if len( modelVersReport.xmlDocument. getElementsByTagNameNS('*', expectedEvent)) == 0: modelTestcases.error( "Generated test case {0} missing expected event {1}" .format(reportName, expectedEvent), "wrn", "missingEvent") modelVersReport.close([]) for i, testcaseElt in enumerate(testcaseElements): variationElement = XmlUtil.addChild( testcaseElt, conformanceNS, "variation", attributes=("id", "_{0:02n}".format(variationID))) XmlUtil.addChild(variationElement, conformanceNS, "name", text=intention) dataElement = XmlUtil.addChild(variationElement, conformanceNS, "data") for schemaURIs, dtsAttr in ((uriFrom, "from"), (uriTo, "to")): for schemaURI in schemaURIs.split(","): XmlUtil.addChild( dataElement, conformanceNS, "schema", attributes=((("dts", dtsAttr), ) + ((("readMeFirst", "true"), ) if i == 0 else ())), text=os.path.basename(schemaURI.strip())) resultElement = XmlUtil.addChild( variationElement, conformanceNS, "result") XmlUtil.addChild( resultElement if i == 0 else dataElement, conformanceNS, "versioningReport", attributes=(("readMeFirst", "true") if i == 1 else ()), text="report/" + reportName) variationID += 1 with open(logMessagesFile, "w") as fh: fh.writelines(self.logMessages) if priorTestcasesDir: for i, testcaseFile in enumerate(testcaseFiles): with open(testcaseFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8") for i, indexFile in enumerate(indexFiles): with open(indexFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def runFromExcel(self, options): #testGenFileName = options.excelfilename testGenFileName = r"C:\Users\Herm Fischer\Documents\mvsl\projects\XBRL.org\conformance-versioning\trunk\versioningReport\conf\creation-index.xls" testGenDir = os.path.dirname(testGenFileName) schemaDir = os.path.dirname(testGenDir) + os.sep + "schema" timeNow = XmlUtil.dateunionValue(datetime.datetime.now()) if options.testfiledate: today = options.testfiledate else: today = XmlUtil.dateunionValue(datetime.date.today()) startedAt = time.time() LogHandler(self) # start logger self.logMessages = [] logMessagesFile = testGenDir + os.sep + 'log-generation-messages.txt' modelTestcases = ModelXbrl.create(self.modelManager, url=testGenFileName, isEntry=True) testcaseIndexBook = xlrd.open_workbook(testGenFileName) testcaseIndexSheet = testcaseIndexBook.sheet_by_index(0) self.addToLog( _("[info] xls loaded in {0:.2} secs at {1}").format( time.time() - startedAt, timeNow)) # start index file indexFiles = [ testGenDir + os.sep + 'creation-testcases-index.xml', testGenDir + os.sep + 'consumption-testcases-index.xml' ] indexDocs = [] testcasesElements = [] for purpose in ("Creation", "Consumption"): file = io.StringIO( #'<?xml version="1.0" encoding="UTF-8"?>' '<!-- XBRL Versioning 1.0 {0} Tests -->' '<!-- Copyright 2011 XBRL International. All Rights Reserved. -->' '<?xml-stylesheet type="text/xsl" href="infrastructure/testcases-index.xsl"?>' '<testcases name="XBRL Versioning 1.0 {0} Tests" ' ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' ' xsi:noNamespaceSchemaLocation="infrastructure/testcases-index.xsd">' '</testcases>'.format(purpose, today)) doc = etree.parse(file) file.close() indexDocs.append(doc) testcasesElements.append(doc.getroot()) priorTestcasesDir = None testcaseFiles = None testcaseDocs = None for iRow in range(1, testcaseIndexSheet.nrows): try: row = testcaseIndexSheet.row(iRow) if (row[0].ctype == xlrd.XL_CELL_EMPTY or # must have directory row[1].ctype == xlrd.XL_CELL_EMPTY or # from row[2].ctype == xlrd.XL_CELL_EMPTY): # to continue testDir = row[0].value uriFrom = row[1].value uriTo = row[2].value overrideReport = row[3].value description = row[4].value if description is None or len(description) == 0: continue # test not ready to run assignment = row[5].value expectedEvents = row[ 6].value # comma space separated if multiple note = row[7].value useCase = row[8].value base = os.path.join(os.path.dirname(testGenFileName), testDir) + os.sep self.addToLog(_("[info] testcase uriFrom {0}").format(uriFrom)) if uriFrom and uriTo and assignment.lower() not in ( "n.a.", "error") and expectedEvents != "N.A.": modelDTSfrom = modelDTSto = None for URIs, msg, isFrom in ((uriFrom, _("loading from DTS"), True), (uriTo, _("loading to DTS"), False)): if ',' not in URIs: modelDTS = ModelXbrl.load(self.modelManager, URIs, msg, base=base) else: modelDTS = ModelXbrl.create( self.modelManager, ModelDocument.Type.DTSENTRIES, self.webCache.normalizeUrl( URIs.replace(", ", "_") + ".dts", base), isEntry=True) DTSdoc = modelDTS.modelDocument DTSdoc.inDTS = True for uri in URIs.split(','): doc = ModelDocument.load(modelDTS, uri.strip(), base=base) if doc is not None: DTSdoc.referencesDocument[ doc] = "import" #fake import doc.inDTS = True if isFrom: modelDTSfrom = modelDTS else: modelDTSto = modelDTS if modelDTSfrom is not None and modelDTSto is not None: # generate differences report reportUri = uriFrom.partition(',')[0] # first file reportDir = os.path.dirname(reportUri) if reportDir: reportDir += os.sep reportName = os.path.basename(reportUri).replace( "from.xsd", "report.xml") reportFile = reportDir + "out" + os.sep + reportName #reportFile = reportDir + "report" + os.sep + reportName reportFullPath = self.webCache.normalizeUrl( reportFile, base) testcasesDir = os.path.dirname( os.path.dirname(reportFullPath)) if testcasesDir != priorTestcasesDir: # close prior report if priorTestcasesDir: for i, testcaseFile in enumerate( testcaseFiles): with open(testcaseFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8") testcaseName = os.path.basename(testcasesDir) testcaseFiles = [ testcasesDir + os.sep + testcaseName + "-creation-testcase.xml", testcasesDir + os.sep + testcaseName + "-consumption-testcase.xml" ] for i, testcaseFile in enumerate(testcaseFiles): etree.SubElement( testcasesElements[i], "testcase", attrib={ "uri": testcaseFile[len(testGenDir) + 1:].replace("\\", "/") }) # start testcase file testcaseDocs = [] testcaseElements = [] testcaseNumber = testcaseName[0:4] if testcaseNumber.isnumeric(): testcaseNumberElement = "<number>{0}</number>".format( testcaseNumber) testcaseName = testcaseName[5:] else: testcaseNumberElement = "" testDirSegments = testDir.split('/') if len(testDirSegments ) >= 2 and '-' in testDirSegments[1]: testedModule = testDirSegments[1][ testDirSegments[1].index('-') + 1:] else: testedModule = '' for purpose in ("Creation", "Consumption"): file = io.StringIO( #'<?xml version="1.0" encoding="UTF-8"?>' '<!-- Copyright 2011 XBRL International. All Rights Reserved. -->' '<?xml-stylesheet type="text/xsl" href="../../../infrastructure/test.xsl"?>' '<testcase ' ' xmlns="http://xbrl.org/2008/conformance"' ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' ' xsi:schemaLocation="http://xbrl.org/2008/conformance ../../../infrastructure/test.xsd">' '<creator>' '<name>Roland Hommes</name>' '<email>[email protected]</email>' '</creator>' '{0}' '<name>{1}</name>' # '<description>{0}</description>' '<reference>' '{2}' '{3}' '</reference>' '</testcase>'.format( testcaseNumberElement, testcaseName, '<name>{0}</name>'.format(testedModule) if testedModule else '', '<id>{0}</id>'.format(useCase) if useCase else '')) doc = etree.parse(file) file.close() testcaseDocs.append(doc) testcaseElements.append(doc.getroot()) priorTestcasesDir = testcasesDir variationSeq = 1 try: os.makedirs(os.path.dirname(reportFullPath)) except WindowsError: pass # dir already exists modelVersReport = ModelVersReport.ModelVersReport( modelTestcases) modelVersReport.diffDTSes(reportFullPath, modelDTSfrom, modelDTSto, assignment=assignment, schemaDir=schemaDir) # check for expected elements if expectedEvents: for expectedEvent in expectedEvents.split(","): if expectedEvent not in ("No change", "N.A."): prefix, sep, localName = expectedEvent.partition( ':') if sep and len( modelVersReport.xmlDocument. findall('//{{{0}}}{1}'.format( XbrlConst.verPrefixNS.get( prefix), localName))) == 0: modelTestcases.warning( "warning", "Generated test case %(reportName)s missing expected event %(event)s", reportName=reportName, event=expectedEvent) modelVersReport.close() uriFromParts = uriFrom.split('_') if len(uriFromParts) >= 2: variationId = uriFromParts[1] else: variationId = "_{0:02n}".format(variationSeq) for i, testcaseElt in enumerate(testcaseElements): variationElement = etree.SubElement( testcaseElt, "{http://xbrl.org/2008/conformance}variation", attrib={"id": variationId}) nameElement = etree.SubElement( variationElement, "{http://xbrl.org/2008/conformance}description" ) nameElement.text = description ''' (removed per RH 2011/10/04 if note: paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}description") paramElement.text = "Note: " + note if useCase: paramElement = etree.SubElement(variationElement, "{http://xbrl.org/2008/conformance}reference") paramElement.set("specification", "versioning-requirements") paramElement.set("useCase", useCase) ''' dataElement = etree.SubElement( variationElement, "{http://xbrl.org/2008/conformance}data") if i == 0: # result is report if expectedEvents: paramElement = etree.SubElement( dataElement, "{http://xbrl.org/2008/conformance}parameter", attrib={ "name": "expectedEvent", "value": expectedEvents.replace(',', ' ') }, nsmap={ "conf": "http://xbrl.org/2008/conformance", None: "" }) if assignment: paramElement = etree.SubElement( dataElement, "{http://xbrl.org/2008/conformance}parameter", attrib={ "name": "assignment", "value": assignment }, nsmap={ "conf": "http://xbrl.org/2008/conformance", None: "" }) for schemaURIs, dtsAttr in ((uriFrom, "from"), (uriTo, "to")): for schemaURI in schemaURIs.split(","): schemaElement = etree.SubElement( dataElement, "{http://xbrl.org/2008/conformance}schema" ) schemaElement.set("dts", dtsAttr) if i == 0: schemaElement.set( "readMeFirst", "true") schemaElement.text = os.path.basename( schemaURI.strip()) resultElement = etree.SubElement( variationElement, "{http://xbrl.org/2008/conformance}result") reportElement = etree.SubElement( resultElement if i == 0 else dataElement, "{http://xbrl.org/2008/conformance}versioningReport" ) if i == 1: reportElement.set("readMeFirst", "true") reportElement.text = "report/" + reportName variationSeq += 1 except Exception as err: modelTestcases.error( "exception", _("Exception: %(error)s, Excel row: %(excelRow)s"), error=err, excelRow=iRow, exc_info=True) # add tests-error-code index files to consumption for testcaseFile in self.testcaseFiles(testGenDir + os.sep + "tests-error-code"): etree.SubElement(testcasesElements[1], "testcase", attrib={ "uri": testcaseFile[len(testGenDir) + 1:].replace( "\\", "/") }) with open(logMessagesFile, "w") as fh: fh.writelines(self.logMessages) if priorTestcasesDir: for i, testcaseFile in enumerate(testcaseFiles): with open(testcaseFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, testcaseDocs[i], encoding="utf-8") for i, indexFile in enumerate(indexFiles): with open(indexFile, "w", encoding="utf-8") as fh: XmlUtil.writexml(fh, indexDocs[i], encoding="utf-8")
def run(self, options, sourceZipStream=None): """Process command line arguments or web service request, such as to load and validate an XBRL document, or start web server. When a web server has been requested, this method may be called multiple times, once for each web service (REST) request that requires processing. Otherwise (when called for a command line request) this method is called only once for the command line arguments request. :param options: OptionParser options from parse_args of main argv arguments (when called from command line) or corresponding arguments from web service (REST) request. :type options: optparse.Values """ if options.showOptions: # debug options for optName, optValue in sorted(options.__dict__.items(), key=lambda optItem: optItem[0]): self.addToLog("Option {0}={1}".format(optName, optValue), messageCode="info") self.addToLog("sys.argv {0}".format(sys.argv), messageCode="info") if options.uiLang: # set current UI Lang (but not config setting) self.setUiLanguage(options.uiLang) if options.proxy: if options.proxy != "show": proxySettings = proxyTuple(options.proxy) self.webCache.resetProxies(proxySettings) self.config["proxySettings"] = proxySettings self.saveConfig() self.addToLog(_("Proxy configuration has been set."), messageCode="info") useOsProxy, urlAddr, urlPort, user, password = self.config.get("proxySettings", proxyTuple("none")) if useOsProxy: self.addToLog(_("Proxy configured to use {0}.").format( _('Microsoft Windows Internet Settings') if sys.platform.startswith("win") else (_('Mac OS X System Configuration') if sys.platform in ("darwin", "macos") else _('environment variables'))), messageCode="info") elif urlAddr: self.addToLog(_("Proxy setting: http://{0}{1}{2}{3}{4}").format( user if user else "", ":****" if password else "", "@" if (user or password) else "", urlAddr, ":{0}".format(urlPort) if urlPort else ""), messageCode="info") else: self.addToLog(_("Proxy is disabled."), messageCode="info") if options.plugins: from arelle import PluginManager resetPlugins = False savePluginChanges = True showPluginModules = False for pluginCmd in options.plugins.split('|'): cmd = pluginCmd.strip() if cmd == "show": showPluginModules = True elif cmd == "temp": savePluginChanges = False elif cmd.startswith("+"): moduleInfo = PluginManager.addPluginModule(cmd[1:]) if moduleInfo: self.addToLog(_("Addition of plug-in {0} successful.").format(moduleInfo.get("name")), messageCode="info", file=moduleInfo.get("moduleURL")) resetPlugins = True else: self.addToLog(_("Unable to load plug-in."), messageCode="info", file=cmd[1:]) elif cmd.startswith("~"): if PluginManager.reloadPluginModule(cmd[1:]): self.addToLog(_("Reload of plug-in successful."), messageCode="info", file=cmd[1:]) resetPlugins = True else: self.addToLog(_("Unable to reload plug-in."), messageCode="info", file=cmd[1:]) elif cmd.startswith("-"): if PluginManager.removePluginModule(cmd[1:]): self.addToLog(_("Deletion of plug-in successful."), messageCode="info", file=cmd[1:]) resetPlugins = True else: self.addToLog(_("Unable to delete plug-in."), messageCode="info", file=cmd[1:]) else: # assume it is a module or package savePluginChanges = False moduleInfo = PluginManager.addPluginModule(cmd) if moduleInfo: self.addToLog(_("Activation of plug-in {0} successful.").format(moduleInfo.get("name")), messageCode="info", file=moduleInfo.get("moduleURL")) resetPlugins = True else: self.addToLog(_("Unable to load {0} as a plug-in or {0} is not recognized as a command. ").format(cmd), messageCode="info", file=cmd) if resetPlugins: PluginManager.reset() if savePluginChanges: PluginManager.save(self) if showPluginModules: self.addToLog(_("Plug-in modules:"), messageCode="info") for i, moduleItem in enumerate(sorted(PluginManager.pluginConfig.get("modules", {}).items())): moduleInfo = moduleItem[1] self.addToLog(_("Plug-in: {0}; author: {1}; version: {2}; status: {3}; date: {4}; description: {5}; license {6}.").format( moduleItem[0], moduleInfo.get("author"), moduleInfo.get("version"), moduleInfo.get("status"), moduleInfo.get("fileDate"), moduleInfo.get("description"), moduleInfo.get("license")), messageCode="info", file=moduleInfo.get("moduleURL")) # run utility command line options that don't depend on entrypoint Files hasUtilityPlugin = False for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Utility.Run"): hasUtilityPlugin = True pluginXbrlMethod(self, options) # if no entrypointFile is applicable, quit now if options.proxy or options.plugins or hasUtilityPlugin: if not options.entrypointFile: return True # success self.username = options.username self.password = options.password self.entrypointFile = options.entrypointFile if self.entrypointFile: filesource = FileSource.openFileSource(self.entrypointFile, self, sourceZipStream) else: filesource = None if options.validateEFM: if options.disclosureSystemName: self.addToLog(_("both --efm and --disclosureSystem validation are requested, proceeding with --efm only"), messageCode="info", file=self.entrypointFile) self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select("efm") elif options.disclosureSystemName: self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select(options.disclosureSystemName) elif options.validateHMRC: self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select("hmrc") else: self.modelManager.disclosureSystem.select(None) # just load ordinary mappings self.modelManager.validateDisclosureSystem = False if options.utrUrl: # override disclosureSystem utrUrl self.modelManager.disclosureSystem.utrUrl = options.utrUrl # can be set now because the utr is first loaded at validation time # disclosure system sets logging filters, override disclosure filters, if specified by command line if options.logLevelFilter: self.setLogLevelFilter(options.logLevelFilter) if options.logCodeFilter: self.setLogCodeFilter(options.logCodeFilter) if options.calcDecimals: if options.calcPrecision: self.addToLog(_("both --calcDecimals and --calcPrecision validation are requested, proceeding with --calcDecimals only"), messageCode="info", file=self.entrypointFile) self.modelManager.validateInferDecimals = True self.modelManager.validateCalcLB = True elif options.calcPrecision: self.modelManager.validateInferDecimals = False self.modelManager.validateCalcLB = True if options.utrValidate: self.modelManager.validateUtr = True if options.infosetValidate: self.modelManager.validateInfoset = True if options.abortOnMajorError: self.modelManager.abortOnMajorError = True if options.collectProfileStats: self.modelManager.collectProfileStats = True if options.internetConnectivity == "offline": self.webCache.workOffline = True elif options.internetConnectivity == "online": self.webCache.workOffline = False if options.internetTimeout is not None: self.webCache.timeout = (options.internetTimeout or None) # use None if zero specified to disable timeout fo = FormulaOptions() if options.parameters: parameterSeparator = (options.parameterSeparator or ',') fo.parameterValues = dict(((qname(key, noPrefixIsNoNamespace=True),(None,value)) for param in options.parameters.split(parameterSeparator) for key,sep,value in (param.partition('='),) ) ) if options.formulaParamExprResult: fo.traceParameterExpressionResult = True if options.formulaParamInputValue: fo.traceParameterInputValue = True if options.formulaCallExprSource: fo.traceCallExpressionSource = True if options.formulaCallExprCode: fo.traceCallExpressionCode = True if options.formulaCallExprEval: fo.traceCallExpressionEvaluation = True if options.formulaCallExprResult: fo.traceCallExpressionResult = True if options.formulaVarSetExprEval: fo.traceVariableSetExpressionEvaluation = True if options.formulaVarSetExprResult: fo.traceVariableSetExpressionResult = True if options.formulaAsserResultCounts: fo.traceAssertionResultCounts = True if options.formulaFormulaRules: fo.traceFormulaRules = True if options.formulaVarsOrder: fo.traceVariablesOrder = True if options.formulaVarExpressionSource: fo.traceVariableExpressionSource = True if options.formulaVarExpressionCode: fo.traceVariableExpressionCode = True if options.formulaVarExpressionEvaluation: fo.traceVariableExpressionEvaluation = True if options.formulaVarExpressionResult: fo.traceVariableExpressionResult = True if options.timeVariableSetEvaluation: fo.timeVariableSetEvaluation = True if options.formulaVarFilterWinnowing: fo.traceVariableFilterWinnowing = True if options.formulaVarFiltersResult: fo.traceVariableFiltersResult = True if options.formulaVarFiltersResult: fo.traceVariableFiltersResult = True self.modelManager.formulaOptions = fo timeNow = XmlUtil.dateunionValue(datetime.datetime.now()) firstStartedAt = startedAt = time.time() modelDiffReport = None success = True modelXbrl = None try: if filesource: modelXbrl = self.modelManager.load(filesource, _("views loading")) except ModelDocument.LoadingException: pass except Exception as err: self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format( err, traceback.format_tb(sys.exc_info()[2]))) success = False # loading errors, don't attempt to utilize loaded DTS if modelXbrl and modelXbrl.modelDocument: loadTime = time.time() - startedAt modelXbrl.profileStat(_("load"), loadTime) self.addToLog(format_string(self.modelManager.locale, _("loaded in %.2f secs at %s"), (loadTime, timeNow)), messageCode="info", file=self.entrypointFile) if options.importFiles: for importFile in options.importFiles.split("|"): fileName = importFile.strip() if sourceZipStream is not None and not (fileName.startswith('http://') or os.path.isabs(fileName)): fileName = os.path.dirname(modelXbrl.uri) + os.sep + fileName # make relative to sourceZipStream ModelDocument.load(modelXbrl, fileName) loadTime = time.time() - startedAt self.addToLog(format_string(self.modelManager.locale, _("import in %.2f secs at %s"), (loadTime, timeNow)), messageCode="info", file=importFile) modelXbrl.profileStat(_("import"), loadTime) if modelXbrl.errors: success = False # loading errors, don't attempt to utilize loaded DTS if modelXbrl.modelDocument.type in ModelDocument.Type.TESTCASETYPES: for pluginXbrlMethod in pluginClassMethods("Testcases.Start"): pluginXbrlMethod(self, options, modelXbrl) else: # not a test case, probably instance or DTS for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Loaded"): pluginXbrlMethod(self, options, modelXbrl) else: success = False if success and options.diffFile and options.versReportFile: try: diffFilesource = FileSource.FileSource(options.diffFile,self) startedAt = time.time() modelXbrl2 = self.modelManager.load(diffFilesource, _("views loading")) if modelXbrl2.errors: if not options.keepOpen: modelXbrl2.close() success = False else: loadTime = time.time() - startedAt modelXbrl.profileStat(_("load"), loadTime) self.addToLog(format_string(self.modelManager.locale, _("diff comparison DTS loaded in %.2f secs"), loadTime), messageCode="info", file=self.entrypointFile) startedAt = time.time() modelDiffReport = self.modelManager.compareDTSes(options.versReportFile) diffTime = time.time() - startedAt modelXbrl.profileStat(_("diff"), diffTime) self.addToLog(format_string(self.modelManager.locale, _("compared in %.2f secs"), diffTime), messageCode="info", file=self.entrypointFile) except ModelDocument.LoadingException: success = False except Exception as err: success = False self.addToLog(_("[Exception] Failed to doad diff file: \n{0} \n{1}").format( err, traceback.format_tb(sys.exc_info()[2]))) if success: try: modelXbrl = self.modelManager.modelXbrl hasFormulae = modelXbrl.hasFormulae if options.validate: startedAt = time.time() if options.formulaAction: # don't automatically run formulas modelXbrl.hasFormulae = False self.modelManager.validate() if options.formulaAction: # restore setting modelXbrl.hasFormulae = hasFormulae self.addToLog(format_string(self.modelManager.locale, _("validated in %.2f secs"), time.time() - startedAt), messageCode="info", file=self.entrypointFile) if options.formulaAction in ("validate", "run"): # do nothing here if "none" from arelle import ValidateXbrlDimensions, ValidateFormula startedAt = time.time() if not options.validate: ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # setup fresh parameters from formula optoins modelXbrl.parameters = fo.typedParameters() ValidateFormula.validate(modelXbrl, compileOnly=(options.formulaAction != "run")) self.addToLog(format_string(self.modelManager.locale, _("formula validation and execution in %.2f secs") if options.formulaAction == "run" else _("formula validation only in %.2f secs"), time.time() - startedAt), messageCode="info", file=self.entrypointFile) if options.testReport: ViewFileTests.viewTests(self.modelManager.modelXbrl, options.testReport, options.testReportCols) if options.rssReport: ViewFileRssFeed.viewRssFeed(self.modelManager.modelXbrl, options.rssReport, options.rssReportCols) if options.DTSFile: ViewFileDTS.viewDTS(modelXbrl, options.DTSFile) if options.factsFile: ViewFileFactList.viewFacts(modelXbrl, options.factsFile, labelrole=options.labelRole, lang=options.labelLang, cols=options.factListCols) if options.factTableFile: ViewFileFactTable.viewFacts(modelXbrl, options.factTableFile, labelrole=options.labelRole, lang=options.labelLang) if options.conceptsFile: ViewFileConcepts.viewConcepts(modelXbrl, options.conceptsFile, labelrole=options.labelRole, lang=options.labelLang) if options.preFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.preFile, "Presentation Linkbase", "http://www.xbrl.org/2003/arcrole/parent-child", labelrole=options.labelRole, lang=options.labelLang) if options.calFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.calFile, "Calculation Linkbase", "http://www.xbrl.org/2003/arcrole/summation-item", labelrole=options.labelRole, lang=options.labelLang) if options.dimFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.dimFile, "Dimensions", "XBRL-dimensions", labelrole=options.labelRole, lang=options.labelLang) if options.formulaeFile: ViewFileFormulae.viewFormulae(modelXbrl, options.formulaeFile, "Formulae", lang=options.labelLang) if options.viewArcrole and options.viewFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.viewFile, os.path.basename(options.viewArcrole), options.viewArcrole, labelrole=options.labelRole, lang=options.labelLang) for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Run"): pluginXbrlMethod(self, options, modelXbrl) except (IOError, EnvironmentError) as err: self.addToLog(_("[IOError] Failed to save output:\n {0}").format(err)) success = False except Exception as err: self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format( err, traceback.format_tb(sys.exc_info()[2]))) success = False if modelXbrl: modelXbrl.profileStat(_("total"), time.time() - firstStartedAt) if options.collectProfileStats and modelXbrl: modelXbrl.logProfileStats() if not options.keepOpen: if modelDiffReport: self.modelManager.close(modelDiffReport) elif modelXbrl: self.modelManager.close(modelXbrl) self.username = self.password = None #dereference password return success
def validateTestcase(self, testcase): self.modelXbrl.info("info", "Testcase", modelDocument=testcase) self.modelXbrl.viewModelObject(testcase.objectId()) if hasattr(testcase, "testcaseVariations"): for modelTestcaseVariation in testcase.testcaseVariations: # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) # is this a versioning report? resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance formulaOutputInstance = None inputDTSes = defaultdict(list) baseForElement = testcase.baseForElement(modelTestcaseVariation) # try to load instance document self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s"), modelObject=modelTestcaseVariation, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, expected=modelTestcaseVariation.expected) for readMeFirstUri in modelTestcaseVariation.readMeFirstUris: if isinstance(readMeFirstUri,tuple): # dtsName is for formula instances, but is from/to dts if versioning dtsName, readMeFirstUri = readMeFirstUri elif resultIsVersioningReport: if inputDTSes: dtsName = "to" else: dtsName = "from" else: dtsName = None if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document if dtsName in inputDTSes: dtsName = inputDTSes[dtsName] else: modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager, ModelDocument.Type.DTSENTRIES, self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement), isEntry=True) DTSdoc = modelXbrl.modelDocument DTSdoc.inDTS = True doc = ModelDocument.load(modelXbrl, readMeFirstUri, base=baseForElement) if doc is not None: DTSdoc.referencesDocument[doc] = "import" #fake import doc.inDTS = True else: # not a multi-schemaRef versioning report modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, readMeFirstUri, _("validating"), base=baseForElement, useFileSource=self.useFileSource) if modelXbrl.modelDocument is None: self.modelXbrl.error("arelle:notLoaded", _("Testcase %(id)s %(name)s document not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) modelXbrl.close() self.determineNotLoadedTestStatus(modelTestcaseVariation) elif resultIsVersioningReport: inputDTSes[dtsName] = modelXbrl elif modelXbrl.modelDocument.type == ModelDocument.Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl) self.determineTestStatus(modelTestcaseVariation, modelXbrl) modelXbrl.close() elif testcase.type == ModelDocument.Type.REGISTRYTESTCASE: self.instValidator.validate(modelXbrl) # required to set up dimensions, etc self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id, modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest) self.determineTestStatus(modelTestcaseVariation, modelXbrl) self.instValidator.close() modelXbrl.close() else: inputDTSes[dtsName].append(modelXbrl) if resultIsVersioningReport and modelXbrl.modelDocument: versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl( modelTestcaseVariation.versioningReportUri, baseForElement) if os.path.exists(versReportFile): #validate existing modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report")) if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == ModelDocument.Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport) self.determineTestStatus(modelTestcaseVariation, modelVersReport) modelVersReport.close() elif len(inputDTSes) == 2: ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes( versReportFile, inputDTSes["from"], inputDTSes["to"]) modelTestcaseVariation.status = "generated" else: self.modelXbrl.error("arelle:notLoaded", _("Testcase %(id)s %(name)s DTSes not loaded, unable to generate versioning report: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) modelTestcaseVariation.status = "failed" for inputDTS in inputDTSes.values(): inputDTS.close() del inputDTSes # dereference elif inputDTSes: # validate schema, linkbase, or instance modelXbrl = inputDTSes[None][0] parameters = modelTestcaseVariation.parameters.copy() for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters if dtsName: parameters[dtsName] = (None, inputDTS) self.instValidator.validate(modelXbrl, parameters) if modelTestcaseVariation.resultIsInfoset: infoset = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultInfosetUri, _("loading result infoset"), base=baseForElement, useFileSource=self.useFileSource) if infoset.modelDocument is None: self.modelXbrl.error("arelle:notLoaded", _("Testcase %(id)s %(name)s result infoset not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstance)) modelTestcaseVariation.status = "result infoset not loadable" else: # check infoset ValidateInfoset.validate(self.instValidator, modelXbrl, infoset) infoset.close() self.determineTestStatus(modelTestcaseVariation, modelXbrl) # include infoset errors in status self.instValidator.close() if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual): # if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost) self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters) self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance) if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance formulaOutputInstance = modelXbrl.formulaOutputInstance modelXbrl.formulaOutputInstance = None # prevent it from being closed now self.instValidator.close() for inputDTSlist in inputDTSes.values(): for inputDTS in inputDTSlist: inputDTS.close() if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument: expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultXbrlInstanceUri, _("loading expected result XBRL instance"), base=baseForElement, useFileSource=self.useFileSource) if expectedInstance.modelDocument is None: self.modelXbrl.error("arelle:notLoaded", _("Testcase %(id)s %(name)s expected result instance not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstance)) modelTestcaseVariation.status = "result not loadable" expectedInstance.close() else: # compare facts if len(expectedInstance.facts) != len(formulaOutputInstance.facts): formulaOutputInstance.error("formula:resultFactCounts", _("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"), modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts), expectedFacts=len(expectedInstance.facts)) else: for fact in expectedInstance.facts: if formulaOutputInstance.matchFact(fact) is None: formulaOutputInstance.error("formula:expectedFactMissing", _("Formula output missing expected fact %(fact)s"), modelXbrl=fact, fact=fact.qname) self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance) formulaOutputInstance.close() formulaOutputInstance = None # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) self.modelXbrl.modelManager.showStatus(_("ready"), 2000)