def backgroundLoadXbrl(self, filesource, importToDTS): startedAt = time.time() try: if importToDTS: action = _("imported") modelXbrl = self.modelManager.modelXbrl if modelXbrl: ModelDocument.load(modelXbrl, filesource.url) else: action = _("loaded") modelXbrl = self.modelManager.load(filesource, _("views loading")) except Exception as err: msg = _("Exception loading {0}: {1}, at {2}").format( filesource.url, err, traceback.format_tb(sys.exc_info()[2])) # not sure if message box can be shown from background thread # tkinter.messagebox.showwarning(_("Exception loading"),msg, parent=self.parent) self.addToLog(msg); return if modelXbrl and modelXbrl.modelDocument: self.addToLog(format_string(self.modelManager.locale, _("%s in %.2f secs"), (action, time.time() - startedAt))) self.showStatus(_("{0}, preparing views").format(action)) self.uiThreadQueue.put((self.showLoadedXbrl, [modelXbrl, importToDTS])) else: self.addToLog(format_string(self.modelManager.locale, _("not successfully %s in %.2f secs"), (action, time.time() - startedAt)))
def logProfileStats(self): """Logs profile stats that were collected """ timeTotal = format_string(self.modelManager.locale, _("%.3f secs"), self.profileStats.get("total", (0,0,0))[1]) timeEFM = format_string(self.modelManager.locale, _("%.3f secs"), self.profileStats.get("validateEFM", (0,0,0))[1]) self.info("info:profileStats", _("Profile statistics \n") + ' \n'.join(format_string(self.modelManager.locale, _("%s %.3f secs, %.0fK"), (statName, statValue[1], statValue[2]), grouping=True) for statName, statValue in sorted(self.profileStats.items(), key=lambda item: item[1])) + " \n", # put instance reference on fresh line in traces modelObject=self.modelXbrl.modelDocument, profileStats=self.profileStats, timeTotal=timeTotal, timeEFM=timeEFM)
def xmlValidate(entryModelDocument): # test of schema validation using lxml (trial experiment, commented out for production use) modelXbrl = entryModelDocument.modelXbrl from arelle import ModelDocument imports = [] importedNamespaces = set() for modelDocument in modelXbrl.urlDocs.values(): if (modelDocument.type == ModelDocument.Type.SCHEMA and modelDocument.targetNamespace not in importedNamespaces): imports.append( '<xsd:import namespace="{0}" schemaLocation="{1}"/>'.format( modelDocument.targetNamespace, modelDocument.filepath.replace("\\", "/"))) importedNamespaces.add(modelDocument.targetNamespace) if entryModelDocument.xmlRootElement.hasAttributeNS( XbrlConst.xsi, "schemaLocation"): ns = None for entry in entryModelDocument.xmlRootElement.getAttributeNS( XbrlConst.xsi, "schemaLocation").split(): if ns is None: ns = entry else: if ns not in importedNamespaces: imports.append( '<xsd:import namespace="{0}" schemaLocation="{1}"/>'. format(ns, entry)) importedNamespaces.add(ns) ns = None schema_root = etree.XML( '<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema">{0}</xsd:schema>' .format(''.join(imports))) import time startedAt = time.time() schema = etree.XMLSchema(schema_root) from arelle.Locale import format_string modelXbrl.modelManager.addToLog( format_string(modelXbrl.modelManager.locale, _("schema loaded in %.2f secs"), time.time() - startedAt)) startedAt = time.time() instDoc = etree.parse(entryModelDocument.filepath) modelXbrl.modelManager.addToLog( format_string(modelXbrl.modelManager.locale, _("instance parsed in %.2f secs"), time.time() - startedAt)) if not schema.validate(instDoc): for error in schema.error_log: modelXbrl.error(str(error), "err", "xmlschema:error")
def storeIntoDB(dbConnection, modelXbrl, rssItem=None): host = port = user = password = db = timeout = dbType = None if isinstance(dbConnection, (list, tuple)): # variable length list if len(dbConnection) > 0: host = dbConnection[0] if len(dbConnection) > 1: port = dbConnection[1] if len(dbConnection) > 2: user = dbConnection[2] if len(dbConnection) > 3: password = dbConnection[3] if len(dbConnection) > 4: db = dbConnection[4] if len(dbConnection) > 5 and dbConnection[5] and dbConnection[5].isdigit(): timeout = int(dbConnection[5]) if len(dbConnection) > 6: dbType = dbConnection[6] startedAt = time.time() if dbType in dbTypes: insertIntoDB = dbTypes[dbType] elif isPostgresPort(host, port): insertIntoDB = insertIntoPostgresDB elif isRexsterPort(host, port): insertIntoDB = insertIntoRexsterDB elif isRdfPort(host, port, db): insertIntoDB = insertIntoRdfDB elif isJsonPort(host, port, db): insertIntoDB = insertIntoJsonDB else: modelXbrl.modelManager.addToLog('Server at "{0}:{1}" is not recognized to be either a Postgres or a Rexter service.'.format(host, port)) return insertIntoDB(modelXbrl, host=host, port=port, user=user, password=password, database=db, timeout=timeout, rssItem=rssItem) modelXbrl.modelManager.addToLog(format_string(modelXbrl.modelManager.locale, _("stored to database in %.2f secs"), time.time() - startedAt), messageCode="info", file=modelXbrl.uri)
def showLoadedXbrl(self, modelXbrl, attach): startedAt = time.time() currentAction = "setting title" try: if attach: modelXbrl.closeViews() self.parent.title(_("arelle\u2122 - {0}").format( os.path.basename(modelXbrl.modelDocument.uri))) self.setValidateTooltipText() if modelXbrl.modelDocument.type in (ModelDocument.Type.TESTCASESINDEX, ModelDocument.Type.TESTCASE, ModelDocument.Type.REGISTRY, ModelDocument.Type.REGISTRYTESTCASE): currentAction = "tree view of tests" ViewWinTests.viewTests(modelXbrl, self.tabWinTopRt) elif modelXbrl.modelDocument.type == ModelDocument.Type.VERSIONINGREPORT: currentAction = "view of versioning report" ViewWinVersReport.viewVersReport(modelXbrl, self.tabWinTopRt) from arelle.ViewWinDiffs import ViewWinDiffs ViewWinDiffs(modelXbrl, self.tabWinBtm, lang=self.lang) elif modelXbrl.modelDocument.type == ModelDocument.Type.RSSFEED: currentAction = "view of RSS feed" ViewWinRssFeed.viewRssFeed(modelXbrl, self.tabWinTopRt) else: currentAction = "tree view of tests" ViewWinDTS.viewDTS(modelXbrl, self.tabWinTopLeft) currentAction = "view of concepts" ViewWinConcepts.viewConcepts(modelXbrl, self.tabWinBtm, "Concepts", lang=self.lang) if modelXbrl.hasEuRendering: # show rendering grid even without any facts ViewWinRenderedGrid.viewRenderedGrid(modelXbrl, self.tabWinTopRt, lang=self.lang) if modelXbrl.modelDocument.type in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL): currentAction = "table view of facts" if not modelXbrl.hasEuRendering: # table view only if not grid rendered view ViewWinFactTable.viewFacts(modelXbrl, self.tabWinTopRt, lang=self.lang) currentAction = "tree/list of facts" ViewWinFactList.viewFacts(modelXbrl, self.tabWinTopRt, lang=self.lang) if modelXbrl.hasFormulae: currentAction = "formulae view" ViewWinFormulae.viewFormulae(modelXbrl, self.tabWinTopRt) currentAction = "presentation linkbase view" ViewWinRelationshipSet.viewRelationshipSet(modelXbrl, self.tabWinTopRt, XbrlConst.parentChild, lang=self.lang) currentAction = "calculation linkbase view" ViewWinRelationshipSet.viewRelationshipSet(modelXbrl, self.tabWinTopRt, XbrlConst.summationItem, lang=self.lang) currentAction = "dimensions relationships view" ViewWinRelationshipSet.viewRelationshipSet(modelXbrl, self.tabWinTopRt, "XBRL-dimensions", lang=self.lang) if modelXbrl.hasEuRendering: currentAction = "rendering view" ViewWinRelationshipSet.viewRelationshipSet(modelXbrl, self.tabWinTopRt, "EU-rendering", lang=self.lang) currentAction = "property grid" ViewWinProperties.viewProperties(modelXbrl, self.tabWinTopLeft) currentAction = "log view creation time" self.addToLog(format_string(self.modelManager.locale, _("views %.2f secs"), time.time() - startedAt)) except Exception as err: msg = _("Exception preparing {0}: {1}, at {2}").format( currentAction, err, traceback.format_tb(sys.exc_info()[2])) tkinter.messagebox.showwarning(_("Exception preparing view"),msg, parent=self.parent) self.addToLog(msg); self.showStatus(_("Ready..."), 2000)
def backgroundCompareDTSes(self, versReportFile): startedAt = time.time() modelVersReport = self.modelManager.compareDTSes(versReportFile) if modelVersReport and modelVersReport.modelDocument: self.addToLog(format_string(self.modelManager.locale, _("compared in %.2f secs"), time.time() - startedAt)) self.uiThreadQueue.put((self.showComparedDTSes, [modelVersReport]))
def xmlValidate(entryModelDocument): # test of schema validation using lxml (trial experiment, commented out for production use) modelXbrl = entryModelDocument.modelXbrl from arelle import ModelDocument imports = [] importedNamespaces = set() for modelDocument in modelXbrl.urlDocs.values(): if (modelDocument.type == ModelDocument.Type.SCHEMA and modelDocument.targetNamespace not in importedNamespaces): imports.append('<xsd:import namespace="{0}" schemaLocation="{1}"/>'.format( modelDocument.targetNamespace, modelDocument.filepath.replace("\\","/"))) importedNamespaces.add(modelDocument.targetNamespace) if entryModelDocument.xmlRootElement.hasAttributeNS(XbrlConst.xsi, "schemaLocation"): ns = None for entry in entryModelDocument.xmlRootElement.getAttributeNS(XbrlConst.xsi, "schemaLocation").split(): if ns is None: ns = entry else: if ns not in importedNamespaces: imports.append('<xsd:import namespace="{0}" schemaLocation="{1}"/>'.format( ns, entry)) importedNamespaces.add(ns) ns = None schema_root = etree.XML( '<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema">{0}</xsd:schema>'.format( ''.join(imports)) ) import time startedAt = time.time() schema = etree.XMLSchema(schema_root) from arelle.Locale import format_string modelXbrl.modelManager.addToLog(format_string(modelXbrl.modelManager.locale, _("schema loaded in %.2f secs"), time.time() - startedAt)) startedAt = time.time() instDoc = etree.parse(entryModelDocument.filepath) modelXbrl.modelManager.addToLog(format_string(modelXbrl.modelManager.locale, _("instance parsed in %.2f secs"), time.time() - startedAt)) if not schema.validate(instDoc): for error in schema.error_log: modelXbrl.error( str(error), "err", "xmlschema:error")
def storeIntoDB(dbConnection, modelXbrl): host, port, user, password, db = dbConnection startedAt = time.time() if isPostgresPort(host, port): insertIntoPostgresDB(modelXbrl, host=host, port=port, user=user, password=password, database=db) elif isRexsterPort(host, port): insertIntoRexsterDB(modelXbrl, host=host, port=port, user=user, password=password, database=db) modelXbrl.modelManager.addToLog(format_string(modelXbrl.modelManager.locale, _("stored to database in %.2f secs"), time.time() - startedAt), messageCode="info", file=modelXbrl.uri)
def backgroundStoreIntoDB(): try: host, port, user, password, db, timeout, dbType = dbConnection product = None if timeout and timeout.isdigit(): timeout = int(timeout) # identify server if dbType in dbTypes: insertIntoDB = dbTypes[dbType] product = dbProduct[dbType] else: cntlr.addToLog(_("Probing host {0} port {1} to determine server database type.") .format(host, port)) if isPostgresPort(host, port): dbType = "postgres" insertIntoDB = insertIntoPostgresDB elif isSemanticSqlPort(host, port): dbType = "pgSemantic" insertIntoDB = insertIntoPostgresDB elif isRexsterPort(host, port): dbType = "rexster" insertIntoDB = insertIntoRexsterDB elif isRdfPort(host, port, db): dbType = "rdfDB" insertIntoDB = insertIntoRdfDB elif isJsonPort(host, port, db): dbType = "json" insertIntoDB = insertIntoJsonDB else: cntlr.addToLog(_("Unable to determine server type!\n ") + _("Probing host {0} port {1} unable to determine server type.") .format(host, port)) cntlr.config["xbrlDBconnection"] = (host, port, user, password, db, timeout, '') # forget type cntlr.saveConfig() return cntlr.addToLog(_("Database type {} identified.").format(dbType)) cntlr.config["xbrlDBconnection"] = (host, port, user, password, db, timeout, dbType) cntlr.saveConfig() startedAt = time.time() insertIntoDB(cntlr.modelManager.modelXbrl, host=host, port=port, user=user, password=password, database=db, timeout=timeout, product=product) cntlr.addToLog(format_string(cntlr.modelManager.locale, _("stored to database in %.2f secs"), time.time() - startedAt)) except Exception as ex: import traceback cntlr.addToLog( _("[xpDB:exception] Loading XBRL DB: %(exception)s: %(error)s \n%(traceback)s") % {"exception": ex.__class__.__name__, "error": str(ex), "exc_info": True, "traceback": traceback.format_tb(sys.exc_info()[2])}) cntlr.config["xbrlDBconnection"] = (host, port, user, password, db, timeout, '') # forget type cntlr.saveConfig()
def initializeParser(modelManager): global isInitialized if not isInitialized: modelManager.showStatus(_("initializing formula xpath2 grammar")) startedAt = time.time() xpathExpr.parseString( "0", parseAll=True ) modelManager.addToLog(format_string(modelManager.locale, _("Formula xpath2 grammar initialized in %.2f secs"), time.time() - startedAt)) modelManager.showStatus(None) isInitialized = True
def storeIntoDB(dbConnection, modelXbrl, rssItem=None, **kwargs): host = port = user = password = db = timeout = dbType = None if isinstance(dbConnection, (list, tuple)): # variable length list if len(dbConnection) > 0: host = dbConnection[0] if len(dbConnection) > 1: port = dbConnection[1] if len(dbConnection) > 2: user = dbConnection[2] if len(dbConnection) > 3: password = dbConnection[3] if len(dbConnection) > 4: db = dbConnection[4] if len(dbConnection ) > 5 and dbConnection[5] and dbConnection[5].isdigit(): timeout = int(dbConnection[5]) if len(dbConnection) > 6: dbType = dbConnection[6] startedAt = time.time() product = None if dbType in dbTypes: insertIntoDB = dbTypes[dbType] product = dbProduct[dbType] elif isPostgresPort(host, port): insertIntoDB = insertIntoPostgresDB elif isSemanticSqlPort(host, port): insertIntoDB = insertIntoSemanticSqlDB elif isRexsterPort(host, port): insertIntoDB = insertIntoRexsterDB elif isRdfPort(host, port, db): insertIntoDB = insertIntoRdfDB elif isJsonPort(host, port, db): insertIntoDB = insertIntoJsonDB else: modelXbrl.modelManager.addToLog( 'Server at "{0}:{1}" is not recognized to be either a Postgres or a Rexter service.' .format(host, port)) return result = insertIntoDB(modelXbrl, host=host, port=port, user=user, password=password, database=db, timeout=timeout, product=product, rssItem=rssItem, **kwargs) if kwargs.get( "logStoredMsg", result ): # if false/None result and no logStoredMsg parameter then skip the message modelXbrl.modelManager.addToLog(format_string( modelXbrl.modelManager.locale, _("stored to database in %.2f secs"), time.time() - startedAt), messageCode="info", file=modelXbrl.uri) return result
def initializeParser(modelManager): global isInitialized if not isInitialized: modelManager.showStatus(_("initializing formula xpath2 grammar")) startedAt = time.time() xpathExpr.parseString("0", parseAll=True) modelManager.addToLog( format_string(modelManager.locale, _("Formula xpath2 grammar initialized in %.2f secs"), time.time() - startedAt)) modelManager.showStatus(None) isInitialized = True
def backgroundValidate(self): startedAt = time.time() modelXbrl = self.modelManager.modelXbrl priorOutputInstance = modelXbrl.formulaOutputInstance modelXbrl.formulaOutputInstance = None # prevent closing on background thread by validateFormula self.modelManager.validate() self.addToLog(format_string(self.modelManager.locale, _("validated in %.2f secs"), time.time() - startedAt)) if modelXbrl and (priorOutputInstance or modelXbrl.formulaOutputInstance): self.uiThreadQueue.put((self.showFormulaOutputInstance, [priorOutputInstance, modelXbrl.formulaOutputInstance])) self.uiThreadQueue.put((self.logSelect, []))
def initializeParser(modelManager): global isInitialized if not isInitialized: modelManager.showStatus(_(u"initializing formula xpath2 grammar")) startedAt = time.time() xpathExpr.parseString( u"0", parseAll=True ) modelManager.addToLog(format_string(modelManager.locale, _(u"Formula xpath2 grammar initialized in %.2f secs"), time.time() - startedAt)) modelManager.showStatus(None) isInitialized = True return True # was initialized on this call return False # had already been initialized
def initializeParser(modelManager): global isInitialized, FunctionIxt if not isInitialized: from arelle import FunctionIxt modelManager.showStatus(_("initializing formula xpath2 grammar")) startedAt = time.time() xpathExpr.parseString( "0", parseAll=True ) modelManager.addToLog(format_string(modelManager.locale, _("Formula xpath2 grammar initialized in %.2f secs"), time.time() - startedAt)) modelManager.showStatus(None) isInitialized = True return True # was initialized on this call return False # had already been initialized
def initializeParser(modelManager): global isInitialized, FunctionIxt if not isInitialized: from arelle import FunctionIxt modelManager.showStatus(_("initializing formula xpath2 grammar")) startedAt = time.time() xpathExpr.parseString( "0", parseAll=True ) if not(modelManager.cntlr.testMode): modelManager.addToLog(format_string(modelManager.locale, _("Formula xpath2 grammar initialized in %.2f secs"), time.time() - startedAt)) modelManager.showStatus(None) isInitialized = True return True # was initialized on this call return False # had already been initialized
def backgroundStoreIntoDB(): try: startedAt = time.time() insertIntoDB(cntlr.modelManager.modelXbrl, host=host, port=port, user=user, password=password, database=db, timeout=timeout) cntlr.addToLog(format_string(cntlr.modelManager.locale, _("stored to database in %.2f secs"), time.time() - startedAt)) except Exception as ex: import traceback cntlr.addToLog( _("[xpDB:exception] Loading XBRL DB: %(exception)s: %(error)s \n%(traceback)s") % {"exception": ex.__class__.__name__, "error": str(ex), "exc_info": True, "traceback": traceback.format_tb(sys.exc_info()[2])})
def storeIntoDB(dbConnection, modelXbrl): host, port, user, password, db = dbConnection startedAt = time.time() if isPostgresPort(host, port): insertIntoPostgresDB(modelXbrl, host=host, port=port, user=user, password=password, database=db) elif isRexsterPort(host, port): insertIntoRexsterDB(modelXbrl, host=host, port=port, user=user, password=password, database=db) modelXbrl.modelManager.addToLog(format_string( modelXbrl.modelManager.locale, _("stored to database in %.2f secs"), time.time() - startedAt), messageCode="info", file=modelXbrl.uri)
def backgroundStoreIntoDB(): try: startedAt = time.time() insertIntoDB(cntlr.modelManager.modelXbrl, host=host, port=port, user=user, password=password, database=db) cntlr.addToLog( format_string(cntlr.modelManager.locale, _("stored to database in %.2f secs"), time.time() - startedAt)) except Exception as ex: import traceback cntlr.addToLog( _("[xpDB:exception] Loading XBRL DB: %(exception)s: %(error)s \n%(traceback)s" ) % { "exception": ex.__class__.__name__, "error": str(ex), "exc_info": True, "traceback": traceback.format_tb(sys.exc_info()[2]) })
def compileCdrGrammar( cntlr, _logMessage ): global isGrammarCompiled, cdrProg, lineno if isGrammarCompiled: return cdrProg global logMessage logMessage = _logMessage debugParsing = False # True cntlr.showStatus(_("Compiling CDR Grammar")) if sys.version[0] >= '3': # python 3 requires modified parser to allow release of global objects when closing DTS from arelle.pyparsing.pyparsing_py3 import (Word, Keyword, alphas, Literal, CaselessLiteral, Combine, Optional, nums, Or, Forward, Group, ZeroOrMore, StringEnd, alphanums, ParserElement, sglQuotedString, delimitedList, Suppress, Regex, FollowedBy, lineno, restOfLine) else: from pyparsing import (Word, Keyword, alphas, Literal, CaselessLiteral, Combine, Optional, nums, Or, Forward, Group, ZeroOrMore, StringEnd, alphanums, ParserElement, sglQuotedString, delimitedList, Suppress, Regex, FollowedBy, lineno, restOfLine) ParserElement.enablePackrat() """ the pyparsing parser constructs are defined in this method to prevent the need to compile the grammar when the plug in is loaded (which is likely to be when setting up GUI menus or command line parser). instead the grammar is compiled the first time that any sphinx needs to be parsed only the sphinxExpression (result below) needs to be global for the parser """ # define grammar periodOffset = Regex("-?P[1-3]?[0-9][YQMD](/-[1]?[0-9]-([1-3]?[0-9]|end))?") qName = Regex("([A-Za-z\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD_]" "[A-Za-z0-9\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\u0300-\u036F\u203F-\u2040\xB7_.-]*:)?" # localname or wildcard-localname part "([A-Za-z\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD_]" "[A-Za-z0-9\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\u0300-\u036F\u203F-\u2040\xB7_.-]*|[*])" ).setName("qName").setDebug(debugParsing) ncName = Regex("([A-Za-z\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD_]" "[A-Za-z0-9\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\u0300-\u036F\u203F-\u2040\xB7_.-]*)" ).setName("ncName").setDebug(debugParsing) decimalPoint = Literal('.') exponentLiteral = CaselessLiteral('e') plusorminusLiteral = Literal('+') | Literal('-') digits = Word(nums) integerLiteral = Combine( Optional(plusorminusLiteral) + digits ) decimalFractionLiteral = Combine( Optional(plusorminusLiteral) + decimalPoint + digits ) infLiteral = Combine( Optional(plusorminusLiteral) + Literal("INF") ) nanLiteral = Literal("NaN") floatLiteral = ( Combine( integerLiteral + ( ( decimalPoint + Optional(digits) + exponentLiteral + integerLiteral ) | ( exponentLiteral + integerLiteral ) | ( decimalPoint + Optional(digits) ) ) ) | Combine( decimalFractionLiteral + exponentLiteral + integerLiteral ) | decimalFractionLiteral | infLiteral | nanLiteral ) #emptySequence = Literal( "(" ) + Literal( ")" ) lParen = Literal( "(" ) rParen = Literal( ")" ) lPred = Literal("[") rPred = Literal("]") refOp = Literal("#") commaOp = Literal(",") neOp = Literal("<>") leOp = Literal("<=") ltOp = Literal("<") geOp = Literal(">=") gtOp = Literal(">") eqOp = Literal("=") eqNeOp = eqOp | neOp compOp = leOp | ltOp | geOp | gtOp plusOp = Literal("+") minusOp = Literal("-") plusMinusOp = plusOp | minusOp expOp = Literal("^") multOp = Literal("*") divOp = Literal("/") multDivOp = multOp | divOp concatOp = Literal("&") andOp = CaselessLiteral("And") orOp = CaselessLiteral("Or") xorOp = CaselessLiteral("Xor") expr = Forward() atom = ( ( refOp + qName ).setParseAction(compileRefExpression) | ( qName + Suppress(lParen) + Optional(delimitedList( expr )) + Suppress(rParen) ).setParseAction(compileFunctionReference) | ( qName.setParseAction(compileQname) + lPred + ( periodOffset | ncName) + rPred).setParseAction(compilePeriodOffsetExpression) | ( floatLiteral ).setParseAction(compileFloatLiteral) | ( integerLiteral ).setParseAction(compileIntegerLiteral) | ( sglQuotedString ).setParseAction(compileStringLiteral) | ( qName ).setParseAction(compileQname) | ( Suppress(lParen) - expr - Optional( commaOp - Optional( expr - ZeroOrMore( commaOp - expr ) ) ) - Suppress(rParen) ).setParseAction(compileBrackets) ) atom.setName("atom").setDebug(debugParsing) valueExpr = atom negationExpr = ( Optional(minusOp) + valueExpr ).setParseAction(compileUnaryOperation) expExpr = ( Optional(expOp) + negationExpr ).setParseAction(compileUnaryOperation) multDivExpr = ( expExpr + Optional( multDivOp + expExpr ) ).setParseAction(compileBinaryOperation) multDivExpr.setName("multDivExpr").setDebug(debugParsing) addSubExpr = ( multDivExpr + ZeroOrMore( plusMinusOp + multDivExpr ) ).setParseAction(compileBinaryOperation) addSubExpr.setName("addSubExpr").setDebug(debugParsing) concatExpr = ( addSubExpr + ZeroOrMore( concatOp + addSubExpr ) ).setParseAction(compileBinaryOperation) comparisonExpr = ( concatExpr + Optional( compOp + concatExpr ) ).setParseAction(compileBinaryOperation) equalityExpr = ( comparisonExpr + Optional( eqNeOp + comparisonExpr ) ).setParseAction(compileBinaryOperation) xorExpr = ( equalityExpr + ZeroOrMore( xorOp + equalityExpr) ).setParseAction(compileBinaryOperation) andExpr = ( xorExpr + ZeroOrMore( andOp + xorExpr ) ).setParseAction(compileBinaryOperation) orExpr = ( andExpr + ZeroOrMore( orOp + andExpr ) ).setParseAction(compileBinaryOperation) orExpr.setName("orExpr").setDebug(debugParsing) expr << orExpr expr.setName("expr").setDebug(debugParsing) cdrProg = expr + StringEnd() expr.setName("cdrProg").setDebug(debugParsing) startedAt = time.time() cntlr.modelManager.showStatus(_("initializing CDR grammar")) cdrProg.parseString( "0", parseAll=True ) from arelle.Locale import format_string _msg = format_string(cntlr.modelManager.locale, _("CDR grammar initialized in %.2f secs"), time.time() - startedAt) logMessage("INFO", "info", _msg) cntlr.modelManager.showStatus(_msg, 5000) isGrammarCompiled = True return cdrProg
def logArguments(self, codes, msg, codedArgs): # determine logCode messageCode = None for argCode in codes if isinstance(codes, tuple) else (codes, ): if (isinstance(argCode, ModelValue.QName) or (self.modelManager.disclosureSystem.EFM and argCode.startswith("EFM")) or (self.modelManager.disclosureSystem.GFM and argCode.startswith("GFM")) or (self.modelManager.disclosureSystem.HMRC and argCode.startswith("HMRC")) or (self.modelManager.disclosureSystem.SBRNL and argCode.startswith("SBR.NL")) or argCode[0:3] not in ("EFM", "GFM", "HMR", "SBR")): messageCode = argCode break # determine message and extra arguments fmtArgs = {} extras = {"messageCode": messageCode} for argName, argValue in codedArgs.items(): if argName in ("modelObject", "modelXbrl", "modelDocument"): try: entryUrl = self.modelDocument.uri except AttributeError: entryUrl = self.entryLoadingUrl refs = [] for arg in (argValue if isinstance(argValue, (tuple, list)) else (argValue, )): if arg is not None: if isinstance(arg, _STR_BASE): objectUrl = arg else: try: objectUrl = arg.modelDocument.uri except AttributeError: try: objectUrl = self.modelDocument.uri except AttributeError: objectUrl = self.entryLoadingUrl file = UrlUtil.relativeUri(entryUrl, objectUrl) ref = {} if isinstance(arg, ModelObject): ref["href"] = file + "#" + XmlUtil.elementFragmentIdentifier( arg) ref["sourceLine"] = arg.sourceline ref["objectId"] = arg.objectId() else: ref["href"] = file refs.append(ref) extras["refs"] = refs elif argName == "sourceLine": if isinstance( argValue, _INT_TYPES): # must be sortable with int's in logger extras["sourceLine"] = argValue elif argName != "exc_info": if isinstance( argValue, (ModelValue.QName, ModelObject, bool, FileNamedStringIO)): fmtArgs[argName] = str(argValue) elif isinstance(argValue, _INT_TYPES): # need locale-dependent formatting fmtArgs[argName] = format_string(self.modelManager.locale, '%i', argValue) elif isinstance(argValue, float): # need locale-dependent formatting fmtArgs[argName] = format_string(self.modelManager.locale, '%f', argValue) else: fmtArgs[argName] = argValue if "refs" not in extras: try: file = os.path.basename(self.modelDocument.uri) except AttributeError: try: file = os.path.basename(self.entryLoadingUrl) except: file = "" extras["refs"] = [{"href": file}] return (messageCode, (msg, fmtArgs) if fmtArgs else (msg, ), extras)
def run(self, options, sourceZipStream=None): """Process command line arguments or web service request, such as to load and validate an XBRL document, or start web server. When a web server has been requested, this method may be called multiple times, once for each web service (REST) request that requires processing. Otherwise (when called for a command line request) this method is called only once for the command line arguments request. :param options: OptionParser options from parse_args of main argv arguments (when called from command line) or corresponding arguments from web service (REST) request. :type options: optparse.Values """ if options.showOptions: # debug options for optName, optValue in sorted(options.__dict__.items(), key=lambda optItem: optItem[0]): self.addToLog("Option {0}={1}".format(optName, optValue), messageCode="info") self.addToLog("sys.argv {0}".format(sys.argv), messageCode="info") if options.uiLang: # set current UI Lang (but not config setting) self.setUiLanguage(options.uiLang) if options.proxy: if options.proxy != "show": proxySettings = proxyTuple(options.proxy) self.webCache.resetProxies(proxySettings) self.config["proxySettings"] = proxySettings self.saveConfig() self.addToLog(_("Proxy configuration has been set."), messageCode="info") useOsProxy, urlAddr, urlPort, user, password = self.config.get("proxySettings", proxyTuple("none")) if useOsProxy: self.addToLog(_("Proxy configured to use {0}.").format( _('Microsoft Windows Internet Settings') if sys.platform.startswith("win") else (_('Mac OS X System Configuration') if sys.platform in ("darwin", "macos") else _('environment variables'))), messageCode="info") elif urlAddr: self.addToLog(_("Proxy setting: http://{0}{1}{2}{3}{4}").format( user if user else "", ":****" if password else "", "@" if (user or password) else "", urlAddr, ":{0}".format(urlPort) if urlPort else ""), messageCode="info") else: self.addToLog(_("Proxy is disabled."), messageCode="info") if options.plugins: from arelle import PluginManager resetPlugins = False savePluginChanges = True showPluginModules = False for pluginCmd in options.plugins.split('|'): cmd = pluginCmd.strip() if cmd == "show": showPluginModules = True elif cmd == "temp": savePluginChanges = False elif cmd.startswith("+"): moduleInfo = PluginManager.addPluginModule(cmd[1:]) if moduleInfo: self.addToLog(_("Addition of plug-in {0} successful.").format(moduleInfo.get("name")), messageCode="info", file=moduleInfo.get("moduleURL")) resetPlugins = True else: self.addToLog(_("Unable to load plug-in."), messageCode="info", file=cmd[1:]) elif cmd.startswith("~"): if PluginManager.reloadPluginModule(cmd[1:]): self.addToLog(_("Reload of plug-in successful."), messageCode="info", file=cmd[1:]) resetPlugins = True else: self.addToLog(_("Unable to reload plug-in."), messageCode="info", file=cmd[1:]) elif cmd.startswith("-"): if PluginManager.removePluginModule(cmd[1:]): self.addToLog(_("Deletion of plug-in successful."), messageCode="info", file=cmd[1:]) resetPlugins = True else: self.addToLog(_("Unable to delete plug-in."), messageCode="info", file=cmd[1:]) else: # assume it is a module or package savePluginChanges = False moduleInfo = PluginManager.addPluginModule(cmd) if moduleInfo: self.addToLog(_("Activation of plug-in {0} successful.").format(moduleInfo.get("name")), messageCode="info", file=moduleInfo.get("moduleURL")) resetPlugins = True else: self.addToLog(_("Unable to load {0} as a plug-in or {0} is not recognized as a command. ").format(cmd), messageCode="info", file=cmd) if resetPlugins: PluginManager.reset() if savePluginChanges: PluginManager.save(self) if showPluginModules: self.addToLog(_("Plug-in modules:"), messageCode="info") for i, moduleItem in enumerate(sorted(PluginManager.pluginConfig.get("modules", {}).items())): moduleInfo = moduleItem[1] self.addToLog(_("Plug-in: {0}; author: {1}; version: {2}; status: {3}; date: {4}; description: {5}; license {6}.").format( moduleItem[0], moduleInfo.get("author"), moduleInfo.get("version"), moduleInfo.get("status"), moduleInfo.get("fileDate"), moduleInfo.get("description"), moduleInfo.get("license")), messageCode="info", file=moduleInfo.get("moduleURL")) # run utility command line options that don't depend on entrypoint Files hasUtilityPlugin = False for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Utility.Run"): hasUtilityPlugin = True pluginXbrlMethod(self, options) # if no entrypointFile is applicable, quit now if options.proxy or options.plugins or hasUtilityPlugin: if not options.entrypointFile: return True # success self.username = options.username self.password = options.password self.entrypointFile = options.entrypointFile if self.entrypointFile: filesource = FileSource.openFileSource(self.entrypointFile, self, sourceZipStream) else: filesource = None if options.validateEFM: if options.disclosureSystemName: self.addToLog(_("both --efm and --disclosureSystem validation are requested, proceeding with --efm only"), messageCode="info", file=self.entrypointFile) self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select("efm") elif options.disclosureSystemName: self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select(options.disclosureSystemName) elif options.validateHMRC: self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select("hmrc") else: self.modelManager.disclosureSystem.select(None) # just load ordinary mappings self.modelManager.validateDisclosureSystem = False if options.utrUrl: # override disclosureSystem utrUrl self.modelManager.disclosureSystem.utrUrl = options.utrUrl # can be set now because the utr is first loaded at validation time # disclosure system sets logging filters, override disclosure filters, if specified by command line if options.logLevelFilter: self.setLogLevelFilter(options.logLevelFilter) if options.logCodeFilter: self.setLogCodeFilter(options.logCodeFilter) if options.calcDecimals: if options.calcPrecision: self.addToLog(_("both --calcDecimals and --calcPrecision validation are requested, proceeding with --calcDecimals only"), messageCode="info", file=self.entrypointFile) self.modelManager.validateInferDecimals = True self.modelManager.validateCalcLB = True elif options.calcPrecision: self.modelManager.validateInferDecimals = False self.modelManager.validateCalcLB = True if options.utrValidate: self.modelManager.validateUtr = True if options.infosetValidate: self.modelManager.validateInfoset = True if options.abortOnMajorError: self.modelManager.abortOnMajorError = True if options.collectProfileStats: self.modelManager.collectProfileStats = True if options.internetConnectivity == "offline": self.webCache.workOffline = True elif options.internetConnectivity == "online": self.webCache.workOffline = False if options.internetTimeout is not None: self.webCache.timeout = (options.internetTimeout or None) # use None if zero specified to disable timeout fo = FormulaOptions() if options.parameters: parameterSeparator = (options.parameterSeparator or ',') fo.parameterValues = dict(((qname(key, noPrefixIsNoNamespace=True),(None,value)) for param in options.parameters.split(parameterSeparator) for key,sep,value in (param.partition('='),) ) ) if options.formulaParamExprResult: fo.traceParameterExpressionResult = True if options.formulaParamInputValue: fo.traceParameterInputValue = True if options.formulaCallExprSource: fo.traceCallExpressionSource = True if options.formulaCallExprCode: fo.traceCallExpressionCode = True if options.formulaCallExprEval: fo.traceCallExpressionEvaluation = True if options.formulaCallExprResult: fo.traceCallExpressionResult = True if options.formulaVarSetExprEval: fo.traceVariableSetExpressionEvaluation = True if options.formulaVarSetExprResult: fo.traceVariableSetExpressionResult = True if options.formulaAsserResultCounts: fo.traceAssertionResultCounts = True if options.formulaFormulaRules: fo.traceFormulaRules = True if options.formulaVarsOrder: fo.traceVariablesOrder = True if options.formulaVarExpressionSource: fo.traceVariableExpressionSource = True if options.formulaVarExpressionCode: fo.traceVariableExpressionCode = True if options.formulaVarExpressionEvaluation: fo.traceVariableExpressionEvaluation = True if options.formulaVarExpressionResult: fo.traceVariableExpressionResult = True if options.timeVariableSetEvaluation: fo.timeVariableSetEvaluation = True if options.formulaVarFilterWinnowing: fo.traceVariableFilterWinnowing = True if options.formulaVarFiltersResult: fo.traceVariableFiltersResult = True if options.formulaVarFiltersResult: fo.traceVariableFiltersResult = True self.modelManager.formulaOptions = fo timeNow = XmlUtil.dateunionValue(datetime.datetime.now()) firstStartedAt = startedAt = time.time() modelDiffReport = None success = True modelXbrl = None try: if filesource: modelXbrl = self.modelManager.load(filesource, _("views loading")) except ModelDocument.LoadingException: pass except Exception as err: self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format( err, traceback.format_tb(sys.exc_info()[2]))) success = False # loading errors, don't attempt to utilize loaded DTS if modelXbrl and modelXbrl.modelDocument: loadTime = time.time() - startedAt modelXbrl.profileStat(_("load"), loadTime) self.addToLog(format_string(self.modelManager.locale, _("loaded in %.2f secs at %s"), (loadTime, timeNow)), messageCode="info", file=self.entrypointFile) if options.importFiles: for importFile in options.importFiles.split("|"): fileName = importFile.strip() if sourceZipStream is not None and not (fileName.startswith('http://') or os.path.isabs(fileName)): fileName = os.path.dirname(modelXbrl.uri) + os.sep + fileName # make relative to sourceZipStream ModelDocument.load(modelXbrl, fileName) loadTime = time.time() - startedAt self.addToLog(format_string(self.modelManager.locale, _("import in %.2f secs at %s"), (loadTime, timeNow)), messageCode="info", file=importFile) modelXbrl.profileStat(_("import"), loadTime) if modelXbrl.errors: success = False # loading errors, don't attempt to utilize loaded DTS if modelXbrl.modelDocument.type in ModelDocument.Type.TESTCASETYPES: for pluginXbrlMethod in pluginClassMethods("Testcases.Start"): pluginXbrlMethod(self, options, modelXbrl) else: # not a test case, probably instance or DTS for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Loaded"): pluginXbrlMethod(self, options, modelXbrl) else: success = False if success and options.diffFile and options.versReportFile: try: diffFilesource = FileSource.FileSource(options.diffFile,self) startedAt = time.time() modelXbrl2 = self.modelManager.load(diffFilesource, _("views loading")) if modelXbrl2.errors: if not options.keepOpen: modelXbrl2.close() success = False else: loadTime = time.time() - startedAt modelXbrl.profileStat(_("load"), loadTime) self.addToLog(format_string(self.modelManager.locale, _("diff comparison DTS loaded in %.2f secs"), loadTime), messageCode="info", file=self.entrypointFile) startedAt = time.time() modelDiffReport = self.modelManager.compareDTSes(options.versReportFile) diffTime = time.time() - startedAt modelXbrl.profileStat(_("diff"), diffTime) self.addToLog(format_string(self.modelManager.locale, _("compared in %.2f secs"), diffTime), messageCode="info", file=self.entrypointFile) except ModelDocument.LoadingException: success = False except Exception as err: success = False self.addToLog(_("[Exception] Failed to doad diff file: \n{0} \n{1}").format( err, traceback.format_tb(sys.exc_info()[2]))) if success: try: modelXbrl = self.modelManager.modelXbrl hasFormulae = modelXbrl.hasFormulae if options.validate: startedAt = time.time() if options.formulaAction: # don't automatically run formulas modelXbrl.hasFormulae = False self.modelManager.validate() if options.formulaAction: # restore setting modelXbrl.hasFormulae = hasFormulae self.addToLog(format_string(self.modelManager.locale, _("validated in %.2f secs"), time.time() - startedAt), messageCode="info", file=self.entrypointFile) if options.formulaAction in ("validate", "run"): # do nothing here if "none" from arelle import ValidateXbrlDimensions, ValidateFormula startedAt = time.time() if not options.validate: ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # setup fresh parameters from formula optoins modelXbrl.parameters = fo.typedParameters() ValidateFormula.validate(modelXbrl, compileOnly=(options.formulaAction != "run")) self.addToLog(format_string(self.modelManager.locale, _("formula validation and execution in %.2f secs") if options.formulaAction == "run" else _("formula validation only in %.2f secs"), time.time() - startedAt), messageCode="info", file=self.entrypointFile) if options.testReport: ViewFileTests.viewTests(self.modelManager.modelXbrl, options.testReport, options.testReportCols) if options.rssReport: ViewFileRssFeed.viewRssFeed(self.modelManager.modelXbrl, options.rssReport, options.rssReportCols) if options.DTSFile: ViewFileDTS.viewDTS(modelXbrl, options.DTSFile) if options.factsFile: ViewFileFactList.viewFacts(modelXbrl, options.factsFile, labelrole=options.labelRole, lang=options.labelLang, cols=options.factListCols) if options.factTableFile: ViewFileFactTable.viewFacts(modelXbrl, options.factTableFile, labelrole=options.labelRole, lang=options.labelLang) if options.conceptsFile: ViewFileConcepts.viewConcepts(modelXbrl, options.conceptsFile, labelrole=options.labelRole, lang=options.labelLang) if options.preFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.preFile, "Presentation Linkbase", "http://www.xbrl.org/2003/arcrole/parent-child", labelrole=options.labelRole, lang=options.labelLang) if options.calFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.calFile, "Calculation Linkbase", "http://www.xbrl.org/2003/arcrole/summation-item", labelrole=options.labelRole, lang=options.labelLang) if options.dimFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.dimFile, "Dimensions", "XBRL-dimensions", labelrole=options.labelRole, lang=options.labelLang) if options.formulaeFile: ViewFileFormulae.viewFormulae(modelXbrl, options.formulaeFile, "Formulae", lang=options.labelLang) if options.viewArcrole and options.viewFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.viewFile, os.path.basename(options.viewArcrole), options.viewArcrole, labelrole=options.labelRole, lang=options.labelLang) for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Run"): pluginXbrlMethod(self, options, modelXbrl) except (IOError, EnvironmentError) as err: self.addToLog(_("[IOError] Failed to save output:\n {0}").format(err)) success = False except Exception as err: self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format( err, traceback.format_tb(sys.exc_info()[2]))) success = False if modelXbrl: modelXbrl.profileStat(_("total"), time.time() - firstStartedAt) if options.collectProfileStats and modelXbrl: modelXbrl.logProfileStats() if not options.keepOpen: if modelDiffReport: self.modelManager.close(modelDiffReport) elif modelXbrl: self.modelManager.close(modelXbrl) self.username = self.password = None #dereference password return success
def run(self, options): if options.logFile: self.messages = [] else: self.messages = None self.filename = options.filename filesource = FileSource.FileSource(self.filename, self) if options.validateEFM: if options.gfmName: self.addToLog( _("[info] both --efm and --gfm validation are requested, proceeding with --efm only" )) self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select("efm") elif options.gfmName: self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select(options.gfmName) else: self.modelManager.disclosureSystem.select( None) # just load ordinary mappings if options.calcDecimals: if options.calcPrecision: self.addToLog( _("[info] both --calcDecimals and --calcPrecision validation are requested, proceeding with --calcDecimals only" )) self.modelManager.validateInferDecimals = True self.modelManager.validateCalcLB = True elif options.calcPrecision: self.modelManager.validateInferDecimals = False self.modelManager.validateCalcLB = True if options.utrValidate: self.modelManager.validateUtr = True fo = FormulaOptions() if options.formulaParamExprResult: fo.traceParameterExpressionResult = True if options.formulaParamInputValue: fo.traceParameterInputValue = True if options.formulaCallExprSource: fo.traceCallExpressionSource = True if options.formulaCallExprCode: fo.traceCallExpressionCode = True if options.formulaCallExprEval: fo.traceCallExpressionEvaluation = True if options.formulaCallExprResult: fo.traceCallExpressionResult = True if options.formulaVarSetExprEval: fo.traceVariableSetExpressionEvaluation = True if options.formulaVarSetExprResult: fo.traceVariableSetExpressionResult = True if options.formulaAsserResultCounts: fo.traceAssertionResultCounts = True if options.formulaFormulaRules: fo.traceFormulaRules = True if options.formulaVarsOrder: fo.traceVariablesOrder = True if options.formulaVarExpressionSource: fo.traceVariableExpressionSource = True if options.formulaVarExpressionCode: fo.traceVariableExpressionCode = True if options.formulaVarExpressionEvaluation: fo.traceVariableExpressionEvaluation = True if options.formulaVarExpressionResult: fo.traceVariableExpressionResult = True if options.formulaVarFiltersResult: fo.traceVariableFiltersResult = True self.modelManager.formulaOptions = fo timeNow = XmlUtil.dateunionValue(datetime.datetime.now()) startedAt = time.time() modelXbrl = self.modelManager.load(filesource, _("views loading")) self.addToLog( format_string(self.modelManager.locale, _("[info] loaded in %.2f secs at %s"), (time.time() - startedAt, timeNow))) if options.diffFilename and options.versReportFilename: diffFilesource = FileSource.FileSource(self.diffFilename, self) startedAt = time.time() modelXbrl = self.modelManager.load(diffFilesource, _("views loading")) self.addToLog( format_string( self.modelManager.locale, _("[info] diff comparison DTS loaded in %.2f secs"), time.time() - startedAt)) startedAt = time.time() self.modelManager.compareDTSes(options.versReportFilename) self.addToLog( format_string(self.modelManager.locale, _("[info] compared in %.2f secs"), time.time() - startedAt)) try: if options.validate: startedAt = time.time() self.modelManager.validate() self.addToLog( format_string(self.modelManager.locale, _("[info] validated in %.2f secs"), time.time() - startedAt)) if (options.csvTestReport and self.modelManager.modelXbrl.modelDocument.type in (ModelDocument.Type.TESTCASESINDEX, ModelDocument.Type.REGISTRY)): ViewCsvTests.viewTests(self.modelManager.modelXbrl, options.csvTestReport) if options.csvDTS: ViewCsvDTS.viewDTS(modelXbrl, options.csvDTS) if options.csvFactList: ViewCsvFactList.viewFacts(modelXbrl, options.csvFactList) if options.csvConcepts: ViewCsvConcepts.viewConcepts(modelXbrl, options.csvConcepts) if options.csvPre: ViewCsvRelationshipSet.viewRelationshipSet( modelXbrl, options.csvPre, "Presentation", "http://www.xbrl.org/2003/arcrole/parent-child") if options.csvCal: ViewCsvRelationshipSet.viewRelationshipSet( modelXbrl, options.csvCal, "Calculation", "http://www.xbrl.org/2003/arcrole/summation-item") if options.csvDim: ViewCsvRelationshipSet.viewRelationshipSet( modelXbrl, options.csvDim, "Dimension", "XBRL-dimensions") except (IOError, EnvironmentError) as err: self.addToLog( _("[IOError] Failed to save output:\n {0}").format(err)) if self.messages: try: with open(options.logFile, "w", encoding="utf-8") as fh: fh.writelines(self.messages) except (IOError, EnvironmentError) as err: print("Unable to save log to file: " + err)
def run(self, options): self.filename = options.filename filesource = FileSource.openFileSource(self.filename,self) if options.validateEFM: if options.gfmName: self.addToLog(_("both --efm and --gfm validation are requested, proceeding with --efm only"), messageCode="info", file=self.filename) self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select("efm") elif options.gfmName: self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select(options.gfmName) else: self.modelManager.disclosureSystem.select(None) # just load ordinary mappings if options.calcDecimals: if options.calcPrecision: self.addToLog(_("both --calcDecimals and --calcPrecision validation are requested, proceeding with --calcDecimals only"), messageCode="info", file=self.filename) self.modelManager.validateInferDecimals = True self.modelManager.validateCalcLB = True elif options.calcPrecision: self.modelManager.validateInferDecimals = False self.modelManager.validateCalcLB = True if options.utrValidate: self.modelManager.validateUtr = True fo = FormulaOptions() if options.formulaParamExprResult: fo.traceParameterExpressionResult = True if options.formulaParamInputValue: fo.traceParameterInputValue = True if options.formulaCallExprSource: fo.traceCallExpressionSource = True if options.formulaCallExprCode: fo.traceCallExpressionCode = True if options.formulaCallExprEval: fo.traceCallExpressionEvaluation = True if options.formulaCallExprResult: fo.traceCallExpressionResult = True if options.formulaVarSetExprEval: fo.traceVariableSetExpressionEvaluation = True if options.formulaVarSetExprResult: fo.traceVariableSetExpressionResult = True if options.formulaAsserResultCounts: fo.traceAssertionResultCounts = True if options.formulaFormulaRules: fo.traceFormulaRules = True if options.formulaVarsOrder: fo.traceVariablesOrder = True if options.formulaVarExpressionSource: fo.traceVariableExpressionSource = True if options.formulaVarExpressionCode: fo.traceVariableExpressionCode = True if options.formulaVarExpressionEvaluation: fo.traceVariableExpressionEvaluation = True if options.formulaVarExpressionResult: fo.traceVariableExpressionResult = True if options.formulaVarFiltersResult: fo.traceVariableFiltersResult = True self.modelManager.formulaOptions = fo timeNow = XmlUtil.dateunionValue(datetime.datetime.now()) startedAt = time.time() modelXbrl = self.modelManager.load(filesource, _("views loading")) self.addToLog(format_string(self.modelManager.locale, _("loaded in %.2f secs at %s"), (time.time() - startedAt, timeNow)), messageCode="info", file=self.filename) if options.diffFilename and options.versReportFilename: diffFilesource = FileSource.FileSource(self.diffFilename,self) startedAt = time.time() modelXbrl = self.modelManager.load(diffFilesource, _("views loading")) self.addToLog(format_string(self.modelManager.locale, _("diff comparison DTS loaded in %.2f secs"), time.time() - startedAt), messageCode="info", file=self.filename) startedAt = time.time() self.modelManager.compareDTSes(options.versReportFilename) self.addToLog(format_string(self.modelManager.locale, _("compared in %.2f secs"), time.time() - startedAt), messageCode="info", file=self.filename) try: if options.validate: startedAt = time.time() self.modelManager.validate() self.addToLog(format_string(self.modelManager.locale, _("validated in %.2f secs"), time.time() - startedAt), messageCode="info", file=self.filename) if (options.csvTestReport and self.modelManager.modelXbrl.modelDocument.type in (ModelDocument.Type.TESTCASESINDEX, ModelDocument.Type.TESTCASE, ModelDocument.Type.REGISTRY)): ViewCsvTests.viewTests(self.modelManager.modelXbrl, options.csvTestReport) if options.csvDTS: ViewCsvDTS.viewDTS(modelXbrl, options.csvDTS) if options.csvFactList: ViewCsvFactList.viewFacts(modelXbrl, options.csvFactList, cols=options.csvFactListCols) if options.csvConcepts: ViewCsvConcepts.viewConcepts(modelXbrl, options.csvConcepts) if options.csvPre: ViewCsvRelationshipSet.viewRelationshipSet(modelXbrl, options.csvPre, "Presentation", "http://www.xbrl.org/2003/arcrole/parent-child") if options.csvCal: ViewCsvRelationshipSet.viewRelationshipSet(modelXbrl, options.csvCal, "Calculation", "http://www.xbrl.org/2003/arcrole/summation-item") if options.csvDim: ViewCsvRelationshipSet.viewRelationshipSet(modelXbrl, options.csvDim, "Dimension", "XBRL-dimensions") except (IOError, EnvironmentError) as err: self.addToLog(_("[IOError] Failed to save output:\n {0}").format(err)) except Exception as err: self.addToLog(_("[Exception] Failed to complete validation: \n{0} \n{1}").format( err, traceback.format_tb(sys.exc_info()[2])))
def run(self, options, sourceZipStream=None): self.entrypointFile = options.entrypointFile filesource = FileSource.openFileSource(self.entrypointFile, self, sourceZipStream) if options.validateEFM: if options.gfmName: self.addToLog(_("both --efm and --gfm validation are requested, proceeding with --efm only"), messageCode="info", file=self.entrypointFile) self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select("efm") elif options.gfmName: self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select(options.gfmName) elif options.validateHMRC: self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select("hmrc") else: self.modelManager.disclosureSystem.select(None) # just load ordinary mappings if options.calcDecimals: if options.calcPrecision: self.addToLog(_("both --calcDecimals and --calcPrecision validation are requested, proceeding with --calcDecimals only"), messageCode="info", file=self.entrypointFile) self.modelManager.validateInferDecimals = True self.modelManager.validateCalcLB = True elif options.calcPrecision: self.modelManager.validateInferDecimals = False self.modelManager.validateCalcLB = True if options.utrValidate: self.modelManager.validateUtr = True fo = FormulaOptions() if options.parameters: fo.parameterValues = dict(((qname(key, noPrefixIsNoNamespace=True),(None,value)) for param in options.parameters.split(',') for key,sep,value in (param.partition('='),) ) ) if options.formulaParamExprResult: fo.traceParameterExpressionResult = True if options.formulaParamInputValue: fo.traceParameterInputValue = True if options.formulaCallExprSource: fo.traceCallExpressionSource = True if options.formulaCallExprCode: fo.traceCallExpressionCode = True if options.formulaCallExprEval: fo.traceCallExpressionEvaluation = True if options.formulaCallExprResult: fo.traceCallExpressionResult = True if options.formulaVarSetExprEval: fo.traceVariableSetExpressionEvaluation = True if options.formulaVarSetExprResult: fo.traceVariableSetExpressionResult = True if options.formulaAsserResultCounts: fo.traceAssertionResultCounts = True if options.formulaFormulaRules: fo.traceFormulaRules = True if options.formulaVarsOrder: fo.traceVariablesOrder = True if options.formulaVarExpressionSource: fo.traceVariableExpressionSource = True if options.formulaVarExpressionCode: fo.traceVariableExpressionCode = True if options.formulaVarExpressionEvaluation: fo.traceVariableExpressionEvaluation = True if options.formulaVarExpressionResult: fo.traceVariableExpressionResult = True if options.formulaVarFilterWinnowing: fo.traceVariableFilterWinnowing = True if options.formulaVarFiltersResult: fo.traceVariableFiltersResult = True self.modelManager.formulaOptions = fo timeNow = XmlUtil.dateunionValue(datetime.datetime.now()) startedAt = time.time() modelDiffReport = None success = True modelXbrl = None try: modelXbrl = self.modelManager.load(filesource, _("views loading")) except Exception as err: self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format( err, traceback.format_tb(sys.exc_info()[2]))) success = False # loading errors, don't attempt to utilize loaded DTS if modelXbrl and modelXbrl.modelDocument: self.addToLog(format_string(self.modelManager.locale, _("loaded in %.2f secs at %s"), (time.time() - startedAt, timeNow)), messageCode="info", file=self.entrypointFile) if options.importFiles: for importFile in options.importFiles.split("|"): ModelDocument.load(modelXbrl, importFile.strip()) self.addToLog(format_string(self.modelManager.locale, _("imported in %.2f secs at %s"), (time.time() - startedAt, timeNow)), messageCode="info", file=importFile) if modelXbrl.errors: success = False # loading errors, don't attempt to utilize loaded DTS else: success = False if success and options.diffFile and options.versReportFile: diffFilesource = FileSource.FileSource(options.diffFile,self) startedAt = time.time() modelXbrl2 = self.modelManager.load(diffFilesource, _("views loading")) if modelXbrl2.errors: if not options.keepOpen: modelXbrl2.close() success = False else: self.addToLog(format_string(self.modelManager.locale, _("diff comparison DTS loaded in %.2f secs"), time.time() - startedAt), messageCode="info", file=self.entrypointFile) startedAt = time.time() modelDiffReport = self.modelManager.compareDTSes(options.versReportFile) self.addToLog(format_string(self.modelManager.locale, _("compared in %.2f secs"), time.time() - startedAt), messageCode="info", file=self.entrypointFile) if success: try: if options.validate: startedAt = time.time() self.modelManager.validate() self.addToLog(format_string(self.modelManager.locale, _("validated in %.2f secs"), time.time() - startedAt), messageCode="info", file=self.entrypointFile) if (options.testReport and self.modelManager.modelXbrl.modelDocument.type in (ModelDocument.Type.TESTCASESINDEX, ModelDocument.Type.TESTCASE, ModelDocument.Type.REGISTRY)): ViewFileTests.viewTests(self.modelManager.modelXbrl, options.testReport) if options.DTSFile: ViewFileDTS.viewDTS(modelXbrl, options.DTSFile) if options.factsFile: ViewFileFactList.viewFacts(modelXbrl, options.factsFile, labelrole=options.labelRole, lang=options.labelLang, cols=options.factListCols) if options.factTableFile: ViewFileFactTable.viewFacts(modelXbrl, options.factTableFile, labelrole=options.labelRole, lang=options.labelLang) if options.conceptsFile: ViewFileConcepts.viewConcepts(modelXbrl, options.conceptsFile, labelrole=options.labelRole, lang=options.labelLang) if options.preFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.preFile, "Presentation Linkbase", "http://www.xbrl.org/2003/arcrole/parent-child", labelrole=options.labelRole, lang=options.labelLang) if options.calFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.calFile, "Calculation Linkbase", "http://www.xbrl.org/2003/arcrole/summation-item", labelrole=options.labelRole, lang=options.labelLang) if options.dimFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.dimFile, "Dimensions", "XBRL-dimensions", labelrole=options.labelRole, lang=options.labelLang) if options.formulaeFile: ViewFileFormulae.viewFormulae(modelXbrl, options.formulaeFile, "Formulae", lang=options.labelLang) for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Run"): pluginXbrlMethod(self, options, modelXbrl) except (IOError, EnvironmentError) as err: self.addToLog(_("[IOError] Failed to save output:\n {0}").format(err)) success = False except Exception as err: self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format( err, traceback.format_tb(sys.exc_info()[2]))) success = False if not options.keepOpen: if modelDiffReport: modelDiffReport.close() elif modelXbrl: modelXbrl.close() return success
def compileSphinxGrammar(cntlr): global isGrammarCompiled, sphinxProg, lineno if isGrammarCompiled: return sphinxProg debugParsing = True cntlr.showStatus(_("Compiling Sphinx Grammar")) if sys.version[0] >= '3': # python 3 requires modified parser to allow release of global objects when closing DTS from arelle.pyparsing.pyparsing_py3 import ( Word, Keyword, alphas, Literal, CaselessLiteral, Combine, Optional, nums, Or, Forward, Group, ZeroOrMore, StringEnd, alphanums, ParserElement, quotedString, delimitedList, Suppress, Regex, FollowedBy, lineno) else: from pyparsing import (Word, Keyword, alphas, Literal, CaselessLiteral, Combine, Optional, nums, Or, Forward, Group, ZeroOrMore, StringEnd, alphanums, ParserElement, quotedString, delimitedList, Suppress, Regex, FollowedBy, lineno) ParserElement.enablePackrat() """ the pyparsing parser constructs are defined in this method to prevent the need to compile the grammar when the plug in is loaded (which is likely to be when setting up GUI menus or command line parser). instead the grammar is compiled the first time that any sphinx needs to be parsed only the sphinxExpression (result below) needs to be global for the parser """ # define grammar sphinxComment = Regex( r"/(?:\*(?:[^*]*\*+)+?/|/[^\n]*(?:\n[^\n]*)*?(?:(?<!\\)|\Z))" ).setParseAction(compileComment) variableRef = Regex( "[$]" # variable prefix # localname part "([A-Za-z\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD_]" "[A-Za-z0-9\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\u0300-\u036F\u203F-\u2040\xB7_.-]*)" ) qName = Regex( "([A-Za-z\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD_]" "[A-Za-z0-9\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\u0300-\u036F\u203F-\u2040\xB7_.-]*:)?" # localname or wildcard-localname part "([A-Za-z\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD_]" "[A-Za-z0-9\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\u0300-\u036F\u203F-\u2040\xB7_.-]*|[*])" ) ncName = Regex( "([A-Za-z\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD_]" "[A-Za-z0-9\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\u0300-\u036F\u203F-\u2040\xB7_.-]*)" ).setName("ncName").setDebug(debugParsing) #annotationName = Word("@",alphanums + '_-.').setName("annotationName").setDebug(debugParsing) annotationName = Regex( "@[A-Za-z\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD_]\w*" ).setName("annotationName").setDebug(debugParsing) decimalPoint = Literal('.') exponentLiteral = CaselessLiteral('e') plusorminusLiteral = Literal('+') | Literal('-') digits = Word(nums) integerLiteral = Combine(Optional(plusorminusLiteral) + digits) decimalFractionLiteral = Combine( Optional(plusorminusLiteral) + decimalPoint + digits) infLiteral = Combine(Optional(plusorminusLiteral) + Literal("INF")) nanLiteral = Literal("NaN") floatLiteral = (Combine(integerLiteral + ( (decimalPoint + Optional(digits) + exponentLiteral + integerLiteral) | (exponentLiteral + integerLiteral) | (decimalPoint + Optional(digits)))) | Combine(decimalFractionLiteral + exponentLiteral + integerLiteral) | decimalFractionLiteral | infLiteral | nanLiteral) #emptySequence = Literal( "(" ) + Literal( ")" ) lParen = Literal("(") rParen = Literal(")") lPred = Literal("[[") | Literal("[") rPred = Literal("]]") | Literal("]") commaOp = Literal(",") ifOp = Keyword("if") elseOp = Keyword("else") forOp = Keyword("for") inOp = Keyword("in") withOp = Keyword("with") notOp = Keyword("not") valuesOp = Keyword("values") andOp = Keyword("and") orOp = Keyword("or") neOp = Literal("!=") leOp = Literal("<=") ltOp = Literal("<") geOp = Literal(">=") gtOp = Literal(">") eqOp = Literal("==") compOp = leOp | ltOp | geOp | gtOp plusOp = Literal("|+|") | Literal("|+") | Literal("+|") | Literal("+") minusOp = Literal("|-|") | Literal("|-") | Literal("-|") | Literal("-") plusMinusOp = (plusOp | minusOp).setParseAction(compileOp) multOp = Literal("*") divOp = Literal("/") varAssign = Literal("=") tagOp = Literal("#") asOp = Keyword("as") whereOp = Keyword("where") wildOp = Literal("**") | Literal("*") methodOp = Literal("::") formulaOp = Literal(":=") namespaceDeclaration = ( Literal("xmlns") + Optional(Suppress(Literal(":")) + ncName) + Suppress(Literal("=")) + quotedString ).setParseAction(compileNamespaceDeclaration).ignore(sphinxComment) annotationDeclaration = ( Suppress(Keyword("annotation")) + ncName + Optional(Suppress(Keyword("as")) + ncName) ).setParseAction(compileAnnotationDeclaration).ignore(sphinxComment) packageDeclaration = ( Suppress(Keyword("package")) + ncName).setParseAction(compilePackageDeclaration).ignore(sphinxComment) severity = (Suppress(Keyword("severity")) + (ncName)).setParseAction(compileSeverity).ignore(sphinxComment) expr = Forward() atom = ((forOp - Suppress(lParen) - ncName - Suppress(inOp) - expr - Suppress(rParen) - expr).setParseAction(compileFor) | (ifOp - Suppress(lParen) - expr - Suppress(rParen) - expr - Suppress(elseOp) - expr).setParseAction(compileIf) | (ncName + Suppress(lParen) + Optional( delimitedList( ZeroOrMore((ncName + Optional(tagOp + Optional(ncName)) + varAssign + expr + Suppress(Literal(";")) ).setParseAction(compileVariableAssignment)) + Optional(ncName + varAssign) + expr)) + Suppress(rParen)).setParseAction(compileFunctionReference) | (floatLiteral).setParseAction(compileFloatLiteral) | (integerLiteral).setParseAction(compileIntegerLiteral) | (quotedString).setParseAction(compileStringLiteral) | (Optional(qName) + lPred + Optional( delimitedList( ((whereOp + expr) | ((qName | variableRef) + Optional(tagOp + Optional(ncName)) + Optional( (varAssign + (wildOp | expr) | (inOp + expr) | (asOp + ncName + varAssign + wildOp + Optional(whereOp + expr))))) ).setParseAction(compileHyperspaceAxis), delim=';')) + rPred).setParseAction(compileHyperspaceExpression) | (variableRef).setParseAction(compileVariableReference) | (qName).setParseAction(compileQname) | (Suppress(lParen) - expr - Optional(commaOp - Optional(expr - ZeroOrMore(commaOp - expr))) - Suppress(rParen)).setParseAction(compileBrackets) ).ignore(sphinxComment) atom.setName("atom").setDebug(debugParsing) valueExpr = atom taggedExpr = (valueExpr - Optional(tagOp - ncName) ).setParseAction(compileTagAssignment).ignore(sphinxComment) methodExpr = ( (methodOp + ncName + ZeroOrMore(methodOp + taggedExpr) ).setParseAction(compileMethodReference) | (ZeroOrMore(taggedExpr + methodOp) + taggedExpr) ).setParseAction(compileMethodReference).ignore(sphinxComment) unaryExpr = ( Optional(plusMinusOp) + methodExpr).setParseAction(compileUnaryOperation).ignore(sphinxComment) negateExpr = ( Optional(notOp) + unaryExpr).setParseAction(compileUnaryOperation).ignore(sphinxComment) valuesExpr = (Optional(valuesOp) + negateExpr).setParseAction( compileValuesIteration).ignore(sphinxComment) method2Expr = (valuesExpr + Optional(methodOp + methodExpr)).setParseAction( compileMethodReference).ignore(sphinxComment) multiplyExpr = (method2Expr + Optional(multOp + method2Expr)).setParseAction( compileBinaryOperation).ignore(sphinxComment) divideExpr = (multiplyExpr + Optional(divOp + multiplyExpr)).setParseAction( compileBinaryOperation).ignore(sphinxComment) addExpr = (divideExpr + Optional(plusOp + divideExpr) ).setParseAction(compileBinaryOperation).ignore(sphinxComment) subtractExpr = (addExpr + Optional(minusOp + addExpr)).setParseAction( compileBinaryOperation).ignore(sphinxComment) equalityExpr = (subtractExpr + Optional(eqOp + subtractExpr)).setParseAction( compileBinaryOperation).ignore(sphinxComment) inequalityExpr = (equalityExpr + Optional(neOp + equalityExpr)).setParseAction( compileBinaryOperation).ignore(sphinxComment) comparisonExpr = (inequalityExpr + Optional(compOp + inequalityExpr)).setParseAction( compileBinaryOperation).ignore(sphinxComment) andExpr = (comparisonExpr + Optional(andOp + comparisonExpr) ).setParseAction(compileBinaryOperation).ignore(sphinxComment) orExpr = (andExpr + Optional(orOp + andExpr) ).setParseAction(compileBinaryOperation).ignore(sphinxComment) formulaExpr = (orExpr + Optional(formulaOp + orExpr)).setParseAction( compileBinaryOperation).ignore(sphinxComment) withExpr = (Optional(withOp + Suppress(lParen) + expr + Suppress(rParen)) + ZeroOrMore( (ncName + Optional(tagOp + Optional(ncName)) + varAssign + expr + Suppress(Literal(";"))).setParseAction( compileVariableAssignment).ignore(sphinxComment)) + formulaExpr).setParseAction(compileWith) #parsedExpr = withExpr #parsedExpr.setName("parsedExpr").setDebug(debugParsing) #expr << parsedExpr expr << withExpr expr.setName("expr").setDebug(debugParsing) annotation = (annotationName + Optional( Suppress(lParen) + Optional(delimitedList(expr)) + Suppress(rParen))).setParseAction(compileAnnotation).ignore( sphinxComment).setName("annotation").setDebug(debugParsing) constant = (Suppress(Keyword("constant")) + ncName + Optional(tagOp + Optional(ncName)) + varAssign + expr).setParseAction(compileConstant).ignore(sphinxComment) functionDeclaration = ( (Keyword("function") | Keyword("macro")) + ncName + lParen + Optional(delimitedList(ncName)) + rParen + expr).setParseAction(compileFunctionDeclaration).ignore(sphinxComment) message = (Suppress(Keyword("message")) + expr).setParseAction(compileMessage) preconditionDeclaration = ( Suppress(Keyword("precondition")) + ncName + expr + Optional( Keyword("otherwise") + Keyword("raise") + ncName + Optional(severity) + Optional(message)) ).setParseAction(compilePreconditionDeclaration).ignore(sphinxComment) assignedExpr = (ncName + Optional(tagOp + Optional(ncName)) + varAssign + expr + Suppress(Literal(";"))).setParseAction( compileVariableAssignment).ignore(sphinxComment) precondition = ( Suppress(Keyword("require")) + delimitedList(ncName)).setParseAction(compilePrecondition).ignore( sphinxComment).setName("precondition").setDebug(debugParsing) formulaRule = (Optional(precondition) + Keyword("formula") + ncName + Optional(severity) + Optional((Keyword("bind") + expr)) + ZeroOrMore(assignedExpr) + expr + Optional(message) ).setParseAction(compileFormulaRule).ignore(sphinxComment) reportRule = (Optional(precondition) + Keyword("report") + ncName + Optional(severity) + ZeroOrMore(assignedExpr) + expr + Optional(message) ).setParseAction(compileReportRule).ignore(sphinxComment) validationRule = (Optional(precondition) + Keyword("raise") + ncName + Optional(severity) + ZeroOrMore(assignedExpr) + expr + Optional(message)).setParseAction( compileValidationRule).ignore(sphinxComment) ruleBase = ( Optional(precondition) + Suppress(Keyword("rule-base")) + ZeroOrMore( (Suppress(Keyword("transform")) + (Keyword("namespace") + expr + Suppress(Keyword("to")) + expr) | (Keyword("qname") + expr + Suppress(Keyword("to")) + expr)).setParseAction(compileTransform)) ).setParseAction(compileRuleBase).ignore(sphinxComment).setName( "ruleBase").setDebug(debugParsing) sphinxProg = ( ZeroOrMore(namespaceDeclaration | sphinxComment) + ZeroOrMore(annotationDeclaration | annotation | constant | preconditionDeclaration | packageDeclaration | functionDeclaration | ruleBase | formulaRule | reportRule | validationRule | sphinxComment)) + StringEnd() sphinxProg.ignore(sphinxComment) startedAt = time.time() cntlr.modelManager.showStatus(_("initializing sphinx grammar")) sphinxProg.parseString("// force initialization\n", parseAll=True) from arelle.Locale import format_string logMessage( "INFO", "info", format_string(cntlr.modelManager.locale, _("Sphinx grammar initialized in %.2f secs"), time.time() - startedAt)) isGrammarCompiled = True return sphinxProg
def logArguments(self, codes, msg, codedArgs): # determine logCode messageCode = None for argCode in codes if isinstance(codes,tuple) else (codes,): if (isinstance(argCode, ModelValue.QName) or (self.modelManager.disclosureSystem.EFM and argCode.startswith("EFM")) or (self.modelManager.disclosureSystem.GFM and argCode.startswith("GFM")) or (self.modelManager.disclosureSystem.HMRC and argCode.startswith("HMRC")) or (self.modelManager.disclosureSystem.SBRNL and argCode.startswith("SBR.NL")) or argCode[0:3] not in ("EFM", "GFM", "HMR", "SBR")): messageCode = argCode break # determine message and extra arguments fmtArgs = {} extras = {"messageCode":messageCode} for argName, argValue in codedArgs.items(): if argName in ("modelObject", "modelXbrl", "modelDocument"): try: entryUrl = self.modelDocument.uri except AttributeError: entryUrl = self.entryLoadingUrl try: objectUrl = argValue.modelDocument.uri except AttributeError: try: objectUrl = self.modelDocument.uri except AttributeError: objectUrl = self.entryLoadingUrl refs = [] for arg in (argValue if isinstance(argValue, (tuple,list)) else (argValue,)): if arg is not None: file = UrlUtil.relativeUri(entryUrl, objectUrl) ref = {} if isinstance(arg,ModelObject): ref["href"] = file + "#" + XmlUtil.elementFragmentIdentifier(arg) ref["sourceLine"] = arg.sourceline ref["objectId"] = arg.objectId() else: ref["href"] = file refs.append(ref) extras["refs"] = refs elif argName == "sourceLine": if isinstance(argValue, _INT_TYPES): # must be sortable with int's in logger extras["sourceLine"] = argValue elif argName != "exc_info": if isinstance(argValue, (ModelValue.QName, ModelObject, bool, FileNamedStringIO)): fmtArgs[argName] = str(argValue) elif isinstance(argValue, _INT_TYPES): # need locale-dependent formatting fmtArgs[argName] = format_string(self.modelManager.locale, '%i', argValue) elif isinstance(argValue,float): # need locale-dependent formatting fmtArgs[argName] = format_string(self.modelManager.locale, '%f', argValue) else: fmtArgs[argName] = argValue if "refs" not in extras: try: file = os.path.basename(self.modelDocument.uri) except AttributeError: try: file = os.path.basename(self.entryLoadingUrl) except: file = "" extras["refs"] = [{"href": file}] return (messageCode, (msg, fmtArgs) if fmtArgs else (msg,), extras)
def compileSphinxGrammar( cntlr ): global isGrammarCompiled, sphinxProg, lineno if isGrammarCompiled: return sphinxProg debugParsing = True cntlr.showStatus(_("Compiling Sphinx Grammar")) if sys.version[0] >= '3': # python 3 requires modified parser to allow release of global objects when closing DTS from arelle.pyparsing.pyparsing_py3 import (Word, Keyword, alphas, Literal, CaselessLiteral, Combine, Optional, nums, Or, Forward, Group, ZeroOrMore, StringEnd, alphanums, ParserElement, quotedString, delimitedList, Suppress, Regex, FollowedBy, lineno) else: from pyparsing import (Word, Keyword, alphas, Literal, CaselessLiteral, Combine, Optional, nums, Or, Forward, Group, ZeroOrMore, StringEnd, alphanums, ParserElement, quotedString, delimitedList, Suppress, Regex, FollowedBy, lineno) ParserElement.enablePackrat() """ the pyparsing parser constructs are defined in this method to prevent the need to compile the grammar when the plug in is loaded (which is likely to be when setting up GUI menus or command line parser). instead the grammar is compiled the first time that any sphinx needs to be parsed only the sphinxExpression (result below) needs to be global for the parser """ # define grammar sphinxComment = Regex(r"/(?:\*(?:[^*]*\*+)+?/|/[^\n]*(?:\n[^\n]*)*?(?:(?<!\\)|\Z))").setParseAction(compileComment) variableRef = Regex("[$]" # variable prefix # localname part "([A-Za-z\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD_]" "[A-Za-z0-9\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\u0300-\u036F\u203F-\u2040\xB7_.-]*)" ) qName = Regex("([A-Za-z\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD_]" "[A-Za-z0-9\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\u0300-\u036F\u203F-\u2040\xB7_.-]*:)?" # localname or wildcard-localname part "([A-Za-z\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD_]" "[A-Za-z0-9\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\u0300-\u036F\u203F-\u2040\xB7_.-]*|[*])" ) ncName = Regex("([A-Za-z\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD_]" "[A-Za-z0-9\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\u0300-\u036F\u203F-\u2040\xB7_.-]*)" ).setName("ncName").setDebug(debugParsing) #annotationName = Word("@",alphanums + '_-.').setName("annotationName").setDebug(debugParsing) annotationName = Regex("@[A-Za-z\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD_]\w*").setName("annotationName").setDebug(debugParsing) decimalPoint = Literal('.') exponentLiteral = CaselessLiteral('e') plusorminusLiteral = Literal('+') | Literal('-') digits = Word(nums) integerLiteral = Combine( Optional(plusorminusLiteral) + digits ) decimalFractionLiteral = Combine( Optional(plusorminusLiteral) + decimalPoint + digits ) infLiteral = Combine( Optional(plusorminusLiteral) + Literal("INF") ) nanLiteral = Literal("NaN") floatLiteral = ( Combine( integerLiteral + ( ( decimalPoint + Optional(digits) + exponentLiteral + integerLiteral ) | ( exponentLiteral + integerLiteral ) | ( decimalPoint + Optional(digits) ) ) ) | Combine( decimalFractionLiteral + exponentLiteral + integerLiteral ) | decimalFractionLiteral | infLiteral | nanLiteral ) #emptySequence = Literal( "(" ) + Literal( ")" ) lParen = Literal( "(" ) rParen = Literal( ")" ) lPred = Literal( "[[" ) | Literal("[") rPred = Literal( "]]" ) | Literal("]") commaOp = Literal(",") ifOp = Keyword("if") elseOp = Keyword("else") forOp = Keyword("for") inOp = Keyword("in") withOp = Keyword("with") notOp = Keyword("not") valuesOp = Keyword("values") andOp = Keyword("and") orOp = Keyword("or") neOp = Literal("!=") leOp = Literal("<=") ltOp = Literal("<") geOp = Literal(">=") gtOp = Literal(">") eqOp = Literal("==") compOp = leOp | ltOp | geOp | gtOp plusOp = Literal("|+|") | Literal("|+") | Literal("+|") | Literal("+") minusOp = Literal("|-|") | Literal("|-") | Literal("-|") | Literal("-") plusMinusOp = ( plusOp | minusOp ).setParseAction(compileOp) multOp = Literal("*") divOp = Literal("/") varAssign = Literal("=") tagOp = Literal("#") asOp = Keyword("as") whereOp = Keyword("where") wildOp = Literal("**") | Literal("*") methodOp = Literal("::") formulaOp = Literal(":=") namespaceDeclaration = (Literal("xmlns") + Optional( Suppress(Literal(":")) + ncName ) + Suppress(Literal("=")) + quotedString ).setParseAction(compileNamespaceDeclaration).ignore(sphinxComment) annotationDeclaration = (Suppress(Keyword("annotation")) + ncName + Optional( Suppress(Keyword("as")) + ncName )).setParseAction(compileAnnotationDeclaration).ignore(sphinxComment) packageDeclaration = (Suppress(Keyword("package")) + ncName ).setParseAction(compilePackageDeclaration).ignore(sphinxComment) severity = ( Suppress(Keyword("severity")) + ( ncName ) ).setParseAction(compileSeverity).ignore(sphinxComment) expr = Forward() atom = ( ( forOp - Suppress(lParen) - ncName - Suppress(inOp) - expr - Suppress(rParen) - expr ).setParseAction(compileFor) | ( ifOp - Suppress(lParen) - expr - Suppress(rParen) - expr - Suppress(elseOp) - expr ).setParseAction(compileIf) | ( ncName + Suppress(lParen) + Optional(delimitedList( ZeroOrMore( ( ncName + Optional( tagOp + Optional(ncName) ) + varAssign + expr + Suppress(Literal(";")) ).setParseAction(compileVariableAssignment) ) + Optional( ncName + varAssign ) + expr )) + Suppress(rParen) ).setParseAction(compileFunctionReference) | ( floatLiteral ).setParseAction(compileFloatLiteral) | ( integerLiteral ).setParseAction(compileIntegerLiteral) | ( quotedString ).setParseAction(compileStringLiteral) | ( Optional(qName) + lPred + Optional(delimitedList( ((whereOp + expr) | ((qName | variableRef) + Optional( tagOp + Optional(ncName) ) + Optional( (varAssign + (wildOp | expr) | (inOp + expr) | (asOp + ncName + varAssign + wildOp + Optional( whereOp + expr ) ) ) ) ) ).setParseAction(compileHyperspaceAxis), delim=';')) + rPred).setParseAction(compileHyperspaceExpression) | ( variableRef ).setParseAction(compileVariableReference) | ( qName ).setParseAction(compileQname) | ( Suppress(lParen) - expr - Optional( commaOp - Optional( expr - ZeroOrMore( commaOp - expr ) ) ) - Suppress(rParen) ).setParseAction(compileBrackets) ).ignore(sphinxComment) atom.setName("atom").setDebug(debugParsing) valueExpr = atom taggedExpr = ( valueExpr - Optional(tagOp - ncName) ).setParseAction(compileTagAssignment).ignore(sphinxComment) methodExpr = ( ( methodOp + ncName + ZeroOrMore(methodOp + taggedExpr) ).setParseAction(compileMethodReference) | ( ZeroOrMore(taggedExpr + methodOp) + taggedExpr )).setParseAction(compileMethodReference).ignore(sphinxComment) unaryExpr = ( Optional(plusMinusOp) + methodExpr ).setParseAction(compileUnaryOperation).ignore(sphinxComment) negateExpr = ( Optional(notOp) + unaryExpr ).setParseAction(compileUnaryOperation).ignore(sphinxComment) valuesExpr = ( Optional(valuesOp) + negateExpr ).setParseAction(compileValuesIteration).ignore(sphinxComment) method2Expr = ( valuesExpr + Optional( methodOp + methodExpr ) ).setParseAction(compileMethodReference).ignore(sphinxComment) multiplyExpr = ( method2Expr + Optional( multOp + method2Expr ) ).setParseAction(compileBinaryOperation).ignore(sphinxComment) divideExpr = ( multiplyExpr + Optional( divOp + multiplyExpr ) ).setParseAction(compileBinaryOperation).ignore(sphinxComment) addExpr = ( divideExpr + Optional( plusOp + divideExpr ) ).setParseAction(compileBinaryOperation).ignore(sphinxComment) subtractExpr = ( addExpr + Optional( minusOp + addExpr ) ).setParseAction(compileBinaryOperation).ignore(sphinxComment) equalityExpr = ( subtractExpr + Optional( eqOp + subtractExpr ) ).setParseAction(compileBinaryOperation).ignore(sphinxComment) inequalityExpr = ( equalityExpr + Optional( neOp + equalityExpr ) ).setParseAction(compileBinaryOperation).ignore(sphinxComment) comparisonExpr = ( inequalityExpr + Optional( compOp + inequalityExpr ) ).setParseAction(compileBinaryOperation).ignore(sphinxComment) andExpr = ( comparisonExpr + Optional( andOp + comparisonExpr ) ).setParseAction(compileBinaryOperation ).ignore(sphinxComment) orExpr = ( andExpr + Optional( orOp + andExpr ) ).setParseAction(compileBinaryOperation).ignore(sphinxComment) formulaExpr = ( orExpr + Optional( formulaOp + orExpr ) ).setParseAction(compileBinaryOperation).ignore(sphinxComment) withExpr = ( Optional( withOp + Suppress(lParen) + expr + Suppress(rParen) ) + ZeroOrMore( ( ncName + Optional( tagOp + Optional(ncName) ) + varAssign + expr + Suppress(Literal(";")) ).setParseAction(compileVariableAssignment).ignore(sphinxComment) ) + formulaExpr ).setParseAction(compileWith) #parsedExpr = withExpr #parsedExpr.setName("parsedExpr").setDebug(debugParsing) #expr << parsedExpr expr << withExpr expr.setName("expr").setDebug(debugParsing) annotation = ( annotationName + Optional( Suppress(lParen) + Optional(delimitedList(expr)) + Suppress(rParen) ) ).setParseAction(compileAnnotation).ignore(sphinxComment).setName("annotation").setDebug(debugParsing) constant = ( Suppress(Keyword("constant")) + ncName + Optional( tagOp + Optional(ncName) ) + varAssign + expr ).setParseAction(compileConstant).ignore(sphinxComment) functionDeclaration = ( (Keyword("function") | Keyword("macro")) + ncName + lParen + Optional(delimitedList(ncName)) + rParen + expr ).setParseAction(compileFunctionDeclaration).ignore(sphinxComment) message = ( Suppress(Keyword("message")) + expr ).setParseAction(compileMessage) preconditionDeclaration = ( Suppress(Keyword("precondition")) + ncName + expr + Optional(Keyword("otherwise") + Keyword("raise") + ncName + Optional( severity ) + Optional( message ) ) ).setParseAction(compilePreconditionDeclaration).ignore(sphinxComment) assignedExpr = ( ncName + Optional( tagOp + Optional(ncName) ) + varAssign + expr + Suppress(Literal(";")) ).setParseAction(compileVariableAssignment).ignore(sphinxComment) precondition = ( Suppress(Keyword("require")) + delimitedList(ncName) ).setParseAction(compilePrecondition).ignore(sphinxComment).setName("precondition").setDebug(debugParsing) formulaRule = ( Optional( precondition ) + Keyword("formula") + ncName + Optional( severity ) + Optional( ( Keyword("bind") + expr ) ) + ZeroOrMore( assignedExpr ) + expr + Optional( message )).setParseAction(compileFormulaRule).ignore(sphinxComment) reportRule = ( Optional( precondition ) + Keyword("report") + ncName + Optional( severity ) + ZeroOrMore( assignedExpr ) + expr + Optional( message )).setParseAction( compileReportRule).ignore(sphinxComment) validationRule = ( Optional( precondition ) + Keyword("raise") + ncName + Optional( severity ) + ZeroOrMore( assignedExpr ) + expr + Optional( message )).setParseAction(compileValidationRule).ignore(sphinxComment) ruleBase = (Optional( precondition ) + Suppress(Keyword("rule-base")) + ZeroOrMore( (Suppress(Keyword("transform")) + (Keyword("namespace") + expr + Suppress(Keyword("to")) + expr) | (Keyword ("qname") + expr + Suppress(Keyword("to")) + expr) ).setParseAction(compileTransform) ) ).setParseAction(compileRuleBase).ignore(sphinxComment).setName("ruleBase").setDebug(debugParsing) sphinxProg = ( ZeroOrMore( namespaceDeclaration | sphinxComment ) + ZeroOrMore( annotationDeclaration | annotation | constant | preconditionDeclaration | packageDeclaration | functionDeclaration | ruleBase | formulaRule | reportRule | validationRule | sphinxComment ) ) + StringEnd() sphinxProg.ignore(sphinxComment) startedAt = time.time() cntlr.modelManager.showStatus(_("initializing sphinx grammar")) sphinxProg.parseString( "// force initialization\n", parseAll=True ) from arelle.Locale import format_string logMessage("INFO", "info", format_string(cntlr.modelManager.locale, _("Sphinx grammar initialized in %.2f secs"), time.time() - startedAt)) isGrammarCompiled = True return sphinxProg
def schemaValidate(modelXbrl): class schemaResolver(etree.Resolver): def resolve(self, url, id, context): if url.startswith("file:///__"): url = importedFilepaths[int(url[10:])] filepath = modelXbrl.modelManager.cntlr.webCache.getfilename(url) return self.resolve_filename(filepath, context) entryDocument = modelXbrl.modelDocument # test of schema validation using lxml (trial experiment, commented out for production use) from arelle import ModelDocument imports = [] importedNamespaces = set() importedFilepaths = [] ''' for mdlSchemaDoc in entryDocument.referencesDocument.keys(): if (mdlSchemaDoc.type == ModelDocument.Type.SCHEMA and mdlSchemaDoc.targetNamespace not in importedNamespaces): # actual file won't pass through properly, fake with table reference imports.append('<xsd:import namespace="{0}" schemaLocation="file:///__{1}"/>'.format( mdlSchemaDoc.targetNamespace, len(importedFilepaths))) importedNamespaces.add(mdlSchemaDoc.targetNamespace) importedFilepaths.append(mdlSchemaDoc.filepath) ''' def importReferences(referencingDocument): for mdlSchemaDoc in referencingDocument.referencesDocument.keys(): if (mdlSchemaDoc.type == ModelDocument.Type.SCHEMA and mdlSchemaDoc.targetNamespace not in importedNamespaces): importedNamespaces.add(mdlSchemaDoc.targetNamespace) importReferences(mdlSchemaDoc) # do dependencies first # actual file won't pass through properly, fake with table reference imports.append('<xsd:import namespace="{0}" schemaLocation="file:///__{1}"/>'.format( mdlSchemaDoc.targetNamespace, len(importedFilepaths))) importedFilepaths.append(mdlSchemaDoc.filepath) importReferences(entryDocument) # add schemas used in xml validation but not DTS discovered for mdlDoc in modelXbrl.urlDocs.values(): if mdlDoc.type in (ModelDocument.Type.INSTANCE, ModelDocument.Type.LINKBASE): schemaLocation = mdlDoc.xmlRootElement.get("{http://www.w3.org/2001/XMLSchema-instance}schemaLocation") if schemaLocation: ns = None for entry in schemaLocation.split(): if ns is None: ns = entry else: if ns not in importedNamespaces: imports.append('<xsd:import namespace="{0}" schemaLocation="file:///__{1}"/>'.format( ns, len(importedFilepaths))) importedNamespaces.add(ns) importedFilepaths.append(entry) ns = None schemaXml = '<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema">\n{0}</xsd:schema>\n'.format( '\n'.join(imports)) # trace schema files referenced with open("c:\\temp\\test.xml", "w") as fh: fh.write(schemaXml) modelXbrl.modelManager.showStatus(_("lxml validator loading xml schema")) schema_root = etree.XML(schemaXml) import time startedAt = time.time() parser = etree.XMLParser() parser.resolvers.add(schemaResolver()) schemaDoc = etree.fromstring(schemaXml, parser=parser, base_url=entryDocument.filepath+"-dummy-import.xsd") schema = etree.XMLSchema(schemaDoc) from arelle.Locale import format_string modelXbrl.info("info:lxmlSchemaValidator", format_string(modelXbrl.modelManager.locale, _("schema loaded in %.2f secs"), time.time() - startedAt)) modelXbrl.modelManager.showStatus(_("lxml schema validating")) # check instance documents and linkbases (sort for inst doc before linkbases, and in file name order) for mdlDoc in sorted(modelXbrl.urlDocs.values(), key=lambda mdlDoc: (-mdlDoc.type, mdlDoc.filepath)): if mdlDoc.type in (ModelDocument.Type.INSTANCE, ModelDocument.Type.LINKBASE): startedAt = time.time() docXmlTree = etree.parse(mdlDoc.filepath) modelXbrl.info("info:lxmlSchemaValidator", format_string(modelXbrl.modelManager.locale, _("schema validated in %.3f secs"), time.time() - startedAt), modelDocument=mdlDoc) if not schema.validate(docXmlTree): for error in schema.error_log: modelXbrl.error("lxmlSchema:{0}".format(error.type_name.lower()), error.message, modelDocument=mdlDoc, sourceLine=error.line) modelXbrl.modelManager.showStatus(_("lxml validation done"), clearAfter=3000)
def logArguments(self, codes, msg, codedArgs): # determine logCode messageCode = None for argCode in codes if isinstance(codes,tuple) else (codes,): if ((self.modelManager.disclosureSystem.EFM and argCode.startswith("EFM")) or (self.modelManager.disclosureSystem.GFM and argCode.startswith("GFM")) or (self.modelManager.disclosureSystem.HMRC and argCode.startswith("HMRC")) or (self.modelManager.disclosureSystem.SBRNL and argCode.startswith("SBR.NL")) or argCode[0:3] not in ("EFM", "GFM", "HMR", "SBR")): messageCode = argCode break # determine message and extra arguments fmtArgs = {} extras = {"messageCode":messageCode} for argName, argValue in codedArgs.items(): if argName in ("modelObject", "modelXbrl", "modelDocument"): try: entryUrl = self.modelDocument.uri except AttributeError: entryUrl = self.entryLoadingUrl try: objectUrl = argValue.modelDocument.uri except AttributeError: try: objectUrl = self.modelDocument.uri except AttributeError: objectUrl = self.entryLoadingUrl file = UrlUtil.relativeUri(entryUrl, objectUrl) extras["file"] = file if isinstance(argValue,ModelObject): extras["href"] = file + "#" + XmlUtil.elementFragmentIdentifier(argValue) extras["sourceLine"] = argValue.sourceline extras["objectId"] = argValue.objectId() else: extras["href"] = file extras["sourceLine"] = "" elif argName == "sourceLine": extras["sourceLine"] = argValue elif argName != "exc_info": if isinstance(argValue, (ModelValue.QName, ModelObject, bool)): fmtArgs[argName] = str(argValue) elif isinstance(argValue,int): # need locale-dependent formatting fmtArgs[argName] = format_string(self.modelManager.locale, '%i', argValue) elif isinstance(argValue,float): # need locale-dependent formatting fmtArgs[argName] = format_string(self.modelManager.locale, '%f', argValue) else: fmtArgs[argName] = argValue if "href" not in extras: try: file = os.path.basename(self.modelDocument.uri) except AttributeError: try: file = os.path.basename(self.entryLoadingUrl) except: file = "" extras["file"] = file extras["href"] = file extras["sourceLine"] = "" return (messageCode, (msg, fmtArgs) if fmtArgs else (msg,), extras)
def schemaValidate(modelXbrl): class schemaResolver(etree.Resolver): def resolve(self, url, id, context): if url.startswith("file:///__"): url = importedFilepaths[int(url[10:])] filepath = modelXbrl.modelManager.cntlr.webCache.getfilename(url) return self.resolve_filename(filepath, context) entryDocument = modelXbrl.modelDocument # test of schema validation using lxml (trial experiment, commented out for production use) from arelle import ModelDocument imports = [] importedNamespaces = set() importedFilepaths = [] ''' for mdlSchemaDoc in entryDocument.referencesDocument.keys(): if (mdlSchemaDoc.type == ModelDocument.Type.SCHEMA and mdlSchemaDoc.targetNamespace not in importedNamespaces): # actual file won't pass through properly, fake with table reference imports.append('<xsd:import namespace="{0}" schemaLocation="file:///__{1}"/>'.format( mdlSchemaDoc.targetNamespace, len(importedFilepaths))) importedNamespaces.add(mdlSchemaDoc.targetNamespace) importedFilepaths.append(mdlSchemaDoc.filepath) ''' def importReferences(referencingDocument): for mdlSchemaDoc in referencingDocument.referencesDocument.keys(): if (mdlSchemaDoc.type == ModelDocument.Type.SCHEMA and mdlSchemaDoc.targetNamespace not in importedNamespaces): importedNamespaces.add(mdlSchemaDoc.targetNamespace) importReferences(mdlSchemaDoc) # do dependencies first # actual file won't pass through properly, fake with table reference imports.append( '<xsd:import namespace="{0}" schemaLocation="file:///__{1}"/>' .format(mdlSchemaDoc.targetNamespace, len(importedFilepaths))) importedFilepaths.append(mdlSchemaDoc.filepath) importReferences(entryDocument) # add schemas used in xml validation but not DTS discovered for mdlDoc in modelXbrl.urlDocs.values(): if mdlDoc.type in (ModelDocument.Type.INSTANCE, ModelDocument.Type.LINKBASE): schemaLocation = mdlDoc.xmlRootElement.get( "{http://www.w3.org/2001/XMLSchema-instance}schemaLocation") if schemaLocation: ns = None for entry in schemaLocation.split(): if ns is None: ns = entry else: if ns not in importedNamespaces: imports.append( '<xsd:import namespace="{0}" schemaLocation="file:///__{1}"/>' .format(ns, len(importedFilepaths))) importedNamespaces.add(ns) importedFilepaths.append(entry) ns = None schemaXml = '<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema">\n{0}</xsd:schema>\n'.format( '\n'.join(imports)) # trace schema files referenced with open("c:\\temp\\test.xml", "w") as fh: fh.write(schemaXml) modelXbrl.modelManager.showStatus(_("lxml validator loading xml schema")) schema_root = etree.XML(schemaXml) import time startedAt = time.time() parser = etree.XMLParser() parser.resolvers.add(schemaResolver()) schemaDoc = etree.fromstring(schemaXml, parser=parser, base_url=entryDocument.filepath + "-dummy-import.xsd") schema = etree.XMLSchema(schemaDoc) from arelle.Locale import format_string modelXbrl.info( "info:lxmlSchemaValidator", format_string(modelXbrl.modelManager.locale, _("schema loaded in %.2f secs"), time.time() - startedAt)) modelXbrl.modelManager.showStatus(_("lxml schema validating")) # check instance documents and linkbases (sort for inst doc before linkbases, and in file name order) for mdlDoc in sorted(modelXbrl.urlDocs.values(), key=lambda mdlDoc: (-mdlDoc.type, mdlDoc.filepath)): if mdlDoc.type in (ModelDocument.Type.INSTANCE, ModelDocument.Type.LINKBASE): startedAt = time.time() docXmlTree = etree.parse(mdlDoc.filepath) modelXbrl.info("info:lxmlSchemaValidator", format_string(modelXbrl.modelManager.locale, _("schema validated in %.3f secs"), time.time() - startedAt), modelDocument=mdlDoc) if not schema.validate(docXmlTree): for error in schema.error_log: modelXbrl.error("lxmlSchema:{0}".format( error.type_name.lower()), error.message, modelDocument=mdlDoc, sourceLine=error.line) modelXbrl.modelManager.showStatus(_("lxml validation done"), clearAfter=3000)
def logArguments(self, codes, msg, codedArgs): """ Prepares arguments for logger function as per info() below. If codes includes EFM, GFM, HMRC, or SBR-coded error then the code chosen (if a sequence) corresponds to whether EFM, GFM, HMRC, or SBR validation is in effect. """ def propValues(properties): # deref objects in properties return [(p[0],str(p[1])) if len(p) == 2 else (p[0],str(p[1]),propValues(p[2])) for p in properties if 2 <= len(p) <= 3] # determine logCode messageCode = None for argCode in codes if isinstance(codes,tuple) else (codes,): if (isinstance(argCode, ModelValue.QName) or (self.modelManager.disclosureSystem.EFM and argCode.startswith("EFM")) or (self.modelManager.disclosureSystem.GFM and argCode.startswith("GFM")) or (self.modelManager.disclosureSystem.HMRC and argCode.startswith("HMRC")) or (self.modelManager.disclosureSystem.SBRNL and argCode.startswith("SBR.NL")) or argCode[0:3] not in ("EFM", "GFM", "HMR", "SBR")): messageCode = argCode break # determine message and extra arguments fmtArgs = {} extras = {"messageCode":messageCode} logHrefObjectProperties = getattr(self.logger, "logHrefObjectProperties", False) for argName, argValue in codedArgs.items(): if argName in ("modelObject", "modelXbrl", "modelDocument"): try: entryUrl = self.modelDocument.uri except AttributeError: entryUrl = self.entryLoadingUrl refs = [] for arg in (argValue if isinstance(argValue, (tuple,list,set)) else (argValue,)): if arg is not None: if isinstance(arg, _STR_BASE): objectUrl = arg else: try: objectUrl = arg.modelDocument.uri except AttributeError: try: objectUrl = self.modelDocument.uri except AttributeError: objectUrl = self.entryLoadingUrl file = UrlUtil.relativeUri(entryUrl, objectUrl) ref = {} if isinstance(arg,ModelObject): ref["href"] = file + "#" + XmlUtil.elementFragmentIdentifier(arg) ref["sourceLine"] = arg.sourceline ref["objectId"] = arg.objectId() if logHrefObjectProperties: try: ref["properties"] = propValues(arg.propertyView) except AttributeError: pass # is a default properties entry appropriate or needed? else: ref["href"] = file try: ref["sourceLine"] = arg.sourceline except AttributeError: pass # arg may not have sourceline, ignore if so refs.append(ref) extras["refs"] = refs elif argName == "sourceLine": if isinstance(argValue, _INT_TYPES): # must be sortable with int's in logger extras["sourceLine"] = argValue elif argName != "exc_info": if isinstance(argValue, (ModelValue.QName, ModelObject, bool, FileNamedStringIO, # might be a set of lxml objects not dereferencable at shutdown tuple, list, set)): fmtArgs[argName] = str(argValue) elif argValue is None: fmtArgs[argName] = "(none)" elif isinstance(argValue, _INT_TYPES): # need locale-dependent formatting fmtArgs[argName] = format_string(self.modelManager.locale, '%i', argValue) elif isinstance(argValue,float): # need locale-dependent formatting fmtArgs[argName] = format_string(self.modelManager.locale, '%f', argValue) else: fmtArgs[argName] = argValue if "refs" not in extras: try: file = os.path.basename(self.modelDocument.uri) except AttributeError: try: file = os.path.basename(self.entryLoadingUrl) except: file = "" extras["refs"] = [{"href": file}] return (messageCode, (msg, fmtArgs) if fmtArgs else (msg,), extras)