def backgroundLoadXbrl(self, filesource, importToDTS):
startedAt = time.time()
try:
if importToDTS:
action = _("imported")
modelXbrl = self.modelManager.modelXbrl
if modelXbrl:
ModelDocument.load(modelXbrl, filesource.url)
else:
action = _("loaded")
modelXbrl = self.modelManager.load(filesource, _("views loading"))
except Exception as err:
msg = _("Exception loading {0}: {1}, at {2}").format(
filesource.url,
err,
traceback.format_tb(sys.exc_info()[2]))
# not sure if message box can be shown from background thread
# tkinter.messagebox.showwarning(_("Exception loading"),msg, parent=self.parent)
self.addToLog(msg);
return
if modelXbrl and modelXbrl.modelDocument:
self.addToLog(format_string(self.modelManager.locale,
_("%s in %.2f secs"),
(action, time.time() - startedAt)))
self.showStatus(_("{0}, preparing views").format(action))
self.uiThreadQueue.put((self.showLoadedXbrl, [modelXbrl, importToDTS]))
else:
self.addToLog(format_string(self.modelManager.locale,
_("not successfully %s in %.2f secs"),
(action, time.time() - startedAt)))
def logProfileStats(self):
"""Logs profile stats that were collected
"""
timeTotal = format_string(self.modelManager.locale, _("%.3f secs"), self.profileStats.get("total", (0,0,0))[1])
timeEFM = format_string(self.modelManager.locale, _("%.3f secs"), self.profileStats.get("validateEFM", (0,0,0))[1])
self.info("info:profileStats",
_("Profile statistics \n") +
' \n'.join(format_string(self.modelManager.locale, _("%s %.3f secs, %.0fK"), (statName, statValue[1], statValue[2]), grouping=True)
for statName, statValue in sorted(self.profileStats.items(), key=lambda item: item[1])) +
" \n", # put instance reference on fresh line in traces
modelObject=self.modelXbrl.modelDocument, profileStats=self.profileStats,
timeTotal=timeTotal, timeEFM=timeEFM)
def xmlValidate(entryModelDocument):
# test of schema validation using lxml (trial experiment, commented out for production use)
modelXbrl = entryModelDocument.modelXbrl
from arelle import ModelDocument
imports = []
importedNamespaces = set()
for modelDocument in modelXbrl.urlDocs.values():
if (modelDocument.type == ModelDocument.Type.SCHEMA
and modelDocument.targetNamespace not in importedNamespaces):
imports.append(
'<xsd:import namespace="{0}" schemaLocation="{1}"/>'.format(
modelDocument.targetNamespace,
modelDocument.filepath.replace("\\", "/")))
importedNamespaces.add(modelDocument.targetNamespace)
if entryModelDocument.xmlRootElement.hasAttributeNS(
XbrlConst.xsi, "schemaLocation"):
ns = None
for entry in entryModelDocument.xmlRootElement.getAttributeNS(
XbrlConst.xsi, "schemaLocation").split():
if ns is None:
ns = entry
else:
if ns not in importedNamespaces:
imports.append(
'<xsd:import namespace="{0}" schemaLocation="{1}"/>'.
format(ns, entry))
importedNamespaces.add(ns)
ns = None
schema_root = etree.XML(
'<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema">{0}</xsd:schema>'
.format(''.join(imports)))
import time
startedAt = time.time()
schema = etree.XMLSchema(schema_root)
from arelle.Locale import format_string
modelXbrl.modelManager.addToLog(
format_string(modelXbrl.modelManager.locale,
_("schema loaded in %.2f secs"),
time.time() - startedAt))
startedAt = time.time()
instDoc = etree.parse(entryModelDocument.filepath)
modelXbrl.modelManager.addToLog(
format_string(modelXbrl.modelManager.locale,
_("instance parsed in %.2f secs"),
time.time() - startedAt))
if not schema.validate(instDoc):
for error in schema.error_log:
modelXbrl.error(str(error), "err", "xmlschema:error")
def storeIntoDB(dbConnection, modelXbrl, rssItem=None):
host = port = user = password = db = timeout = dbType = None
if isinstance(dbConnection, (list, tuple)): # variable length list
if len(dbConnection) > 0: host = dbConnection[0]
if len(dbConnection) > 1: port = dbConnection[1]
if len(dbConnection) > 2: user = dbConnection[2]
if len(dbConnection) > 3: password = dbConnection[3]
if len(dbConnection) > 4: db = dbConnection[4]
if len(dbConnection) > 5 and dbConnection[5] and dbConnection[5].isdigit():
timeout = int(dbConnection[5])
if len(dbConnection) > 6: dbType = dbConnection[6]
startedAt = time.time()
if dbType in dbTypes:
insertIntoDB = dbTypes[dbType]
elif isPostgresPort(host, port):
insertIntoDB = insertIntoPostgresDB
elif isRexsterPort(host, port):
insertIntoDB = insertIntoRexsterDB
elif isRdfPort(host, port, db):
insertIntoDB = insertIntoRdfDB
elif isJsonPort(host, port, db):
insertIntoDB = insertIntoJsonDB
else:
modelXbrl.modelManager.addToLog('Server at "{0}:{1}" is not recognized to be either a Postgres or a Rexter service.'.format(host, port))
return
insertIntoDB(modelXbrl, host=host, port=port, user=user, password=password, database=db, timeout=timeout, rssItem=rssItem)
modelXbrl.modelManager.addToLog(format_string(modelXbrl.modelManager.locale,
_("stored to database in %.2f secs"),
time.time() - startedAt), messageCode="info", file=modelXbrl.uri)
def showLoadedXbrl(self, modelXbrl, attach):
startedAt = time.time()
currentAction = "setting title"
try:
if attach:
modelXbrl.closeViews()
self.parent.title(_("arelle\u2122 - {0}").format(
os.path.basename(modelXbrl.modelDocument.uri)))
self.setValidateTooltipText()
if modelXbrl.modelDocument.type in (ModelDocument.Type.TESTCASESINDEX,
ModelDocument.Type.TESTCASE, ModelDocument.Type.REGISTRY, ModelDocument.Type.REGISTRYTESTCASE):
currentAction = "tree view of tests"
ViewWinTests.viewTests(modelXbrl, self.tabWinTopRt)
elif modelXbrl.modelDocument.type == ModelDocument.Type.VERSIONINGREPORT:
currentAction = "view of versioning report"
ViewWinVersReport.viewVersReport(modelXbrl, self.tabWinTopRt)
from arelle.ViewWinDiffs import ViewWinDiffs
ViewWinDiffs(modelXbrl, self.tabWinBtm, lang=self.lang)
elif modelXbrl.modelDocument.type == ModelDocument.Type.RSSFEED:
currentAction = "view of RSS feed"
ViewWinRssFeed.viewRssFeed(modelXbrl, self.tabWinTopRt)
else:
currentAction = "tree view of tests"
ViewWinDTS.viewDTS(modelXbrl, self.tabWinTopLeft)
currentAction = "view of concepts"
ViewWinConcepts.viewConcepts(modelXbrl, self.tabWinBtm, "Concepts", lang=self.lang)
if modelXbrl.hasEuRendering: # show rendering grid even without any facts
ViewWinRenderedGrid.viewRenderedGrid(modelXbrl, self.tabWinTopRt, lang=self.lang)
if modelXbrl.modelDocument.type in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL):
currentAction = "table view of facts"
if not modelXbrl.hasEuRendering: # table view only if not grid rendered view
ViewWinFactTable.viewFacts(modelXbrl, self.tabWinTopRt, lang=self.lang)
currentAction = "tree/list of facts"
ViewWinFactList.viewFacts(modelXbrl, self.tabWinTopRt, lang=self.lang)
if modelXbrl.hasFormulae:
currentAction = "formulae view"
ViewWinFormulae.viewFormulae(modelXbrl, self.tabWinTopRt)
currentAction = "presentation linkbase view"
ViewWinRelationshipSet.viewRelationshipSet(modelXbrl, self.tabWinTopRt, XbrlConst.parentChild, lang=self.lang)
currentAction = "calculation linkbase view"
ViewWinRelationshipSet.viewRelationshipSet(modelXbrl, self.tabWinTopRt, XbrlConst.summationItem, lang=self.lang)
currentAction = "dimensions relationships view"
ViewWinRelationshipSet.viewRelationshipSet(modelXbrl, self.tabWinTopRt, "XBRL-dimensions", lang=self.lang)
if modelXbrl.hasEuRendering:
currentAction = "rendering view"
ViewWinRelationshipSet.viewRelationshipSet(modelXbrl, self.tabWinTopRt, "EU-rendering", lang=self.lang)
currentAction = "property grid"
ViewWinProperties.viewProperties(modelXbrl, self.tabWinTopLeft)
currentAction = "log view creation time"
self.addToLog(format_string(self.modelManager.locale,
_("views %.2f secs"),
time.time() - startedAt))
except Exception as err:
msg = _("Exception preparing {0}: {1}, at {2}").format(
currentAction,
err,
traceback.format_tb(sys.exc_info()[2]))
tkinter.messagebox.showwarning(_("Exception preparing view"),msg, parent=self.parent)
self.addToLog(msg);
self.showStatus(_("Ready..."), 2000)
def backgroundCompareDTSes(self, versReportFile):
startedAt = time.time()
modelVersReport = self.modelManager.compareDTSes(versReportFile)
if modelVersReport and modelVersReport.modelDocument:
self.addToLog(format_string(self.modelManager.locale,
_("compared in %.2f secs"),
time.time() - startedAt))
self.uiThreadQueue.put((self.showComparedDTSes, [modelVersReport]))
def xmlValidate(entryModelDocument):
# test of schema validation using lxml (trial experiment, commented out for production use)
modelXbrl = entryModelDocument.modelXbrl
from arelle import ModelDocument
imports = []
importedNamespaces = set()
for modelDocument in modelXbrl.urlDocs.values():
if (modelDocument.type == ModelDocument.Type.SCHEMA and
modelDocument.targetNamespace not in importedNamespaces):
imports.append('<xsd:import namespace="{0}" schemaLocation="{1}"/>'.format(
modelDocument.targetNamespace, modelDocument.filepath.replace("\\","/")))
importedNamespaces.add(modelDocument.targetNamespace)
if entryModelDocument.xmlRootElement.hasAttributeNS(XbrlConst.xsi, "schemaLocation"):
ns = None
for entry in entryModelDocument.xmlRootElement.getAttributeNS(XbrlConst.xsi, "schemaLocation").split():
if ns is None:
ns = entry
else:
if ns not in importedNamespaces:
imports.append('<xsd:import namespace="{0}" schemaLocation="{1}"/>'.format(
ns, entry))
importedNamespaces.add(ns)
ns = None
schema_root = etree.XML(
'<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema">{0}</xsd:schema>'.format(
''.join(imports))
)
import time
startedAt = time.time()
schema = etree.XMLSchema(schema_root)
from arelle.Locale import format_string
modelXbrl.modelManager.addToLog(format_string(modelXbrl.modelManager.locale,
_("schema loaded in %.2f secs"),
time.time() - startedAt))
startedAt = time.time()
instDoc = etree.parse(entryModelDocument.filepath)
modelXbrl.modelManager.addToLog(format_string(modelXbrl.modelManager.locale,
_("instance parsed in %.2f secs"),
time.time() - startedAt))
if not schema.validate(instDoc):
for error in schema.error_log:
modelXbrl.error(
str(error),
"err", "xmlschema:error")
def storeIntoDB(dbConnection, modelXbrl):
host, port, user, password, db = dbConnection
startedAt = time.time()
if isPostgresPort(host, port):
insertIntoPostgresDB(modelXbrl, host=host, port=port, user=user, password=password, database=db)
elif isRexsterPort(host, port):
insertIntoRexsterDB(modelXbrl, host=host, port=port, user=user, password=password, database=db)
modelXbrl.modelManager.addToLog(format_string(modelXbrl.modelManager.locale,
_("stored to database in %.2f secs"),
time.time() - startedAt), messageCode="info", file=modelXbrl.uri)
def backgroundStoreIntoDB():
try:
host, port, user, password, db, timeout, dbType = dbConnection
product = None
if timeout and timeout.isdigit():
timeout = int(timeout)
# identify server
if dbType in dbTypes:
insertIntoDB = dbTypes[dbType]
product = dbProduct[dbType]
else:
cntlr.addToLog(_("Probing host {0} port {1} to determine server database type.")
.format(host, port))
if isPostgresPort(host, port):
dbType = "postgres"
insertIntoDB = insertIntoPostgresDB
elif isSemanticSqlPort(host, port):
dbType = "pgSemantic"
insertIntoDB = insertIntoPostgresDB
elif isRexsterPort(host, port):
dbType = "rexster"
insertIntoDB = insertIntoRexsterDB
elif isRdfPort(host, port, db):
dbType = "rdfDB"
insertIntoDB = insertIntoRdfDB
elif isJsonPort(host, port, db):
dbType = "json"
insertIntoDB = insertIntoJsonDB
else:
cntlr.addToLog(_("Unable to determine server type!\n ") +
_("Probing host {0} port {1} unable to determine server type.")
.format(host, port))
cntlr.config["xbrlDBconnection"] = (host, port, user, password, db, timeout, '') # forget type
cntlr.saveConfig()
return
cntlr.addToLog(_("Database type {} identified.").format(dbType))
cntlr.config["xbrlDBconnection"] = (host, port, user, password, db, timeout, dbType)
cntlr.saveConfig()
startedAt = time.time()
insertIntoDB(cntlr.modelManager.modelXbrl,
host=host, port=port, user=user, password=password, database=db, timeout=timeout,
product=product)
cntlr.addToLog(format_string(cntlr.modelManager.locale,
_("stored to database in %.2f secs"),
time.time() - startedAt))
except Exception as ex:
import traceback
cntlr.addToLog(
_("[xpDB:exception] Loading XBRL DB: %(exception)s: %(error)s \n%(traceback)s") %
{"exception": ex.__class__.__name__,
"error": str(ex),
"exc_info": True,
"traceback": traceback.format_tb(sys.exc_info()[2])})
cntlr.config["xbrlDBconnection"] = (host, port, user, password, db, timeout, '') # forget type
cntlr.saveConfig()
def backgroundStoreIntoDB():
try:
host, port, user, password, db, timeout, dbType = dbConnection
product = None
if timeout and timeout.isdigit():
timeout = int(timeout)
# identify server
if dbType in dbTypes:
insertIntoDB = dbTypes[dbType]
product = dbProduct[dbType]
else:
cntlr.addToLog(_("Probing host {0} port {1} to determine server database type.")
.format(host, port))
if isPostgresPort(host, port):
dbType = "postgres"
insertIntoDB = insertIntoPostgresDB
elif isSemanticSqlPort(host, port):
dbType = "pgSemantic"
insertIntoDB = insertIntoPostgresDB
elif isRexsterPort(host, port):
dbType = "rexster"
insertIntoDB = insertIntoRexsterDB
elif isRdfPort(host, port, db):
dbType = "rdfDB"
insertIntoDB = insertIntoRdfDB
elif isJsonPort(host, port, db):
dbType = "json"
insertIntoDB = insertIntoJsonDB
else:
cntlr.addToLog(_("Unable to determine server type!\n ") +
_("Probing host {0} port {1} unable to determine server type.")
.format(host, port))
cntlr.config["xbrlDBconnection"] = (host, port, user, password, db, timeout, '') # forget type
cntlr.saveConfig()
return
cntlr.addToLog(_("Database type {} identified.").format(dbType))
cntlr.config["xbrlDBconnection"] = (host, port, user, password, db, timeout, dbType)
cntlr.saveConfig()
startedAt = time.time()
insertIntoDB(cntlr.modelManager.modelXbrl,
host=host, port=port, user=user, password=password, database=db, timeout=timeout,
product=product)
cntlr.addToLog(format_string(cntlr.modelManager.locale,
_("stored to database in %.2f secs"),
time.time() - startedAt))
except Exception as ex:
import traceback
cntlr.addToLog(
_("[xpDB:exception] Loading XBRL DB: %(exception)s: %(error)s \n%(traceback)s") %
{"exception": ex.__class__.__name__,
"error": str(ex),
"exc_info": True,
"traceback": traceback.format_tb(sys.exc_info()[2])})
cntlr.config["xbrlDBconnection"] = (host, port, user, password, db, timeout, '') # forget type
cntlr.saveConfig()
def initializeParser(modelManager):
global isInitialized
if not isInitialized:
modelManager.showStatus(_("initializing formula xpath2 grammar"))
startedAt = time.time()
xpathExpr.parseString( "0", parseAll=True )
modelManager.addToLog(format_string(modelManager.locale,
_("Formula xpath2 grammar initialized in %.2f secs"),
time.time() - startedAt))
modelManager.showStatus(None)
isInitialized = True
def storeIntoDB(dbConnection, modelXbrl, rssItem=None, **kwargs):
host = port = user = password = db = timeout = dbType = None
if isinstance(dbConnection, (list, tuple)): # variable length list
if len(dbConnection) > 0: host = dbConnection[0]
if len(dbConnection) > 1: port = dbConnection[1]
if len(dbConnection) > 2: user = dbConnection[2]
if len(dbConnection) > 3: password = dbConnection[3]
if len(dbConnection) > 4: db = dbConnection[4]
if len(dbConnection
) > 5 and dbConnection[5] and dbConnection[5].isdigit():
timeout = int(dbConnection[5])
if len(dbConnection) > 6: dbType = dbConnection[6]
startedAt = time.time()
product = None
if dbType in dbTypes:
insertIntoDB = dbTypes[dbType]
product = dbProduct[dbType]
elif isPostgresPort(host, port):
insertIntoDB = insertIntoPostgresDB
elif isSemanticSqlPort(host, port):
insertIntoDB = insertIntoSemanticSqlDB
elif isRexsterPort(host, port):
insertIntoDB = insertIntoRexsterDB
elif isRdfPort(host, port, db):
insertIntoDB = insertIntoRdfDB
elif isJsonPort(host, port, db):
insertIntoDB = insertIntoJsonDB
else:
modelXbrl.modelManager.addToLog(
'Server at "{0}:{1}" is not recognized to be either a Postgres or a Rexter service.'
.format(host, port))
return
result = insertIntoDB(modelXbrl,
host=host,
port=port,
user=user,
password=password,
database=db,
timeout=timeout,
product=product,
rssItem=rssItem,
**kwargs)
if kwargs.get(
"logStoredMsg", result
): # if false/None result and no logStoredMsg parameter then skip the message
modelXbrl.modelManager.addToLog(format_string(
modelXbrl.modelManager.locale,
_("stored to database in %.2f secs"),
time.time() - startedAt),
messageCode="info",
file=modelXbrl.uri)
return result
def initializeParser(modelManager):
global isInitialized
if not isInitialized:
modelManager.showStatus(_("initializing formula xpath2 grammar"))
startedAt = time.time()
xpathExpr.parseString("0", parseAll=True)
modelManager.addToLog(
format_string(modelManager.locale,
_("Formula xpath2 grammar initialized in %.2f secs"),
time.time() - startedAt))
modelManager.showStatus(None)
isInitialized = True
def backgroundValidate(self):
startedAt = time.time()
modelXbrl = self.modelManager.modelXbrl
priorOutputInstance = modelXbrl.formulaOutputInstance
modelXbrl.formulaOutputInstance = None # prevent closing on background thread by validateFormula
self.modelManager.validate()
self.addToLog(format_string(self.modelManager.locale,
_("validated in %.2f secs"),
time.time() - startedAt))
if modelXbrl and (priorOutputInstance or modelXbrl.formulaOutputInstance):
self.uiThreadQueue.put((self.showFormulaOutputInstance, [priorOutputInstance, modelXbrl.formulaOutputInstance]))
self.uiThreadQueue.put((self.logSelect, []))
def initializeParser(modelManager):
global isInitialized
if not isInitialized:
modelManager.showStatus(_(u"initializing formula xpath2 grammar"))
startedAt = time.time()
xpathExpr.parseString( u"0", parseAll=True )
modelManager.addToLog(format_string(modelManager.locale,
_(u"Formula xpath2 grammar initialized in %.2f secs"),
time.time() - startedAt))
modelManager.showStatus(None)
isInitialized = True
return True # was initialized on this call
return False # had already been initialized
def initializeParser(modelManager):
global isInitialized, FunctionIxt
if not isInitialized:
from arelle import FunctionIxt
modelManager.showStatus(_("initializing formula xpath2 grammar"))
startedAt = time.time()
xpathExpr.parseString( "0", parseAll=True )
modelManager.addToLog(format_string(modelManager.locale,
_("Formula xpath2 grammar initialized in %.2f secs"),
time.time() - startedAt))
modelManager.showStatus(None)
isInitialized = True
return True # was initialized on this call
return False # had already been initialized
def initializeParser(modelManager):
global isInitialized, FunctionIxt
if not isInitialized:
from arelle import FunctionIxt
modelManager.showStatus(_("initializing formula xpath2 grammar"))
startedAt = time.time()
xpathExpr.parseString( "0", parseAll=True )
if not(modelManager.cntlr.testMode):
modelManager.addToLog(format_string(modelManager.locale,
_("Formula xpath2 grammar initialized in %.2f secs"),
time.time() - startedAt))
modelManager.showStatus(None)
isInitialized = True
return True # was initialized on this call
return False # had already been initialized
def backgroundStoreIntoDB():
try:
startedAt = time.time()
insertIntoDB(cntlr.modelManager.modelXbrl,
host=host, port=port, user=user, password=password, database=db, timeout=timeout)
cntlr.addToLog(format_string(cntlr.modelManager.locale,
_("stored to database in %.2f secs"),
time.time() - startedAt))
except Exception as ex:
import traceback
cntlr.addToLog(
_("[xpDB:exception] Loading XBRL DB: %(exception)s: %(error)s \n%(traceback)s") %
{"exception": ex.__class__.__name__,
"error": str(ex),
"exc_info": True,
"traceback": traceback.format_tb(sys.exc_info()[2])})
def storeIntoDB(dbConnection, modelXbrl):
host, port, user, password, db = dbConnection
startedAt = time.time()
if isPostgresPort(host, port):
insertIntoPostgresDB(modelXbrl,
host=host,
port=port,
user=user,
password=password,
database=db)
elif isRexsterPort(host, port):
insertIntoRexsterDB(modelXbrl,
host=host,
port=port,
user=user,
password=password,
database=db)
modelXbrl.modelManager.addToLog(format_string(
modelXbrl.modelManager.locale, _("stored to database in %.2f secs"),
time.time() - startedAt),
messageCode="info",
file=modelXbrl.uri)
def backgroundStoreIntoDB():
try:
startedAt = time.time()
insertIntoDB(cntlr.modelManager.modelXbrl,
host=host,
port=port,
user=user,
password=password,
database=db)
cntlr.addToLog(
format_string(cntlr.modelManager.locale,
_("stored to database in %.2f secs"),
time.time() - startedAt))
except Exception as ex:
import traceback
cntlr.addToLog(
_("[xpDB:exception] Loading XBRL DB: %(exception)s: %(error)s \n%(traceback)s"
) % {
"exception": ex.__class__.__name__,
"error": str(ex),
"exc_info": True,
"traceback": traceback.format_tb(sys.exc_info()[2])
})
def compileCdrGrammar( cntlr, _logMessage ):
global isGrammarCompiled, cdrProg, lineno
if isGrammarCompiled:
return cdrProg
global logMessage
logMessage = _logMessage
debugParsing = False # True
cntlr.showStatus(_("Compiling CDR Grammar"))
if sys.version[0] >= '3':
# python 3 requires modified parser to allow release of global objects when closing DTS
from arelle.pyparsing.pyparsing_py3 import (Word, Keyword, alphas,
Literal, CaselessLiteral,
Combine, Optional, nums, Or, Forward, Group, ZeroOrMore, StringEnd, alphanums,
ParserElement, sglQuotedString, delimitedList, Suppress, Regex, FollowedBy,
lineno, restOfLine)
else:
from pyparsing import (Word, Keyword, alphas,
Literal, CaselessLiteral,
Combine, Optional, nums, Or, Forward, Group, ZeroOrMore, StringEnd, alphanums,
ParserElement, sglQuotedString, delimitedList, Suppress, Regex, FollowedBy,
lineno, restOfLine)
ParserElement.enablePackrat()
"""
the pyparsing parser constructs are defined in this method to prevent the need to compile
the grammar when the plug in is loaded (which is likely to be when setting up GUI
menus or command line parser).
instead the grammar is compiled the first time that any sphinx needs to be parsed
only the sphinxExpression (result below) needs to be global for the parser
"""
# define grammar
periodOffset = Regex("-?P[1-3]?[0-9][YQMD](/-[1]?[0-9]-([1-3]?[0-9]|end))?")
qName = Regex("([A-Za-z\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD_]"
"[A-Za-z0-9\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\u0300-\u036F\u203F-\u2040\xB7_.-]*:)?"
# localname or wildcard-localname part
"([A-Za-z\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD_]"
"[A-Za-z0-9\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\u0300-\u036F\u203F-\u2040\xB7_.-]*|[*])"
).setName("qName").setDebug(debugParsing)
ncName = Regex("([A-Za-z\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD_]"
"[A-Za-z0-9\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\u0300-\u036F\u203F-\u2040\xB7_.-]*)"
).setName("ncName").setDebug(debugParsing)
decimalPoint = Literal('.')
exponentLiteral = CaselessLiteral('e')
plusorminusLiteral = Literal('+') | Literal('-')
digits = Word(nums)
integerLiteral = Combine( Optional(plusorminusLiteral) + digits )
decimalFractionLiteral = Combine( Optional(plusorminusLiteral) + decimalPoint + digits )
infLiteral = Combine( Optional(plusorminusLiteral) + Literal("INF") )
nanLiteral = Literal("NaN")
floatLiteral = ( Combine( integerLiteral +
( ( decimalPoint + Optional(digits) + exponentLiteral + integerLiteral ) |
( exponentLiteral + integerLiteral ) |
( decimalPoint + Optional(digits) ) )
) |
Combine( decimalFractionLiteral + exponentLiteral + integerLiteral ) |
decimalFractionLiteral |
infLiteral | nanLiteral )
#emptySequence = Literal( "(" ) + Literal( ")" )
lParen = Literal( "(" )
rParen = Literal( ")" )
lPred = Literal("[")
rPred = Literal("]")
refOp = Literal("#")
commaOp = Literal(",")
neOp = Literal("<>")
leOp = Literal("<=")
ltOp = Literal("<")
geOp = Literal(">=")
gtOp = Literal(">")
eqOp = Literal("=")
eqNeOp = eqOp | neOp
compOp = leOp | ltOp | geOp | gtOp
plusOp = Literal("+")
minusOp = Literal("-")
plusMinusOp = plusOp | minusOp
expOp = Literal("^")
multOp = Literal("*")
divOp = Literal("/")
multDivOp = multOp | divOp
concatOp = Literal("&")
andOp = CaselessLiteral("And")
orOp = CaselessLiteral("Or")
xorOp = CaselessLiteral("Xor")
expr = Forward()
atom = (
( refOp + qName ).setParseAction(compileRefExpression) |
( qName + Suppress(lParen) + Optional(delimitedList( expr )) + Suppress(rParen) ).setParseAction(compileFunctionReference) |
( qName.setParseAction(compileQname) + lPred + ( periodOffset | ncName)
+ rPred).setParseAction(compilePeriodOffsetExpression) |
( floatLiteral ).setParseAction(compileFloatLiteral) |
( integerLiteral ).setParseAction(compileIntegerLiteral) |
( sglQuotedString ).setParseAction(compileStringLiteral) |
( qName ).setParseAction(compileQname) |
( Suppress(lParen) - expr - Optional( commaOp - Optional( expr - ZeroOrMore( commaOp - expr ) ) ) - Suppress(rParen) ).setParseAction(compileBrackets)
)
atom.setName("atom").setDebug(debugParsing)
valueExpr = atom
negationExpr = ( Optional(minusOp) + valueExpr ).setParseAction(compileUnaryOperation)
expExpr = ( Optional(expOp) + negationExpr ).setParseAction(compileUnaryOperation)
multDivExpr = ( expExpr + Optional( multDivOp + expExpr ) ).setParseAction(compileBinaryOperation)
multDivExpr.setName("multDivExpr").setDebug(debugParsing)
addSubExpr = ( multDivExpr + ZeroOrMore( plusMinusOp + multDivExpr ) ).setParseAction(compileBinaryOperation)
addSubExpr.setName("addSubExpr").setDebug(debugParsing)
concatExpr = ( addSubExpr + ZeroOrMore( concatOp + addSubExpr ) ).setParseAction(compileBinaryOperation)
comparisonExpr = ( concatExpr + Optional( compOp + concatExpr ) ).setParseAction(compileBinaryOperation)
equalityExpr = ( comparisonExpr + Optional( eqNeOp + comparisonExpr ) ).setParseAction(compileBinaryOperation)
xorExpr = ( equalityExpr + ZeroOrMore( xorOp + equalityExpr) ).setParseAction(compileBinaryOperation)
andExpr = ( xorExpr + ZeroOrMore( andOp + xorExpr ) ).setParseAction(compileBinaryOperation)
orExpr = ( andExpr + ZeroOrMore( orOp + andExpr ) ).setParseAction(compileBinaryOperation)
orExpr.setName("orExpr").setDebug(debugParsing)
expr << orExpr
expr.setName("expr").setDebug(debugParsing)
cdrProg = expr + StringEnd()
expr.setName("cdrProg").setDebug(debugParsing)
startedAt = time.time()
cntlr.modelManager.showStatus(_("initializing CDR grammar"))
cdrProg.parseString( "0", parseAll=True )
from arelle.Locale import format_string
_msg = format_string(cntlr.modelManager.locale,
_("CDR grammar initialized in %.2f secs"),
time.time() - startedAt)
logMessage("INFO", "info", _msg)
cntlr.modelManager.showStatus(_msg, 5000)
isGrammarCompiled = True
return cdrProg
def logArguments(self, codes, msg, codedArgs):
# determine logCode
messageCode = None
for argCode in codes if isinstance(codes, tuple) else (codes, ):
if (isinstance(argCode, ModelValue.QName)
or (self.modelManager.disclosureSystem.EFM
and argCode.startswith("EFM"))
or (self.modelManager.disclosureSystem.GFM
and argCode.startswith("GFM"))
or (self.modelManager.disclosureSystem.HMRC
and argCode.startswith("HMRC"))
or (self.modelManager.disclosureSystem.SBRNL
and argCode.startswith("SBR.NL"))
or argCode[0:3] not in ("EFM", "GFM", "HMR", "SBR")):
messageCode = argCode
break
# determine message and extra arguments
fmtArgs = {}
extras = {"messageCode": messageCode}
for argName, argValue in codedArgs.items():
if argName in ("modelObject", "modelXbrl", "modelDocument"):
try:
entryUrl = self.modelDocument.uri
except AttributeError:
entryUrl = self.entryLoadingUrl
refs = []
for arg in (argValue if isinstance(argValue,
(tuple, list)) else
(argValue, )):
if arg is not None:
if isinstance(arg, _STR_BASE):
objectUrl = arg
else:
try:
objectUrl = arg.modelDocument.uri
except AttributeError:
try:
objectUrl = self.modelDocument.uri
except AttributeError:
objectUrl = self.entryLoadingUrl
file = UrlUtil.relativeUri(entryUrl, objectUrl)
ref = {}
if isinstance(arg, ModelObject):
ref["href"] = file + "#" + XmlUtil.elementFragmentIdentifier(
arg)
ref["sourceLine"] = arg.sourceline
ref["objectId"] = arg.objectId()
else:
ref["href"] = file
refs.append(ref)
extras["refs"] = refs
elif argName == "sourceLine":
if isinstance(
argValue,
_INT_TYPES): # must be sortable with int's in logger
extras["sourceLine"] = argValue
elif argName != "exc_info":
if isinstance(
argValue,
(ModelValue.QName, ModelObject, bool, FileNamedStringIO)):
fmtArgs[argName] = str(argValue)
elif isinstance(argValue, _INT_TYPES):
# need locale-dependent formatting
fmtArgs[argName] = format_string(self.modelManager.locale,
'%i', argValue)
elif isinstance(argValue, float):
# need locale-dependent formatting
fmtArgs[argName] = format_string(self.modelManager.locale,
'%f', argValue)
else:
fmtArgs[argName] = argValue
if "refs" not in extras:
try:
file = os.path.basename(self.modelDocument.uri)
except AttributeError:
try:
file = os.path.basename(self.entryLoadingUrl)
except:
file = ""
extras["refs"] = [{"href": file}]
return (messageCode, (msg, fmtArgs) if fmtArgs else (msg, ), extras)
def run(self, options, sourceZipStream=None):
"""Process command line arguments or web service request, such as to load and validate an XBRL document, or start web server.
When a web server has been requested, this method may be called multiple times, once for each web service (REST) request that requires processing.
Otherwise (when called for a command line request) this method is called only once for the command line arguments request.
:param options: OptionParser options from parse_args of main argv arguments (when called from command line) or corresponding arguments from web service (REST) request.
:type options: optparse.Values
"""
if options.showOptions: # debug options
for optName, optValue in sorted(options.__dict__.items(), key=lambda optItem: optItem[0]):
self.addToLog("Option {0}={1}".format(optName, optValue), messageCode="info")
self.addToLog("sys.argv {0}".format(sys.argv), messageCode="info")
if options.uiLang: # set current UI Lang (but not config setting)
self.setUiLanguage(options.uiLang)
if options.proxy:
if options.proxy != "show":
proxySettings = proxyTuple(options.proxy)
self.webCache.resetProxies(proxySettings)
self.config["proxySettings"] = proxySettings
self.saveConfig()
self.addToLog(_("Proxy configuration has been set."), messageCode="info")
useOsProxy, urlAddr, urlPort, user, password = self.config.get("proxySettings", proxyTuple("none"))
if useOsProxy:
self.addToLog(_("Proxy configured to use {0}.").format(
_('Microsoft Windows Internet Settings') if sys.platform.startswith("win")
else (_('Mac OS X System Configuration') if sys.platform in ("darwin", "macos")
else _('environment variables'))), messageCode="info")
elif urlAddr:
self.addToLog(_("Proxy setting: http://{0}{1}{2}{3}{4}").format(
user if user else "",
":****" if password else "",
"@" if (user or password) else "",
urlAddr,
":{0}".format(urlPort) if urlPort else ""), messageCode="info")
else:
self.addToLog(_("Proxy is disabled."), messageCode="info")
if options.plugins:
from arelle import PluginManager
resetPlugins = False
savePluginChanges = True
showPluginModules = False
for pluginCmd in options.plugins.split('|'):
cmd = pluginCmd.strip()
if cmd == "show":
showPluginModules = True
elif cmd == "temp":
savePluginChanges = False
elif cmd.startswith("+"):
moduleInfo = PluginManager.addPluginModule(cmd[1:])
if moduleInfo:
self.addToLog(_("Addition of plug-in {0} successful.").format(moduleInfo.get("name")),
messageCode="info", file=moduleInfo.get("moduleURL"))
resetPlugins = True
else:
self.addToLog(_("Unable to load plug-in."), messageCode="info", file=cmd[1:])
elif cmd.startswith("~"):
if PluginManager.reloadPluginModule(cmd[1:]):
self.addToLog(_("Reload of plug-in successful."), messageCode="info", file=cmd[1:])
resetPlugins = True
else:
self.addToLog(_("Unable to reload plug-in."), messageCode="info", file=cmd[1:])
elif cmd.startswith("-"):
if PluginManager.removePluginModule(cmd[1:]):
self.addToLog(_("Deletion of plug-in successful."), messageCode="info", file=cmd[1:])
resetPlugins = True
else:
self.addToLog(_("Unable to delete plug-in."), messageCode="info", file=cmd[1:])
else: # assume it is a module or package
savePluginChanges = False
moduleInfo = PluginManager.addPluginModule(cmd)
if moduleInfo:
self.addToLog(_("Activation of plug-in {0} successful.").format(moduleInfo.get("name")),
messageCode="info", file=moduleInfo.get("moduleURL"))
resetPlugins = True
else:
self.addToLog(_("Unable to load {0} as a plug-in or {0} is not recognized as a command. ").format(cmd), messageCode="info", file=cmd)
if resetPlugins:
PluginManager.reset()
if savePluginChanges:
PluginManager.save(self)
if showPluginModules:
self.addToLog(_("Plug-in modules:"), messageCode="info")
for i, moduleItem in enumerate(sorted(PluginManager.pluginConfig.get("modules", {}).items())):
moduleInfo = moduleItem[1]
self.addToLog(_("Plug-in: {0}; author: {1}; version: {2}; status: {3}; date: {4}; description: {5}; license {6}.").format(
moduleItem[0], moduleInfo.get("author"), moduleInfo.get("version"), moduleInfo.get("status"),
moduleInfo.get("fileDate"), moduleInfo.get("description"), moduleInfo.get("license")),
messageCode="info", file=moduleInfo.get("moduleURL"))
# run utility command line options that don't depend on entrypoint Files
hasUtilityPlugin = False
for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Utility.Run"):
hasUtilityPlugin = True
pluginXbrlMethod(self, options)
# if no entrypointFile is applicable, quit now
if options.proxy or options.plugins or hasUtilityPlugin:
if not options.entrypointFile:
return True # success
self.username = options.username
self.password = options.password
self.entrypointFile = options.entrypointFile
if self.entrypointFile:
filesource = FileSource.openFileSource(self.entrypointFile, self, sourceZipStream)
else:
filesource = None
if options.validateEFM:
if options.disclosureSystemName:
self.addToLog(_("both --efm and --disclosureSystem validation are requested, proceeding with --efm only"),
messageCode="info", file=self.entrypointFile)
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select("efm")
elif options.disclosureSystemName:
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select(options.disclosureSystemName)
elif options.validateHMRC:
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select("hmrc")
else:
self.modelManager.disclosureSystem.select(None) # just load ordinary mappings
self.modelManager.validateDisclosureSystem = False
if options.utrUrl: # override disclosureSystem utrUrl
self.modelManager.disclosureSystem.utrUrl = options.utrUrl
# can be set now because the utr is first loaded at validation time
# disclosure system sets logging filters, override disclosure filters, if specified by command line
if options.logLevelFilter:
self.setLogLevelFilter(options.logLevelFilter)
if options.logCodeFilter:
self.setLogCodeFilter(options.logCodeFilter)
if options.calcDecimals:
if options.calcPrecision:
self.addToLog(_("both --calcDecimals and --calcPrecision validation are requested, proceeding with --calcDecimals only"),
messageCode="info", file=self.entrypointFile)
self.modelManager.validateInferDecimals = True
self.modelManager.validateCalcLB = True
elif options.calcPrecision:
self.modelManager.validateInferDecimals = False
self.modelManager.validateCalcLB = True
if options.utrValidate:
self.modelManager.validateUtr = True
if options.infosetValidate:
self.modelManager.validateInfoset = True
if options.abortOnMajorError:
self.modelManager.abortOnMajorError = True
if options.collectProfileStats:
self.modelManager.collectProfileStats = True
if options.internetConnectivity == "offline":
self.webCache.workOffline = True
elif options.internetConnectivity == "online":
self.webCache.workOffline = False
if options.internetTimeout is not None:
self.webCache.timeout = (options.internetTimeout or None) # use None if zero specified to disable timeout
fo = FormulaOptions()
if options.parameters:
parameterSeparator = (options.parameterSeparator or ',')
fo.parameterValues = dict(((qname(key, noPrefixIsNoNamespace=True),(None,value))
for param in options.parameters.split(parameterSeparator)
for key,sep,value in (param.partition('='),) ) )
if options.formulaParamExprResult:
fo.traceParameterExpressionResult = True
if options.formulaParamInputValue:
fo.traceParameterInputValue = True
if options.formulaCallExprSource:
fo.traceCallExpressionSource = True
if options.formulaCallExprCode:
fo.traceCallExpressionCode = True
if options.formulaCallExprEval:
fo.traceCallExpressionEvaluation = True
if options.formulaCallExprResult:
fo.traceCallExpressionResult = True
if options.formulaVarSetExprEval:
fo.traceVariableSetExpressionEvaluation = True
if options.formulaVarSetExprResult:
fo.traceVariableSetExpressionResult = True
if options.formulaAsserResultCounts:
fo.traceAssertionResultCounts = True
if options.formulaFormulaRules:
fo.traceFormulaRules = True
if options.formulaVarsOrder:
fo.traceVariablesOrder = True
if options.formulaVarExpressionSource:
fo.traceVariableExpressionSource = True
if options.formulaVarExpressionCode:
fo.traceVariableExpressionCode = True
if options.formulaVarExpressionEvaluation:
fo.traceVariableExpressionEvaluation = True
if options.formulaVarExpressionResult:
fo.traceVariableExpressionResult = True
if options.timeVariableSetEvaluation:
fo.timeVariableSetEvaluation = True
if options.formulaVarFilterWinnowing:
fo.traceVariableFilterWinnowing = True
if options.formulaVarFiltersResult:
fo.traceVariableFiltersResult = True
if options.formulaVarFiltersResult:
fo.traceVariableFiltersResult = True
self.modelManager.formulaOptions = fo
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
firstStartedAt = startedAt = time.time()
modelDiffReport = None
success = True
modelXbrl = None
try:
if filesource:
modelXbrl = self.modelManager.load(filesource, _("views loading"))
except ModelDocument.LoadingException:
pass
except Exception as err:
self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format(
err,
traceback.format_tb(sys.exc_info()[2])))
success = False # loading errors, don't attempt to utilize loaded DTS
if modelXbrl and modelXbrl.modelDocument:
loadTime = time.time() - startedAt
modelXbrl.profileStat(_("load"), loadTime)
self.addToLog(format_string(self.modelManager.locale,
_("loaded in %.2f secs at %s"),
(loadTime, timeNow)),
messageCode="info", file=self.entrypointFile)
if options.importFiles:
for importFile in options.importFiles.split("|"):
fileName = importFile.strip()
if sourceZipStream is not None and not (fileName.startswith('http://') or os.path.isabs(fileName)):
fileName = os.path.dirname(modelXbrl.uri) + os.sep + fileName # make relative to sourceZipStream
ModelDocument.load(modelXbrl, fileName)
loadTime = time.time() - startedAt
self.addToLog(format_string(self.modelManager.locale,
_("import in %.2f secs at %s"),
(loadTime, timeNow)),
messageCode="info", file=importFile)
modelXbrl.profileStat(_("import"), loadTime)
if modelXbrl.errors:
success = False # loading errors, don't attempt to utilize loaded DTS
if modelXbrl.modelDocument.type in ModelDocument.Type.TESTCASETYPES:
for pluginXbrlMethod in pluginClassMethods("Testcases.Start"):
pluginXbrlMethod(self, options, modelXbrl)
else: # not a test case, probably instance or DTS
for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Loaded"):
pluginXbrlMethod(self, options, modelXbrl)
else:
success = False
if success and options.diffFile and options.versReportFile:
try:
diffFilesource = FileSource.FileSource(options.diffFile,self)
startedAt = time.time()
modelXbrl2 = self.modelManager.load(diffFilesource, _("views loading"))
if modelXbrl2.errors:
if not options.keepOpen:
modelXbrl2.close()
success = False
else:
loadTime = time.time() - startedAt
modelXbrl.profileStat(_("load"), loadTime)
self.addToLog(format_string(self.modelManager.locale,
_("diff comparison DTS loaded in %.2f secs"),
loadTime),
messageCode="info", file=self.entrypointFile)
startedAt = time.time()
modelDiffReport = self.modelManager.compareDTSes(options.versReportFile)
diffTime = time.time() - startedAt
modelXbrl.profileStat(_("diff"), diffTime)
self.addToLog(format_string(self.modelManager.locale,
_("compared in %.2f secs"),
diffTime),
messageCode="info", file=self.entrypointFile)
except ModelDocument.LoadingException:
success = False
except Exception as err:
success = False
self.addToLog(_("[Exception] Failed to doad diff file: \n{0} \n{1}").format(
err,
traceback.format_tb(sys.exc_info()[2])))
if success:
try:
modelXbrl = self.modelManager.modelXbrl
hasFormulae = modelXbrl.hasFormulae
if options.validate:
startedAt = time.time()
if options.formulaAction: # don't automatically run formulas
modelXbrl.hasFormulae = False
self.modelManager.validate()
if options.formulaAction: # restore setting
modelXbrl.hasFormulae = hasFormulae
self.addToLog(format_string(self.modelManager.locale,
_("validated in %.2f secs"),
time.time() - startedAt),
messageCode="info", file=self.entrypointFile)
if options.formulaAction in ("validate", "run"): # do nothing here if "none"
from arelle import ValidateXbrlDimensions, ValidateFormula
startedAt = time.time()
if not options.validate:
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
# setup fresh parameters from formula optoins
modelXbrl.parameters = fo.typedParameters()
ValidateFormula.validate(modelXbrl, compileOnly=(options.formulaAction != "run"))
self.addToLog(format_string(self.modelManager.locale,
_("formula validation and execution in %.2f secs")
if options.formulaAction == "run"
else _("formula validation only in %.2f secs"),
time.time() - startedAt),
messageCode="info", file=self.entrypointFile)
if options.testReport:
ViewFileTests.viewTests(self.modelManager.modelXbrl, options.testReport, options.testReportCols)
if options.rssReport:
ViewFileRssFeed.viewRssFeed(self.modelManager.modelXbrl, options.rssReport, options.rssReportCols)
if options.DTSFile:
ViewFileDTS.viewDTS(modelXbrl, options.DTSFile)
if options.factsFile:
ViewFileFactList.viewFacts(modelXbrl, options.factsFile, labelrole=options.labelRole, lang=options.labelLang, cols=options.factListCols)
if options.factTableFile:
ViewFileFactTable.viewFacts(modelXbrl, options.factTableFile, labelrole=options.labelRole, lang=options.labelLang)
if options.conceptsFile:
ViewFileConcepts.viewConcepts(modelXbrl, options.conceptsFile, labelrole=options.labelRole, lang=options.labelLang)
if options.preFile:
ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.preFile, "Presentation Linkbase", "http://www.xbrl.org/2003/arcrole/parent-child", labelrole=options.labelRole, lang=options.labelLang)
if options.calFile:
ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.calFile, "Calculation Linkbase", "http://www.xbrl.org/2003/arcrole/summation-item", labelrole=options.labelRole, lang=options.labelLang)
if options.dimFile:
ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.dimFile, "Dimensions", "XBRL-dimensions", labelrole=options.labelRole, lang=options.labelLang)
if options.formulaeFile:
ViewFileFormulae.viewFormulae(modelXbrl, options.formulaeFile, "Formulae", lang=options.labelLang)
if options.viewArcrole and options.viewFile:
ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.viewFile, os.path.basename(options.viewArcrole), options.viewArcrole, labelrole=options.labelRole, lang=options.labelLang)
for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Run"):
pluginXbrlMethod(self, options, modelXbrl)
except (IOError, EnvironmentError) as err:
self.addToLog(_("[IOError] Failed to save output:\n {0}").format(err))
success = False
except Exception as err:
self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format(
err,
traceback.format_tb(sys.exc_info()[2])))
success = False
if modelXbrl:
modelXbrl.profileStat(_("total"), time.time() - firstStartedAt)
if options.collectProfileStats and modelXbrl:
modelXbrl.logProfileStats()
if not options.keepOpen:
if modelDiffReport:
self.modelManager.close(modelDiffReport)
elif modelXbrl:
self.modelManager.close(modelXbrl)
self.username = self.password = None #dereference password
return success
def run(self, options):
if options.logFile:
self.messages = []
else:
self.messages = None
self.filename = options.filename
filesource = FileSource.FileSource(self.filename, self)
if options.validateEFM:
if options.gfmName:
self.addToLog(
_("[info] both --efm and --gfm validation are requested, proceeding with --efm only"
))
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select("efm")
elif options.gfmName:
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select(options.gfmName)
else:
self.modelManager.disclosureSystem.select(
None) # just load ordinary mappings
if options.calcDecimals:
if options.calcPrecision:
self.addToLog(
_("[info] both --calcDecimals and --calcPrecision validation are requested, proceeding with --calcDecimals only"
))
self.modelManager.validateInferDecimals = True
self.modelManager.validateCalcLB = True
elif options.calcPrecision:
self.modelManager.validateInferDecimals = False
self.modelManager.validateCalcLB = True
if options.utrValidate:
self.modelManager.validateUtr = True
fo = FormulaOptions()
if options.formulaParamExprResult:
fo.traceParameterExpressionResult = True
if options.formulaParamInputValue:
fo.traceParameterInputValue = True
if options.formulaCallExprSource:
fo.traceCallExpressionSource = True
if options.formulaCallExprCode:
fo.traceCallExpressionCode = True
if options.formulaCallExprEval:
fo.traceCallExpressionEvaluation = True
if options.formulaCallExprResult:
fo.traceCallExpressionResult = True
if options.formulaVarSetExprEval:
fo.traceVariableSetExpressionEvaluation = True
if options.formulaVarSetExprResult:
fo.traceVariableSetExpressionResult = True
if options.formulaAsserResultCounts:
fo.traceAssertionResultCounts = True
if options.formulaFormulaRules:
fo.traceFormulaRules = True
if options.formulaVarsOrder:
fo.traceVariablesOrder = True
if options.formulaVarExpressionSource:
fo.traceVariableExpressionSource = True
if options.formulaVarExpressionCode:
fo.traceVariableExpressionCode = True
if options.formulaVarExpressionEvaluation:
fo.traceVariableExpressionEvaluation = True
if options.formulaVarExpressionResult:
fo.traceVariableExpressionResult = True
if options.formulaVarFiltersResult:
fo.traceVariableFiltersResult = True
self.modelManager.formulaOptions = fo
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
startedAt = time.time()
modelXbrl = self.modelManager.load(filesource, _("views loading"))
self.addToLog(
format_string(self.modelManager.locale,
_("[info] loaded in %.2f secs at %s"),
(time.time() - startedAt, timeNow)))
if options.diffFilename and options.versReportFilename:
diffFilesource = FileSource.FileSource(self.diffFilename, self)
startedAt = time.time()
modelXbrl = self.modelManager.load(diffFilesource,
_("views loading"))
self.addToLog(
format_string(
self.modelManager.locale,
_("[info] diff comparison DTS loaded in %.2f secs"),
time.time() - startedAt))
startedAt = time.time()
self.modelManager.compareDTSes(options.versReportFilename)
self.addToLog(
format_string(self.modelManager.locale,
_("[info] compared in %.2f secs"),
time.time() - startedAt))
try:
if options.validate:
startedAt = time.time()
self.modelManager.validate()
self.addToLog(
format_string(self.modelManager.locale,
_("[info] validated in %.2f secs"),
time.time() - startedAt))
if (options.csvTestReport
and self.modelManager.modelXbrl.modelDocument.type
in (ModelDocument.Type.TESTCASESINDEX,
ModelDocument.Type.REGISTRY)):
ViewCsvTests.viewTests(self.modelManager.modelXbrl,
options.csvTestReport)
if options.csvDTS:
ViewCsvDTS.viewDTS(modelXbrl, options.csvDTS)
if options.csvFactList:
ViewCsvFactList.viewFacts(modelXbrl, options.csvFactList)
if options.csvConcepts:
ViewCsvConcepts.viewConcepts(modelXbrl, options.csvConcepts)
if options.csvPre:
ViewCsvRelationshipSet.viewRelationshipSet(
modelXbrl, options.csvPre, "Presentation",
"http://www.xbrl.org/2003/arcrole/parent-child")
if options.csvCal:
ViewCsvRelationshipSet.viewRelationshipSet(
modelXbrl, options.csvCal, "Calculation",
"http://www.xbrl.org/2003/arcrole/summation-item")
if options.csvDim:
ViewCsvRelationshipSet.viewRelationshipSet(
modelXbrl, options.csvDim, "Dimension", "XBRL-dimensions")
except (IOError, EnvironmentError) as err:
self.addToLog(
_("[IOError] Failed to save output:\n {0}").format(err))
if self.messages:
try:
with open(options.logFile, "w", encoding="utf-8") as fh:
fh.writelines(self.messages)
except (IOError, EnvironmentError) as err:
print("Unable to save log to file: " + err)
def run(self, options):
self.filename = options.filename
filesource = FileSource.openFileSource(self.filename,self)
if options.validateEFM:
if options.gfmName:
self.addToLog(_("both --efm and --gfm validation are requested, proceeding with --efm only"),
messageCode="info", file=self.filename)
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select("efm")
elif options.gfmName:
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select(options.gfmName)
else:
self.modelManager.disclosureSystem.select(None) # just load ordinary mappings
if options.calcDecimals:
if options.calcPrecision:
self.addToLog(_("both --calcDecimals and --calcPrecision validation are requested, proceeding with --calcDecimals only"),
messageCode="info", file=self.filename)
self.modelManager.validateInferDecimals = True
self.modelManager.validateCalcLB = True
elif options.calcPrecision:
self.modelManager.validateInferDecimals = False
self.modelManager.validateCalcLB = True
if options.utrValidate:
self.modelManager.validateUtr = True
fo = FormulaOptions()
if options.formulaParamExprResult:
fo.traceParameterExpressionResult = True
if options.formulaParamInputValue:
fo.traceParameterInputValue = True
if options.formulaCallExprSource:
fo.traceCallExpressionSource = True
if options.formulaCallExprCode:
fo.traceCallExpressionCode = True
if options.formulaCallExprEval:
fo.traceCallExpressionEvaluation = True
if options.formulaCallExprResult:
fo.traceCallExpressionResult = True
if options.formulaVarSetExprEval:
fo.traceVariableSetExpressionEvaluation = True
if options.formulaVarSetExprResult:
fo.traceVariableSetExpressionResult = True
if options.formulaAsserResultCounts:
fo.traceAssertionResultCounts = True
if options.formulaFormulaRules:
fo.traceFormulaRules = True
if options.formulaVarsOrder:
fo.traceVariablesOrder = True
if options.formulaVarExpressionSource:
fo.traceVariableExpressionSource = True
if options.formulaVarExpressionCode:
fo.traceVariableExpressionCode = True
if options.formulaVarExpressionEvaluation:
fo.traceVariableExpressionEvaluation = True
if options.formulaVarExpressionResult:
fo.traceVariableExpressionResult = True
if options.formulaVarFiltersResult:
fo.traceVariableFiltersResult = True
self.modelManager.formulaOptions = fo
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
startedAt = time.time()
modelXbrl = self.modelManager.load(filesource, _("views loading"))
self.addToLog(format_string(self.modelManager.locale,
_("loaded in %.2f secs at %s"),
(time.time() - startedAt, timeNow)),
messageCode="info", file=self.filename)
if options.diffFilename and options.versReportFilename:
diffFilesource = FileSource.FileSource(self.diffFilename,self)
startedAt = time.time()
modelXbrl = self.modelManager.load(diffFilesource, _("views loading"))
self.addToLog(format_string(self.modelManager.locale,
_("diff comparison DTS loaded in %.2f secs"),
time.time() - startedAt),
messageCode="info", file=self.filename)
startedAt = time.time()
self.modelManager.compareDTSes(options.versReportFilename)
self.addToLog(format_string(self.modelManager.locale,
_("compared in %.2f secs"),
time.time() - startedAt),
messageCode="info", file=self.filename)
try:
if options.validate:
startedAt = time.time()
self.modelManager.validate()
self.addToLog(format_string(self.modelManager.locale,
_("validated in %.2f secs"),
time.time() - startedAt),
messageCode="info", file=self.filename)
if (options.csvTestReport and
self.modelManager.modelXbrl.modelDocument.type in
(ModelDocument.Type.TESTCASESINDEX,
ModelDocument.Type.TESTCASE,
ModelDocument.Type.REGISTRY)):
ViewCsvTests.viewTests(self.modelManager.modelXbrl, options.csvTestReport)
if options.csvDTS:
ViewCsvDTS.viewDTS(modelXbrl, options.csvDTS)
if options.csvFactList:
ViewCsvFactList.viewFacts(modelXbrl, options.csvFactList, cols=options.csvFactListCols)
if options.csvConcepts:
ViewCsvConcepts.viewConcepts(modelXbrl, options.csvConcepts)
if options.csvPre:
ViewCsvRelationshipSet.viewRelationshipSet(modelXbrl, options.csvPre, "Presentation", "http://www.xbrl.org/2003/arcrole/parent-child")
if options.csvCal:
ViewCsvRelationshipSet.viewRelationshipSet(modelXbrl, options.csvCal, "Calculation", "http://www.xbrl.org/2003/arcrole/summation-item")
if options.csvDim:
ViewCsvRelationshipSet.viewRelationshipSet(modelXbrl, options.csvDim, "Dimension", "XBRL-dimensions")
except (IOError, EnvironmentError) as err:
self.addToLog(_("[IOError] Failed to save output:\n {0}").format(err))
except Exception as err:
self.addToLog(_("[Exception] Failed to complete validation: \n{0} \n{1}").format(
err,
traceback.format_tb(sys.exc_info()[2])))
def run(self, options, sourceZipStream=None):
self.entrypointFile = options.entrypointFile
filesource = FileSource.openFileSource(self.entrypointFile, self, sourceZipStream)
if options.validateEFM:
if options.gfmName:
self.addToLog(_("both --efm and --gfm validation are requested, proceeding with --efm only"),
messageCode="info", file=self.entrypointFile)
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select("efm")
elif options.gfmName:
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select(options.gfmName)
elif options.validateHMRC:
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select("hmrc")
else:
self.modelManager.disclosureSystem.select(None) # just load ordinary mappings
if options.calcDecimals:
if options.calcPrecision:
self.addToLog(_("both --calcDecimals and --calcPrecision validation are requested, proceeding with --calcDecimals only"),
messageCode="info", file=self.entrypointFile)
self.modelManager.validateInferDecimals = True
self.modelManager.validateCalcLB = True
elif options.calcPrecision:
self.modelManager.validateInferDecimals = False
self.modelManager.validateCalcLB = True
if options.utrValidate:
self.modelManager.validateUtr = True
fo = FormulaOptions()
if options.parameters:
fo.parameterValues = dict(((qname(key, noPrefixIsNoNamespace=True),(None,value))
for param in options.parameters.split(',')
for key,sep,value in (param.partition('='),) ) )
if options.formulaParamExprResult:
fo.traceParameterExpressionResult = True
if options.formulaParamInputValue:
fo.traceParameterInputValue = True
if options.formulaCallExprSource:
fo.traceCallExpressionSource = True
if options.formulaCallExprCode:
fo.traceCallExpressionCode = True
if options.formulaCallExprEval:
fo.traceCallExpressionEvaluation = True
if options.formulaCallExprResult:
fo.traceCallExpressionResult = True
if options.formulaVarSetExprEval:
fo.traceVariableSetExpressionEvaluation = True
if options.formulaVarSetExprResult:
fo.traceVariableSetExpressionResult = True
if options.formulaAsserResultCounts:
fo.traceAssertionResultCounts = True
if options.formulaFormulaRules:
fo.traceFormulaRules = True
if options.formulaVarsOrder:
fo.traceVariablesOrder = True
if options.formulaVarExpressionSource:
fo.traceVariableExpressionSource = True
if options.formulaVarExpressionCode:
fo.traceVariableExpressionCode = True
if options.formulaVarExpressionEvaluation:
fo.traceVariableExpressionEvaluation = True
if options.formulaVarExpressionResult:
fo.traceVariableExpressionResult = True
if options.formulaVarFilterWinnowing:
fo.traceVariableFilterWinnowing = True
if options.formulaVarFiltersResult:
fo.traceVariableFiltersResult = True
self.modelManager.formulaOptions = fo
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
startedAt = time.time()
modelDiffReport = None
success = True
modelXbrl = None
try:
modelXbrl = self.modelManager.load(filesource, _("views loading"))
except Exception as err:
self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format(
err,
traceback.format_tb(sys.exc_info()[2])))
success = False # loading errors, don't attempt to utilize loaded DTS
if modelXbrl and modelXbrl.modelDocument:
self.addToLog(format_string(self.modelManager.locale,
_("loaded in %.2f secs at %s"),
(time.time() - startedAt, timeNow)),
messageCode="info", file=self.entrypointFile)
if options.importFiles:
for importFile in options.importFiles.split("|"):
ModelDocument.load(modelXbrl, importFile.strip())
self.addToLog(format_string(self.modelManager.locale,
_("imported in %.2f secs at %s"),
(time.time() - startedAt, timeNow)),
messageCode="info", file=importFile)
if modelXbrl.errors:
success = False # loading errors, don't attempt to utilize loaded DTS
else:
success = False
if success and options.diffFile and options.versReportFile:
diffFilesource = FileSource.FileSource(options.diffFile,self)
startedAt = time.time()
modelXbrl2 = self.modelManager.load(diffFilesource, _("views loading"))
if modelXbrl2.errors:
if not options.keepOpen:
modelXbrl2.close()
success = False
else:
self.addToLog(format_string(self.modelManager.locale,
_("diff comparison DTS loaded in %.2f secs"),
time.time() - startedAt),
messageCode="info", file=self.entrypointFile)
startedAt = time.time()
modelDiffReport = self.modelManager.compareDTSes(options.versReportFile)
self.addToLog(format_string(self.modelManager.locale,
_("compared in %.2f secs"),
time.time() - startedAt),
messageCode="info", file=self.entrypointFile)
if success:
try:
if options.validate:
startedAt = time.time()
self.modelManager.validate()
self.addToLog(format_string(self.modelManager.locale,
_("validated in %.2f secs"),
time.time() - startedAt),
messageCode="info", file=self.entrypointFile)
if (options.testReport and
self.modelManager.modelXbrl.modelDocument.type in
(ModelDocument.Type.TESTCASESINDEX,
ModelDocument.Type.TESTCASE,
ModelDocument.Type.REGISTRY)):
ViewFileTests.viewTests(self.modelManager.modelXbrl, options.testReport)
if options.DTSFile:
ViewFileDTS.viewDTS(modelXbrl, options.DTSFile)
if options.factsFile:
ViewFileFactList.viewFacts(modelXbrl, options.factsFile, labelrole=options.labelRole, lang=options.labelLang, cols=options.factListCols)
if options.factTableFile:
ViewFileFactTable.viewFacts(modelXbrl, options.factTableFile, labelrole=options.labelRole, lang=options.labelLang)
if options.conceptsFile:
ViewFileConcepts.viewConcepts(modelXbrl, options.conceptsFile, labelrole=options.labelRole, lang=options.labelLang)
if options.preFile:
ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.preFile, "Presentation Linkbase", "http://www.xbrl.org/2003/arcrole/parent-child", labelrole=options.labelRole, lang=options.labelLang)
if options.calFile:
ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.calFile, "Calculation Linkbase", "http://www.xbrl.org/2003/arcrole/summation-item", labelrole=options.labelRole, lang=options.labelLang)
if options.dimFile:
ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.dimFile, "Dimensions", "XBRL-dimensions", labelrole=options.labelRole, lang=options.labelLang)
if options.formulaeFile:
ViewFileFormulae.viewFormulae(modelXbrl, options.formulaeFile, "Formulae", lang=options.labelLang)
for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Run"):
pluginXbrlMethod(self, options, modelXbrl)
except (IOError, EnvironmentError) as err:
self.addToLog(_("[IOError] Failed to save output:\n {0}").format(err))
success = False
except Exception as err:
self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format(
err,
traceback.format_tb(sys.exc_info()[2])))
success = False
if not options.keepOpen:
if modelDiffReport:
modelDiffReport.close()
elif modelXbrl:
modelXbrl.close()
return success
def compileSphinxGrammar(cntlr):
global isGrammarCompiled, sphinxProg, lineno
if isGrammarCompiled:
return sphinxProg
debugParsing = True
cntlr.showStatus(_("Compiling Sphinx Grammar"))
if sys.version[0] >= '3':
# python 3 requires modified parser to allow release of global objects when closing DTS
from arelle.pyparsing.pyparsing_py3 import (
Word, Keyword, alphas, Literal, CaselessLiteral, Combine, Optional,
nums, Or, Forward, Group, ZeroOrMore, StringEnd, alphanums,
ParserElement, quotedString, delimitedList, Suppress, Regex,
FollowedBy, lineno)
else:
from pyparsing import (Word, Keyword, alphas, Literal, CaselessLiteral,
Combine, Optional, nums, Or, Forward, Group,
ZeroOrMore, StringEnd, alphanums, ParserElement,
quotedString, delimitedList, Suppress, Regex,
FollowedBy, lineno)
ParserElement.enablePackrat()
"""
the pyparsing parser constructs are defined in this method to prevent the need to compile
the grammar when the plug in is loaded (which is likely to be when setting up GUI
menus or command line parser).
instead the grammar is compiled the first time that any sphinx needs to be parsed
only the sphinxExpression (result below) needs to be global for the parser
"""
# define grammar
sphinxComment = Regex(
r"/(?:\*(?:[^*]*\*+)+?/|/[^\n]*(?:\n[^\n]*)*?(?:(?<!\\)|\Z))"
).setParseAction(compileComment)
variableRef = Regex(
"[$]" # variable prefix
# localname part
"([A-Za-z\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD_]"
"[A-Za-z0-9\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\u0300-\u036F\u203F-\u2040\xB7_.-]*)"
)
qName = Regex(
"([A-Za-z\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD_]"
"[A-Za-z0-9\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\u0300-\u036F\u203F-\u2040\xB7_.-]*:)?"
# localname or wildcard-localname part
"([A-Za-z\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD_]"
"[A-Za-z0-9\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\u0300-\u036F\u203F-\u2040\xB7_.-]*|[*])"
)
ncName = Regex(
"([A-Za-z\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD_]"
"[A-Za-z0-9\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\u0300-\u036F\u203F-\u2040\xB7_.-]*)"
).setName("ncName").setDebug(debugParsing)
#annotationName = Word("@",alphanums + '_-.').setName("annotationName").setDebug(debugParsing)
annotationName = Regex(
"@[A-Za-z\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD_]\w*"
).setName("annotationName").setDebug(debugParsing)
decimalPoint = Literal('.')
exponentLiteral = CaselessLiteral('e')
plusorminusLiteral = Literal('+') | Literal('-')
digits = Word(nums)
integerLiteral = Combine(Optional(plusorminusLiteral) + digits)
decimalFractionLiteral = Combine(
Optional(plusorminusLiteral) + decimalPoint + digits)
infLiteral = Combine(Optional(plusorminusLiteral) + Literal("INF"))
nanLiteral = Literal("NaN")
floatLiteral = (Combine(integerLiteral + (
(decimalPoint + Optional(digits) + exponentLiteral + integerLiteral) |
(exponentLiteral + integerLiteral) |
(decimalPoint + Optional(digits))))
| Combine(decimalFractionLiteral + exponentLiteral +
integerLiteral) | decimalFractionLiteral
| infLiteral | nanLiteral)
#emptySequence = Literal( "(" ) + Literal( ")" )
lParen = Literal("(")
rParen = Literal(")")
lPred = Literal("[[") | Literal("[")
rPred = Literal("]]") | Literal("]")
commaOp = Literal(",")
ifOp = Keyword("if")
elseOp = Keyword("else")
forOp = Keyword("for")
inOp = Keyword("in")
withOp = Keyword("with")
notOp = Keyword("not")
valuesOp = Keyword("values")
andOp = Keyword("and")
orOp = Keyword("or")
neOp = Literal("!=")
leOp = Literal("<=")
ltOp = Literal("<")
geOp = Literal(">=")
gtOp = Literal(">")
eqOp = Literal("==")
compOp = leOp | ltOp | geOp | gtOp
plusOp = Literal("|+|") | Literal("|+") | Literal("+|") | Literal("+")
minusOp = Literal("|-|") | Literal("|-") | Literal("-|") | Literal("-")
plusMinusOp = (plusOp | minusOp).setParseAction(compileOp)
multOp = Literal("*")
divOp = Literal("/")
varAssign = Literal("=")
tagOp = Literal("#")
asOp = Keyword("as")
whereOp = Keyword("where")
wildOp = Literal("**") | Literal("*")
methodOp = Literal("::")
formulaOp = Literal(":=")
namespaceDeclaration = (
Literal("xmlns") + Optional(Suppress(Literal(":")) + ncName) +
Suppress(Literal("=")) + quotedString
).setParseAction(compileNamespaceDeclaration).ignore(sphinxComment)
annotationDeclaration = (
Suppress(Keyword("annotation")) + ncName +
Optional(Suppress(Keyword("as")) + ncName)
).setParseAction(compileAnnotationDeclaration).ignore(sphinxComment)
packageDeclaration = (
Suppress(Keyword("package")) +
ncName).setParseAction(compilePackageDeclaration).ignore(sphinxComment)
severity = (Suppress(Keyword("severity")) +
(ncName)).setParseAction(compileSeverity).ignore(sphinxComment)
expr = Forward()
atom = ((forOp - Suppress(lParen) - ncName - Suppress(inOp) - expr -
Suppress(rParen) - expr).setParseAction(compileFor) |
(ifOp - Suppress(lParen) - expr - Suppress(rParen) - expr -
Suppress(elseOp) - expr).setParseAction(compileIf) |
(ncName + Suppress(lParen) + Optional(
delimitedList(
ZeroOrMore((ncName + Optional(tagOp + Optional(ncName)) +
varAssign + expr + Suppress(Literal(";"))
).setParseAction(compileVariableAssignment)) +
Optional(ncName + varAssign) + expr)) +
Suppress(rParen)).setParseAction(compileFunctionReference) |
(floatLiteral).setParseAction(compileFloatLiteral) |
(integerLiteral).setParseAction(compileIntegerLiteral) |
(quotedString).setParseAction(compileStringLiteral) |
(Optional(qName) + lPred + Optional(
delimitedList(
((whereOp + expr) |
((qName | variableRef) +
Optional(tagOp + Optional(ncName)) + Optional(
(varAssign + (wildOp | expr) | (inOp + expr) |
(asOp + ncName + varAssign + wildOp +
Optional(whereOp + expr)))))
).setParseAction(compileHyperspaceAxis),
delim=';')) +
rPred).setParseAction(compileHyperspaceExpression) |
(variableRef).setParseAction(compileVariableReference) |
(qName).setParseAction(compileQname) |
(Suppress(lParen) - expr -
Optional(commaOp - Optional(expr - ZeroOrMore(commaOp - expr))) -
Suppress(rParen)).setParseAction(compileBrackets)
).ignore(sphinxComment)
atom.setName("atom").setDebug(debugParsing)
valueExpr = atom
taggedExpr = (valueExpr - Optional(tagOp - ncName)
).setParseAction(compileTagAssignment).ignore(sphinxComment)
methodExpr = (
(methodOp + ncName + ZeroOrMore(methodOp + taggedExpr)
).setParseAction(compileMethodReference) |
(ZeroOrMore(taggedExpr + methodOp) + taggedExpr)
).setParseAction(compileMethodReference).ignore(sphinxComment)
unaryExpr = (
Optional(plusMinusOp) +
methodExpr).setParseAction(compileUnaryOperation).ignore(sphinxComment)
negateExpr = (
Optional(notOp) +
unaryExpr).setParseAction(compileUnaryOperation).ignore(sphinxComment)
valuesExpr = (Optional(valuesOp) + negateExpr).setParseAction(
compileValuesIteration).ignore(sphinxComment)
method2Expr = (valuesExpr +
Optional(methodOp + methodExpr)).setParseAction(
compileMethodReference).ignore(sphinxComment)
multiplyExpr = (method2Expr +
Optional(multOp + method2Expr)).setParseAction(
compileBinaryOperation).ignore(sphinxComment)
divideExpr = (multiplyExpr +
Optional(divOp + multiplyExpr)).setParseAction(
compileBinaryOperation).ignore(sphinxComment)
addExpr = (divideExpr + Optional(plusOp + divideExpr)
).setParseAction(compileBinaryOperation).ignore(sphinxComment)
subtractExpr = (addExpr + Optional(minusOp + addExpr)).setParseAction(
compileBinaryOperation).ignore(sphinxComment)
equalityExpr = (subtractExpr +
Optional(eqOp + subtractExpr)).setParseAction(
compileBinaryOperation).ignore(sphinxComment)
inequalityExpr = (equalityExpr +
Optional(neOp + equalityExpr)).setParseAction(
compileBinaryOperation).ignore(sphinxComment)
comparisonExpr = (inequalityExpr +
Optional(compOp + inequalityExpr)).setParseAction(
compileBinaryOperation).ignore(sphinxComment)
andExpr = (comparisonExpr + Optional(andOp + comparisonExpr)
).setParseAction(compileBinaryOperation).ignore(sphinxComment)
orExpr = (andExpr + Optional(orOp + andExpr)
).setParseAction(compileBinaryOperation).ignore(sphinxComment)
formulaExpr = (orExpr + Optional(formulaOp + orExpr)).setParseAction(
compileBinaryOperation).ignore(sphinxComment)
withExpr = (Optional(withOp + Suppress(lParen) + expr + Suppress(rParen)) +
ZeroOrMore(
(ncName + Optional(tagOp + Optional(ncName)) + varAssign +
expr + Suppress(Literal(";"))).setParseAction(
compileVariableAssignment).ignore(sphinxComment)) +
formulaExpr).setParseAction(compileWith)
#parsedExpr = withExpr
#parsedExpr.setName("parsedExpr").setDebug(debugParsing)
#expr << parsedExpr
expr << withExpr
expr.setName("expr").setDebug(debugParsing)
annotation = (annotationName + Optional(
Suppress(lParen) + Optional(delimitedList(expr)) +
Suppress(rParen))).setParseAction(compileAnnotation).ignore(
sphinxComment).setName("annotation").setDebug(debugParsing)
constant = (Suppress(Keyword("constant")) + ncName +
Optional(tagOp + Optional(ncName)) + varAssign +
expr).setParseAction(compileConstant).ignore(sphinxComment)
functionDeclaration = (
(Keyword("function") | Keyword("macro")) + ncName + lParen +
Optional(delimitedList(ncName)) + rParen +
expr).setParseAction(compileFunctionDeclaration).ignore(sphinxComment)
message = (Suppress(Keyword("message")) +
expr).setParseAction(compileMessage)
preconditionDeclaration = (
Suppress(Keyword("precondition")) + ncName + expr + Optional(
Keyword("otherwise") + Keyword("raise") + ncName +
Optional(severity) + Optional(message))
).setParseAction(compilePreconditionDeclaration).ignore(sphinxComment)
assignedExpr = (ncName + Optional(tagOp + Optional(ncName)) + varAssign +
expr + Suppress(Literal(";"))).setParseAction(
compileVariableAssignment).ignore(sphinxComment)
precondition = (
Suppress(Keyword("require")) +
delimitedList(ncName)).setParseAction(compilePrecondition).ignore(
sphinxComment).setName("precondition").setDebug(debugParsing)
formulaRule = (Optional(precondition) + Keyword("formula") + ncName +
Optional(severity) + Optional((Keyword("bind") + expr)) +
ZeroOrMore(assignedExpr) + expr + Optional(message)
).setParseAction(compileFormulaRule).ignore(sphinxComment)
reportRule = (Optional(precondition) + Keyword("report") + ncName +
Optional(severity) + ZeroOrMore(assignedExpr) + expr +
Optional(message)
).setParseAction(compileReportRule).ignore(sphinxComment)
validationRule = (Optional(precondition) + Keyword("raise") + ncName +
Optional(severity) + ZeroOrMore(assignedExpr) + expr +
Optional(message)).setParseAction(
compileValidationRule).ignore(sphinxComment)
ruleBase = (
Optional(precondition) + Suppress(Keyword("rule-base")) + ZeroOrMore(
(Suppress(Keyword("transform")) +
(Keyword("namespace") + expr + Suppress(Keyword("to")) + expr) |
(Keyword("qname") + expr + Suppress(Keyword("to")) +
expr)).setParseAction(compileTransform))
).setParseAction(compileRuleBase).ignore(sphinxComment).setName(
"ruleBase").setDebug(debugParsing)
sphinxProg = (
ZeroOrMore(namespaceDeclaration | sphinxComment) +
ZeroOrMore(annotationDeclaration | annotation | constant
| preconditionDeclaration | packageDeclaration
| functionDeclaration | ruleBase | formulaRule | reportRule
| validationRule | sphinxComment)) + StringEnd()
sphinxProg.ignore(sphinxComment)
startedAt = time.time()
cntlr.modelManager.showStatus(_("initializing sphinx grammar"))
sphinxProg.parseString("// force initialization\n", parseAll=True)
from arelle.Locale import format_string
logMessage(
"INFO", "info",
format_string(cntlr.modelManager.locale,
_("Sphinx grammar initialized in %.2f secs"),
time.time() - startedAt))
isGrammarCompiled = True
return sphinxProg
def logArguments(self, codes, msg, codedArgs):
# determine logCode
messageCode = None
for argCode in codes if isinstance(codes,tuple) else (codes,):
if (isinstance(argCode, ModelValue.QName) or
(self.modelManager.disclosureSystem.EFM and argCode.startswith("EFM")) or
(self.modelManager.disclosureSystem.GFM and argCode.startswith("GFM")) or
(self.modelManager.disclosureSystem.HMRC and argCode.startswith("HMRC")) or
(self.modelManager.disclosureSystem.SBRNL and argCode.startswith("SBR.NL")) or
argCode[0:3] not in ("EFM", "GFM", "HMR", "SBR")):
messageCode = argCode
break
# determine message and extra arguments
fmtArgs = {}
extras = {"messageCode":messageCode}
for argName, argValue in codedArgs.items():
if argName in ("modelObject", "modelXbrl", "modelDocument"):
try:
entryUrl = self.modelDocument.uri
except AttributeError:
entryUrl = self.entryLoadingUrl
try:
objectUrl = argValue.modelDocument.uri
except AttributeError:
try:
objectUrl = self.modelDocument.uri
except AttributeError:
objectUrl = self.entryLoadingUrl
refs = []
for arg in (argValue if isinstance(argValue, (tuple,list)) else (argValue,)):
if arg is not None:
file = UrlUtil.relativeUri(entryUrl, objectUrl)
ref = {}
if isinstance(arg,ModelObject):
ref["href"] = file + "#" + XmlUtil.elementFragmentIdentifier(arg)
ref["sourceLine"] = arg.sourceline
ref["objectId"] = arg.objectId()
else:
ref["href"] = file
refs.append(ref)
extras["refs"] = refs
elif argName == "sourceLine":
if isinstance(argValue, _INT_TYPES): # must be sortable with int's in logger
extras["sourceLine"] = argValue
elif argName != "exc_info":
if isinstance(argValue, (ModelValue.QName, ModelObject, bool, FileNamedStringIO)):
fmtArgs[argName] = str(argValue)
elif isinstance(argValue, _INT_TYPES):
# need locale-dependent formatting
fmtArgs[argName] = format_string(self.modelManager.locale, '%i', argValue)
elif isinstance(argValue,float):
# need locale-dependent formatting
fmtArgs[argName] = format_string(self.modelManager.locale, '%f', argValue)
else:
fmtArgs[argName] = argValue
if "refs" not in extras:
try:
file = os.path.basename(self.modelDocument.uri)
except AttributeError:
try:
file = os.path.basename(self.entryLoadingUrl)
except:
file = ""
extras["refs"] = [{"href": file}]
return (messageCode,
(msg, fmtArgs) if fmtArgs else (msg,),
extras)
def compileSphinxGrammar( cntlr ):
global isGrammarCompiled, sphinxProg, lineno
if isGrammarCompiled:
return sphinxProg
debugParsing = True
cntlr.showStatus(_("Compiling Sphinx Grammar"))
if sys.version[0] >= '3':
# python 3 requires modified parser to allow release of global objects when closing DTS
from arelle.pyparsing.pyparsing_py3 import (Word, Keyword, alphas,
Literal, CaselessLiteral,
Combine, Optional, nums, Or, Forward, Group, ZeroOrMore, StringEnd, alphanums,
ParserElement, quotedString, delimitedList, Suppress, Regex, FollowedBy,
lineno)
else:
from pyparsing import (Word, Keyword, alphas,
Literal, CaselessLiteral,
Combine, Optional, nums, Or, Forward, Group, ZeroOrMore, StringEnd, alphanums,
ParserElement, quotedString, delimitedList, Suppress, Regex, FollowedBy,
lineno)
ParserElement.enablePackrat()
"""
the pyparsing parser constructs are defined in this method to prevent the need to compile
the grammar when the plug in is loaded (which is likely to be when setting up GUI
menus or command line parser).
instead the grammar is compiled the first time that any sphinx needs to be parsed
only the sphinxExpression (result below) needs to be global for the parser
"""
# define grammar
sphinxComment = Regex(r"/(?:\*(?:[^*]*\*+)+?/|/[^\n]*(?:\n[^\n]*)*?(?:(?<!\\)|\Z))").setParseAction(compileComment)
variableRef = Regex("[$]" # variable prefix
# localname part
"([A-Za-z\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD_]"
"[A-Za-z0-9\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\u0300-\u036F\u203F-\u2040\xB7_.-]*)"
)
qName = Regex("([A-Za-z\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD_]"
"[A-Za-z0-9\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\u0300-\u036F\u203F-\u2040\xB7_.-]*:)?"
# localname or wildcard-localname part
"([A-Za-z\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD_]"
"[A-Za-z0-9\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\u0300-\u036F\u203F-\u2040\xB7_.-]*|[*])"
)
ncName = Regex("([A-Za-z\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD_]"
"[A-Za-z0-9\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\u0300-\u036F\u203F-\u2040\xB7_.-]*)"
).setName("ncName").setDebug(debugParsing)
#annotationName = Word("@",alphanums + '_-.').setName("annotationName").setDebug(debugParsing)
annotationName = Regex("@[A-Za-z\xC0-\xD6\xD8-\xF6\xF8-\xFF\u0100-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD_]\w*").setName("annotationName").setDebug(debugParsing)
decimalPoint = Literal('.')
exponentLiteral = CaselessLiteral('e')
plusorminusLiteral = Literal('+') | Literal('-')
digits = Word(nums)
integerLiteral = Combine( Optional(plusorminusLiteral) + digits )
decimalFractionLiteral = Combine( Optional(plusorminusLiteral) + decimalPoint + digits )
infLiteral = Combine( Optional(plusorminusLiteral) + Literal("INF") )
nanLiteral = Literal("NaN")
floatLiteral = ( Combine( integerLiteral +
( ( decimalPoint + Optional(digits) + exponentLiteral + integerLiteral ) |
( exponentLiteral + integerLiteral ) |
( decimalPoint + Optional(digits) ) )
) |
Combine( decimalFractionLiteral + exponentLiteral + integerLiteral ) |
decimalFractionLiteral |
infLiteral | nanLiteral )
#emptySequence = Literal( "(" ) + Literal( ")" )
lParen = Literal( "(" )
rParen = Literal( ")" )
lPred = Literal( "[[" ) | Literal("[")
rPred = Literal( "]]" ) | Literal("]")
commaOp = Literal(",")
ifOp = Keyword("if")
elseOp = Keyword("else")
forOp = Keyword("for")
inOp = Keyword("in")
withOp = Keyword("with")
notOp = Keyword("not")
valuesOp = Keyword("values")
andOp = Keyword("and")
orOp = Keyword("or")
neOp = Literal("!=")
leOp = Literal("<=")
ltOp = Literal("<")
geOp = Literal(">=")
gtOp = Literal(">")
eqOp = Literal("==")
compOp = leOp | ltOp | geOp | gtOp
plusOp = Literal("|+|") | Literal("|+") | Literal("+|") | Literal("+")
minusOp = Literal("|-|") | Literal("|-") | Literal("-|") | Literal("-")
plusMinusOp = ( plusOp | minusOp ).setParseAction(compileOp)
multOp = Literal("*")
divOp = Literal("/")
varAssign = Literal("=")
tagOp = Literal("#")
asOp = Keyword("as")
whereOp = Keyword("where")
wildOp = Literal("**") | Literal("*")
methodOp = Literal("::")
formulaOp = Literal(":=")
namespaceDeclaration = (Literal("xmlns") + Optional( Suppress(Literal(":")) + ncName ) + Suppress(Literal("=")) + quotedString ).setParseAction(compileNamespaceDeclaration).ignore(sphinxComment)
annotationDeclaration = (Suppress(Keyword("annotation")) + ncName + Optional( Suppress(Keyword("as")) + ncName )).setParseAction(compileAnnotationDeclaration).ignore(sphinxComment)
packageDeclaration = (Suppress(Keyword("package")) + ncName ).setParseAction(compilePackageDeclaration).ignore(sphinxComment)
severity = ( Suppress(Keyword("severity")) + ( ncName ) ).setParseAction(compileSeverity).ignore(sphinxComment)
expr = Forward()
atom = (
( forOp - Suppress(lParen) - ncName - Suppress(inOp) - expr - Suppress(rParen) - expr ).setParseAction(compileFor) |
( ifOp - Suppress(lParen) - expr - Suppress(rParen) - expr - Suppress(elseOp) - expr ).setParseAction(compileIf) |
( ncName + Suppress(lParen) + Optional(delimitedList( ZeroOrMore( ( ncName + Optional( tagOp + Optional(ncName) ) + varAssign + expr + Suppress(Literal(";")) ).setParseAction(compileVariableAssignment) ) +
Optional( ncName + varAssign ) + expr
)) + Suppress(rParen) ).setParseAction(compileFunctionReference) |
( floatLiteral ).setParseAction(compileFloatLiteral) |
( integerLiteral ).setParseAction(compileIntegerLiteral) |
( quotedString ).setParseAction(compileStringLiteral) |
( Optional(qName) + lPred + Optional(delimitedList( ((whereOp + expr) |
((qName | variableRef) + Optional( tagOp + Optional(ncName) ) +
Optional( (varAssign + (wildOp | expr) |
(inOp + expr) |
(asOp + ncName + varAssign + wildOp + Optional( whereOp + expr ) ) ) ) )
).setParseAction(compileHyperspaceAxis),
delim=';')) + rPred).setParseAction(compileHyperspaceExpression) |
( variableRef ).setParseAction(compileVariableReference) |
( qName ).setParseAction(compileQname) |
( Suppress(lParen) - expr - Optional( commaOp - Optional( expr - ZeroOrMore( commaOp - expr ) ) ) - Suppress(rParen) ).setParseAction(compileBrackets)
).ignore(sphinxComment)
atom.setName("atom").setDebug(debugParsing)
valueExpr = atom
taggedExpr = ( valueExpr - Optional(tagOp - ncName) ).setParseAction(compileTagAssignment).ignore(sphinxComment)
methodExpr = ( ( methodOp + ncName + ZeroOrMore(methodOp + taggedExpr) ).setParseAction(compileMethodReference) |
( ZeroOrMore(taggedExpr + methodOp) + taggedExpr )).setParseAction(compileMethodReference).ignore(sphinxComment)
unaryExpr = ( Optional(plusMinusOp) + methodExpr ).setParseAction(compileUnaryOperation).ignore(sphinxComment)
negateExpr = ( Optional(notOp) + unaryExpr ).setParseAction(compileUnaryOperation).ignore(sphinxComment)
valuesExpr = ( Optional(valuesOp) + negateExpr ).setParseAction(compileValuesIteration).ignore(sphinxComment)
method2Expr = ( valuesExpr + Optional( methodOp + methodExpr ) ).setParseAction(compileMethodReference).ignore(sphinxComment)
multiplyExpr = ( method2Expr + Optional( multOp + method2Expr ) ).setParseAction(compileBinaryOperation).ignore(sphinxComment)
divideExpr = ( multiplyExpr + Optional( divOp + multiplyExpr ) ).setParseAction(compileBinaryOperation).ignore(sphinxComment)
addExpr = ( divideExpr + Optional( plusOp + divideExpr ) ).setParseAction(compileBinaryOperation).ignore(sphinxComment)
subtractExpr = ( addExpr + Optional( minusOp + addExpr ) ).setParseAction(compileBinaryOperation).ignore(sphinxComment)
equalityExpr = ( subtractExpr + Optional( eqOp + subtractExpr ) ).setParseAction(compileBinaryOperation).ignore(sphinxComment)
inequalityExpr = ( equalityExpr + Optional( neOp + equalityExpr ) ).setParseAction(compileBinaryOperation).ignore(sphinxComment)
comparisonExpr = ( inequalityExpr + Optional( compOp + inequalityExpr ) ).setParseAction(compileBinaryOperation).ignore(sphinxComment)
andExpr = ( comparisonExpr + Optional( andOp + comparisonExpr ) ).setParseAction(compileBinaryOperation ).ignore(sphinxComment)
orExpr = ( andExpr + Optional( orOp + andExpr ) ).setParseAction(compileBinaryOperation).ignore(sphinxComment)
formulaExpr = ( orExpr + Optional( formulaOp + orExpr ) ).setParseAction(compileBinaryOperation).ignore(sphinxComment)
withExpr = ( Optional( withOp + Suppress(lParen) + expr + Suppress(rParen) ) +
ZeroOrMore( ( ncName + Optional( tagOp + Optional(ncName) ) + varAssign + expr + Suppress(Literal(";")) ).setParseAction(compileVariableAssignment).ignore(sphinxComment) ) +
formulaExpr ).setParseAction(compileWith)
#parsedExpr = withExpr
#parsedExpr.setName("parsedExpr").setDebug(debugParsing)
#expr << parsedExpr
expr << withExpr
expr.setName("expr").setDebug(debugParsing)
annotation = ( annotationName + Optional( Suppress(lParen) + Optional(delimitedList(expr)) + Suppress(rParen) ) ).setParseAction(compileAnnotation).ignore(sphinxComment).setName("annotation").setDebug(debugParsing)
constant = ( Suppress(Keyword("constant")) + ncName + Optional( tagOp + Optional(ncName) ) + varAssign + expr ).setParseAction(compileConstant).ignore(sphinxComment)
functionDeclaration = ( (Keyword("function") | Keyword("macro")) + ncName + lParen + Optional(delimitedList(ncName)) + rParen + expr ).setParseAction(compileFunctionDeclaration).ignore(sphinxComment)
message = ( Suppress(Keyword("message")) + expr ).setParseAction(compileMessage)
preconditionDeclaration = ( Suppress(Keyword("precondition")) + ncName + expr +
Optional(Keyword("otherwise") + Keyword("raise") + ncName + Optional( severity ) + Optional( message ) )
).setParseAction(compilePreconditionDeclaration).ignore(sphinxComment)
assignedExpr = ( ncName + Optional( tagOp + Optional(ncName) ) + varAssign + expr + Suppress(Literal(";")) ).setParseAction(compileVariableAssignment).ignore(sphinxComment)
precondition = ( Suppress(Keyword("require")) + delimitedList(ncName) ).setParseAction(compilePrecondition).ignore(sphinxComment).setName("precondition").setDebug(debugParsing)
formulaRule = ( Optional( precondition ) +
Keyword("formula") + ncName +
Optional( severity ) +
Optional( ( Keyword("bind") + expr ) ) +
ZeroOrMore( assignedExpr ) +
expr +
Optional( message )).setParseAction(compileFormulaRule).ignore(sphinxComment)
reportRule = ( Optional( precondition ) +
Keyword("report") + ncName +
Optional( severity ) +
ZeroOrMore( assignedExpr ) +
expr +
Optional( message )).setParseAction( compileReportRule).ignore(sphinxComment)
validationRule = ( Optional( precondition ) +
Keyword("raise") + ncName +
Optional( severity ) +
ZeroOrMore( assignedExpr ) +
expr +
Optional( message )).setParseAction(compileValidationRule).ignore(sphinxComment)
ruleBase = (Optional( precondition ) +
Suppress(Keyword("rule-base")) +
ZeroOrMore( (Suppress(Keyword("transform")) +
(Keyword("namespace") + expr + Suppress(Keyword("to")) + expr) |
(Keyword ("qname") + expr + Suppress(Keyword("to")) + expr)
).setParseAction(compileTransform)
)
).setParseAction(compileRuleBase).ignore(sphinxComment).setName("ruleBase").setDebug(debugParsing)
sphinxProg = ( ZeroOrMore( namespaceDeclaration | sphinxComment ) +
ZeroOrMore( annotationDeclaration |
annotation |
constant |
preconditionDeclaration |
packageDeclaration |
functionDeclaration |
ruleBase |
formulaRule | reportRule | validationRule |
sphinxComment
)
) + StringEnd()
sphinxProg.ignore(sphinxComment)
startedAt = time.time()
cntlr.modelManager.showStatus(_("initializing sphinx grammar"))
sphinxProg.parseString( "// force initialization\n", parseAll=True )
from arelle.Locale import format_string
logMessage("INFO", "info",
format_string(cntlr.modelManager.locale,
_("Sphinx grammar initialized in %.2f secs"),
time.time() - startedAt))
isGrammarCompiled = True
return sphinxProg
def schemaValidate(modelXbrl):
class schemaResolver(etree.Resolver):
def resolve(self, url, id, context):
if url.startswith("file:///__"):
url = importedFilepaths[int(url[10:])]
filepath = modelXbrl.modelManager.cntlr.webCache.getfilename(url)
return self.resolve_filename(filepath, context)
entryDocument = modelXbrl.modelDocument
# test of schema validation using lxml (trial experiment, commented out for production use)
from arelle import ModelDocument
imports = []
importedNamespaces = set()
importedFilepaths = []
'''
for mdlSchemaDoc in entryDocument.referencesDocument.keys():
if (mdlSchemaDoc.type == ModelDocument.Type.SCHEMA and
mdlSchemaDoc.targetNamespace not in importedNamespaces):
# actual file won't pass through properly, fake with table reference
imports.append('<xsd:import namespace="{0}" schemaLocation="file:///__{1}"/>'.format(
mdlSchemaDoc.targetNamespace, len(importedFilepaths)))
importedNamespaces.add(mdlSchemaDoc.targetNamespace)
importedFilepaths.append(mdlSchemaDoc.filepath)
'''
def importReferences(referencingDocument):
for mdlSchemaDoc in referencingDocument.referencesDocument.keys():
if (mdlSchemaDoc.type == ModelDocument.Type.SCHEMA and
mdlSchemaDoc.targetNamespace not in importedNamespaces):
importedNamespaces.add(mdlSchemaDoc.targetNamespace)
importReferences(mdlSchemaDoc) # do dependencies first
# actual file won't pass through properly, fake with table reference
imports.append('<xsd:import namespace="{0}" schemaLocation="file:///__{1}"/>'.format(
mdlSchemaDoc.targetNamespace, len(importedFilepaths)))
importedFilepaths.append(mdlSchemaDoc.filepath)
importReferences(entryDocument)
# add schemas used in xml validation but not DTS discovered
for mdlDoc in modelXbrl.urlDocs.values():
if mdlDoc.type in (ModelDocument.Type.INSTANCE, ModelDocument.Type.LINKBASE):
schemaLocation = mdlDoc.xmlRootElement.get("{http://www.w3.org/2001/XMLSchema-instance}schemaLocation")
if schemaLocation:
ns = None
for entry in schemaLocation.split():
if ns is None:
ns = entry
else:
if ns not in importedNamespaces:
imports.append('<xsd:import namespace="{0}" schemaLocation="file:///__{1}"/>'.format(
ns, len(importedFilepaths)))
importedNamespaces.add(ns)
importedFilepaths.append(entry)
ns = None
schemaXml = '<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema">\n{0}</xsd:schema>\n'.format(
'\n'.join(imports))
# trace schema files referenced
with open("c:\\temp\\test.xml", "w") as fh:
fh.write(schemaXml)
modelXbrl.modelManager.showStatus(_("lxml validator loading xml schema"))
schema_root = etree.XML(schemaXml)
import time
startedAt = time.time()
parser = etree.XMLParser()
parser.resolvers.add(schemaResolver())
schemaDoc = etree.fromstring(schemaXml, parser=parser, base_url=entryDocument.filepath+"-dummy-import.xsd")
schema = etree.XMLSchema(schemaDoc)
from arelle.Locale import format_string
modelXbrl.info("info:lxmlSchemaValidator", format_string(modelXbrl.modelManager.locale,
_("schema loaded in %.2f secs"),
time.time() - startedAt))
modelXbrl.modelManager.showStatus(_("lxml schema validating"))
# check instance documents and linkbases (sort for inst doc before linkbases, and in file name order)
for mdlDoc in sorted(modelXbrl.urlDocs.values(), key=lambda mdlDoc: (-mdlDoc.type, mdlDoc.filepath)):
if mdlDoc.type in (ModelDocument.Type.INSTANCE, ModelDocument.Type.LINKBASE):
startedAt = time.time()
docXmlTree = etree.parse(mdlDoc.filepath)
modelXbrl.info("info:lxmlSchemaValidator", format_string(modelXbrl.modelManager.locale,
_("schema validated in %.3f secs"),
time.time() - startedAt),
modelDocument=mdlDoc)
if not schema.validate(docXmlTree):
for error in schema.error_log:
modelXbrl.error("lxmlSchema:{0}".format(error.type_name.lower()),
error.message,
modelDocument=mdlDoc,
sourceLine=error.line)
modelXbrl.modelManager.showStatus(_("lxml validation done"), clearAfter=3000)
def logArguments(self, codes, msg, codedArgs):
# determine logCode
messageCode = None
for argCode in codes if isinstance(codes,tuple) else (codes,):
if ((self.modelManager.disclosureSystem.EFM and argCode.startswith("EFM")) or
(self.modelManager.disclosureSystem.GFM and argCode.startswith("GFM")) or
(self.modelManager.disclosureSystem.HMRC and argCode.startswith("HMRC")) or
(self.modelManager.disclosureSystem.SBRNL and argCode.startswith("SBR.NL")) or
argCode[0:3] not in ("EFM", "GFM", "HMR", "SBR")):
messageCode = argCode
break
# determine message and extra arguments
fmtArgs = {}
extras = {"messageCode":messageCode}
for argName, argValue in codedArgs.items():
if argName in ("modelObject", "modelXbrl", "modelDocument"):
try:
entryUrl = self.modelDocument.uri
except AttributeError:
entryUrl = self.entryLoadingUrl
try:
objectUrl = argValue.modelDocument.uri
except AttributeError:
try:
objectUrl = self.modelDocument.uri
except AttributeError:
objectUrl = self.entryLoadingUrl
file = UrlUtil.relativeUri(entryUrl, objectUrl)
extras["file"] = file
if isinstance(argValue,ModelObject):
extras["href"] = file + "#" + XmlUtil.elementFragmentIdentifier(argValue)
extras["sourceLine"] = argValue.sourceline
extras["objectId"] = argValue.objectId()
else:
extras["href"] = file
extras["sourceLine"] = ""
elif argName == "sourceLine":
extras["sourceLine"] = argValue
elif argName != "exc_info":
if isinstance(argValue, (ModelValue.QName, ModelObject, bool)):
fmtArgs[argName] = str(argValue)
elif isinstance(argValue,int):
# need locale-dependent formatting
fmtArgs[argName] = format_string(self.modelManager.locale, '%i', argValue)
elif isinstance(argValue,float):
# need locale-dependent formatting
fmtArgs[argName] = format_string(self.modelManager.locale, '%f', argValue)
else:
fmtArgs[argName] = argValue
if "href" not in extras:
try:
file = os.path.basename(self.modelDocument.uri)
except AttributeError:
try:
file = os.path.basename(self.entryLoadingUrl)
except:
file = ""
extras["file"] = file
extras["href"] = file
extras["sourceLine"] = ""
return (messageCode,
(msg, fmtArgs) if fmtArgs else (msg,),
extras)
def schemaValidate(modelXbrl):
class schemaResolver(etree.Resolver):
def resolve(self, url, id, context):
if url.startswith("file:///__"):
url = importedFilepaths[int(url[10:])]
filepath = modelXbrl.modelManager.cntlr.webCache.getfilename(url)
return self.resolve_filename(filepath, context)
entryDocument = modelXbrl.modelDocument
# test of schema validation using lxml (trial experiment, commented out for production use)
from arelle import ModelDocument
imports = []
importedNamespaces = set()
importedFilepaths = []
'''
for mdlSchemaDoc in entryDocument.referencesDocument.keys():
if (mdlSchemaDoc.type == ModelDocument.Type.SCHEMA and
mdlSchemaDoc.targetNamespace not in importedNamespaces):
# actual file won't pass through properly, fake with table reference
imports.append('<xsd:import namespace="{0}" schemaLocation="file:///__{1}"/>'.format(
mdlSchemaDoc.targetNamespace, len(importedFilepaths)))
importedNamespaces.add(mdlSchemaDoc.targetNamespace)
importedFilepaths.append(mdlSchemaDoc.filepath)
'''
def importReferences(referencingDocument):
for mdlSchemaDoc in referencingDocument.referencesDocument.keys():
if (mdlSchemaDoc.type == ModelDocument.Type.SCHEMA and
mdlSchemaDoc.targetNamespace not in importedNamespaces):
importedNamespaces.add(mdlSchemaDoc.targetNamespace)
importReferences(mdlSchemaDoc) # do dependencies first
# actual file won't pass through properly, fake with table reference
imports.append(
'<xsd:import namespace="{0}" schemaLocation="file:///__{1}"/>'
.format(mdlSchemaDoc.targetNamespace,
len(importedFilepaths)))
importedFilepaths.append(mdlSchemaDoc.filepath)
importReferences(entryDocument)
# add schemas used in xml validation but not DTS discovered
for mdlDoc in modelXbrl.urlDocs.values():
if mdlDoc.type in (ModelDocument.Type.INSTANCE,
ModelDocument.Type.LINKBASE):
schemaLocation = mdlDoc.xmlRootElement.get(
"{http://www.w3.org/2001/XMLSchema-instance}schemaLocation")
if schemaLocation:
ns = None
for entry in schemaLocation.split():
if ns is None:
ns = entry
else:
if ns not in importedNamespaces:
imports.append(
'<xsd:import namespace="{0}" schemaLocation="file:///__{1}"/>'
.format(ns, len(importedFilepaths)))
importedNamespaces.add(ns)
importedFilepaths.append(entry)
ns = None
schemaXml = '<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema">\n{0}</xsd:schema>\n'.format(
'\n'.join(imports))
# trace schema files referenced
with open("c:\\temp\\test.xml", "w") as fh:
fh.write(schemaXml)
modelXbrl.modelManager.showStatus(_("lxml validator loading xml schema"))
schema_root = etree.XML(schemaXml)
import time
startedAt = time.time()
parser = etree.XMLParser()
parser.resolvers.add(schemaResolver())
schemaDoc = etree.fromstring(schemaXml,
parser=parser,
base_url=entryDocument.filepath +
"-dummy-import.xsd")
schema = etree.XMLSchema(schemaDoc)
from arelle.Locale import format_string
modelXbrl.info(
"info:lxmlSchemaValidator",
format_string(modelXbrl.modelManager.locale,
_("schema loaded in %.2f secs"),
time.time() - startedAt))
modelXbrl.modelManager.showStatus(_("lxml schema validating"))
# check instance documents and linkbases (sort for inst doc before linkbases, and in file name order)
for mdlDoc in sorted(modelXbrl.urlDocs.values(),
key=lambda mdlDoc: (-mdlDoc.type, mdlDoc.filepath)):
if mdlDoc.type in (ModelDocument.Type.INSTANCE,
ModelDocument.Type.LINKBASE):
startedAt = time.time()
docXmlTree = etree.parse(mdlDoc.filepath)
modelXbrl.info("info:lxmlSchemaValidator",
format_string(modelXbrl.modelManager.locale,
_("schema validated in %.3f secs"),
time.time() - startedAt),
modelDocument=mdlDoc)
if not schema.validate(docXmlTree):
for error in schema.error_log:
modelXbrl.error("lxmlSchema:{0}".format(
error.type_name.lower()),
error.message,
modelDocument=mdlDoc,
sourceLine=error.line)
modelXbrl.modelManager.showStatus(_("lxml validation done"),
clearAfter=3000)
def logArguments(self, codes, msg, codedArgs):
""" Prepares arguments for logger function as per info() below.
If codes includes EFM, GFM, HMRC, or SBR-coded error then the code chosen (if a sequence)
corresponds to whether EFM, GFM, HMRC, or SBR validation is in effect.
"""
def propValues(properties):
# deref objects in properties
return [(p[0],str(p[1])) if len(p) == 2 else (p[0],str(p[1]),propValues(p[2]))
for p in properties if 2 <= len(p) <= 3]
# determine logCode
messageCode = None
for argCode in codes if isinstance(codes,tuple) else (codes,):
if (isinstance(argCode, ModelValue.QName) or
(self.modelManager.disclosureSystem.EFM and argCode.startswith("EFM")) or
(self.modelManager.disclosureSystem.GFM and argCode.startswith("GFM")) or
(self.modelManager.disclosureSystem.HMRC and argCode.startswith("HMRC")) or
(self.modelManager.disclosureSystem.SBRNL and argCode.startswith("SBR.NL")) or
argCode[0:3] not in ("EFM", "GFM", "HMR", "SBR")):
messageCode = argCode
break
# determine message and extra arguments
fmtArgs = {}
extras = {"messageCode":messageCode}
logHrefObjectProperties = getattr(self.logger, "logHrefObjectProperties", False)
for argName, argValue in codedArgs.items():
if argName in ("modelObject", "modelXbrl", "modelDocument"):
try:
entryUrl = self.modelDocument.uri
except AttributeError:
entryUrl = self.entryLoadingUrl
refs = []
for arg in (argValue if isinstance(argValue, (tuple,list,set)) else (argValue,)):
if arg is not None:
if isinstance(arg, _STR_BASE):
objectUrl = arg
else:
try:
objectUrl = arg.modelDocument.uri
except AttributeError:
try:
objectUrl = self.modelDocument.uri
except AttributeError:
objectUrl = self.entryLoadingUrl
file = UrlUtil.relativeUri(entryUrl, objectUrl)
ref = {}
if isinstance(arg,ModelObject):
ref["href"] = file + "#" + XmlUtil.elementFragmentIdentifier(arg)
ref["sourceLine"] = arg.sourceline
ref["objectId"] = arg.objectId()
if logHrefObjectProperties:
try:
ref["properties"] = propValues(arg.propertyView)
except AttributeError:
pass # is a default properties entry appropriate or needed?
else:
ref["href"] = file
try:
ref["sourceLine"] = arg.sourceline
except AttributeError:
pass # arg may not have sourceline, ignore if so
refs.append(ref)
extras["refs"] = refs
elif argName == "sourceLine":
if isinstance(argValue, _INT_TYPES): # must be sortable with int's in logger
extras["sourceLine"] = argValue
elif argName != "exc_info":
if isinstance(argValue, (ModelValue.QName, ModelObject, bool, FileNamedStringIO,
# might be a set of lxml objects not dereferencable at shutdown
tuple, list, set)):
fmtArgs[argName] = str(argValue)
elif argValue is None:
fmtArgs[argName] = "(none)"
elif isinstance(argValue, _INT_TYPES):
# need locale-dependent formatting
fmtArgs[argName] = format_string(self.modelManager.locale, '%i', argValue)
elif isinstance(argValue,float):
# need locale-dependent formatting
fmtArgs[argName] = format_string(self.modelManager.locale, '%f', argValue)
else:
fmtArgs[argName] = argValue
if "refs" not in extras:
try:
file = os.path.basename(self.modelDocument.uri)
except AttributeError:
try:
file = os.path.basename(self.entryLoadingUrl)
except:
file = ""
extras["refs"] = [{"href": file}]
return (messageCode,
(msg, fmtArgs) if fmtArgs else (msg,),
extras)
