def run(self, options, sourceZipStream=None):
"""Process command line arguments or web service request, such as to load and validate an XBRL document, or start web server.
When a web server has been requested, this method may be called multiple times, once for each web service (REST) request that requires processing.
Otherwise (when called for a command line request) this method is called only once for the command line arguments request.
:param options: OptionParser options from parse_args of main argv arguments (when called from command line) or corresponding arguments from web service (REST) request.
:type options: optparse.Values
"""
if options.showOptions: # debug options
for optName, optValue in sorted(options.__dict__.items(), key=lambda optItem: optItem[0]):
self.addToLog("Option {0}={1}".format(optName, optValue), messageCode="info")
self.addToLog("sys.argv {0}".format(sys.argv), messageCode="info")
if options.uiLang: # set current UI Lang (but not config setting)
self.setUiLanguage(options.uiLang)
if options.proxy:
if options.proxy != "show":
proxySettings = proxyTuple(options.proxy)
self.webCache.resetProxies(proxySettings)
self.config["proxySettings"] = proxySettings
self.saveConfig()
self.addToLog(_("Proxy configuration has been set."), messageCode="info")
useOsProxy, urlAddr, urlPort, user, password = self.config.get("proxySettings", proxyTuple("none"))
if useOsProxy:
self.addToLog(_("Proxy configured to use {0}.").format(
_('Microsoft Windows Internet Settings') if sys.platform.startswith("win")
else (_('Mac OS X System Configuration') if sys.platform in ("darwin", "macos")
else _('environment variables'))), messageCode="info")
elif urlAddr:
self.addToLog(_("Proxy setting: http://{0}{1}{2}{3}{4}").format(
user if user else "",
":****" if password else "",
"@" if (user or password) else "",
urlAddr,
":{0}".format(urlPort) if urlPort else ""), messageCode="info")
else:
self.addToLog(_("Proxy is disabled."), messageCode="info")
if options.plugins:
from arelle import PluginManager
resetPlugins = False
savePluginChanges = True
showPluginModules = False
for pluginCmd in options.plugins.split('|'):
cmd = pluginCmd.strip()
if cmd == "show":
showPluginModules = True
elif cmd == "temp":
savePluginChanges = False
elif cmd.startswith("+"):
moduleInfo = PluginManager.addPluginModule(cmd[1:])
if moduleInfo:
self.addToLog(_("Addition of plug-in {0} successful.").format(moduleInfo.get("name")),
messageCode="info", file=moduleInfo.get("moduleURL"))
resetPlugins = True
else:
self.addToLog(_("Unable to load plug-in."), messageCode="info", file=cmd[1:])
elif cmd.startswith("~"):
if PluginManager.reloadPluginModule(cmd[1:]):
self.addToLog(_("Reload of plug-in successful."), messageCode="info", file=cmd[1:])
resetPlugins = True
else:
self.addToLog(_("Unable to reload plug-in."), messageCode="info", file=cmd[1:])
elif cmd.startswith("-"):
if PluginManager.removePluginModule(cmd[1:]):
self.addToLog(_("Deletion of plug-in successful."), messageCode="info", file=cmd[1:])
resetPlugins = True
else:
self.addToLog(_("Unable to delete plug-in."), messageCode="info", file=cmd[1:])
else: # assume it is a module or package
savePluginChanges = False
moduleInfo = PluginManager.addPluginModule(cmd)
if moduleInfo:
self.addToLog(_("Activation of plug-in {0} successful.").format(moduleInfo.get("name")),
messageCode="info", file=moduleInfo.get("moduleURL"))
resetPlugins = True
else:
self.addToLog(_("Unable to load {0} as a plug-in or {0} is not recognized as a command. ").format(cmd), messageCode="info", file=cmd)
if resetPlugins:
PluginManager.reset()
if savePluginChanges:
PluginManager.save(self)
if showPluginModules:
self.addToLog(_("Plug-in modules:"), messageCode="info")
for i, moduleItem in enumerate(sorted(PluginManager.pluginConfig.get("modules", {}).items())):
moduleInfo = moduleItem[1]
self.addToLog(_("Plug-in: {0}; author: {1}; version: {2}; status: {3}; date: {4}; description: {5}; license {6}.").format(
moduleItem[0], moduleInfo.get("author"), moduleInfo.get("version"), moduleInfo.get("status"),
moduleInfo.get("fileDate"), moduleInfo.get("description"), moduleInfo.get("license")),
messageCode="info", file=moduleInfo.get("moduleURL"))
# run utility command line options that don't depend on entrypoint Files
hasUtilityPlugin = False
for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Utility.Run"):
hasUtilityPlugin = True
pluginXbrlMethod(self, options)
# if no entrypointFile is applicable, quit now
if options.proxy or options.plugins or hasUtilityPlugin:
if not options.entrypointFile:
return True # success
self.username = options.username
self.password = options.password
self.entrypointFile = options.entrypointFile
if self.entrypointFile:
filesource = FileSource.openFileSource(self.entrypointFile, self, sourceZipStream)
else:
filesource = None
if options.validateEFM:
if options.disclosureSystemName:
self.addToLog(_("both --efm and --disclosureSystem validation are requested, proceeding with --efm only"),
messageCode="info", file=self.entrypointFile)
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select("efm")
elif options.disclosureSystemName:
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select(options.disclosureSystemName)
elif options.validateHMRC:
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select("hmrc")
else:
self.modelManager.disclosureSystem.select(None) # just load ordinary mappings
self.modelManager.validateDisclosureSystem = False
if options.utrUrl: # override disclosureSystem utrUrl
self.modelManager.disclosureSystem.utrUrl = options.utrUrl
# can be set now because the utr is first loaded at validation time
# disclosure system sets logging filters, override disclosure filters, if specified by command line
if options.logLevelFilter:
self.setLogLevelFilter(options.logLevelFilter)
if options.logCodeFilter:
self.setLogCodeFilter(options.logCodeFilter)
if options.calcDecimals:
if options.calcPrecision:
self.addToLog(_("both --calcDecimals and --calcPrecision validation are requested, proceeding with --calcDecimals only"),
messageCode="info", file=self.entrypointFile)
self.modelManager.validateInferDecimals = True
self.modelManager.validateCalcLB = True
elif options.calcPrecision:
self.modelManager.validateInferDecimals = False
self.modelManager.validateCalcLB = True
if options.utrValidate:
self.modelManager.validateUtr = True
if options.infosetValidate:
self.modelManager.validateInfoset = True
if options.abortOnMajorError:
self.modelManager.abortOnMajorError = True
if options.collectProfileStats:
self.modelManager.collectProfileStats = True
if options.internetConnectivity == "offline":
self.webCache.workOffline = True
elif options.internetConnectivity == "online":
self.webCache.workOffline = False
if options.internetTimeout is not None:
self.webCache.timeout = (options.internetTimeout or None) # use None if zero specified to disable timeout
fo = FormulaOptions()
if options.parameters:
parameterSeparator = (options.parameterSeparator or ',')
fo.parameterValues = dict(((qname(key, noPrefixIsNoNamespace=True),(None,value))
for param in options.parameters.split(parameterSeparator)
for key,sep,value in (param.partition('='),) ) )
if options.formulaParamExprResult:
fo.traceParameterExpressionResult = True
if options.formulaParamInputValue:
fo.traceParameterInputValue = True
if options.formulaCallExprSource:
fo.traceCallExpressionSource = True
if options.formulaCallExprCode:
fo.traceCallExpressionCode = True
if options.formulaCallExprEval:
fo.traceCallExpressionEvaluation = True
if options.formulaCallExprResult:
fo.traceCallExpressionResult = True
if options.formulaVarSetExprEval:
fo.traceVariableSetExpressionEvaluation = True
if options.formulaVarSetExprResult:
fo.traceVariableSetExpressionResult = True
if options.formulaAsserResultCounts:
fo.traceAssertionResultCounts = True
if options.formulaFormulaRules:
fo.traceFormulaRules = True
if options.formulaVarsOrder:
fo.traceVariablesOrder = True
if options.formulaVarExpressionSource:
fo.traceVariableExpressionSource = True
if options.formulaVarExpressionCode:
fo.traceVariableExpressionCode = True
if options.formulaVarExpressionEvaluation:
fo.traceVariableExpressionEvaluation = True
if options.formulaVarExpressionResult:
fo.traceVariableExpressionResult = True
if options.timeVariableSetEvaluation:
fo.timeVariableSetEvaluation = True
if options.formulaVarFilterWinnowing:
fo.traceVariableFilterWinnowing = True
if options.formulaVarFiltersResult:
fo.traceVariableFiltersResult = True
if options.formulaVarFiltersResult:
fo.traceVariableFiltersResult = True
self.modelManager.formulaOptions = fo
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
firstStartedAt = startedAt = time.time()
modelDiffReport = None
success = True
modelXbrl = None
try:
if filesource:
modelXbrl = self.modelManager.load(filesource, _("views loading"))
except ModelDocument.LoadingException:
pass
except Exception as err:
self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format(
err,
traceback.format_tb(sys.exc_info()[2])))
success = False # loading errors, don't attempt to utilize loaded DTS
if modelXbrl and modelXbrl.modelDocument:
loadTime = time.time() - startedAt
modelXbrl.profileStat(_("load"), loadTime)
self.addToLog(format_string(self.modelManager.locale,
_("loaded in %.2f secs at %s"),
(loadTime, timeNow)),
messageCode="info", file=self.entrypointFile)
if options.importFiles:
for importFile in options.importFiles.split("|"):
fileName = importFile.strip()
if sourceZipStream is not None and not (fileName.startswith('http://') or os.path.isabs(fileName)):
fileName = os.path.dirname(modelXbrl.uri) + os.sep + fileName # make relative to sourceZipStream
ModelDocument.load(modelXbrl, fileName)
loadTime = time.time() - startedAt
self.addToLog(format_string(self.modelManager.locale,
_("import in %.2f secs at %s"),
(loadTime, timeNow)),
messageCode="info", file=importFile)
modelXbrl.profileStat(_("import"), loadTime)
if modelXbrl.errors:
success = False # loading errors, don't attempt to utilize loaded DTS
if modelXbrl.modelDocument.type in ModelDocument.Type.TESTCASETYPES:
for pluginXbrlMethod in pluginClassMethods("Testcases.Start"):
pluginXbrlMethod(self, options, modelXbrl)
else: # not a test case, probably instance or DTS
for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Loaded"):
pluginXbrlMethod(self, options, modelXbrl)
else:
success = False
if success and options.diffFile and options.versReportFile:
try:
diffFilesource = FileSource.FileSource(options.diffFile,self)
startedAt = time.time()
modelXbrl2 = self.modelManager.load(diffFilesource, _("views loading"))
if modelXbrl2.errors:
if not options.keepOpen:
modelXbrl2.close()
success = False
else:
loadTime = time.time() - startedAt
modelXbrl.profileStat(_("load"), loadTime)
self.addToLog(format_string(self.modelManager.locale,
_("diff comparison DTS loaded in %.2f secs"),
loadTime),
messageCode="info", file=self.entrypointFile)
startedAt = time.time()
modelDiffReport = self.modelManager.compareDTSes(options.versReportFile)
diffTime = time.time() - startedAt
modelXbrl.profileStat(_("diff"), diffTime)
self.addToLog(format_string(self.modelManager.locale,
_("compared in %.2f secs"),
diffTime),
messageCode="info", file=self.entrypointFile)
except ModelDocument.LoadingException:
success = False
except Exception as err:
success = False
self.addToLog(_("[Exception] Failed to doad diff file: \n{0} \n{1}").format(
err,
traceback.format_tb(sys.exc_info()[2])))
if success:
try:
modelXbrl = self.modelManager.modelXbrl
hasFormulae = modelXbrl.hasFormulae
if options.validate:
startedAt = time.time()
if options.formulaAction: # don't automatically run formulas
modelXbrl.hasFormulae = False
self.modelManager.validate()
if options.formulaAction: # restore setting
modelXbrl.hasFormulae = hasFormulae
self.addToLog(format_string(self.modelManager.locale,
_("validated in %.2f secs"),
time.time() - startedAt),
messageCode="info", file=self.entrypointFile)
if options.formulaAction in ("validate", "run"): # do nothing here if "none"
from arelle import ValidateXbrlDimensions, ValidateFormula
startedAt = time.time()
if not options.validate:
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
# setup fresh parameters from formula optoins
modelXbrl.parameters = fo.typedParameters()
ValidateFormula.validate(modelXbrl, compileOnly=(options.formulaAction != "run"))
self.addToLog(format_string(self.modelManager.locale,
_("formula validation and execution in %.2f secs")
if options.formulaAction == "run"
else _("formula validation only in %.2f secs"),
time.time() - startedAt),
messageCode="info", file=self.entrypointFile)
if options.testReport:
ViewFileTests.viewTests(self.modelManager.modelXbrl, options.testReport, options.testReportCols)
if options.rssReport:
ViewFileRssFeed.viewRssFeed(self.modelManager.modelXbrl, options.rssReport, options.rssReportCols)
if options.DTSFile:
ViewFileDTS.viewDTS(modelXbrl, options.DTSFile)
if options.factsFile:
ViewFileFactList.viewFacts(modelXbrl, options.factsFile, labelrole=options.labelRole, lang=options.labelLang, cols=options.factListCols)
if options.factTableFile:
ViewFileFactTable.viewFacts(modelXbrl, options.factTableFile, labelrole=options.labelRole, lang=options.labelLang)
if options.conceptsFile:
ViewFileConcepts.viewConcepts(modelXbrl, options.conceptsFile, labelrole=options.labelRole, lang=options.labelLang)
if options.preFile:
ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.preFile, "Presentation Linkbase", "http://www.xbrl.org/2003/arcrole/parent-child", labelrole=options.labelRole, lang=options.labelLang)
if options.calFile:
ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.calFile, "Calculation Linkbase", "http://www.xbrl.org/2003/arcrole/summation-item", labelrole=options.labelRole, lang=options.labelLang)
if options.dimFile:
ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.dimFile, "Dimensions", "XBRL-dimensions", labelrole=options.labelRole, lang=options.labelLang)
if options.formulaeFile:
ViewFileFormulae.viewFormulae(modelXbrl, options.formulaeFile, "Formulae", lang=options.labelLang)
if options.viewArcrole and options.viewFile:
ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.viewFile, os.path.basename(options.viewArcrole), options.viewArcrole, labelrole=options.labelRole, lang=options.labelLang)
for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Run"):
pluginXbrlMethod(self, options, modelXbrl)
except (IOError, EnvironmentError) as err:
self.addToLog(_("[IOError] Failed to save output:\n {0}").format(err))
success = False
except Exception as err:
self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format(
err,
traceback.format_tb(sys.exc_info()[2])))
success = False
if modelXbrl:
modelXbrl.profileStat(_("total"), time.time() - firstStartedAt)
if options.collectProfileStats and modelXbrl:
modelXbrl.logProfileStats()
if not options.keepOpen:
if modelDiffReport:
self.modelManager.close(modelDiffReport)
elif modelXbrl:
self.modelManager.close(modelXbrl)
self.username = self.password = None #dereference password
return success
def run(self, options, sourceZipStream=None):
self.entrypointFile = options.entrypointFile
filesource = FileSource.openFileSource(self.entrypointFile, self, sourceZipStream)
if options.validateEFM:
if options.gfmName:
self.addToLog(_("both --efm and --gfm validation are requested, proceeding with --efm only"),
messageCode="info", file=self.entrypointFile)
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select("efm")
elif options.gfmName:
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select(options.gfmName)
elif options.validateHMRC:
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select("hmrc")
else:
self.modelManager.disclosureSystem.select(None) # just load ordinary mappings
if options.calcDecimals:
if options.calcPrecision:
self.addToLog(_("both --calcDecimals and --calcPrecision validation are requested, proceeding with --calcDecimals only"),
messageCode="info", file=self.entrypointFile)
self.modelManager.validateInferDecimals = True
self.modelManager.validateCalcLB = True
elif options.calcPrecision:
self.modelManager.validateInferDecimals = False
self.modelManager.validateCalcLB = True
if options.utrValidate:
self.modelManager.validateUtr = True
fo = FormulaOptions()
if options.parameters:
fo.parameterValues = dict(((qname(key, noPrefixIsNoNamespace=True),(None,value))
for param in options.parameters.split(',')
for key,sep,value in (param.partition('='),) ) )
if options.formulaParamExprResult:
fo.traceParameterExpressionResult = True
if options.formulaParamInputValue:
fo.traceParameterInputValue = True
if options.formulaCallExprSource:
fo.traceCallExpressionSource = True
if options.formulaCallExprCode:
fo.traceCallExpressionCode = True
if options.formulaCallExprEval:
fo.traceCallExpressionEvaluation = True
if options.formulaCallExprResult:
fo.traceCallExpressionResult = True
if options.formulaVarSetExprEval:
fo.traceVariableSetExpressionEvaluation = True
if options.formulaVarSetExprResult:
fo.traceVariableSetExpressionResult = True
if options.formulaAsserResultCounts:
fo.traceAssertionResultCounts = True
if options.formulaFormulaRules:
fo.traceFormulaRules = True
if options.formulaVarsOrder:
fo.traceVariablesOrder = True
if options.formulaVarExpressionSource:
fo.traceVariableExpressionSource = True
if options.formulaVarExpressionCode:
fo.traceVariableExpressionCode = True
if options.formulaVarExpressionEvaluation:
fo.traceVariableExpressionEvaluation = True
if options.formulaVarExpressionResult:
fo.traceVariableExpressionResult = True
if options.formulaVarFilterWinnowing:
fo.traceVariableFilterWinnowing = True
if options.formulaVarFiltersResult:
fo.traceVariableFiltersResult = True
self.modelManager.formulaOptions = fo
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
startedAt = time.time()
modelDiffReport = None
success = True
modelXbrl = None
try:
modelXbrl = self.modelManager.load(filesource, _("views loading"))
except Exception as err:
self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format(
err,
traceback.format_tb(sys.exc_info()[2])))
success = False # loading errors, don't attempt to utilize loaded DTS
if modelXbrl and modelXbrl.modelDocument:
self.addToLog(format_string(self.modelManager.locale,
_("loaded in %.2f secs at %s"),
(time.time() - startedAt, timeNow)),
messageCode="info", file=self.entrypointFile)
if options.importFiles:
for importFile in options.importFiles.split("|"):
ModelDocument.load(modelXbrl, importFile.strip())
self.addToLog(format_string(self.modelManager.locale,
_("imported in %.2f secs at %s"),
(time.time() - startedAt, timeNow)),
messageCode="info", file=importFile)
if modelXbrl.errors:
success = False # loading errors, don't attempt to utilize loaded DTS
else:
success = False
if success and options.diffFile and options.versReportFile:
diffFilesource = FileSource.FileSource(options.diffFile,self)
startedAt = time.time()
modelXbrl2 = self.modelManager.load(diffFilesource, _("views loading"))
if modelXbrl2.errors:
if not options.keepOpen:
modelXbrl2.close()
success = False
else:
self.addToLog(format_string(self.modelManager.locale,
_("diff comparison DTS loaded in %.2f secs"),
time.time() - startedAt),
messageCode="info", file=self.entrypointFile)
startedAt = time.time()
modelDiffReport = self.modelManager.compareDTSes(options.versReportFile)
self.addToLog(format_string(self.modelManager.locale,
_("compared in %.2f secs"),
time.time() - startedAt),
messageCode="info", file=self.entrypointFile)
if success:
try:
if options.validate:
startedAt = time.time()
self.modelManager.validate()
self.addToLog(format_string(self.modelManager.locale,
_("validated in %.2f secs"),
time.time() - startedAt),
messageCode="info", file=self.entrypointFile)
if (options.testReport and
self.modelManager.modelXbrl.modelDocument.type in
(ModelDocument.Type.TESTCASESINDEX,
ModelDocument.Type.TESTCASE,
ModelDocument.Type.REGISTRY)):
ViewFileTests.viewTests(self.modelManager.modelXbrl, options.testReport)
if options.DTSFile:
ViewFileDTS.viewDTS(modelXbrl, options.DTSFile)
if options.factsFile:
ViewFileFactList.viewFacts(modelXbrl, options.factsFile, labelrole=options.labelRole, lang=options.labelLang, cols=options.factListCols)
if options.factTableFile:
ViewFileFactTable.viewFacts(modelXbrl, options.factTableFile, labelrole=options.labelRole, lang=options.labelLang)
if options.conceptsFile:
ViewFileConcepts.viewConcepts(modelXbrl, options.conceptsFile, labelrole=options.labelRole, lang=options.labelLang)
if options.preFile:
ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.preFile, "Presentation Linkbase", "http://www.xbrl.org/2003/arcrole/parent-child", labelrole=options.labelRole, lang=options.labelLang)
if options.calFile:
ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.calFile, "Calculation Linkbase", "http://www.xbrl.org/2003/arcrole/summation-item", labelrole=options.labelRole, lang=options.labelLang)
if options.dimFile:
ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.dimFile, "Dimensions", "XBRL-dimensions", labelrole=options.labelRole, lang=options.labelLang)
if options.formulaeFile:
ViewFileFormulae.viewFormulae(modelXbrl, options.formulaeFile, "Formulae", lang=options.labelLang)
for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Run"):
pluginXbrlMethod(self, options, modelXbrl)
except (IOError, EnvironmentError) as err:
self.addToLog(_("[IOError] Failed to save output:\n {0}").format(err))
success = False
except Exception as err:
self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format(
err,
traceback.format_tb(sys.exc_info()[2])))
success = False
if not options.keepOpen:
if modelDiffReport:
modelDiffReport.close()
elif modelXbrl:
modelXbrl.close()
return success