def run(self, options, sourceZipStream=None): """Process command line arguments or web service request, such as to load and validate an XBRL document, or start web server. When a web server has been requested, this method may be called multiple times, once for each web service (REST) request that requires processing. Otherwise (when called for a command line request) this method is called only once for the command line arguments request. :param options: OptionParser options from parse_args of main argv arguments (when called from command line) or corresponding arguments from web service (REST) request. :type options: optparse.Values """ if options.showOptions: # debug options for optName, optValue in sorted(options.__dict__.items(), key=lambda optItem: optItem[0]): self.addToLog("Option {0}={1}".format(optName, optValue), messageCode="info") self.addToLog("sys.argv {0}".format(sys.argv), messageCode="info") if options.uiLang: # set current UI Lang (but not config setting) self.setUiLanguage(options.uiLang) if options.proxy: if options.proxy != "show": proxySettings = proxyTuple(options.proxy) self.webCache.resetProxies(proxySettings) self.config["proxySettings"] = proxySettings self.saveConfig() self.addToLog(_("Proxy configuration has been set."), messageCode="info") useOsProxy, urlAddr, urlPort, user, password = self.config.get("proxySettings", proxyTuple("none")) if useOsProxy: self.addToLog(_("Proxy configured to use {0}.").format( _('Microsoft Windows Internet Settings') if sys.platform.startswith("win") else (_('Mac OS X System Configuration') if sys.platform in ("darwin", "macos") else _('environment variables'))), messageCode="info") elif urlAddr: self.addToLog(_("Proxy setting: http://{0}{1}{2}{3}{4}").format( user if user else "", ":****" if password else "", "@" if (user or password) else "", urlAddr, ":{0}".format(urlPort) if urlPort else ""), messageCode="info") else: self.addToLog(_("Proxy is disabled."), messageCode="info") if options.plugins: from arelle import PluginManager resetPlugins = False savePluginChanges = True showPluginModules = False for pluginCmd in options.plugins.split('|'): cmd = pluginCmd.strip() if cmd == "show": showPluginModules = True elif cmd == "temp": savePluginChanges = False elif cmd.startswith("+"): moduleInfo = PluginManager.addPluginModule(cmd[1:]) if moduleInfo: self.addToLog(_("Addition of plug-in {0} successful.").format(moduleInfo.get("name")), messageCode="info", file=moduleInfo.get("moduleURL")) resetPlugins = True else: self.addToLog(_("Unable to load plug-in."), messageCode="info", file=cmd[1:]) elif cmd.startswith("~"): if PluginManager.reloadPluginModule(cmd[1:]): self.addToLog(_("Reload of plug-in successful."), messageCode="info", file=cmd[1:]) resetPlugins = True else: self.addToLog(_("Unable to reload plug-in."), messageCode="info", file=cmd[1:]) elif cmd.startswith("-"): if PluginManager.removePluginModule(cmd[1:]): self.addToLog(_("Deletion of plug-in successful."), messageCode="info", file=cmd[1:]) resetPlugins = True else: self.addToLog(_("Unable to delete plug-in."), messageCode="info", file=cmd[1:]) else: # assume it is a module or package savePluginChanges = False moduleInfo = PluginManager.addPluginModule(cmd) if moduleInfo: self.addToLog(_("Activation of plug-in {0} successful.").format(moduleInfo.get("name")), messageCode="info", file=moduleInfo.get("moduleURL")) resetPlugins = True else: self.addToLog(_("Unable to load {0} as a plug-in or {0} is not recognized as a command. ").format(cmd), messageCode="info", file=cmd) if resetPlugins: PluginManager.reset() if savePluginChanges: PluginManager.save(self) if showPluginModules: self.addToLog(_("Plug-in modules:"), messageCode="info") for i, moduleItem in enumerate(sorted(PluginManager.pluginConfig.get("modules", {}).items())): moduleInfo = moduleItem[1] self.addToLog(_("Plug-in: {0}; author: {1}; version: {2}; status: {3}; date: {4}; description: {5}; license {6}.").format( moduleItem[0], moduleInfo.get("author"), moduleInfo.get("version"), moduleInfo.get("status"), moduleInfo.get("fileDate"), moduleInfo.get("description"), moduleInfo.get("license")), messageCode="info", file=moduleInfo.get("moduleURL")) # run utility command line options that don't depend on entrypoint Files hasUtilityPlugin = False for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Utility.Run"): hasUtilityPlugin = True pluginXbrlMethod(self, options) # if no entrypointFile is applicable, quit now if options.proxy or options.plugins or hasUtilityPlugin: if not options.entrypointFile: return True # success self.username = options.username self.password = options.password self.entrypointFile = options.entrypointFile if self.entrypointFile: filesource = FileSource.openFileSource(self.entrypointFile, self, sourceZipStream) else: filesource = None if options.validateEFM: if options.disclosureSystemName: self.addToLog(_("both --efm and --disclosureSystem validation are requested, proceeding with --efm only"), messageCode="info", file=self.entrypointFile) self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select("efm") elif options.disclosureSystemName: self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select(options.disclosureSystemName) elif options.validateHMRC: self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select("hmrc") else: self.modelManager.disclosureSystem.select(None) # just load ordinary mappings self.modelManager.validateDisclosureSystem = False if options.utrUrl: # override disclosureSystem utrUrl self.modelManager.disclosureSystem.utrUrl = options.utrUrl # can be set now because the utr is first loaded at validation time # disclosure system sets logging filters, override disclosure filters, if specified by command line if options.logLevelFilter: self.setLogLevelFilter(options.logLevelFilter) if options.logCodeFilter: self.setLogCodeFilter(options.logCodeFilter) if options.calcDecimals: if options.calcPrecision: self.addToLog(_("both --calcDecimals and --calcPrecision validation are requested, proceeding with --calcDecimals only"), messageCode="info", file=self.entrypointFile) self.modelManager.validateInferDecimals = True self.modelManager.validateCalcLB = True elif options.calcPrecision: self.modelManager.validateInferDecimals = False self.modelManager.validateCalcLB = True if options.utrValidate: self.modelManager.validateUtr = True if options.infosetValidate: self.modelManager.validateInfoset = True if options.abortOnMajorError: self.modelManager.abortOnMajorError = True if options.collectProfileStats: self.modelManager.collectProfileStats = True if options.internetConnectivity == "offline": self.webCache.workOffline = True elif options.internetConnectivity == "online": self.webCache.workOffline = False if options.internetTimeout is not None: self.webCache.timeout = (options.internetTimeout or None) # use None if zero specified to disable timeout fo = FormulaOptions() if options.parameters: parameterSeparator = (options.parameterSeparator or ',') fo.parameterValues = dict(((qname(key, noPrefixIsNoNamespace=True),(None,value)) for param in options.parameters.split(parameterSeparator) for key,sep,value in (param.partition('='),) ) ) if options.formulaParamExprResult: fo.traceParameterExpressionResult = True if options.formulaParamInputValue: fo.traceParameterInputValue = True if options.formulaCallExprSource: fo.traceCallExpressionSource = True if options.formulaCallExprCode: fo.traceCallExpressionCode = True if options.formulaCallExprEval: fo.traceCallExpressionEvaluation = True if options.formulaCallExprResult: fo.traceCallExpressionResult = True if options.formulaVarSetExprEval: fo.traceVariableSetExpressionEvaluation = True if options.formulaVarSetExprResult: fo.traceVariableSetExpressionResult = True if options.formulaAsserResultCounts: fo.traceAssertionResultCounts = True if options.formulaFormulaRules: fo.traceFormulaRules = True if options.formulaVarsOrder: fo.traceVariablesOrder = True if options.formulaVarExpressionSource: fo.traceVariableExpressionSource = True if options.formulaVarExpressionCode: fo.traceVariableExpressionCode = True if options.formulaVarExpressionEvaluation: fo.traceVariableExpressionEvaluation = True if options.formulaVarExpressionResult: fo.traceVariableExpressionResult = True if options.timeVariableSetEvaluation: fo.timeVariableSetEvaluation = True if options.formulaVarFilterWinnowing: fo.traceVariableFilterWinnowing = True if options.formulaVarFiltersResult: fo.traceVariableFiltersResult = True if options.formulaVarFiltersResult: fo.traceVariableFiltersResult = True self.modelManager.formulaOptions = fo timeNow = XmlUtil.dateunionValue(datetime.datetime.now()) firstStartedAt = startedAt = time.time() modelDiffReport = None success = True modelXbrl = None try: if filesource: modelXbrl = self.modelManager.load(filesource, _("views loading")) except ModelDocument.LoadingException: pass except Exception as err: self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format( err, traceback.format_tb(sys.exc_info()[2]))) success = False # loading errors, don't attempt to utilize loaded DTS if modelXbrl and modelXbrl.modelDocument: loadTime = time.time() - startedAt modelXbrl.profileStat(_("load"), loadTime) self.addToLog(format_string(self.modelManager.locale, _("loaded in %.2f secs at %s"), (loadTime, timeNow)), messageCode="info", file=self.entrypointFile) if options.importFiles: for importFile in options.importFiles.split("|"): fileName = importFile.strip() if sourceZipStream is not None and not (fileName.startswith('http://') or os.path.isabs(fileName)): fileName = os.path.dirname(modelXbrl.uri) + os.sep + fileName # make relative to sourceZipStream ModelDocument.load(modelXbrl, fileName) loadTime = time.time() - startedAt self.addToLog(format_string(self.modelManager.locale, _("import in %.2f secs at %s"), (loadTime, timeNow)), messageCode="info", file=importFile) modelXbrl.profileStat(_("import"), loadTime) if modelXbrl.errors: success = False # loading errors, don't attempt to utilize loaded DTS if modelXbrl.modelDocument.type in ModelDocument.Type.TESTCASETYPES: for pluginXbrlMethod in pluginClassMethods("Testcases.Start"): pluginXbrlMethod(self, options, modelXbrl) else: # not a test case, probably instance or DTS for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Loaded"): pluginXbrlMethod(self, options, modelXbrl) else: success = False if success and options.diffFile and options.versReportFile: try: diffFilesource = FileSource.FileSource(options.diffFile,self) startedAt = time.time() modelXbrl2 = self.modelManager.load(diffFilesource, _("views loading")) if modelXbrl2.errors: if not options.keepOpen: modelXbrl2.close() success = False else: loadTime = time.time() - startedAt modelXbrl.profileStat(_("load"), loadTime) self.addToLog(format_string(self.modelManager.locale, _("diff comparison DTS loaded in %.2f secs"), loadTime), messageCode="info", file=self.entrypointFile) startedAt = time.time() modelDiffReport = self.modelManager.compareDTSes(options.versReportFile) diffTime = time.time() - startedAt modelXbrl.profileStat(_("diff"), diffTime) self.addToLog(format_string(self.modelManager.locale, _("compared in %.2f secs"), diffTime), messageCode="info", file=self.entrypointFile) except ModelDocument.LoadingException: success = False except Exception as err: success = False self.addToLog(_("[Exception] Failed to doad diff file: \n{0} \n{1}").format( err, traceback.format_tb(sys.exc_info()[2]))) if success: try: modelXbrl = self.modelManager.modelXbrl hasFormulae = modelXbrl.hasFormulae if options.validate: startedAt = time.time() if options.formulaAction: # don't automatically run formulas modelXbrl.hasFormulae = False self.modelManager.validate() if options.formulaAction: # restore setting modelXbrl.hasFormulae = hasFormulae self.addToLog(format_string(self.modelManager.locale, _("validated in %.2f secs"), time.time() - startedAt), messageCode="info", file=self.entrypointFile) if options.formulaAction in ("validate", "run"): # do nothing here if "none" from arelle import ValidateXbrlDimensions, ValidateFormula startedAt = time.time() if not options.validate: ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # setup fresh parameters from formula optoins modelXbrl.parameters = fo.typedParameters() ValidateFormula.validate(modelXbrl, compileOnly=(options.formulaAction != "run")) self.addToLog(format_string(self.modelManager.locale, _("formula validation and execution in %.2f secs") if options.formulaAction == "run" else _("formula validation only in %.2f secs"), time.time() - startedAt), messageCode="info", file=self.entrypointFile) if options.testReport: ViewFileTests.viewTests(self.modelManager.modelXbrl, options.testReport, options.testReportCols) if options.rssReport: ViewFileRssFeed.viewRssFeed(self.modelManager.modelXbrl, options.rssReport, options.rssReportCols) if options.DTSFile: ViewFileDTS.viewDTS(modelXbrl, options.DTSFile) if options.factsFile: ViewFileFactList.viewFacts(modelXbrl, options.factsFile, labelrole=options.labelRole, lang=options.labelLang, cols=options.factListCols) if options.factTableFile: ViewFileFactTable.viewFacts(modelXbrl, options.factTableFile, labelrole=options.labelRole, lang=options.labelLang) if options.conceptsFile: ViewFileConcepts.viewConcepts(modelXbrl, options.conceptsFile, labelrole=options.labelRole, lang=options.labelLang) if options.preFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.preFile, "Presentation Linkbase", "http://www.xbrl.org/2003/arcrole/parent-child", labelrole=options.labelRole, lang=options.labelLang) if options.calFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.calFile, "Calculation Linkbase", "http://www.xbrl.org/2003/arcrole/summation-item", labelrole=options.labelRole, lang=options.labelLang) if options.dimFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.dimFile, "Dimensions", "XBRL-dimensions", labelrole=options.labelRole, lang=options.labelLang) if options.formulaeFile: ViewFileFormulae.viewFormulae(modelXbrl, options.formulaeFile, "Formulae", lang=options.labelLang) if options.viewArcrole and options.viewFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.viewFile, os.path.basename(options.viewArcrole), options.viewArcrole, labelrole=options.labelRole, lang=options.labelLang) for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Run"): pluginXbrlMethod(self, options, modelXbrl) except (IOError, EnvironmentError) as err: self.addToLog(_("[IOError] Failed to save output:\n {0}").format(err)) success = False except Exception as err: self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format( err, traceback.format_tb(sys.exc_info()[2]))) success = False if modelXbrl: modelXbrl.profileStat(_("total"), time.time() - firstStartedAt) if options.collectProfileStats and modelXbrl: modelXbrl.logProfileStats() if not options.keepOpen: if modelDiffReport: self.modelManager.close(modelDiffReport) elif modelXbrl: self.modelManager.close(modelXbrl) self.username = self.password = None #dereference password return success
def run(self, options, sourceZipStream=None): self.entrypointFile = options.entrypointFile filesource = FileSource.openFileSource(self.entrypointFile, self, sourceZipStream) if options.validateEFM: if options.gfmName: self.addToLog(_("both --efm and --gfm validation are requested, proceeding with --efm only"), messageCode="info", file=self.entrypointFile) self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select("efm") elif options.gfmName: self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select(options.gfmName) elif options.validateHMRC: self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select("hmrc") else: self.modelManager.disclosureSystem.select(None) # just load ordinary mappings if options.calcDecimals: if options.calcPrecision: self.addToLog(_("both --calcDecimals and --calcPrecision validation are requested, proceeding with --calcDecimals only"), messageCode="info", file=self.entrypointFile) self.modelManager.validateInferDecimals = True self.modelManager.validateCalcLB = True elif options.calcPrecision: self.modelManager.validateInferDecimals = False self.modelManager.validateCalcLB = True if options.utrValidate: self.modelManager.validateUtr = True fo = FormulaOptions() if options.parameters: fo.parameterValues = dict(((qname(key, noPrefixIsNoNamespace=True),(None,value)) for param in options.parameters.split(',') for key,sep,value in (param.partition('='),) ) ) if options.formulaParamExprResult: fo.traceParameterExpressionResult = True if options.formulaParamInputValue: fo.traceParameterInputValue = True if options.formulaCallExprSource: fo.traceCallExpressionSource = True if options.formulaCallExprCode: fo.traceCallExpressionCode = True if options.formulaCallExprEval: fo.traceCallExpressionEvaluation = True if options.formulaCallExprResult: fo.traceCallExpressionResult = True if options.formulaVarSetExprEval: fo.traceVariableSetExpressionEvaluation = True if options.formulaVarSetExprResult: fo.traceVariableSetExpressionResult = True if options.formulaAsserResultCounts: fo.traceAssertionResultCounts = True if options.formulaFormulaRules: fo.traceFormulaRules = True if options.formulaVarsOrder: fo.traceVariablesOrder = True if options.formulaVarExpressionSource: fo.traceVariableExpressionSource = True if options.formulaVarExpressionCode: fo.traceVariableExpressionCode = True if options.formulaVarExpressionEvaluation: fo.traceVariableExpressionEvaluation = True if options.formulaVarExpressionResult: fo.traceVariableExpressionResult = True if options.formulaVarFilterWinnowing: fo.traceVariableFilterWinnowing = True if options.formulaVarFiltersResult: fo.traceVariableFiltersResult = True self.modelManager.formulaOptions = fo timeNow = XmlUtil.dateunionValue(datetime.datetime.now()) startedAt = time.time() modelDiffReport = None success = True modelXbrl = None try: modelXbrl = self.modelManager.load(filesource, _("views loading")) except Exception as err: self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format( err, traceback.format_tb(sys.exc_info()[2]))) success = False # loading errors, don't attempt to utilize loaded DTS if modelXbrl and modelXbrl.modelDocument: self.addToLog(format_string(self.modelManager.locale, _("loaded in %.2f secs at %s"), (time.time() - startedAt, timeNow)), messageCode="info", file=self.entrypointFile) if options.importFiles: for importFile in options.importFiles.split("|"): ModelDocument.load(modelXbrl, importFile.strip()) self.addToLog(format_string(self.modelManager.locale, _("imported in %.2f secs at %s"), (time.time() - startedAt, timeNow)), messageCode="info", file=importFile) if modelXbrl.errors: success = False # loading errors, don't attempt to utilize loaded DTS else: success = False if success and options.diffFile and options.versReportFile: diffFilesource = FileSource.FileSource(options.diffFile,self) startedAt = time.time() modelXbrl2 = self.modelManager.load(diffFilesource, _("views loading")) if modelXbrl2.errors: if not options.keepOpen: modelXbrl2.close() success = False else: self.addToLog(format_string(self.modelManager.locale, _("diff comparison DTS loaded in %.2f secs"), time.time() - startedAt), messageCode="info", file=self.entrypointFile) startedAt = time.time() modelDiffReport = self.modelManager.compareDTSes(options.versReportFile) self.addToLog(format_string(self.modelManager.locale, _("compared in %.2f secs"), time.time() - startedAt), messageCode="info", file=self.entrypointFile) if success: try: if options.validate: startedAt = time.time() self.modelManager.validate() self.addToLog(format_string(self.modelManager.locale, _("validated in %.2f secs"), time.time() - startedAt), messageCode="info", file=self.entrypointFile) if (options.testReport and self.modelManager.modelXbrl.modelDocument.type in (ModelDocument.Type.TESTCASESINDEX, ModelDocument.Type.TESTCASE, ModelDocument.Type.REGISTRY)): ViewFileTests.viewTests(self.modelManager.modelXbrl, options.testReport) if options.DTSFile: ViewFileDTS.viewDTS(modelXbrl, options.DTSFile) if options.factsFile: ViewFileFactList.viewFacts(modelXbrl, options.factsFile, labelrole=options.labelRole, lang=options.labelLang, cols=options.factListCols) if options.factTableFile: ViewFileFactTable.viewFacts(modelXbrl, options.factTableFile, labelrole=options.labelRole, lang=options.labelLang) if options.conceptsFile: ViewFileConcepts.viewConcepts(modelXbrl, options.conceptsFile, labelrole=options.labelRole, lang=options.labelLang) if options.preFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.preFile, "Presentation Linkbase", "http://www.xbrl.org/2003/arcrole/parent-child", labelrole=options.labelRole, lang=options.labelLang) if options.calFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.calFile, "Calculation Linkbase", "http://www.xbrl.org/2003/arcrole/summation-item", labelrole=options.labelRole, lang=options.labelLang) if options.dimFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.dimFile, "Dimensions", "XBRL-dimensions", labelrole=options.labelRole, lang=options.labelLang) if options.formulaeFile: ViewFileFormulae.viewFormulae(modelXbrl, options.formulaeFile, "Formulae", lang=options.labelLang) for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Run"): pluginXbrlMethod(self, options, modelXbrl) except (IOError, EnvironmentError) as err: self.addToLog(_("[IOError] Failed to save output:\n {0}").format(err)) success = False except Exception as err: self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format( err, traceback.format_tb(sys.exc_info()[2]))) success = False if not options.keepOpen: if modelDiffReport: modelDiffReport.close() elif modelXbrl: modelXbrl.close() return success