コード例 #1
0
ファイル: CntlrCmdLine.py プロジェクト: GuoHuiChen/Arelle
    def run(self, options, sourceZipStream=None):
        """Process command line arguments or web service request, such as to load and validate an XBRL document, or start web server.
        
        When a web server has been requested, this method may be called multiple times, once for each web service (REST) request that requires processing.
        Otherwise (when called for a command line request) this method is called only once for the command line arguments request.
           
        :param options: OptionParser options from parse_args of main argv arguments (when called from command line) or corresponding arguments from web service (REST) request.
        :type options: optparse.Values
        """
        if options.showOptions: # debug options
            for optName, optValue in sorted(options.__dict__.items(), key=lambda optItem: optItem[0]):
                self.addToLog("Option {0}={1}".format(optName, optValue), messageCode="info")
            self.addToLog("sys.argv {0}".format(sys.argv), messageCode="info")
        if options.uiLang: # set current UI Lang (but not config setting)
            self.setUiLanguage(options.uiLang)
        if options.proxy:
            if options.proxy != "show":
                proxySettings = proxyTuple(options.proxy)
                self.webCache.resetProxies(proxySettings)
                self.config["proxySettings"] = proxySettings
                self.saveConfig()
                self.addToLog(_("Proxy configuration has been set."), messageCode="info")
            useOsProxy, urlAddr, urlPort, user, password = self.config.get("proxySettings", proxyTuple("none"))
            if useOsProxy:
                self.addToLog(_("Proxy configured to use {0}.").format(
                    _('Microsoft Windows Internet Settings') if sys.platform.startswith("win")
                    else (_('Mac OS X System Configuration') if sys.platform in ("darwin", "macos")
                          else _('environment variables'))), messageCode="info")
            elif urlAddr:
                self.addToLog(_("Proxy setting: http://{0}{1}{2}{3}{4}").format(
                    user if user else "",
                    ":****" if password else "",
                    "@" if (user or password) else "",
                    urlAddr,
                    ":{0}".format(urlPort) if urlPort else ""), messageCode="info")
            else:
                self.addToLog(_("Proxy is disabled."), messageCode="info")
        if options.plugins:
            from arelle import PluginManager
            resetPlugins = False
            savePluginChanges = True
            showPluginModules = False
            for pluginCmd in options.plugins.split('|'):
                cmd = pluginCmd.strip()
                if cmd == "show":
                    showPluginModules = True
                elif cmd == "temp":
                    savePluginChanges = False
                elif cmd.startswith("+"):
                    moduleInfo = PluginManager.addPluginModule(cmd[1:])
                    if moduleInfo:
                        self.addToLog(_("Addition of plug-in {0} successful.").format(moduleInfo.get("name")), 
                                      messageCode="info", file=moduleInfo.get("moduleURL"))
                        resetPlugins = True
                    else:
                        self.addToLog(_("Unable to load plug-in."), messageCode="info", file=cmd[1:])
                elif cmd.startswith("~"):
                    if PluginManager.reloadPluginModule(cmd[1:]):
                        self.addToLog(_("Reload of plug-in successful."), messageCode="info", file=cmd[1:])
                        resetPlugins = True
                    else:
                        self.addToLog(_("Unable to reload plug-in."), messageCode="info", file=cmd[1:])
                elif cmd.startswith("-"):
                    if PluginManager.removePluginModule(cmd[1:]):
                        self.addToLog(_("Deletion of plug-in successful."), messageCode="info", file=cmd[1:])
                        resetPlugins = True
                    else:
                        self.addToLog(_("Unable to delete plug-in."), messageCode="info", file=cmd[1:])
                else: # assume it is a module or package
                    savePluginChanges = False
                    moduleInfo = PluginManager.addPluginModule(cmd)
                    if moduleInfo:
                        self.addToLog(_("Activation of plug-in {0} successful.").format(moduleInfo.get("name")), 
                                      messageCode="info", file=moduleInfo.get("moduleURL"))
                        resetPlugins = True
                    else:
                        self.addToLog(_("Unable to load {0} as a plug-in or {0} is not recognized as a command. ").format(cmd), messageCode="info", file=cmd)
                if resetPlugins:
                    PluginManager.reset()
                    if savePluginChanges:
                        PluginManager.save(self)
            if showPluginModules:
                self.addToLog(_("Plug-in modules:"), messageCode="info")
                for i, moduleItem in enumerate(sorted(PluginManager.pluginConfig.get("modules", {}).items())):
                    moduleInfo = moduleItem[1]
                    self.addToLog(_("Plug-in: {0}; author: {1}; version: {2}; status: {3}; date: {4}; description: {5}; license {6}.").format(
                                  moduleItem[0], moduleInfo.get("author"), moduleInfo.get("version"), moduleInfo.get("status"),
                                  moduleInfo.get("fileDate"), moduleInfo.get("description"), moduleInfo.get("license")),
                                  messageCode="info", file=moduleInfo.get("moduleURL"))
                
        if options.packages:
            from arelle import PackageManager
            savePackagesChanges = True
            showPackages = False
            for packageCmd in options.packages.split('|'):
                cmd = packageCmd.strip()
                if cmd == "show":
                    showPackages = True
                elif cmd == "temp":
                    savePackagesChanges = False
                elif cmd.startswith("+"):
                    packageInfo = PackageManager.addPackage(cmd[1:])
                    if packageInfo:
                        self.addToLog(_("Addition of package {0} successful.").format(packageInfo.get("name")), 
                                      messageCode="info", file=packageInfo.get("URL"))
                    else:
                        self.addToLog(_("Unable to load plug-in."), messageCode="info", file=cmd[1:])
                elif cmd.startswith("~"):
                    if PackageManager.reloadPackageModule(cmd[1:]):
                        self.addToLog(_("Reload of package successful."), messageCode="info", file=cmd[1:])
                    else:
                        self.addToLog(_("Unable to reload package."), messageCode="info", file=cmd[1:])
                elif cmd.startswith("-"):
                    if PackageManager.removePackageModule(cmd[1:]):
                        self.addToLog(_("Deletion of package successful."), messageCode="info", file=cmd[1:])
                    else:
                        self.addToLog(_("Unable to delete package."), messageCode="info", file=cmd[1:])
                else: # assume it is a module or package
                    savePackagesChanges = False
                    packageInfo = PackageManager.addPackage(cmd)
                    if packageInfo:
                        self.addToLog(_("Activation of package {0} successful.").format(packageInfo.get("name")), 
                                      messageCode="info", file=packageInfo.get("URL"))
                        resetPlugins = True
                    else:
                        self.addToLog(_("Unable to load {0} as a package or {0} is not recognized as a command. ").format(cmd), messageCode="info", file=cmd)
            if savePackagesChanges:
                PackageManager.save(self)
            if showPackages:
                self.addToLog(_("Taxonomy packages:"), messageCode="info")
                for packageInfo in PackageManager.orderedPackagesConfig()["packages"]:
                    self.addToLog(_("Package: {0}; version: {1}; status: {2}; date: {3}; description: {4}.").format(
                                  packageInfo.get("name"), packageInfo.get("version"), packageInfo.get("status"),
                                  packageInfo.get("fileDate"), packageInfo.get("description")),
                                  messageCode="info", file=packageInfo.get("URL"))
                
        # run utility command line options that don't depend on entrypoint Files
        hasUtilityPlugin = False
        for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Utility.Run"):
            hasUtilityPlugin = True
            pluginXbrlMethod(self, options)
            
        # if no entrypointFile is applicable, quit now
        if options.proxy or options.plugins or hasUtilityPlugin:
            if not options.entrypointFile:
                return True # success
        self.username = options.username
        self.password = options.password
        self.entrypointFile = options.entrypointFile
        if self.entrypointFile:
            filesource = FileSource.openFileSource(self.entrypointFile, self, sourceZipStream)
        else:
            filesource = None
        if options.validateEFM:
            if options.disclosureSystemName:
                self.addToLog(_("both --efm and --disclosureSystem validation are requested, proceeding with --efm only"),
                              messageCode="info", file=self.entrypointFile)
            self.modelManager.validateDisclosureSystem = True
            self.modelManager.disclosureSystem.select("efm")
        elif options.disclosureSystemName:
            self.modelManager.validateDisclosureSystem = True
            self.modelManager.disclosureSystem.select(options.disclosureSystemName)
        elif options.validateHMRC:
            self.modelManager.validateDisclosureSystem = True
            self.modelManager.disclosureSystem.select("hmrc")
        else:
            self.modelManager.disclosureSystem.select(None) # just load ordinary mappings
            self.modelManager.validateDisclosureSystem = False
        if options.utrUrl:  # override disclosureSystem utrUrl
            self.modelManager.disclosureSystem.utrUrl = options.utrUrl
            # can be set now because the utr is first loaded at validation time 
            
        # disclosure system sets logging filters, override disclosure filters, if specified by command line
        if options.logLevelFilter:
            self.setLogLevelFilter(options.logLevelFilter)
        if options.logCodeFilter:
            self.setLogCodeFilter(options.logCodeFilter)
        if options.calcDecimals:
            if options.calcPrecision:
                self.addToLog(_("both --calcDecimals and --calcPrecision validation are requested, proceeding with --calcDecimals only"),
                              messageCode="info", file=self.entrypointFile)
            self.modelManager.validateInferDecimals = True
            self.modelManager.validateCalcLB = True
        elif options.calcPrecision:
            self.modelManager.validateInferDecimals = False
            self.modelManager.validateCalcLB = True
        if options.utrValidate:
            self.modelManager.validateUtr = True
        if options.infosetValidate:
            self.modelManager.validateInfoset = True
        if options.abortOnMajorError:
            self.modelManager.abortOnMajorError = True
        if options.collectProfileStats:
            self.modelManager.collectProfileStats = True
        if options.internetConnectivity == "offline":
            self.webCache.workOffline = True
        elif options.internetConnectivity == "online":
            self.webCache.workOffline = False
        if options.internetTimeout is not None:
            self.webCache.timeout = (options.internetTimeout or None)  # use None if zero specified to disable timeout
        fo = FormulaOptions()
        if options.parameters:
            parameterSeparator = (options.parameterSeparator or ',')
            fo.parameterValues = dict(((qname(key, noPrefixIsNoNamespace=True),(None,value)) 
                                       for param in options.parameters.split(parameterSeparator) 
                                       for key,sep,value in (param.partition('='),) ) )   
        if options.formulaParamExprResult:
            fo.traceParameterExpressionResult = True
        if options.formulaParamInputValue:
            fo.traceParameterInputValue = True
        if options.formulaCallExprSource:
            fo.traceCallExpressionSource = True
        if options.formulaCallExprCode:
            fo.traceCallExpressionCode = True
        if options.formulaCallExprEval:
            fo.traceCallExpressionEvaluation = True
        if options.formulaCallExprResult:
            fo.traceCallExpressionResult = True
        if options.formulaVarSetExprEval:
            fo.traceVariableSetExpressionEvaluation = True
        if options.formulaVarSetExprResult:
            fo.traceVariableSetExpressionResult = True
        if options.formulaAsserResultCounts:
            fo.traceAssertionResultCounts = True
        if options.formulaFormulaRules:
            fo.traceFormulaRules = True
        if options.formulaVarsOrder:
            fo.traceVariablesOrder = True
        if options.formulaVarExpressionSource:
            fo.traceVariableExpressionSource = True
        if options.formulaVarExpressionCode:
            fo.traceVariableExpressionCode = True
        if options.formulaVarExpressionEvaluation:
            fo.traceVariableExpressionEvaluation = True
        if options.formulaVarExpressionResult:
            fo.traceVariableExpressionResult = True
        if options.timeVariableSetEvaluation:
            fo.timeVariableSetEvaluation = True
        if options.formulaVarFilterWinnowing:
            fo.traceVariableFilterWinnowing = True
        if options.formulaVarFiltersResult:
            fo.traceVariableFiltersResult = True
        if options.formulaVarFiltersResult:
            fo.traceVariableFiltersResult = True
        self.modelManager.formulaOptions = fo
        timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
        firstStartedAt = startedAt = time.time()
        modelDiffReport = None
        success = True
        modelXbrl = None
        try:
            if filesource:
                modelXbrl = self.modelManager.load(filesource, _("views loading"))
        except ModelDocument.LoadingException:
            pass
        except Exception as err:
            self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format(
                        err,
                        traceback.format_tb(sys.exc_info()[2])))
            success = False    # loading errors, don't attempt to utilize loaded DTS
        if modelXbrl and modelXbrl.modelDocument:
            loadTime = time.time() - startedAt
            modelXbrl.profileStat(_("load"), loadTime)
            self.addToLog(format_string(self.modelManager.locale, 
                                        _("loaded in %.2f secs at %s"), 
                                        (loadTime, timeNow)), 
                                        messageCode="info", file=self.entrypointFile)
            if options.importFiles:
                for importFile in options.importFiles.split("|"):
                    fileName = importFile.strip()
                    if sourceZipStream is not None and not (fileName.startswith('http://') or os.path.isabs(fileName)):
                        fileName = os.path.dirname(modelXbrl.uri) + os.sep + fileName # make relative to sourceZipStream
                    ModelDocument.load(modelXbrl, fileName)
                    loadTime = time.time() - startedAt
                    self.addToLog(format_string(self.modelManager.locale, 
                                                _("import in %.2f secs at %s"), 
                                                (loadTime, timeNow)), 
                                                messageCode="info", file=importFile)
                    modelXbrl.profileStat(_("import"), loadTime)
                if modelXbrl.errors:
                    success = False    # loading errors, don't attempt to utilize loaded DTS
            if modelXbrl.modelDocument.type in ModelDocument.Type.TESTCASETYPES:
                for pluginXbrlMethod in pluginClassMethods("Testcases.Start"):
                    pluginXbrlMethod(self, options, modelXbrl)
            else: # not a test case, probably instance or DTS
                for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Loaded"):
                    pluginXbrlMethod(self, options, modelXbrl)
        else:
            success = False
        if success and options.diffFile and options.versReportFile:
            try:
                diffFilesource = FileSource.FileSource(options.diffFile,self)
                startedAt = time.time()
                modelXbrl2 = self.modelManager.load(diffFilesource, _("views loading"))
                if modelXbrl2.errors:
                    if not options.keepOpen:
                        modelXbrl2.close()
                    success = False
                else:
                    loadTime = time.time() - startedAt
                    modelXbrl.profileStat(_("load"), loadTime)
                    self.addToLog(format_string(self.modelManager.locale, 
                                                _("diff comparison DTS loaded in %.2f secs"), 
                                                loadTime), 
                                                messageCode="info", file=self.entrypointFile)
                    startedAt = time.time()
                    modelDiffReport = self.modelManager.compareDTSes(options.versReportFile)
                    diffTime = time.time() - startedAt
                    modelXbrl.profileStat(_("diff"), diffTime)
                    self.addToLog(format_string(self.modelManager.locale, 
                                                _("compared in %.2f secs"), 
                                                diffTime), 
                                                messageCode="info", file=self.entrypointFile)
            except ModelDocument.LoadingException:
                success = False
            except Exception as err:
                success = False
                self.addToLog(_("[Exception] Failed to doad diff file: \n{0} \n{1}").format(
                            err,
                            traceback.format_tb(sys.exc_info()[2])))
        if success:
            try:
                modelXbrl = self.modelManager.modelXbrl
                hasFormulae = modelXbrl.hasFormulae
                if options.validate:
                    startedAt = time.time()
                    if options.formulaAction: # don't automatically run formulas
                        modelXbrl.hasFormulae = False
                    self.modelManager.validate()
                    if options.formulaAction: # restore setting
                        modelXbrl.hasFormulae = hasFormulae
                    self.addToLog(format_string(self.modelManager.locale, 
                                                _("validated in %.2f secs"), 
                                                time.time() - startedAt),
                                                messageCode="info", file=self.entrypointFile)
                if options.formulaAction in ("validate", "run"):  # do nothing here if "none"
                    from arelle import ValidateXbrlDimensions, ValidateFormula
                    startedAt = time.time()
                    if not options.validate:
                        ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
                    # setup fresh parameters from formula optoins
                    modelXbrl.parameters = fo.typedParameters()
                    ValidateFormula.validate(modelXbrl, compileOnly=(options.formulaAction != "run"))
                    self.addToLog(format_string(self.modelManager.locale, 
                                                _("formula validation and execution in %.2f secs")
                                                if options.formulaAction == "run"
                                                else _("formula validation only in %.2f secs"), 
                                                time.time() - startedAt),
                                                messageCode="info", file=self.entrypointFile)
                    

                if options.testReport:
                    ViewFileTests.viewTests(self.modelManager.modelXbrl, options.testReport, options.testReportCols)
                    
                if options.rssReport:
                    ViewFileRssFeed.viewRssFeed(self.modelManager.modelXbrl, options.rssReport, options.rssReportCols)
                    
                if options.DTSFile:
                    ViewFileDTS.viewDTS(modelXbrl, options.DTSFile)
                if options.factsFile:
                    ViewFileFactList.viewFacts(modelXbrl, options.factsFile, labelrole=options.labelRole, lang=options.labelLang, cols=options.factListCols)
                if options.factTableFile:
                    ViewFileFactTable.viewFacts(modelXbrl, options.factTableFile, labelrole=options.labelRole, lang=options.labelLang)
                if options.conceptsFile:
                    ViewFileConcepts.viewConcepts(modelXbrl, options.conceptsFile, labelrole=options.labelRole, lang=options.labelLang)
                if options.preFile:
                    ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.preFile, "Presentation Linkbase", "http://www.xbrl.org/2003/arcrole/parent-child", labelrole=options.labelRole, lang=options.labelLang)
                if options.calFile:
                    ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.calFile, "Calculation Linkbase", "http://www.xbrl.org/2003/arcrole/summation-item", labelrole=options.labelRole, lang=options.labelLang)
                if options.dimFile:
                    ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.dimFile, "Dimensions", "XBRL-dimensions", labelrole=options.labelRole, lang=options.labelLang)
                if options.formulaeFile:
                    ViewFileFormulae.viewFormulae(modelXbrl, options.formulaeFile, "Formulae", lang=options.labelLang)
                if options.viewArcrole and options.viewFile:
                    ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.viewFile, os.path.basename(options.viewArcrole), options.viewArcrole, labelrole=options.labelRole, lang=options.labelLang)
                if options.roleTypesFile:
                    ViewFileRoleTypes.viewRoleTypes(modelXbrl, options.roleTypesFile, "Role Types", isArcrole=False, lang=options.labelLang)
                if options.arcroleTypesFile:
                    ViewFileRoleTypes.viewRoleTypes(modelXbrl, options.arcroleTypesFile, "Arcrole Types", isArcrole=True, lang=options.labelLang)
                for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Run"):
                    pluginXbrlMethod(self, options, modelXbrl)
                                        
            except (IOError, EnvironmentError) as err:
                self.addToLog(_("[IOError] Failed to save output:\n {0}").format(err))
                success = False
            except Exception as err:
                self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format(
                            err,
                            traceback.format_tb(sys.exc_info()[2])))
                success = False
        if modelXbrl:
            modelXbrl.profileStat(_("total"), time.time() - firstStartedAt)
            if options.collectProfileStats and modelXbrl:
                modelXbrl.logProfileStats()
            if not options.keepOpen:
                if modelDiffReport:
                    self.modelManager.close(modelDiffReport)
                elif modelXbrl:
                    self.modelManager.close(modelXbrl)
        self.username = self.password = None #dereference password
        return success