def save_taxonomy_config(taxonomies_dir, controller=None):
if controller is None:
controller = Cntlr(logFileName='logToStdErr')
PackageManager.init(controller)
for taxonomy_zip in os.listdir(taxonomies_dir):
if taxonomy_zip.endswith('.zip'):
taxonomy_zip_path = os.path.join(taxonomies_dir, taxonomy_zip)
PackageManager.addPackage(controller, taxonomy_zip_path)
PackageManager.save(controller)
def loadPackagesFromDir(self, directory):
packages = glob.glob(os.path.join(directory, "*.zip"))
for p in packages:
pi = PackageManager.addPackage(self, p)
if pi:
self.addToLog("Package added", messageCode="info", file=pi.get("URL"))
else:
self.addToLog("Failed to load package", messageCode="error", file=p)
def load(self,
filesource,
nextaction=None,
taxonomyPackages=None,
**kwargs):
"""Load an entry point modelDocument object(s), which in turn load documents they discover
(for the case of instance, taxonomies, and versioning reports), but defer loading instances
for test case and RSS feeds.
The modelXbrl that is loaded is 'stacked', by this class, so that any modelXbrl operations such as validate,
and close, operate on the most recently loaded modelXbrl, and compareDTSes operates on the two
most recently loaded modelXbrl's.
:param filesource: may be a FileSource object, with the entry point selected, or string file name (or web URL).
:type filesource: FileSource or str
:param nextAction: status line text string, if any, to show upon completion
:type nextAction: str
:param taxonomyPackages: array of URLs of taxonomy packages required for load operation
"""
if taxonomyPackages:
resetPackageMappings = False
for pkgUrl in taxonomyPackages:
if PackageManager.addPackage(self.cntlr, pkgUrl):
resetPackageMappings = True
if resetPackageMappings:
PackageManager.rebuildRemappings(self.cntlr)
try:
if filesource.url.startswith(
"urn:uuid:"): # request for an open modelXbrl
for modelXbrl in self.loadedModelXbrls:
if not modelXbrl.isClosed and modelXbrl.uuid == filesource.url:
return modelXbrl
raise IOError(
_("Open file handle is not open: {0}").format(
filesource.url))
except AttributeError:
pass # filesource may be a string, which has no url attribute
self.filesource = filesource
modelXbrl = None # loaded modelXbrl
for customLoader in pluginClassMethods("ModelManager.Load"):
modelXbrl = customLoader(self, filesource)
if modelXbrl is not None:
break # custom loader did the loading
if modelXbrl is None: # use default xbrl loader
modelXbrl = ModelXbrl.load(self, filesource, nextaction, **kwargs)
self.modelXbrl = modelXbrl
self.loadedModelXbrls.append(self.modelXbrl)
return self.modelXbrl
def open_package_file(self, file_name):
"""Open a taxonomy package in the rule set
:param file_name: the name of the package file in the rule set
:type file_name: str
:return: The package information. This is the return from Arelle when activating the package
"""
package_info = PackageManager.addPackage(self._cntlr, file_name)
if package_info:
# print("Activation of package {0} successful.".format(package_info.get("name")))
self._cntlr.addToLog(_("Activation of package {0} successful.").format(package_info.get("name")),
messageCode="info", file=package_info.get("URL"))
else:
# print("Unable to load package \"{}\". ".format(file_name))
self._cntlr.addToLog(_("Unable to load package \"%(name)s\". "),
messageCode="arelle:packageLoadingError",
level=logging.ERROR)
return package_info
def open_package_file(self, file_name):
"""Open a taxonomy package in the rule set
:param file_name: the name of the package file in the rule set
:type file_name: str
:return: The package information. This is the return from Arelle when activating the package
"""
package_info = PackageManager.addPackage(self._cntlr, file_name)
if package_info:
# print("Activation of package {0} successful.".format(package_info.get("name")))
self._cntlr.addToLog(_("Activation of package {0} successful.").format(package_info.get("name")),
messageCode="info", file=package_info.get("URL"))
else:
# print("Unable to load package \"{}\". ".format(file_name))
self._cntlr.addToLog(_("Unable to load package \"%(name)s\". "),
messageCode="arelle:packageLoadingError",
level=logging.ERROR)
return package_info
def get_packages_info(self):
"""Get a list of taxonomy packages in the rule set
:returns: List of package information. The package information is returned when activating the package in Arelle
:rtype: list
"""
results = []
temp_dir = tempfile.TemporaryDirectory()
#Using arelle file source object. This will handle files from the web.
file_object = self._get_rule_set_file_object()
try:
with zipfile.ZipFile(file_object, 'r') as zf:
for package_file_name in zf.namelist():
if package_file_name.startswith('packages/'):
package_file = zf.extract(package_file_name, temp_dir.name)
package_info = PackageManager.addPackage(self._cntlr, package_file)
results.append(package_info)
finally:
file_object.close()
return results
def get_packages_info(self):
"""Get a list of taxonomy packages in the rule set
:returns: List of package information. The package information is returned when activating the package in Arelle
:rtype: list
"""
results = []
temp_dir = tempfile.TemporaryDirectory()
#Using arelle file source object. This will handle files from the web.
file_object = self._get_rule_set_file_object()
try:
with zipfile.ZipFile(file_object, 'r') as zf:
for package_file_name in zf.namelist():
if package_file_name.startswith('packages/'):
package_file = zf.extract(package_file_name, temp_dir.name)
package_info = PackageManager.addPackage(self._cntlr, package_file)
results.append(package_info)
finally:
file_object.close()
return results
def run(self, options, sourceZipStream=None):
"""Process command line arguments or web service request, such as to load and validate an XBRL document, or start web server.
When a web server has been requested, this method may be called multiple times, once for each web service (REST) request that requires processing.
Otherwise (when called for a command line request) this method is called only once for the command line arguments request.
:param options: OptionParser options from parse_args of main argv arguments (when called from command line) or corresponding arguments from web service (REST) request.
:type options: optparse.Values
"""
if options.showOptions: # debug options
for optName, optValue in sorted(options.__dict__.items(), key=lambda optItem: optItem[0]):
self.addToLog("Option {0}={1}".format(optName, optValue), messageCode="info")
self.addToLog("sys.argv {0}".format(sys.argv), messageCode="info")
if options.uiLang: # set current UI Lang (but not config setting)
self.setUiLanguage(options.uiLang)
if options.proxy:
if options.proxy != "show":
proxySettings = proxyTuple(options.proxy)
self.webCache.resetProxies(proxySettings)
self.config["proxySettings"] = proxySettings
self.saveConfig()
self.addToLog(_("Proxy configuration has been set."), messageCode="info")
useOsProxy, urlAddr, urlPort, user, password = self.config.get("proxySettings", proxyTuple("none"))
if useOsProxy:
self.addToLog(_("Proxy configured to use {0}.").format(
_('Microsoft Windows Internet Settings') if sys.platform.startswith("win")
else (_('Mac OS X System Configuration') if sys.platform in ("darwin", "macos")
else _('environment variables'))), messageCode="info")
elif urlAddr:
self.addToLog(_("Proxy setting: http://{0}{1}{2}{3}{4}").format(
user if user else "",
":****" if password else "",
"@" if (user or password) else "",
urlAddr,
":{0}".format(urlPort) if urlPort else ""), messageCode="info")
else:
self.addToLog(_("Proxy is disabled."), messageCode="info")
if options.plugins:
from arelle import PluginManager
resetPlugins = False
savePluginChanges = True
showPluginModules = False
for pluginCmd in options.plugins.split('|'):
cmd = pluginCmd.strip()
if cmd == "show":
showPluginModules = True
elif cmd == "temp":
savePluginChanges = False
elif cmd.startswith("+"):
moduleInfo = PluginManager.addPluginModule(cmd[1:])
if moduleInfo:
self.addToLog(_("Addition of plug-in {0} successful.").format(moduleInfo.get("name")),
messageCode="info", file=moduleInfo.get("moduleURL"))
resetPlugins = True
else:
self.addToLog(_("Unable to load plug-in."), messageCode="info", file=cmd[1:])
elif cmd.startswith("~"):
if PluginManager.reloadPluginModule(cmd[1:]):
self.addToLog(_("Reload of plug-in successful."), messageCode="info", file=cmd[1:])
resetPlugins = True
else:
self.addToLog(_("Unable to reload plug-in."), messageCode="info", file=cmd[1:])
elif cmd.startswith("-"):
if PluginManager.removePluginModule(cmd[1:]):
self.addToLog(_("Deletion of plug-in successful."), messageCode="info", file=cmd[1:])
resetPlugins = True
else:
self.addToLog(_("Unable to delete plug-in."), messageCode="info", file=cmd[1:])
else: # assume it is a module or package
savePluginChanges = False
moduleInfo = PluginManager.addPluginModule(cmd)
if moduleInfo:
self.addToLog(_("Activation of plug-in {0} successful.").format(moduleInfo.get("name")),
messageCode="info", file=moduleInfo.get("moduleURL"))
resetPlugins = True
else:
self.addToLog(_("Unable to load {0} as a plug-in or {0} is not recognized as a command. ").format(cmd), messageCode="info", file=cmd)
if resetPlugins:
PluginManager.reset()
if savePluginChanges:
PluginManager.save(self)
if showPluginModules:
self.addToLog(_("Plug-in modules:"), messageCode="info")
for i, moduleItem in enumerate(sorted(PluginManager.pluginConfig.get("modules", {}).items())):
moduleInfo = moduleItem[1]
self.addToLog(_("Plug-in: {0}; author: {1}; version: {2}; status: {3}; date: {4}; description: {5}; license {6}.").format(
moduleItem[0], moduleInfo.get("author"), moduleInfo.get("version"), moduleInfo.get("status"),
moduleInfo.get("fileDate"), moduleInfo.get("description"), moduleInfo.get("license")),
messageCode="info", file=moduleInfo.get("moduleURL"))
if options.packages:
from arelle import PackageManager
savePackagesChanges = True
showPackages = False
for packageCmd in options.packages.split('|'):
cmd = packageCmd.strip()
if cmd == "show":
showPackages = True
elif cmd == "temp":
savePackagesChanges = False
elif cmd.startswith("+"):
packageInfo = PackageManager.addPackage(cmd[1:])
if packageInfo:
self.addToLog(_("Addition of package {0} successful.").format(packageInfo.get("name")),
messageCode="info", file=packageInfo.get("URL"))
else:
self.addToLog(_("Unable to load plug-in."), messageCode="info", file=cmd[1:])
elif cmd.startswith("~"):
if PackageManager.reloadPackageModule(cmd[1:]):
self.addToLog(_("Reload of package successful."), messageCode="info", file=cmd[1:])
else:
self.addToLog(_("Unable to reload package."), messageCode="info", file=cmd[1:])
elif cmd.startswith("-"):
if PackageManager.removePackageModule(cmd[1:]):
self.addToLog(_("Deletion of package successful."), messageCode="info", file=cmd[1:])
else:
self.addToLog(_("Unable to delete package."), messageCode="info", file=cmd[1:])
else: # assume it is a module or package
savePackagesChanges = False
packageInfo = PackageManager.addPackage(cmd)
if packageInfo:
self.addToLog(_("Activation of package {0} successful.").format(packageInfo.get("name")),
messageCode="info", file=packageInfo.get("URL"))
resetPlugins = True
else:
self.addToLog(_("Unable to load {0} as a package or {0} is not recognized as a command. ").format(cmd), messageCode="info", file=cmd)
if savePackagesChanges:
PackageManager.save(self)
if showPackages:
self.addToLog(_("Taxonomy packages:"), messageCode="info")
for packageInfo in PackageManager.orderedPackagesConfig()["packages"]:
self.addToLog(_("Package: {0}; version: {1}; status: {2}; date: {3}; description: {4}.").format(
packageInfo.get("name"), packageInfo.get("version"), packageInfo.get("status"),
packageInfo.get("fileDate"), packageInfo.get("description")),
messageCode="info", file=packageInfo.get("URL"))
# run utility command line options that don't depend on entrypoint Files
hasUtilityPlugin = False
for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Utility.Run"):
hasUtilityPlugin = True
pluginXbrlMethod(self, options)
# if no entrypointFile is applicable, quit now
if options.proxy or options.plugins or hasUtilityPlugin:
if not options.entrypointFile:
return True # success
self.username = options.username
self.password = options.password
self.entrypointFile = options.entrypointFile
if self.entrypointFile:
filesource = FileSource.openFileSource(self.entrypointFile, self, sourceZipStream)
else:
filesource = None
if options.validateEFM:
if options.disclosureSystemName:
self.addToLog(_("both --efm and --disclosureSystem validation are requested, proceeding with --efm only"),
messageCode="info", file=self.entrypointFile)
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select("efm")
elif options.disclosureSystemName:
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select(options.disclosureSystemName)
elif options.validateHMRC:
self.modelManager.validateDisclosureSystem = True
self.modelManager.disclosureSystem.select("hmrc")
else:
self.modelManager.disclosureSystem.select(None) # just load ordinary mappings
self.modelManager.validateDisclosureSystem = False
if options.utrUrl: # override disclosureSystem utrUrl
self.modelManager.disclosureSystem.utrUrl = options.utrUrl
# can be set now because the utr is first loaded at validation time
# disclosure system sets logging filters, override disclosure filters, if specified by command line
if options.logLevelFilter:
self.setLogLevelFilter(options.logLevelFilter)
if options.logCodeFilter:
self.setLogCodeFilter(options.logCodeFilter)
if options.calcDecimals:
if options.calcPrecision:
self.addToLog(_("both --calcDecimals and --calcPrecision validation are requested, proceeding with --calcDecimals only"),
messageCode="info", file=self.entrypointFile)
self.modelManager.validateInferDecimals = True
self.modelManager.validateCalcLB = True
elif options.calcPrecision:
self.modelManager.validateInferDecimals = False
self.modelManager.validateCalcLB = True
if options.utrValidate:
self.modelManager.validateUtr = True
if options.infosetValidate:
self.modelManager.validateInfoset = True
if options.abortOnMajorError:
self.modelManager.abortOnMajorError = True
if options.collectProfileStats:
self.modelManager.collectProfileStats = True
if options.internetConnectivity == "offline":
self.webCache.workOffline = True
elif options.internetConnectivity == "online":
self.webCache.workOffline = False
if options.internetTimeout is not None:
self.webCache.timeout = (options.internetTimeout or None) # use None if zero specified to disable timeout
fo = FormulaOptions()
if options.parameters:
parameterSeparator = (options.parameterSeparator or ',')
fo.parameterValues = dict(((qname(key, noPrefixIsNoNamespace=True),(None,value))
for param in options.parameters.split(parameterSeparator)
for key,sep,value in (param.partition('='),) ) )
if options.formulaParamExprResult:
fo.traceParameterExpressionResult = True
if options.formulaParamInputValue:
fo.traceParameterInputValue = True
if options.formulaCallExprSource:
fo.traceCallExpressionSource = True
if options.formulaCallExprCode:
fo.traceCallExpressionCode = True
if options.formulaCallExprEval:
fo.traceCallExpressionEvaluation = True
if options.formulaCallExprResult:
fo.traceCallExpressionResult = True
if options.formulaVarSetExprEval:
fo.traceVariableSetExpressionEvaluation = True
if options.formulaVarSetExprResult:
fo.traceVariableSetExpressionResult = True
if options.formulaAsserResultCounts:
fo.traceAssertionResultCounts = True
if options.formulaFormulaRules:
fo.traceFormulaRules = True
if options.formulaVarsOrder:
fo.traceVariablesOrder = True
if options.formulaVarExpressionSource:
fo.traceVariableExpressionSource = True
if options.formulaVarExpressionCode:
fo.traceVariableExpressionCode = True
if options.formulaVarExpressionEvaluation:
fo.traceVariableExpressionEvaluation = True
if options.formulaVarExpressionResult:
fo.traceVariableExpressionResult = True
if options.timeVariableSetEvaluation:
fo.timeVariableSetEvaluation = True
if options.formulaVarFilterWinnowing:
fo.traceVariableFilterWinnowing = True
if options.formulaVarFiltersResult:
fo.traceVariableFiltersResult = True
if options.formulaVarFiltersResult:
fo.traceVariableFiltersResult = True
self.modelManager.formulaOptions = fo
timeNow = XmlUtil.dateunionValue(datetime.datetime.now())
firstStartedAt = startedAt = time.time()
modelDiffReport = None
success = True
modelXbrl = None
try:
if filesource:
modelXbrl = self.modelManager.load(filesource, _("views loading"))
except ModelDocument.LoadingException:
pass
except Exception as err:
self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format(
err,
traceback.format_tb(sys.exc_info()[2])))
success = False # loading errors, don't attempt to utilize loaded DTS
if modelXbrl and modelXbrl.modelDocument:
loadTime = time.time() - startedAt
modelXbrl.profileStat(_("load"), loadTime)
self.addToLog(format_string(self.modelManager.locale,
_("loaded in %.2f secs at %s"),
(loadTime, timeNow)),
messageCode="info", file=self.entrypointFile)
if options.importFiles:
for importFile in options.importFiles.split("|"):
fileName = importFile.strip()
if sourceZipStream is not None and not (fileName.startswith('http://') or os.path.isabs(fileName)):
fileName = os.path.dirname(modelXbrl.uri) + os.sep + fileName # make relative to sourceZipStream
ModelDocument.load(modelXbrl, fileName)
loadTime = time.time() - startedAt
self.addToLog(format_string(self.modelManager.locale,
_("import in %.2f secs at %s"),
(loadTime, timeNow)),
messageCode="info", file=importFile)
modelXbrl.profileStat(_("import"), loadTime)
if modelXbrl.errors:
success = False # loading errors, don't attempt to utilize loaded DTS
if modelXbrl.modelDocument.type in ModelDocument.Type.TESTCASETYPES:
for pluginXbrlMethod in pluginClassMethods("Testcases.Start"):
pluginXbrlMethod(self, options, modelXbrl)
else: # not a test case, probably instance or DTS
for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Loaded"):
pluginXbrlMethod(self, options, modelXbrl)
else:
success = False
if success and options.diffFile and options.versReportFile:
try:
diffFilesource = FileSource.FileSource(options.diffFile,self)
startedAt = time.time()
modelXbrl2 = self.modelManager.load(diffFilesource, _("views loading"))
if modelXbrl2.errors:
if not options.keepOpen:
modelXbrl2.close()
success = False
else:
loadTime = time.time() - startedAt
modelXbrl.profileStat(_("load"), loadTime)
self.addToLog(format_string(self.modelManager.locale,
_("diff comparison DTS loaded in %.2f secs"),
loadTime),
messageCode="info", file=self.entrypointFile)
startedAt = time.time()
modelDiffReport = self.modelManager.compareDTSes(options.versReportFile)
diffTime = time.time() - startedAt
modelXbrl.profileStat(_("diff"), diffTime)
self.addToLog(format_string(self.modelManager.locale,
_("compared in %.2f secs"),
diffTime),
messageCode="info", file=self.entrypointFile)
except ModelDocument.LoadingException:
success = False
except Exception as err:
success = False
self.addToLog(_("[Exception] Failed to doad diff file: \n{0} \n{1}").format(
err,
traceback.format_tb(sys.exc_info()[2])))
if success:
try:
modelXbrl = self.modelManager.modelXbrl
hasFormulae = modelXbrl.hasFormulae
if options.validate:
startedAt = time.time()
if options.formulaAction: # don't automatically run formulas
modelXbrl.hasFormulae = False
self.modelManager.validate()
if options.formulaAction: # restore setting
modelXbrl.hasFormulae = hasFormulae
self.addToLog(format_string(self.modelManager.locale,
_("validated in %.2f secs"),
time.time() - startedAt),
messageCode="info", file=self.entrypointFile)
if options.formulaAction in ("validate", "run"): # do nothing here if "none"
from arelle import ValidateXbrlDimensions, ValidateFormula
startedAt = time.time()
if not options.validate:
ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl)
# setup fresh parameters from formula optoins
modelXbrl.parameters = fo.typedParameters()
ValidateFormula.validate(modelXbrl, compileOnly=(options.formulaAction != "run"))
self.addToLog(format_string(self.modelManager.locale,
_("formula validation and execution in %.2f secs")
if options.formulaAction == "run"
else _("formula validation only in %.2f secs"),
time.time() - startedAt),
messageCode="info", file=self.entrypointFile)
if options.testReport:
ViewFileTests.viewTests(self.modelManager.modelXbrl, options.testReport, options.testReportCols)
if options.rssReport:
ViewFileRssFeed.viewRssFeed(self.modelManager.modelXbrl, options.rssReport, options.rssReportCols)
if options.DTSFile:
ViewFileDTS.viewDTS(modelXbrl, options.DTSFile)
if options.factsFile:
ViewFileFactList.viewFacts(modelXbrl, options.factsFile, labelrole=options.labelRole, lang=options.labelLang, cols=options.factListCols)
if options.factTableFile:
ViewFileFactTable.viewFacts(modelXbrl, options.factTableFile, labelrole=options.labelRole, lang=options.labelLang)
if options.conceptsFile:
ViewFileConcepts.viewConcepts(modelXbrl, options.conceptsFile, labelrole=options.labelRole, lang=options.labelLang)
if options.preFile:
ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.preFile, "Presentation Linkbase", "http://www.xbrl.org/2003/arcrole/parent-child", labelrole=options.labelRole, lang=options.labelLang)
if options.calFile:
ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.calFile, "Calculation Linkbase", "http://www.xbrl.org/2003/arcrole/summation-item", labelrole=options.labelRole, lang=options.labelLang)
if options.dimFile:
ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.dimFile, "Dimensions", "XBRL-dimensions", labelrole=options.labelRole, lang=options.labelLang)
if options.formulaeFile:
ViewFileFormulae.viewFormulae(modelXbrl, options.formulaeFile, "Formulae", lang=options.labelLang)
if options.viewArcrole and options.viewFile:
ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.viewFile, os.path.basename(options.viewArcrole), options.viewArcrole, labelrole=options.labelRole, lang=options.labelLang)
if options.roleTypesFile:
ViewFileRoleTypes.viewRoleTypes(modelXbrl, options.roleTypesFile, "Role Types", isArcrole=False, lang=options.labelLang)
if options.arcroleTypesFile:
ViewFileRoleTypes.viewRoleTypes(modelXbrl, options.arcroleTypesFile, "Arcrole Types", isArcrole=True, lang=options.labelLang)
for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Run"):
pluginXbrlMethod(self, options, modelXbrl)
except (IOError, EnvironmentError) as err:
self.addToLog(_("[IOError] Failed to save output:\n {0}").format(err))
success = False
except Exception as err:
self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format(
err,
traceback.format_tb(sys.exc_info()[2])))
success = False
if modelXbrl:
modelXbrl.profileStat(_("total"), time.time() - firstStartedAt)
if options.collectProfileStats and modelXbrl:
modelXbrl.logProfileStats()
if not options.keepOpen:
if modelDiffReport:
self.modelManager.close(modelDiffReport)
elif modelXbrl:
self.modelManager.close(modelXbrl)
self.username = self.password = None #dereference password
return success
