def _get_utr_namespaces(self):
# Get UTR
modelManager = ModelManager.initialize(self.cntlr)
modelUtr = modelManager.create()
loadUtr(modelUtr)
utr_namespaces = collections.defaultdict(set)
for type_name, entries in modelManager.disclosureSystem.utrItemTypeEntries.items(
):
for entry_name, entry in entries.items():
if entry.isSimple:
utr_namespaces[entry.nsUnit].add(entry.unitId)
return utr_namespaces
def main():
if IS_EXTRACTED:
pass
else:
edinet_zip_dir = os.path.join(EDINET_ROOT_DIR, "zip")
extract_files_from_zip(edinet_zip_dir,
dest_dir_root=EDINET_ROOT_DIR,
unzip_members_regrep="|".join([
f"XBRL/PublicDoc/.*\.{extension}"
for extension in ["xbrl", "xsd", "xml"]
]))
# XBRLから情報取得
xbrl_file_regrex = os.path.join(EDINET_ROOT_DIR, EDINET_XBRL_REGREX)
xbrl_files = glob.glob(xbrl_file_regrex)
ctrl = Cntlr.Cntlr()
model_manager = ModelManager.initialize(ctrl)
for index, xbrl_file in enumerate(xbrl_files):
print(xbrl_file, ":", index + 1, "/", len(xbrl_files))
export_facts(model_manager, xbrl_file)
print(f"{'-'*10} XBRL出力 完了 {'-'*10}")
def _get_taxonomy_model(self, taxonomy_url, namespace):
"""Get an xbrl model of the entry_point file.
"""
start = datetime.datetime.today()
rules_taxonomy_filesource = FileSource.openFileSource(
taxonomy_url, self.cntlr)
modelManager = ModelManager.initialize(self.cntlr)
modelXbrl = modelManager.load(rules_taxonomy_filesource)
if len({'IOerror', 'FileNotLoadable'} & set(modelXbrl.errors)) > 0:
modelXbrl.error(
"TaxonomyLoadError",
"Cannot open file {} with namespace {}.".format(
taxonomy_url, namespace))
else:
end = datetime.datetime.today()
print("Taxonomy {namespace} loaded in {time}. {entry}".format(
namespace=modelXbrl.modelDocument.targetNamespace,
time=end - start,
entry=taxonomy_url))
return modelXbrl
def main():
if IS_EXTRACTED:
pass
else:
edinet_zip_dir = os.path.join(EDINET_ROOT_DIR, "zip")
extract_files_from_zip(
edinet_zip_dir,
dest_dir_root=EDINET_ROOT_DIR,
unzip_members_regrep="|".join(
[f"XBRL/PublicDoc/.*\.{extension}" for extension in ["xbrl", "xsd", "xml"]]
)
)
# XBRLから情報取得
xbrl_file_regrex = os.path.join(EDINET_ROOT_DIR, EDINET_XBRL_REGREX)
xbrl_files = glob.glob(xbrl_file_regrex)
list_df_facts = []
ctrl = Cntlr.Cntlr()
model_manager = ModelManager.initialize(ctrl)
for index, xbrl_file in enumerate(xbrl_files):
print(xbrl_file, ":", index + 1, "/", len(xbrl_files))
df_facts = get_facts(model_manager, xbrl_file)
if df_facts is not None:
list_df_facts.append(df_facts)
if list_df_facts:
df_xbrl = pd.concat(list_df_facts, axis=0, sort=False)
# Edinetコードリストの情報をマージ
df_edinetcd_info = get_edinetcd_info(EDINETCDDLINFO_COLS)
df_xbrl = df_edinetcd_info.merge(df_xbrl, on=EDINETCD_COL, how="right")
df_xbrl.to_csv(
os.path.join(EDINET_ROOT_DIR, OUTPUT_FILE_NAME),
index=False,
encoding="cp932"
)
print(f"{'-'*10} 情報抽出 完了 {'-'*10}")
else:
print("処理対象のデータはありませんでした。")
def __init__(self, hasGui=False, logFileName=None, logFileMode=None, logFileEncoding=None, logFormat=None):
self.hasWin32gui = False
self.hasGui = hasGui
self.hasFileSystem = True # no file system on Google App Engine servers
self.isGAE = False
self.systemWordSize = int(round(math.log(sys.maxsize, 2)) + 1) # e.g., 32 or 64
self.moduleDir = os.path.dirname(__file__)
# for python 3.2 remove __pycache__
if self.moduleDir.endswith("__pycache__"):
self.moduleDir = os.path.dirname(self.moduleDir)
if self.moduleDir.endswith("python32.zip/arelle"):
'''
distZipFile = os.path.dirname(self.moduleDir)
d = os.path.join(self.userAppDir, "arelle")
self.configDir = os.path.join(d, "config")
self.imagesDir = os.path.join(d, "images")
import zipfile
distZip = zipfile.ZipFile(distZipFile, mode="r")
distNames = distZip.namelist()
distZip.extractall(path=self.userAppDir,
members=[f for f in distNames if "/config/" in f or "/images/" in f]
)
distZip.close()
'''
resources = os.path.dirname(os.path.dirname(os.path.dirname(self.moduleDir)))
self.configDir = os.path.join(resources, "config")
self.imagesDir = os.path.join(resources, "images")
self.localeDir = os.path.join(resources, "locale")
self.pluginDir = os.path.join(resources, "plugin")
elif self.moduleDir.endswith("library.zip\\arelle") or self.moduleDir.endswith("library.zip/arelle"): # cx_Freexe
resources = os.path.dirname(os.path.dirname(self.moduleDir))
self.configDir = os.path.join(resources, "config")
self.imagesDir = os.path.join(resources, "images")
self.localeDir = os.path.join(resources, "locale")
self.pluginDir = os.path.join(resources, "plugin")
else:
self.configDir = os.path.join(self.moduleDir, "config")
self.imagesDir = os.path.join(self.moduleDir, "images")
self.localeDir = os.path.join(self.moduleDir, "locale")
self.pluginDir = os.path.join(self.moduleDir, "plugin")
serverSoftware = os.getenv("SERVER_SOFTWARE", "")
if serverSoftware.startswith("Google App Engine/") or serverSoftware.startswith("Development/"):
self.hasFileSystem = False # no file system, userAppDir does not exist
self.isGAE = True
configHomeDir = None # look for path configDir/CONFIG_HOME in argv and environment parameters
for i, arg in enumerate(sys.argv): # check if config specified in a argv
if arg.startswith("--xdgConfigHome="):
configHomeDir = arg[16:]
break
elif arg == "--xdgConfigHome" and i + 1 < len(sys.argv):
configHomeDir = sys.argv[i + 1]
break
if not configHomeDir: # not in argv, may be an environment parameter
configHomeDir = os.getenv('XDG_CONFIG_HOME')
if not configHomeDir: # look for path configDir/CONFIG_HOME
configHomeDirFile = os.path.join(self.configDir, "XDG_CONFIG_HOME")
if os.path.exists(configHomeDirFile):
try:
with io.open(configHomeDirFile, 'rt', encoding='utf-8') as f:
configHomeDir = f.read().strip()
if configHomeDir and not os.path.isabs(configHomeDir):
configHomeDir = os.path.abspath(configHomeDir) # make into a full path if relative
except EnvironmentError:
configHomeDir = None
if self.hasFileSystem and configHomeDir and os.path.exists(configHomeDir):
# check if a cache exists in this directory (e.g. from XPE or other tool)
impliedAppDir = os.path.join(configHomeDir, "arelle")
if os.path.exists(impliedAppDir):
self.userAppDir = impliedAppDir
elif os.path.exists(os.path.join(configHomeDir, "cache")):
self.userAppDir = configHomeDir # use the XDG_CONFIG_HOME because cache is already a subdirectory
else:
self.userAppDir = impliedAppDir
if sys.platform == "darwin":
self.isMac = True
self.isMSW = False
if self.hasFileSystem and not configHomeDir:
self.userAppDir = os.path.expanduser("~") + "/Library/Application Support/Arelle"
# note that cache is in ~/Library/Caches/Arelle
self.contextMenuClick = "<Button-2>"
self.hasClipboard = hasGui # clipboard always only if Gui (not command line mode)
self.updateURL = "http://arelle.org/downloads/8"
elif sys.platform.startswith("win"):
self.isMac = False
self.isMSW = True
if self.hasFileSystem and not configHomeDir:
tempDir = tempfile.gettempdir()
if tempDir.endswith('local\\temp'):
impliedAppDir = tempDir[:-10] + 'local'
else:
impliedAppDir = tempDir
self.userAppDir = os.path.join( impliedAppDir, "Arelle")
if hasGui:
try:
import win32clipboard
self.hasClipboard = True
except ImportError:
self.hasClipboard = False
try:
import win32gui
self.hasWin32gui = True # active state for open file dialogs
except ImportError:
pass
else:
self.hasClipboard = False
self.contextMenuClick = "<Button-3>"
if "64 bit" in sys.version:
self.updateURL = "http://arelle.org/downloads/9"
else: # 32 bit
self.updateURL = "http://arelle.org/downloads/10"
else: # Unix/Linux
self.isMac = False
self.isMSW = False
if self.hasFileSystem and not configHomeDir:
self.userAppDir = os.path.join( os.path.expanduser("~/.config"), "arelle")
if hasGui:
try:
import gtk
self.hasClipboard = True
except ImportError:
self.hasClipboard = False
else:
self.hasClipboard = False
self.contextMenuClick = "<Button-3>"
try:
from arelle import webserver
self.hasWebServer = True
except ImportError:
self.hasWebServer = False
# assert that app dir must exist
self.config = None
if self.hasFileSystem:
if not os.path.exists(self.userAppDir):
os.makedirs(self.userAppDir)
# load config if it exists
self.configJsonFile = self.userAppDir + os.sep + "config.json"
if os.path.exists(self.configJsonFile):
try:
with io.open(self.configJsonFile, 'rt', encoding='utf-8') as f:
self.config = json.load(f)
except Exception as ex:
self.config = None # restart with a new config
if not self.config:
self.config = {
'fileHistory': [],
'windowGeometry': "{0}x{1}+{2}+{3}".format(800, 500, 200, 100),
}
# start language translation for domain
self.setUiLanguage(self.config.get("userInterfaceLangOverride",None), fallbackToDefault=True)
from arelle.WebCache import WebCache
self.webCache = WebCache(self, self.config.get("proxySettings"))
self.modelManager = ModelManager.initialize(self)
# start plug in server (requres web cache initialized, but not logger)
PluginManager.init(self)
# start taxonomy package server (requres web cache initialized, but not logger)
PackageManager.init(self)
self.startLogging(logFileName, logFileMode, logFileEncoding, logFormat)
# Cntlr.Init after logging started
for pluginMethod in PluginManager.pluginClassMethods("Cntlr.Init"):
pluginMethod(self)
def xuleCmdUtilityRun(cntlr, options, **kwargs):
# Save the controller and options in the module global variable
global _cntlr
_cntlr = cntlr
global _options
_options = options
cntlr.addToLog("Xule version: %s" % __version__, 'info')
# check option combinations
parser = OptionParser()
if getattr(options, "xule_version", False):
cntlr.addToLog("Xule version: %s" % __version__, 'xule')
cntlr.close()
if getattr(options, "xule_cpu", None) is not None and not getattr(options, 'xule_multi', None):
parser.error(_("--xule-multi is required with --xule_cpu."))
if getattr(options, "xule_server", None) is not None and not getattr(options, 'xule_rule_set', None):
parser.error(_("--xule-rule-set is required with --xule_server."))
from os import name
if getattr(options, "xule_multi", False) and name == 'nt':
parser.error(_("--xule-multi can't be used in Windows"))
if getattr(options, "xule-numthreads", None) == None:
setattr(options, "xule-numthreads", 1)
if getattr(options, 'xule_add_packages', None) is not None and not getattr(options, 'xule_rule_set', None):
parser.error(_("--xule-rule-set is required with --xule-add-packages."))
if getattr(options, 'xule_remove_packages', None) is not None and not getattr(options, 'xule_rule_set', None):
parser.error(_("--xule-rule-set is required with --xule-remove-packages."))
if getattr(options, 'xule_show_packages', None) is not None and not getattr(options, 'xule_rule_set', None):
parser.error(_("--xule-rule-set is required with --xule-show-packages."))
if len([x for x in (getattr(options, "xule_update_rule_set_map", False),
getattr(options, "xule_replace_rule_set_map", False),
getattr(options, "xule_reset_rule_set_map", False)) if x]) > 1:
parser.error(_("Cannot use --xule-update-rule-set-map or --xule-replace-rule-set-map or --xule-reset-rule-set-map the same time."))
if getattr(options, 'xule_validate', None) is not None and getattr(options, 'xule_rule_set', None) is None:
parser.error(_("--xule-validate requires a Xule ruleset. Use option --xule-rule-set."))
if getattr(options, 'xule_filing_list', None) is not None and getattr(options, 'entrypointFile', None) is not None:
parser.error(_("--xule-filing-list cannot be used with -f"))
# compile rules
if getattr(options, "xule_compile", None):
compile_destination = getattr(options, "xule_rule_set", "xuleRules")
xp.parseRules(options.xule_compile.split("|"), compile_destination, getattr(options, "xule_compile_type"))
# add packages
if getattr(options, "xule_add_packages", None):
rule_set = xr.XuleRuleSet(cntlr)
rule_set.open(getattr(options, "xule_rule_set"), open_packages=False, open_files=False)
packages = options.xule_add_packages.split('|')
rule_set.manage_packages(packages, 'add')
# remove packages
if getattr(options, "xule_remove_packages", None):
rule_set = xr.XuleRuleSet(cntlr)
rule_set.open(getattr(options, "xule_rule_set"), open_packages=False, open_files=False)
packages = options.xule_remove_packages.split('|')
rule_set.manage_packages(packages, 'del')
# show packages
if getattr(options, "xule_show_packages", False):
rule_set = xr.XuleRuleSet(cntlr)
rule_set.open(getattr(options, "xule_rule_set"), open_packages=False, open_files=False)
print("Packages in rule set:")
for package_info in rule_set.get_packages_info():
print('\t' + package_info.get('name') + ' (' + os.path.basename(package_info.get('URL')) + ')')
# update rule set map
if getattr(options, 'xule_update_rule_set_map', None):
xu.update_rule_set_map(cntlr, getattr(options, 'xule_update_rule_set_map'), _xule_rule_set_map_name)
# replace rule set map
if getattr(options, 'xule_replace_rule_set_map', None):
xu.update_rule_set_map(cntlr, getattr(options, 'xule_replace_rule_set_map'), _xule_rule_set_map_name, overwrite=True)
# reset rule set map
if getattr(options, 'xule_reset_rule_set_map', False):
xu.reset_rule_set_map(cntlr, _xule_rule_set_map_name)
# display the rule set map
if getattr(options, 'xule_display_rule_set_map', False):
displayValidatorRulesetMap(cntlr, 'Xule', _xule_rule_set_map_name)
# validate ruleset
if getattr(options, 'xule_validate', False):
rule_set = xr.XuleRuleSet(cntlr)
rule_set.open(options.xule_rule_set, open_packages=not getattr(options, 'xule_bypass_packages', False))
xv.XuleValidate(cntlr, rule_set, options.xule_rule_set)
if getattr(options, "xule_server", None):
from threading import Thread
try:
rule_set = xr.XuleRuleSet()
rule_set.open(options.xule_rule_set, False)
except xr.XuleRuleSetError:
raise
# Create global Context
global_context = XuleGlobalContext(rule_set, cntlr=cntlr, options=options)
global_context.message_queue.print("Using %d processors" % (global_context.num_processors))
# Start Output message queue
if getattr(options, "xule_multi", False):
t = Thread(target=xm.output_message_queue, args=(global_context,))
t.start()
global_context.message_queue.logging("Building Constant and Rule Groups")
global_context.all_constants = rule_set.get_grouped_constants()
global_context.all_rules = rule_set.get_grouped_rules()
for g in global_context.all_constants:
global_context.message_queue.logging("Constants: %s - %d" % (g, len(global_context.all_constants[g])))
for g in global_context.all_rules:
global_context.message_queue.logging("Rules: %s - %d" % (g, len(global_context.all_rules[g])))
# evaluate valid constants (no dependency, rules taxonomy)
global_context.message_queue.logging("Calculating and Storing Constants")
xm.run_constant_group(global_context, 'c', 'rtc')
# Add precalculated information to the cntlr to pass to XuleServer
setattr(cntlr, "xule_options", options)
setattr(cntlr, "rule_set", global_context.rule_set)
setattr(cntlr, "constant_list", global_context._constants)
setattr(cntlr, "all_constants", global_context.all_constants)
setattr(cntlr, "all_rules", global_context.all_rules)
global_context.message_queue.logging("Finished Server Initialization")
# stop message_queue
global_context.message_queue.stop()
if getattr(options, "xule_multi", False):
t.join()
else:
if getattr(options, 'xule_filing_list', None) is not None:
# process filing list
if getattr(options, "xule_filing_list", None):
try:
with open(options.xule_filing_list, 'r') as filing_list_file:
# Try json
try:
filing_list = json.load(filing_list_file, object_pairs_hook=collections.OrderedDict)
except:
# Try a flat list of file names
try:
# reset the file pointer
filing_list_file.seek(0)
filing_list = [{"file": file_name} for file_name in filing_list_file]
except:
cntlr.addToLog(_("Unable to open Filing listing file '%s'." % options.xule_filing_list), 'xule')
raise
except FileNotFoundError:
cntlr.addToLog(_("Filing listing file '%s' is not found" % options.xule_filing_list), 'xule')
raise
if isinstance(filing_list, list):
for file_info in filing_list:
if isinstance(file_info, dict):
input_file_name = file_info.get('file')
if input_file_name is not None:
input_file_name = input_file_name.strip()
print("Processing filing", input_file_name)
filing_filesource = FileSource.openFileSource(input_file_name, cntlr)
modelManager = ModelManager.initialize(cntlr)
modelXbrl = modelManager.load(filing_filesource)
# Update options
new_options = copy.copy(options)
delattr(new_options, 'xule_filing_list')
for k, v in file_info.items():
if k != 'file' and k.strip().lower().startswith('xule'): # Only change xule options
setattr(new_options, k.strip().lower(), v)
if getattr(new_options, 'xule_run'):
xuleCmdXbrlLoaded(cntlr, new_options, modelXbrl)
elif getattr(new_options, 'validate'):
for xule_validator in _xule_validators:
runXule(_cntlr, new_options, modelXbrl, xule_validator['map_name'])
modelXbrl.close()
else:
if options.entrypointFile is None:
# try running the xule processor - This is when rules are run without an instance document
xuleCmdXbrlLoaded(cntlr, options, None)
# Only register xule as a validator if the xule plugin was directly added in the --plugin options.
if isXuleDirect():
xuleRegisterValidators('Xule', _xule_rule_set_map_name)
def xuleCmdUtilityRun(cntlr, options, **kwargs):
# Save the controller and options in the module global variable
global _cntlr
_cntlr = cntlr
global _options
_options = options
cntlr.addToLog("Xule version: %s" % __version__, 'info')
# check option combinations
parser = optparse.OptionParser()
if getattr(options, "xule_version", False):
cntlr.addToLog("Xule version: %s" % __version__, 'xule')
cntlr.close()
if getattr(options, "xule_cpu",
None) is not None and not getattr(options, 'xule_multi', None):
parser.error(_("--xule-multi is required with --xule_cpu."))
if getattr(options, "xule_server", None) is not None and not getattr(
options, 'xule_rule_set', None):
parser.error(_("--xule-rule-set is required with --xule_server."))
from os import name
if getattr(options, "xule_multi", False) and name == 'nt':
parser.error(_("--xule-multi can't be used in Windows"))
if getattr(options, "xule-numthreads", None) == None:
setattr(options, "xule-numthreads", 1)
if getattr(options, 'xule_add_packages', None) is not None and not getattr(
options, 'xule_rule_set', None):
parser.error(
_("--xule-rule-set is required with --xule-add-packages."))
if getattr(
options, 'xule_remove_packages',
None) is not None and not getattr(options, 'xule_rule_set', None):
parser.error(
_("--xule-rule-set is required with --xule-remove-packages."))
if getattr(
options, 'xule_show_packages',
None) is not None and not getattr(options, 'xule_rule_set', None):
parser.error(
_("--xule-rule-set is required with --xule-show-packages."))
if len([
x
for x in (getattr(options, "xule_update_rule_set_map", False),
getattr(options, "xule_replace_rule_set_map", False),
getattr(options, "xule_reset_rule_set_map", False)) if x
]) > 1:
parser.error(
_("Cannot use --xule-update-rule-set-map or --xule-replace-rule-set-map or --xule-reset-rule-set-map the same time."
))
if getattr(options, 'xule_validate', None) is not None and getattr(
options, 'xule_rule_set', None) is None:
parser.error(
_("--xule-validate requires a Xule ruleset. Use option --xule-rule-set."
))
if getattr(options, 'xule_filing_list', None) is not None and getattr(
options, 'entrypointFile', None) is not None:
parser.error(_("--xule-filing-list cannot be used with -f"))
# compile rules
if getattr(options, "xule_compile", None):
compile_destination = getattr(options, "xule_rule_set", "xuleRules")
xp.parseRules(options.xule_compile.split("|"), compile_destination,
getattr(options, "xule_compile_type"))
# add packages
if getattr(options, "xule_add_packages", None):
rule_set = xr.XuleRuleSet(cntlr)
rule_set.open(getattr(options, "xule_rule_set"),
open_packages=False,
open_files=False)
packages = options.xule_add_packages.split('|')
rule_set.manage_packages(packages, 'add')
# remove packages
if getattr(options, "xule_remove_packages", None):
rule_set = xr.XuleRuleSet(cntlr)
rule_set.open(getattr(options, "xule_rule_set"),
open_packages=False,
open_files=False)
packages = options.xule_remove_packages.split('|')
rule_set.manage_packages(packages, 'del')
# show packages
if getattr(options, "xule_show_packages", False):
rule_set = xr.XuleRuleSet(cntlr)
rule_set.open(getattr(options, "xule_rule_set"),
open_packages=False,
open_files=False)
print("Packages in rule set:")
for package_info in rule_set.get_packages_info():
print('\t' + package_info.get('name') + ' (' +
os.path.basename(package_info.get('URL')) + ')')
# update rule set map
if getattr(options, 'xule_update_rule_set_map', None):
xu.update_rule_set_map(cntlr,
getattr(options, 'xule_update_rule_set_map'),
_xule_rule_set_map_name)
# replace rule set map
if getattr(options, 'xule_replace_rule_set_map', None):
xu.update_rule_set_map(cntlr,
getattr(options, 'xule_replace_rule_set_map'),
_xule_rule_set_map_name,
overwrite=True)
# reset rule set map
if getattr(options, 'xule_reset_rule_set_map', False):
xu.reset_rule_set_map(cntlr, _xule_rule_set_map_name)
# display the rule set map
if getattr(options, 'xule_display_rule_set_map', False):
displayValidatorRulesetMap(cntlr, 'Xule', _xule_rule_set_map_name)
# validate ruleset
if getattr(options, 'xule_validate', False):
rule_set = xr.XuleRuleSet(cntlr)
rule_set.open(
options.xule_rule_set,
open_packages=not getattr(options, 'xule_bypass_packages', False))
xv.XuleValidate(cntlr, rule_set, options.xule_rule_set)
if getattr(options, "xule_server", None):
from threading import Thread
try:
rule_set = xr.XuleRuleSet()
rule_set.open(options.xule_rule_set, False)
except xr.XuleRuleSetError:
raise
# Create global Context
global_context = XuleGlobalContext(rule_set,
cntlr=cntlr,
options=options)
global_context.message_queue.print("Using %d processors" %
(global_context.num_processors))
# Start Output message queue
if getattr(options, "xule_multi", False):
t = Thread(target=xm.output_message_queue, args=(global_context, ))
t.start()
global_context.message_queue.logging(
"Building Constant and Rule Groups")
global_context.all_constants = rule_set.get_grouped_constants()
global_context.all_rules = rule_set.get_grouped_rules()
for g in global_context.all_constants:
global_context.message_queue.logging(
"Constants: %s - %d" %
(g, len(global_context.all_constants[g])))
for g in global_context.all_rules:
global_context.message_queue.logging(
"Rules: %s - %d" % (g, len(global_context.all_rules[g])))
# evaluate valid constants (no dependency, rules taxonomy)
global_context.message_queue.logging(
"Calculating and Storing Constants")
xm.run_constant_group(global_context, 'c', 'rtc')
# Add precalculated information to the cntlr to pass to XuleServer
setattr(cntlr, "xule_options", options)
setattr(cntlr, "rule_set", global_context.rule_set)
setattr(cntlr, "constant_list", global_context._constants)
setattr(cntlr, "all_constants", global_context.all_constants)
setattr(cntlr, "all_rules", global_context.all_rules)
global_context.message_queue.logging("Finished Server Initialization")
# stop message_queue
global_context.message_queue.stop()
if getattr(options, "xule_multi", False):
t.join()
else:
if getattr(options, 'xule_filing_list', None) is not None:
# process filing list
if getattr(options, "xule_filing_list", None):
try:
with open(options.xule_filing_list,
'r') as filing_list_file:
# Try json
try:
filing_list = json.load(
filing_list_file,
object_pairs_hook=collections.OrderedDict)
except:
# Try a flat list of file names
try:
# reset the file pointer
filing_list_file.seek(0)
filing_list = [{
"file": file_name
} for file_name in filing_list_file]
except:
cntlr.addToLog(
_("Unable to open Filing listing file '%s'."
% options.xule_filing_list), 'xule')
raise
except FileNotFoundError:
cntlr.addToLog(
_("Filing listing file '%s' is not found" %
options.xule_filing_list), 'xule')
raise
if isinstance(filing_list, list):
for file_info in filing_list:
if isinstance(file_info, dict):
input_file_name = file_info.get('file')
if input_file_name is not None:
input_file_name = input_file_name.strip()
print("Processing filing", input_file_name)
filing_filesource = FileSource.openFileSource(
input_file_name, cntlr)
modelManager = ModelManager.initialize(cntlr)
modelXbrl = modelManager.load(
filing_filesource)
# Update options
new_options = copy.copy(options)
delattr(new_options, 'xule_filing_list')
for k, v in file_info.items():
if k != 'file' and k.strip().lower(
).startswith('xule'
): # Only change xule options
setattr(new_options,
k.strip().lower(), v)
if getattr(new_options, 'xule_run'):
xuleCmdXbrlLoaded(cntlr, new_options,
modelXbrl)
elif getattr(new_options, 'validate'):
for xule_validator in _xule_validators:
runXule(_cntlr, new_options, modelXbrl,
xule_validator['map_name'])
modelXbrl.close()
else:
if options.entrypointFile is None:
# try running the xule processor - This is when rules are run without an instance document
xuleCmdXbrlLoaded(cntlr, options, None)
# Only register xule as a validator if the xule plugin was directly added in the --plugin options.
if isXuleDirect():
xuleRegisterValidators('Xule', _xule_rule_set_map_name)
def xuleCmdUtilityRun(cntlr, options, **kwargs):
# Save the controller and options in the module global variable
global _cntlr
_cntlr = cntlr
global _options
_options = options
#check option combinations
parser = OptionParser()
if getattr(options, "xule_version", False):
cntlr.addToLog("Xule version: %s" % __version__)
cntlr.close()
if getattr(options, "xule_cpu",
None) is not None and not getattr(options, 'xule_multi', None):
parser.error(_("--xule-multi is required with --xule_cpu."))
# if getattr(options, "xule_run", None) is not None and not getattr(options, 'xule_rule_set', None):
# parser.error(_("--xule-rule-set is required with --xule-run."))
if getattr(options, "xule_server", None) is not None and not getattr(
options, 'xule_rule_set', None):
parser.error(_("--xule-rule-set is required with --xule_server."))
if getattr(options, "xule-numthreads", None) == None:
setattr(options, "xule-numthreads", 1)
if getattr(options, 'xule_add_packages', None) is not None and not getattr(
options, 'xule_rule_set', None):
parser.error(
_("--xule-rule-set is required with --xule-add-packages."))
if getattr(
options, 'xule_remove_packages',
None) is not None and not getattr(options, 'xule_rule_set', None):
parser.error(
_("--xule-rule-set is required with --xule-remove-packages."))
if getattr(
options, 'xule_show_packages',
None) is not None and not getattr(options, 'xule_rule_set', None):
parser.error(
_("--xule-rule-set is required with --xule-show-packages."))
from os import name
if getattr(options, "xule_multi", False) and name == 'nt':
parser.error(_("--xule-multi can't be used in Windows"))
if not getattr(options, "xule_multi", False) and getattr(
options, "xule_cpu", None) is not None:
parser.error(
_("--xule-cpu can only be used with --xule-multi enabled"))
if len([
x
for x in (getattr(options, "xule_update_rule_set_map", False),
getattr(options, "xule_replace_rule_set_map", False),
getattr(options, "xule_reset_rule_set_map", False)) if x
]) > 1:
parser.error(
_("Cannot use --xule-update-rule-set-map or --xule-replace-rule-set-map or --xule-reset-rule-set-map the same time."
))
#compile rules
if getattr(options, "xule_compile", None):
compile_destination = getattr(options, "xule_rule_set", "xuleRules")
from .XuleParser import parseRules
parseRules(options.xule_compile.split("|"), compile_destination)
#add packages
if getattr(options, "xule_add_packages", None):
rule_set = xr.XuleRuleSet(cntlr)
rule_set.open(getattr(options, "xule_rule_set"),
open_packages=False,
open_files=False)
packages = options.xule_add_packages.split('|')
rule_set.manage_packages(packages, 'add')
#remove packages
if getattr(options, "xule_remove_packages", None):
rule_set = xr.XuleRuleSet(cntlr)
rule_set.open(getattr(options, "xule_rule_set"),
open_packages=False,
open_files=False)
packages = options.xule_remove_packages.split('|')
rule_set.manage_packages(packages, 'del')
#show packages
if getattr(options, "xule_show_packages", False):
rule_set = xr.XuleRuleSet(cntlr)
rule_set.open(getattr(options, "xule_rule_set"),
open_packages=False,
open_files=False)
print("Packages in rule set:")
for package_info in rule_set.get_packages_info():
print('\t' + package_info.get('name') + ' (' +
os.path.basename(package_info.get('URL')) + ')')
#update rule set map
if getattr(options, 'xule_update_rule_set_map', None):
xu.update_rule_set_map(cntlr,
getattr(options, 'xule_update_rule_set_map'))
#replace rule set map
if getattr(options, 'xule_replace_rule_set_map', None):
xu.update_rule_set_map(cntlr,
getattr(options, 'xule_replace_rule_set_map'),
overwrite=True)
#reset rule set map
if getattr(options, 'xule_reset_rule_set_map', False):
xu.reset_rule_set_map(cntlr)
if getattr(options, "xule_server", None):
from .XuleMultiProcessing import run_constant_group, output_message_queue
from threading import Thread
try:
rule_set = xr.XuleRuleSet()
rule_set.open(options.xule_rule_set, False)
except xr.XuleRuleSetError:
raise
# Create global Context
global_context = XuleGlobalContext(
rule_set,
cntlr=cntlr,
options=options #,
#multi=getattr(options, "xule_multi", False),
#async=getattr(options, "xule_async", False),
#cpunum=getattr(options, "xule_cpu", None))
)
#global_context.show_timing = getattr(options, "xule_time", None)
#global_context.show_debug = getattr(options, "xule_debug", False)
#global_context.show_debug_table = getattr(options, "xule_debug_table", False)
#global_context.show_trace = getattr(options, "xule_trace", None)
#global_context.crash_on_error = getattr(options, "xule_crash", False)
global_context.message_queue.print("Using %d processors" %
(global_context.num_processors))
# Start Output message queue
if getattr(options, "xule_multi", False):
t = Thread(target=output_message_queue, args=(global_context, ))
t.start()
# #load rules taxonomy
# global_context.message_queue.logging("Loading rules taxonomy")
# global_context.get_rules_dts()
# # rules_dts = global_context.get_rules_dts()
# # from .XuleProcessor import load_networks
# # load_networks(rules_dts)
global_context.message_queue.logging(
"Building Constant and Rule Groups")
global_context.all_constants = rule_set.get_grouped_constants()
global_context.all_rules = rule_set.get_grouped_rules()
for g in global_context.all_constants:
global_context.message_queue.logging(
"Constants: %s - %d" %
(g, len(global_context.all_constants[g])))
# for c in global_context.all_constants[g]:
# print(" -- %s" % (c))
for g in global_context.all_rules:
global_context.message_queue.logging(
"Rules: %s - %d" % (g, len(global_context.all_rules[g])))
#for c in global_context.all_rules[g]:
# print(" -- %s" % (c))
# evaluate valid constants (no dependency, rules taxonomy)
global_context.message_queue.logging(
"Calculating and Storing Constants")
run_constant_group(global_context, 'c', 'rtc')
# Add precalculated information to the cntlr to pass to XuleServer
setattr(cntlr, "xule_options", options)
setattr(cntlr, "rule_set", global_context.rule_set)
setattr(cntlr, "constant_list", global_context._constants)
setattr(cntlr, "all_constants", global_context.all_constants)
setattr(cntlr, "all_rules", global_context.all_rules)
global_context.message_queue.logging("Finished Server Initialization")
# stop message_queue
global_context.message_queue.stop()
if getattr(options, "xule_multi", False):
t.join()
else:
if options.entrypointFile is None:
#try running the xule processor
xuleCmdXbrlLoaded(cntlr, options, None)
#process filing list
if getattr(options, "xule_filing_list", None):
try:
with open(options.xule_filing_list, "r") as filing_list:
for line in filing_list:
filing = line.strip()
print("Processing filing", filing)
filing_filesource = FileSource.openFileSource(
filing, cntlr)
modelManager = ModelManager.initialize(cntlr)
modelXbrl = modelManager.load(filing_filesource)
xuleCmdXbrlLoaded(cntlr, options, modelXbrl)
modelXbrl.close()
except FileNotFoundError:
print("Filing listing file '%s' is not found" %
options.xule_filing_list)
def xuleCmdUtilityRun(cntlr, options, **kwargs):
#check option combinations
parser = OptionParser()
if getattr(options, "xule_version", False):
cntlr.addToLog("Xule version: %s" % __version__)
cntlr.close()
if getattr(options, "xule_cpu", None) is not None and not getattr(options, 'xule_multi', None):
parser.error(_("--xule-multi is required with --xule_cpu."))
if getattr(options, "xule_run", None) is not None and not getattr(options, 'xule_rule_set', None):
parser.error(_("--xule-rule-set is required with --xule-run."))
if getattr(options, "xule_server", None) is not None and not getattr(options, 'xule_rule_set', None):
parser.error(_("--xule-rule-set is required with --xule_server."))
if getattr(options, "xule-numthreads", None) == None:
setattr(options, "xule-numthreads", 1)
if getattr(options, 'xule_add_packages', None) is not None and not getattr(options, 'xule_rule_set', None):
parser.error(_("--xule-rule-set is required with --xule-add-packages."))
if getattr(options, 'xule_remove_packages', None) is not None and not getattr(options, 'xule_rule_set', None):
parser.error(_("--xule-rule-set is required with --xule-remove-packages."))
if getattr(options, 'xule_show_packages', None) is not None and not getattr(options, 'xule_rule_set', None):
parser.error(_("--xule-rule-set is required with --xule-show-packages."))
from os import name
if getattr(options, "xule_multi", False) and name == 'nt':
parser.error(_("--xule-multi can't be used in Windows"))
if not getattr(options, "xule_multi", False) and getattr(options, "xule_cpu", None) is not None:
parser.error(_("--xule-cpu can only be used with --xule-multi enabled"))
#compile rules
if getattr(options, "xule_compile", None):
compile_destination = getattr(options, "xule_rule_set", "xuleRules")
from .XuleParser import parseRules
parseRules(options.xule_compile.split("|"),compile_destination)
#add packages
if getattr(options, "xule_add_packages", None):
rule_set = xr.XuleRuleSet(cntlr)
rule_set.open(getattr(options, "xule_rule_set"), open_packages=False, open_files=False)
packages = options.xule_add_packages.split('|')
rule_set.manage_packages(packages, 'add')
#remove packages
if getattr(options, "xule_remove_packages", None):
rule_set = xr.XuleRuleSet(cntlr)
rule_set.open(getattr(options, "xule_rule_set"), open_packages=False, open_files=False)
packages = options.xule_remove_packages.split('|')
rule_set.manage_packages(packages, 'del')
#show packages
if getattr(options, "xule_show_packages", False):
rule_set = xr.XuleRuleSet(cntlr)
rule_set.open(getattr(options, "xule_rule_set"), open_packages=False, open_files=False)
print("Packages in rule set:")
for package_info in rule_set.get_packages_info():
print('\t' + package_info.get('name') + ' (' + os.path.basename(package_info.get('URL')) + ')' )
if getattr(options, "xule_server", None):
from .XuleMultiProcessing import run_constant_group, output_message_queue
from threading import Thread
try:
rule_set = xr.XuleRuleSet()
rule_set.open(options.xule_rule_set, False)
except xr.XuleRuleSetError:
raise
# Create global Context
global_context = XuleGlobalContext(rule_set, cntlr=cntlr, options=options#,
#multi=getattr(options, "xule_multi", False),
#async=getattr(options, "xule_async", False),
#cpunum=getattr(options, "xule_cpu", None))
)
#global_context.show_timing = getattr(options, "xule_time", None)
#global_context.show_debug = getattr(options, "xule_debug", False)
#global_context.show_debug_table = getattr(options, "xule_debug_table", False)
#global_context.show_trace = getattr(options, "xule_trace", None)
#global_context.crash_on_error = getattr(options, "xule_crash", False)
global_context.message_queue.print("Using %d processors" % (global_context.num_processors))
# Start Output message queue
if getattr(options, "xule_multi", False):
t = Thread(target=output_message_queue, args=(global_context,))
t.start()
# #load rules taxonomy
# global_context.message_queue.logging("Loading rules taxonomy")
# global_context.get_rules_dts()
# # rules_dts = global_context.get_rules_dts()
# # from .XuleProcessor import load_networks
# # load_networks(rules_dts)
global_context.message_queue.logging("Building Constant and Rule Groups")
global_context.all_constants = rule_set.get_grouped_constants()
global_context.all_rules = rule_set.get_grouped_rules()
for g in global_context.all_constants:
global_context.message_queue.logging("Constants: %s - %d" % (g, len(global_context.all_constants[g])))
# for c in global_context.all_constants[g]:
# print(" -- %s" % (c))
for g in global_context.all_rules:
global_context.message_queue.logging("Rules: %s - %d" % (g, len(global_context.all_rules[g])))
#for c in global_context.all_rules[g]:
# print(" -- %s" % (c))
# evaluate valid constants (no dependency, rules taxonomy)
global_context.message_queue.logging("Calculating and Storing Constants")
run_constant_group(global_context, 'c', 'rtc')
# Add precalculated information to the cntlr to pass to XuleServer
setattr(cntlr, "xule_options", options)
setattr(cntlr, "rule_set", global_context.rule_set)
setattr(cntlr, "constant_list", global_context._constants)
setattr(cntlr, "all_constants", global_context.all_constants)
setattr(cntlr, "all_rules", global_context.all_rules)
global_context.message_queue.logging("Finished Server Initialization")
# stop message_queue
global_context.message_queue.stop()
if getattr(options, "xule_multi", False):
t.join()
else:
if options.entrypointFile is None:
#try running the xule processor
xuleCmdXbrlLoaded(cntlr, options, None)
#process filing list
if getattr(options, "xule_filing_list", None):
try:
with open(options.xule_filing_list, "r") as filing_list:
for line in filing_list:
filing = line.strip()
print("Processing filing", filing)
filing_filesource = FileSource.openFileSource(filing, cntlr)
modelManager = ModelManager.initialize(cntlr)
modelXbrl = modelManager.load(filing_filesource)
xuleCmdXbrlLoaded(cntlr, options, modelXbrl)
modelXbrl.close()
except FileNotFoundError:
print("Filing listing file '%s' is not found" % options.xule_filing_list)
def __init__(self,
hasGui=False,
logFileName=None,
logFileMode=None,
logFileEncoding=None,
logFormat=None):
self.hasWin32gui = False
self.hasGui = hasGui
self.hasFileSystem = True # no file system on Google App Engine servers
self.isGAE = False
self.isCGI = False
self.systemWordSize = int(round(math.log(sys.maxsize, 2)) +
1) # e.g., 32 or 64
_resourcesDir = resourcesDir()
self.configDir = os.path.join(_resourcesDir, "config")
self.imagesDir = os.path.join(_resourcesDir, "images")
self.localeDir = os.path.join(_resourcesDir, "locale")
self.pluginDir = os.path.join(_resourcesDir, "plugin")
_mplDir = os.path.join(_resourcesDir, "mpl-data")
if os.path.exists(
_mplDir
): # set matplotlibdata for cx_Freeze with local directory
os.environ["MATPLOTLIBDATA"] = _mplDir
serverSoftware = os.getenv("SERVER_SOFTWARE", "")
if serverSoftware.startswith(
"Google App Engine/") or serverSoftware.startswith(
"Development/"):
self.hasFileSystem = False # no file system, userAppDir does not exist
self.isGAE = True
else:
gatewayInterface = os.getenv("GATEWAY_INTERFACE", "")
if gatewayInterface.startswith("CGI/"):
self.isCGI = True
configHomeDir = None # look for path configDir/CONFIG_HOME in argv and environment parameters
for i, arg in enumerate(
sys.argv): # check if config specified in a argv
if arg.startswith("--xdgConfigHome="):
configHomeDir = arg[16:]
break
elif arg == "--xdgConfigHome" and i + 1 < len(sys.argv):
configHomeDir = sys.argv[i + 1]
break
if not configHomeDir: # not in argv, may be an environment parameter
configHomeDir = os.getenv('XDG_CONFIG_HOME')
if not configHomeDir: # look for path configDir/CONFIG_HOME
configHomeDirFile = os.path.join(self.configDir, "XDG_CONFIG_HOME")
if os.path.exists(configHomeDirFile):
try:
with io.open(configHomeDirFile, 'rt',
encoding='utf-8') as f:
configHomeDir = f.read().strip()
if configHomeDir and not os.path.isabs(configHomeDir):
configHomeDir = os.path.abspath(
configHomeDir) # make into a full path if relative
except EnvironmentError:
configHomeDir = None
if self.hasFileSystem and configHomeDir and os.path.exists(
configHomeDir):
# check if a cache exists in this directory (e.g. from XPE or other tool)
impliedAppDir = os.path.join(configHomeDir, "arelle")
if os.path.exists(impliedAppDir):
self.userAppDir = impliedAppDir
elif os.path.exists(os.path.join(configHomeDir, "cache")):
self.userAppDir = configHomeDir # use the XDG_CONFIG_HOME because cache is already a subdirectory
else:
self.userAppDir = impliedAppDir
if sys.platform == "darwin":
self.isMac = True
self.isMSW = False
if self.hasFileSystem and not configHomeDir:
self.userAppDir = os.path.expanduser(
"~") + "/Library/Application Support/Arelle"
# note that cache is in ~/Library/Caches/Arelle
self.contextMenuClick = "<Button-2>"
self.hasClipboard = hasGui # clipboard always only if Gui (not command line mode)
self.updateURL = "http://arelle.org/downloads/8"
elif sys.platform.startswith("win"):
self.isMac = False
self.isMSW = True
if self.hasFileSystem and not configHomeDir:
tempDir = tempfile.gettempdir()
if tempDir.lower().endswith('local\\temp'):
impliedAppDir = tempDir[:-10] + 'local'
else:
impliedAppDir = tempDir
self.userAppDir = os.path.join(impliedAppDir, "Arelle")
if hasGui:
try:
import win32clipboard
self.hasClipboard = True
except ImportError:
self.hasClipboard = False
try:
import win32gui
self.hasWin32gui = True # active state for open file dialogs
except ImportError:
pass
else:
self.hasClipboard = False
self.contextMenuClick = "<Button-3>"
if "64 bit" in sys.version:
self.updateURL = "http://arelle.org/downloads/9"
else: # 32 bit
self.updateURL = "http://arelle.org/downloads/10"
else: # Unix/Linux
self.isMac = False
self.isMSW = False
if self.hasFileSystem and not configHomeDir:
self.userAppDir = os.path.join(os.path.expanduser("~/.config"),
"arelle")
if hasGui:
try:
import gtk
self.hasClipboard = True
except ImportError:
self.hasClipboard = False
else:
self.hasClipboard = False
self.contextMenuClick = "<Button-3>"
try:
from arelle import webserver
self.hasWebServer = True
except ImportError:
self.hasWebServer = False
# assert that app dir must exist
self.config = None
if self.hasFileSystem:
if not os.path.exists(self.userAppDir):
os.makedirs(self.userAppDir)
# load config if it exists
self.configJsonFile = self.userAppDir + os.sep + "config.json"
if os.path.exists(self.configJsonFile):
try:
with io.open(self.configJsonFile, 'rt',
encoding='utf-8') as f:
self.config = json.load(f)
except Exception as ex:
self.config = None # restart with a new config
if not self.config:
self.config = {
'fileHistory': [],
'windowGeometry': "{0}x{1}+{2}+{3}".format(800, 500, 200, 100),
}
# start language translation for domain
self.setUiLanguage(self.config.get("userInterfaceLangOverride", None),
fallbackToDefault=True)
from arelle.WebCache import WebCache
self.webCache = WebCache(self, self.config.get("proxySettings"))
# start plug in server (requres web cache initialized, but not logger)
PluginManager.init(self, loadPluginConfig=hasGui)
# requires plug ins initialized
self.modelManager = ModelManager.initialize(self)
# start taxonomy package server (requres web cache initialized, but not logger)
PackageManager.init(self, loadPackagesConfig=hasGui)
self.startLogging(logFileName, logFileMode, logFileEncoding, logFormat)
# Cntlr.Init after logging started
for pluginMethod in PluginManager.pluginClassMethods("Cntlr.Init"):
pluginMethod(self)
def __init__(self, logFileName=None, logFileMode=None, logFileEncoding=None, logFormat=None):
self.hasWin32gui = False
self.hasGui = False
if sys.platform == "darwin":
self.isMac = True
self.isMSW = False
self.userAppDir = os.path.expanduser("~") + "/Library/Application Support/Arelle"
self.contextMenuClick = "<Button-2>"
self.hasClipboard = True
self.updateURL = "http://arelle.org/downloads/8"
elif sys.platform.startswith("win"):
self.isMac = False
self.isMSW = True
tempDir = tempfile.gettempdir()
if tempDir.endswith('local\\temp'):
impliedAppDir = tempDir[:-10] + 'local'
else:
impliedAppDir = tempDir
self.userAppDir = os.path.join(
os.getenv('XDG_CONFIG_HOME', impliedAppDir),
"Arelle")
try:
import win32clipboard
self.hasClipboard = True
except ImportError:
self.hasClipboard = False
try:
import win32gui
self.hasWin32gui = True # active state for open file dialogs
except ImportError:
pass
self.contextMenuClick = "<Button-3>"
if "64 bit" in sys.version:
self.updateURL = "http://arelle.org/downloads/9"
else: # 32 bit
self.updateURL = "http://arelle.org/downloads/10"
else: # Unix/Linux
self.isMac = False
self.isMSW = False
self.userAppDir = os.path.join(
os.getenv('XDG_CONFIG_HOME', os.path.expanduser("~/.config")),
"arelle")
try:
import gtk
self.hasClipboard = True
except ImportError:
self.hasClipboard = False
self.contextMenuClick = "<Button-3>"
self.moduleDir = os.path.dirname(__file__)
# for python 3.2 remove __pycache__
if self.moduleDir.endswith("__pycache__"):
self.moduleDir = os.path.dirname(self.moduleDir)
if self.moduleDir.endswith("python32.zip/arelle"):
'''
distZipFile = os.path.dirname(self.moduleDir)
d = os.path.join(self.userAppDir, "arelle")
self.configDir = os.path.join(d, "config")
self.imagesDir = os.path.join(d, "images")
import zipfile
distZip = zipfile.ZipFile(distZipFile, mode="r")
distNames = distZip.namelist()
distZip.extractall(path=self.userAppDir,
members=[f for f in distNames if "/config/" in f or "/images/" in f]
)
distZip.close()
'''
resources = os.path.dirname(os.path.dirname(os.path.dirname(self.moduleDir)))
self.configDir = os.path.join(resources, "config")
self.imagesDir = os.path.join(resources, "images")
self.localeDir = os.path.join(resources, "locale")
elif self.moduleDir.endswith("library.zip\\arelle") or self.moduleDir.endswith("library.zip/arelle"): # cx_Freexe
resources = os.path.dirname(os.path.dirname(self.moduleDir))
self.configDir = os.path.join(resources, "config")
self.imagesDir = os.path.join(resources, "images")
self.localeDir = os.path.join(resources, "locale")
else:
self.configDir = os.path.join(self.moduleDir, "config")
self.imagesDir = os.path.join(self.moduleDir, "images")
self.localeDir = os.path.join(self.moduleDir, "locale")
try:
from arelle import webserver
self.hasWebServer = True
except ImportError:
self.hasWebServer = False
# assert that app dir must exist
if not os.path.exists(self.userAppDir):
os.makedirs(self.userAppDir)
# load config if it exists
self.configJsonFile = self.userAppDir + os.sep + "config.json"
self.config = None
if os.path.exists(self.configJsonFile):
try:
with io.open(self.configJsonFile, 'rt', encoding='utf-8') as f:
self.config = json.load(f)
except Exception as ex:
self.config = None # restart with a new config
if not self.config:
self.config = {
'fileHistory': [],
'windowGeometry': "{0}x{1}+{2}+{3}".format(800, 500, 200, 100),
}
# start language translation for domain
self.setUiLanguage(self.config.get("userInterfaceLangOverride",None), fallbackToDefault=True)
from arelle.WebCache import WebCache
self.webCache = WebCache(self, self.config.get("proxySettings"))
self.modelManager = ModelManager.initialize(self)
# start plug in server (requres web cache initialized
PluginManager.init(self)
self.startLogging(logFileName, logFileMode, logFileEncoding, logFormat)
def __init__(self, logFileName=None, logFileMode=None, logFileEncoding=None, logFormat=None):
self.hasWin32gui = False
self.hasGui = False
if sys.platform == "darwin":
self.isMac = True
self.isMSW = False
self.userAppDir = os.path.expanduser("~") + "/Library/Application Support/Arelle"
self.contextMenuClick = "<Button-2>"
self.hasClipboard = True
self.updateURL = "http://arelle.org/downloads/8"
elif sys.platform.startswith("win"):
self.isMac = False
self.isMSW = True
tempDir = tempfile.gettempdir()
if tempDir.endswith('local\\temp'):
self.userAppDir = tempDir[:-10] + 'local\\Arelle'
else:
self.userAppDir = tempDir + os.sep + 'arelle'
try:
import win32clipboard
self.hasClipboard = True
except ImportError:
self.hasClipboard = False
try:
import win32gui
self.hasWin32gui = True # active state for open file dialogs
except ImportError:
pass
self.contextMenuClick = "<Button-3>"
if "64 bit" in sys.version:
self.updateURL = "http://arelle.org/downloads/9"
else: # 32 bit
self.updateURL = "http://arelle.org/downloads/10"
else: # Unix/Linux
self.isMac = False
self.isMSW = False
self.userAppDir = os.path.join(
os.getenv('XDG_CONFIG_HOME', os.path.expanduser("~/.config")),
"arelle")
try:
import gtk
self.hasClipboard = True
except ImportError:
self.hasClipboard = False
self.contextMenuClick = "<Button-3>"
self.moduleDir = os.path.dirname(__file__)
# for python 3.2 remove __pycache__
if self.moduleDir.endswith("__pycache__"):
self.moduleDir = os.path.dirname(self.moduleDir)
if self.moduleDir.endswith("python32.zip/arelle"):
'''
distZipFile = os.path.dirname(self.moduleDir)
d = os.path.join(self.userAppDir, "arelle")
self.configDir = os.path.join(d, "config")
self.imagesDir = os.path.join(d, "images")
import zipfile
distZip = zipfile.ZipFile(distZipFile, mode="r")
distNames = distZip.namelist()
distZip.extractall(path=self.userAppDir,
members=[f for f in distNames if "/config/" in f or "/images/" in f]
)
distZip.close()
'''
resources = os.path.dirname(os.path.dirname(os.path.dirname(self.moduleDir)))
self.configDir = os.path.join(resources, "config")
self.imagesDir = os.path.join(resources, "images")
self.localeDir = os.path.join(resources, "locale")
elif self.moduleDir.endswith("library.zip\\arelle") or self.moduleDir.endswith("library.zip/arelle"): # cx_Freexe
resources = os.path.dirname(os.path.dirname(self.moduleDir))
self.configDir = os.path.join(resources, "config")
self.imagesDir = os.path.join(resources, "images")
self.localeDir = os.path.join(resources, "locale")
else:
self.configDir = os.path.join(self.moduleDir, "config")
self.imagesDir = os.path.join(self.moduleDir, "images")
self.localeDir = os.path.join(self.moduleDir, "locale")
try:
from arelle import webserver
self.hasWebServer = True
except ImportError:
self.hasWebServer = False
# assert that app dir must exist
if not os.path.exists(self.userAppDir):
os.makedirs(self.userAppDir)
# load config if it exists
self.configJsonFile = self.userAppDir + os.sep + "config.json"
self.config = None
if os.path.exists(self.configJsonFile):
try:
with io.open(self.configJsonFile, 'rt', encoding='utf-8') as f:
self.config = json.load(f)
except Exception as ex:
self.config = None # restart with a new config
if not self.config:
self.config = {
'fileHistory': [],
'windowGeometry': "{0}x{1}+{2}+{3}".format(800, 500, 200, 100),
}
# start language translation for domain
try:
gettext.translation("arelle",
self.localeDir,
getLanguageCodes(self.config.get("userInterfaceLangOverride",None))).install()
except Exception as msg:
gettext.install("arelle",
self.localeDir)
from arelle.WebCache import WebCache
self.webCache = WebCache(self, self.config.get("proxySettings"))
self.modelManager = ModelManager.initialize(self)
# start plug in server (requres web cache initialized
PluginManager.init(self)
self.startLogging(logFileName, logFileMode, logFileEncoding, logFormat)
def __init__(self, logFileName=None, logFileMode=None, logFileEncoding=None, logFormat=None):
self.hasWin32gui = False
if sys.platform == "darwin":
self.isMac = True
self.isMSW = False
self.userAppDir = os.path.expanduser("~") + "/Library/Application Support/Arelle"
self.contextMenuClick = "<Button-2>"
self.hasClipboard = True
self.updateURL = "http://arelle.org/downloads/8"
elif sys.platform.startswith("win"):
self.isMac = False
self.isMSW = True
tempDir = tempfile.gettempdir()
if tempDir.endswith("local\\temp"):
self.userAppDir = tempDir[:-10] + "local\\Arelle"
else:
self.userAppDir = tempDir + os.sep + "arelle"
try:
import win32clipboard
self.hasClipboard = True
except ImportError:
self.hasClipboard = False
try:
import win32gui
self.hasWin32gui = True # active state for open file dialogs
except ImportError:
pass
self.contextMenuClick = "<Button-3>"
if "64 bit" in sys.version:
self.updateURL = "http://arelle.org/downloads/9"
else: # 32 bit
self.updateURL = "http://arelle.org/downloads/10"
else: # Unix/Linux
self.isMac = False
self.isMSW = False
self.userAppDir = os.path.join(os.getenv("XDG_CONFIG_HOME", os.path.expanduser("~/.config")), "arelle")
try:
import gtk
self.hasClipboard = True
except ImportError:
self.hasClipboard = False
self.contextMenuClick = "<Button-3>"
self.moduleDir = os.path.dirname(__file__)
# for python 3.2 remove __pycache__
if self.moduleDir.endswith("__pycache__"):
self.moduleDir = os.path.dirname(self.moduleDir)
if self.moduleDir.endswith("python32.zip/arelle"):
"""
distZipFile = os.path.dirname(self.moduleDir)
d = os.path.join(self.userAppDir, "arelle")
self.configDir = os.path.join(d, "config")
self.imagesDir = os.path.join(d, "images")
import zipfile
distZip = zipfile.ZipFile(distZipFile, mode="r")
distNames = distZip.namelist()
distZip.extractall(path=self.userAppDir,
members=[f for f in distNames if "/config/" in f or "/images/" in f]
)
distZip.close()
"""
resources = os.path.dirname(os.path.dirname(os.path.dirname(self.moduleDir)))
self.configDir = os.path.join(resources, "config")
self.imagesDir = os.path.join(resources, "images")
self.localeDir = os.path.join(resources, "locale")
elif self.moduleDir.endswith("library.zip\\arelle"): # cx_Freexe
resources = os.path.dirname(os.path.dirname(self.moduleDir))
self.configDir = os.path.join(resources, "config")
self.imagesDir = os.path.join(resources, "images")
self.localeDir = os.path.join(resources, "locale")
else:
self.configDir = os.path.join(self.moduleDir, "config")
self.imagesDir = os.path.join(self.moduleDir, "images")
self.localeDir = os.path.join(self.moduleDir, "locale")
# assert that app dir must exist
if not os.path.exists(self.userAppDir):
os.makedirs(self.userAppDir)
# load config if it exists
self.configPickleFile = self.userAppDir + os.sep + "config.pickle"
self.config = None
if os.path.exists(self.configPickleFile):
try:
with open(self.configPickleFile, "rb") as f:
self.config = pickle.load(f)
except Exception as ex:
self.config = None # restart with a new config
if not self.config:
self.config = {"fileHistory": [], "windowGeometry": "{0}x{1}+{2}+{3}".format(800, 500, 200, 100)}
# start language translation for domain
try:
gettext.translation("arelle", self.localeDir, getLanguageCodes()).install()
except Exception as msg:
gettext.install("arelle", self.localeDir)
from arelle.WebCache import WebCache
self.webCache = WebCache(self, self.config.get("proxySettings"))
self.modelManager = ModelManager.initialize(self)
if logFileName: # use default logging
self.logger = logging.getLogger("arelle")
if logFileName == "logToPrint":
self.logHandler = LogToPrintHandler()
else:
self.logHandler = logging.FileHandler(
filename=logFileName,
mode=logFileMode if logFileMode else "w",
encoding=logFileEncoding if logFileEncoding else "utf-8",
)
self.logHandler.level = logging.DEBUG
self.logHandler.setFormatter(
logging.Formatter(
logFormat if logFormat else "%(asctime)s [%(messageCode)s] %(message)s - %(file)s %(sourceLine)s \n"
)
)
self.logger.addHandler(self.logHandler)
else:
self.logger = None
def load_edinet():
edinet_list = []
# xbrl 読み込み
xbrl_files = glob.glob(const.FOLDER_XBRL + '*.xbrl')
# EdinetcodeDlInfo.csv 読み込み
edinetcodedata = pd.read_csv(const.FILE_EDINETCODE,
skiprows=1,
encoding='cp932')
for index, xbrl_file in enumerate(xbrl_files):
ctrl = Cntlr.Cntlr()
model_manager = ModelManager.initialize(ctrl)
model_xbrl = model_manager.load(xbrl_file)
print(xbrl_file, ":", index + 1, "/", len(xbrl_files))
# XBRLをDataFrameにセット
factData = pd.DataFrame(data=[
(fact.concept.qname.localName, fact.value, fact.isNumeric,
fact.contextID, fact.decimals) for fact in model_xbrl.facts
],
columns=[
'element_id', 'value', 'isNumeric',
'contextID', 'decimals'
])
# 取得したい要素
datalist = []
datalist_ren = []
flg_ren = False
for key, value in dict_cols.items():
contextRef = value['contextRef']
element_ids = value['element_id']
if len(element_ids) > 0:
for element_id in element_ids:
# data1 = factData[factData['element_id'].str.contains(element_id)]
data1 = []
data2 = []
data3 = []
data1 = factData[factData['element_id'] == element_id]
if len(data1) == 1: break
data2 = data1[data1['contextID'] == (
contextRef + '_NonConsolidatedMember')]
data3 = data1[data1['contextID'] == contextRef]
if len(data2) >= 1 or len(data3) >= 1: break
data, data_ren, unitRef = '', '', ''
if len(data1) == 1:
data = data1['value'].values[0]
data_ren = data
print(key + ' : ' + data)
if len(data2) >= 1 and len(data3) == 0:
data = data2['value'].values[0]
data_ren = data
print(key + ' : ' + data)
if len(data3) >= 1:
data = ''
data_ren = data3['value'].values[0]
print(key + '(連結) : ' + data_ren)
if len(data1) == 0:
print(key + ' : ' + 'NoData ')
data = ''
datalist.append(data)
if flg_ren:
datalist_ren.append(data_ren)
if key == 'EDINETコード':
company = edinetcodedata[edinetcodedata['ＥＤＩＮＥＴコード'] ==
data]
if len(company) == 1:
print('提出者業種 : ' + company['提出者業種'].values[0])
data = company['提出者業種'].values[0]
flg_ren = False
if company['連結の有無'].values[0] == '有':
print('連結の有無 : ' + company['連結の有無'].values[0])
flg_ren = True
else:
print('提出者業種 : ' + 'NoData ')
data = ''
datalist.append(data)
datalist.append('連結' if flg_ren else '単独')
if flg_ren:
# 連結用にデータをコピー
datalist_ren = datalist[:]
if flg_ren:
edinet_list.append(datalist_ren)
else:
edinet_list.append(datalist)
return edinet_list
def __init__(self):
if sys.platform == "darwin":
self.userAppDir = os.path.expanduser("~") + "/Library/Application Support/Arelle"
self.contextMenuClick = "<Button-2>"
self.hasClipboard = True
self.updateURL = "http://arelle.org/downloads/5"
elif sys.platform.startswith("win"):
tempDir = tempfile.gettempdir()
if tempDir.endswith('local\\temp'):
self.userAppDir = tempDir[:-10] + 'local\\Arelle'
else:
self.userAppDir = tempDir + os.sep + 'arelle'
try:
import win32clipboard
self.hasClipboard = True
except ImportError:
self.hasClipboard = False
self.contextMenuClick = "<Button-3>"
if "64 bit" in sys.version:
self.updateURL = "http://arelle.org/downloads/6"
else: # 32 bit
self.updateURL = "http://arelle.org/downloads/7"
else: # Unix/Linux
self.userAppDir = os.path.join(
os.getenv('XDG_CONFIG_HOME', os.path.expanduser("~/.config")),
"arelle")
try:
import gtk
self.hasClipboard = True
except ImportError:
self.hasClipboard = False
self.contextMenuClick = "<Button-3>"
self.moduleDir = os.path.dirname(__file__)
# for python 3.2 remove __pycache__
if self.moduleDir.endswith("__pycache__"):
self.moduleDir = os.path.dirname(self.moduleDir)
if self.moduleDir.endswith("python32.zip/arelle"):
'''
distZipFile = os.path.dirname(self.moduleDir)
d = os.path.join(self.userAppDir, "arelle")
self.configDir = os.path.join(d, "config")
self.imagesDir = os.path.join(d, "images")
import zipfile
distZip = zipfile.ZipFile(distZipFile, mode="r")
distNames = distZip.namelist()
distZip.extractall(path=self.userAppDir,
members=[f for f in distNames if "/config/" in f or "/images/" in f]
)
distZip.close()
'''
resources = os.path.dirname(os.path.dirname(os.path.dirname(self.moduleDir)))
self.configDir = os.path.join(resources, "config")
self.imagesDir = os.path.join(resources, "images")
elif self.moduleDir.endswith("library.zip\\arelle"): # cx_Freexe
resources = os.path.dirname(os.path.dirname(self.moduleDir))
self.configDir = os.path.join(resources, "config")
self.imagesDir = os.path.join(resources, "images")
else:
self.configDir = os.path.join(self.moduleDir, "config")
self.imagesDir = os.path.join(self.moduleDir, "images")
# assert that app dir must exist
if not os.path.exists(self.userAppDir):
os.makedirs(self.userAppDir)
# load config if it exists
self.configPickleFile = self.userAppDir + os.sep + "config.pickle"
self.config = None
if os.path.exists(self.configPickleFile):
try:
with open(self.configPickleFile, 'rb') as f:
self.config = pickle.load(f)
except Exception as ex:
self.config = None # restart with a new config
if not self.config:
self.config = {
'fileHistory': [],
'windowGeometry': "{0}x{1}+{2}+{3}".format(800, 500, 200, 100),
}
from arelle.WebCache import WebCache
self.webCache = WebCache(self, self.config.get("proxySettings"))
self.modelManager = ModelManager.initialize(self)
def __init__(self):
if sys.platform == "darwin":
self.userAppDir = os.path.expanduser(
"~") + "/Library/Application Support/Arelle"
self.contextMenuClick = "<Button-2>"
self.hasClipboard = True
self.updateURL = "http://arelle.org/downloads/5"
elif sys.platform.startswith("win"):
tempDir = tempfile.gettempdir()
if tempDir.endswith('local\\temp'):
self.userAppDir = tempDir[:-10] + 'local\\Arelle'
else:
self.userAppDir = tempDir + os.sep + 'arelle'
try:
import win32clipboard
self.hasClipboard = True
except ImportError:
self.hasClipboard = False
self.contextMenuClick = "<Button-3>"
if "64 bit" in sys.version:
self.updateURL = "http://arelle.org/downloads/6"
else: # 32 bit
self.updateURL = "http://arelle.org/downloads/7"
else: # Unix/Linux
self.userAppDir = os.path.join(
os.getenv('XDG_CONFIG_HOME', os.path.expanduser("~/.config")),
"arelle")
try:
import gtk
self.hasClipboard = True
except ImportError:
self.hasClipboard = False
self.contextMenuClick = "<Button-3>"
self.moduleDir = os.path.dirname(__file__)
# for python 3.2 remove __pycache__
if self.moduleDir.endswith("__pycache__"):
self.moduleDir = os.path.dirname(self.moduleDir)
if self.moduleDir.endswith("python32.zip/arelle"):
'''
distZipFile = os.path.dirname(self.moduleDir)
d = os.path.join(self.userAppDir, "arelle")
self.configDir = os.path.join(d, "config")
self.imagesDir = os.path.join(d, "images")
import zipfile
distZip = zipfile.ZipFile(distZipFile, mode="r")
distNames = distZip.namelist()
distZip.extractall(path=self.userAppDir,
members=[f for f in distNames if "/config/" in f or "/images/" in f]
)
distZip.close()
'''
resources = os.path.dirname(
os.path.dirname(os.path.dirname(self.moduleDir)))
self.configDir = os.path.join(resources, "config")
self.imagesDir = os.path.join(resources, "images")
elif self.moduleDir.endswith("library.zip\\arelle"): # cx_Freexe
resources = os.path.dirname(os.path.dirname(self.moduleDir))
self.configDir = os.path.join(resources, "config")
self.imagesDir = os.path.join(resources, "images")
else:
self.configDir = os.path.join(self.moduleDir, "config")
self.imagesDir = os.path.join(self.moduleDir, "images")
# assert that app dir must exist
if not os.path.exists(self.userAppDir):
os.makedirs(self.userAppDir)
# load config if it exists
self.configPickleFile = self.userAppDir + os.sep + "config.pickle"
self.config = None
if os.path.exists(self.configPickleFile):
try:
with open(self.configPickleFile, 'rb') as f:
self.config = pickle.load(f)
except Exception as ex:
self.config = None # restart with a new config
if not self.config:
self.config = {
'fileHistory': [],
'windowGeometry': "{0}x{1}+{2}+{3}".format(800, 500, 200, 100),
}
from arelle.WebCache import WebCache
self.webCache = WebCache(self, self.config.get("proxySettings"))
self.modelManager = ModelManager.initialize(self)
def __init__(self,
hasGui=False,
logFileName=None,
logFileMode=None,
logFileEncoding=None,
logFormat=None):
self.hasWin32gui = False
self.hasGui = hasGui
self.hasFileSystem = True # no file system on Google App Engine servers
self.moduleDir = os.path.dirname(__file__)
# for python 3.2 remove __pycache__
if self.moduleDir.endswith("__pycache__"):
self.moduleDir = os.path.dirname(self.moduleDir)
if self.moduleDir.endswith("python32.zip/arelle"):
'''
distZipFile = os.path.dirname(self.moduleDir)
d = os.path.join(self.userAppDir, "arelle")
self.configDir = os.path.join(d, "config")
self.imagesDir = os.path.join(d, "images")
import zipfile
distZip = zipfile.ZipFile(distZipFile, mode="r")
distNames = distZip.namelist()
distZip.extractall(path=self.userAppDir,
members=[f for f in distNames if "/config/" in f or "/images/" in f]
)
distZip.close()
'''
resources = os.path.dirname(
os.path.dirname(os.path.dirname(self.moduleDir)))
self.configDir = os.path.join(resources, "config")
self.imagesDir = os.path.join(resources, "images")
self.localeDir = os.path.join(resources, "locale")
self.pluginDir = os.path.join(resources, "plugin")
elif self.moduleDir.endswith(
"library.zip\\arelle") or self.moduleDir.endswith(
"library.zip/arelle"): # cx_Freexe
resources = os.path.dirname(os.path.dirname(self.moduleDir))
self.configDir = os.path.join(resources, "config")
self.imagesDir = os.path.join(resources, "images")
self.localeDir = os.path.join(resources, "locale")
self.pluginDir = os.path.join(resources, "plugin")
else:
self.configDir = os.path.join(self.moduleDir, "config")
self.imagesDir = os.path.join(self.moduleDir, "images")
self.localeDir = os.path.join(self.moduleDir, "locale")
self.pluginDir = os.path.join(self.moduleDir, "plugin")
configHomeDir = os.getenv('XDG_CONFIG_HOME')
if not configHomeDir: # look for path configDir/CONFIG_HOME
configHomeDirFile = os.path.join(self.configDir, "XDG_CONFIG_HOME")
if os.path.exists(configHomeDirFile):
try:
with io.open(configHomeDirFile, 'rt',
encoding='utf-8') as f:
configHomeDir = f.read().strip()
if configHomeDir and not os.path.isabs(configHomeDir):
configHomeDir = os.path.abspath(
configHomeDir) # make into a full path if relative
except EnvironmentError:
configHomeDir = None
if configHomeDir and os.path.exists(configHomeDir):
# check if a cache exists in this directory (e.g. from XPE or other tool)
impliedAppDir = os.path.join(configHomeDir, "arelle")
if os.path.exists(impliedAppDir):
self.userAppDir = impliedAppDir
elif os.path.exists(os.path.join(configHomeDir, "cache")):
self.userAppDir = configHomeDir # use the XDG_CONFIG_HOME because cache is already a subdirectory
else:
self.userAppDir = impliedAppDir
if sys.platform == "darwin":
self.isMac = True
self.isMSW = False
self.isGAE = False
if not configHomeDir:
self.userAppDir = os.path.expanduser(
"~") + "/Library/Application Support/Arelle"
# note that cache is in ~/Library/Caches/Arelle
self.contextMenuClick = "<Button-2>"
self.hasClipboard = hasGui # clipboard always only if Gui (not command line mode)
self.updateURL = "http://arelle.org/downloads/8"
elif sys.platform.startswith("win"):
self.isMac = False
self.isMSW = True
self.isGAE = False
if not configHomeDir:
tempDir = tempfile.gettempdir()
if tempDir.endswith('local\\temp'):
impliedAppDir = tempDir[:-10] + 'local'
else:
impliedAppDir = tempDir
self.userAppDir = os.path.join(impliedAppDir, "Arelle")
if hasGui:
try:
import win32clipboard
self.hasClipboard = True
except ImportError:
self.hasClipboard = False
try:
import win32gui
self.hasWin32gui = True # active state for open file dialogs
except ImportError:
pass
else:
self.hasClipboard = False
self.contextMenuClick = "<Button-3>"
if "64 bit" in sys.version:
self.updateURL = "http://arelle.org/downloads/9"
else: # 32 bit
self.updateURL = "http://arelle.org/downloads/10"
else: # Unix/Linux
self.isMac = False
self.isMSW = False
serverSoftware = os.getenv("SERVER_SOFTWARE", "")
if serverSoftware.startswith(
"Google App Engine/") or serverSoftware.startswith(
"Development/"):
self.hasFileSystem = False # no file system, userAppDir does not exist
self.isGAE = True
elif not configHomeDir:
self.userAppDir = os.path.join(os.path.expanduser("~/.config"),
"arelle")
self.isGAE = False
if hasGui:
try:
import gtk
self.hasClipboard = True
except ImportError:
self.hasClipboard = False
else:
self.hasClipboard = False
self.contextMenuClick = "<Button-3>"
try:
from arelle import webserver
self.hasWebServer = True
except ImportError:
self.hasWebServer = False
# assert that app dir must exist
self.config = None
if self.hasFileSystem:
if not os.path.exists(self.userAppDir):
os.makedirs(self.userAppDir)
# load config if it exists
self.configJsonFile = self.userAppDir + os.sep + "config.json"
if os.path.exists(self.configJsonFile):
try:
with io.open(self.configJsonFile, 'rt',
encoding='utf-8') as f:
self.config = json.load(f)
except Exception as ex:
self.config = None # restart with a new config
if not self.config:
self.config = {
'fileHistory': [],
'windowGeometry': "{0}x{1}+{2}+{3}".format(800, 500, 200, 100),
}
# start language translation for domain
self.setUiLanguage(self.config.get("userInterfaceLangOverride", None),
fallbackToDefault=True)
from arelle.WebCache import WebCache
self.webCache = WebCache(self, self.config.get("proxySettings"))
self.modelManager = ModelManager.initialize(self)
# start plug in server (requres web cache initialized, but not logger)
PluginManager.init(self)
self.startLogging(logFileName, logFileMode, logFileEncoding, logFormat)
# Cntlr.Init after logging started
for pluginMethod in PluginManager.pluginClassMethods("Cntlr.Init"):
pluginMethod(self)
