예제 #1
0
    def _get_utr_namespaces(self):
        # Get UTR
        modelManager = ModelManager.initialize(self.cntlr)
        modelUtr = modelManager.create()
        loadUtr(modelUtr)

        utr_namespaces = collections.defaultdict(set)
        for type_name, entries in modelManager.disclosureSystem.utrItemTypeEntries.items(
        ):
            for entry_name, entry in entries.items():
                if entry.isSimple:
                    utr_namespaces[entry.nsUnit].add(entry.unitId)

        return utr_namespaces
예제 #2
0
def main():
    if IS_EXTRACTED:
        pass
    else:
        edinet_zip_dir = os.path.join(EDINET_ROOT_DIR, "zip")
        extract_files_from_zip(edinet_zip_dir,
                               dest_dir_root=EDINET_ROOT_DIR,
                               unzip_members_regrep="|".join([
                                   f"XBRL/PublicDoc/.*\.{extension}"
                                   for extension in ["xbrl", "xsd", "xml"]
                               ]))
    # XBRLから情報取得
    xbrl_file_regrex = os.path.join(EDINET_ROOT_DIR, EDINET_XBRL_REGREX)
    xbrl_files = glob.glob(xbrl_file_regrex)
    ctrl = Cntlr.Cntlr()
    model_manager = ModelManager.initialize(ctrl)
    for index, xbrl_file in enumerate(xbrl_files):
        print(xbrl_file, ":", index + 1, "/", len(xbrl_files))
        export_facts(model_manager, xbrl_file)

    print(f"{'-'*10} XBRL出力 完了 {'-'*10}")
예제 #3
0
    def _get_taxonomy_model(self, taxonomy_url, namespace):
        """Get an xbrl model of the entry_point file.
        """
        start = datetime.datetime.today()
        rules_taxonomy_filesource = FileSource.openFileSource(
            taxonomy_url, self.cntlr)
        modelManager = ModelManager.initialize(self.cntlr)
        modelXbrl = modelManager.load(rules_taxonomy_filesource)
        if len({'IOerror', 'FileNotLoadable'} & set(modelXbrl.errors)) > 0:
            modelXbrl.error(
                "TaxonomyLoadError",
                "Cannot open file {} with namespace {}.".format(
                    taxonomy_url, namespace))
        else:
            end = datetime.datetime.today()
            print("Taxonomy {namespace} loaded in {time}. {entry}".format(
                namespace=modelXbrl.modelDocument.targetNamespace,
                time=end - start,
                entry=taxonomy_url))

        return modelXbrl
예제 #4
0
def main():
    if IS_EXTRACTED:
        pass
    else:
        edinet_zip_dir = os.path.join(EDINET_ROOT_DIR, "zip")
        extract_files_from_zip(
            edinet_zip_dir,
            dest_dir_root=EDINET_ROOT_DIR,
            unzip_members_regrep="|".join(
                [f"XBRL/PublicDoc/.*\.{extension}" for extension in ["xbrl", "xsd", "xml"]]
            )
        )
    
    # XBRLから情報取得
    xbrl_file_regrex = os.path.join(EDINET_ROOT_DIR, EDINET_XBRL_REGREX)
    xbrl_files = glob.glob(xbrl_file_regrex)
    list_df_facts = []
    ctrl = Cntlr.Cntlr()
    model_manager = ModelManager.initialize(ctrl)
    for index, xbrl_file in enumerate(xbrl_files):
        print(xbrl_file, ":", index + 1, "/", len(xbrl_files))
        df_facts = get_facts(model_manager, xbrl_file)
        if df_facts is not None:
            list_df_facts.append(df_facts)
    if list_df_facts:
        df_xbrl = pd.concat(list_df_facts, axis=0, sort=False)
        # Edinetコードリストの情報をマージ
        df_edinetcd_info = get_edinetcd_info(EDINETCDDLINFO_COLS)
        df_xbrl = df_edinetcd_info.merge(df_xbrl, on=EDINETCD_COL, how="right")
        df_xbrl.to_csv(
            os.path.join(EDINET_ROOT_DIR, OUTPUT_FILE_NAME),
            index=False,
            encoding="cp932"
        )
        print(f"{'-'*10} 情報抽出 完了 {'-'*10}")
    else:
        print("処理対象のデータはありませんでした。")
예제 #5
0
파일: Cntlr.py 프로젝트: irjudson/Arelle
    def __init__(self, hasGui=False, logFileName=None, logFileMode=None, logFileEncoding=None, logFormat=None):
        self.hasWin32gui = False
        self.hasGui = hasGui
        self.hasFileSystem = True # no file system on Google App Engine servers
        self.isGAE = False
        self.systemWordSize = int(round(math.log(sys.maxsize, 2)) + 1) # e.g., 32 or 64

        self.moduleDir = os.path.dirname(__file__)
        # for python 3.2 remove __pycache__
        if self.moduleDir.endswith("__pycache__"):
            self.moduleDir = os.path.dirname(self.moduleDir)
        if self.moduleDir.endswith("python32.zip/arelle"):
            '''
            distZipFile = os.path.dirname(self.moduleDir)
            d = os.path.join(self.userAppDir, "arelle")
            self.configDir = os.path.join(d, "config")
            self.imagesDir = os.path.join(d, "images")
            import zipfile
            distZip = zipfile.ZipFile(distZipFile, mode="r")
            distNames = distZip.namelist()
            distZip.extractall(path=self.userAppDir,
                               members=[f for f in distNames if "/config/" in f or "/images/" in f]
                               )
            distZip.close()
            '''
            resources = os.path.dirname(os.path.dirname(os.path.dirname(self.moduleDir)))
            self.configDir = os.path.join(resources, "config")
            self.imagesDir = os.path.join(resources, "images")
            self.localeDir = os.path.join(resources, "locale")
            self.pluginDir = os.path.join(resources, "plugin")
        elif self.moduleDir.endswith("library.zip\\arelle") or self.moduleDir.endswith("library.zip/arelle"): # cx_Freexe
            resources = os.path.dirname(os.path.dirname(self.moduleDir))
            self.configDir = os.path.join(resources, "config")
            self.imagesDir = os.path.join(resources, "images")
            self.localeDir = os.path.join(resources, "locale")
            self.pluginDir = os.path.join(resources, "plugin")
        else:
            self.configDir = os.path.join(self.moduleDir, "config")
            self.imagesDir = os.path.join(self.moduleDir, "images")
            self.localeDir = os.path.join(self.moduleDir, "locale")
            self.pluginDir = os.path.join(self.moduleDir, "plugin")
        
        serverSoftware = os.getenv("SERVER_SOFTWARE", "")
        if serverSoftware.startswith("Google App Engine/") or serverSoftware.startswith("Development/"):
            self.hasFileSystem = False # no file system, userAppDir does not exist
            self.isGAE = True
            
        configHomeDir = None  # look for path configDir/CONFIG_HOME in argv and environment parameters
        for i, arg in enumerate(sys.argv):  # check if config specified in a argv 
            if arg.startswith("--xdgConfigHome="):
                configHomeDir = arg[16:]
                break
            elif arg == "--xdgConfigHome" and i + 1 < len(sys.argv):
                configHomeDir = sys.argv[i + 1]
                break
        if not configHomeDir: # not in argv, may be an environment parameter
            configHomeDir = os.getenv('XDG_CONFIG_HOME')
        if not configHomeDir:  # look for path configDir/CONFIG_HOME
            configHomeDirFile = os.path.join(self.configDir, "XDG_CONFIG_HOME")
            if os.path.exists(configHomeDirFile):
                try:
                    with io.open(configHomeDirFile, 'rt', encoding='utf-8') as f:
                        configHomeDir = f.read().strip()
                    if configHomeDir and not os.path.isabs(configHomeDir):
                        configHomeDir = os.path.abspath(configHomeDir)  # make into a full path if relative
                except EnvironmentError:
                    configHomeDir = None
        if self.hasFileSystem and configHomeDir and os.path.exists(configHomeDir):
            # check if a cache exists in this directory (e.g. from XPE or other tool)
            impliedAppDir = os.path.join(configHomeDir, "arelle")
            if os.path.exists(impliedAppDir):
                self.userAppDir = impliedAppDir
            elif os.path.exists(os.path.join(configHomeDir, "cache")):
                self.userAppDir = configHomeDir # use the XDG_CONFIG_HOME because cache is already a subdirectory
            else:
                self.userAppDir = impliedAppDir
        if sys.platform == "darwin":
            self.isMac = True
            self.isMSW = False
            if self.hasFileSystem and not configHomeDir:
                self.userAppDir = os.path.expanduser("~") + "/Library/Application Support/Arelle"
            # note that cache is in ~/Library/Caches/Arelle
            self.contextMenuClick = "<Button-2>"
            self.hasClipboard = hasGui  # clipboard always only if Gui (not command line mode)
            self.updateURL = "http://arelle.org/downloads/8"
        elif sys.platform.startswith("win"):
            self.isMac = False
            self.isMSW = True
            if self.hasFileSystem and not configHomeDir:
                tempDir = tempfile.gettempdir()
                if tempDir.endswith('local\\temp'):
                    impliedAppDir = tempDir[:-10] + 'local'
                else:
                    impliedAppDir = tempDir
                self.userAppDir = os.path.join( impliedAppDir, "Arelle")
            if hasGui:
                try:
                    import win32clipboard
                    self.hasClipboard = True
                except ImportError:
                    self.hasClipboard = False
                try:
                    import win32gui
                    self.hasWin32gui = True # active state for open file dialogs
                except ImportError:
                    pass
            else:
                self.hasClipboard = False
            self.contextMenuClick = "<Button-3>"
            if "64 bit" in sys.version:
                self.updateURL = "http://arelle.org/downloads/9"
            else: # 32 bit
                self.updateURL = "http://arelle.org/downloads/10"
        else: # Unix/Linux
            self.isMac = False
            self.isMSW = False
            if self.hasFileSystem and not configHomeDir:
                    self.userAppDir = os.path.join( os.path.expanduser("~/.config"), "arelle")
            if hasGui:
                try:
                    import gtk
                    self.hasClipboard = True
                except ImportError:
                    self.hasClipboard = False
            else:
                self.hasClipboard = False
            self.contextMenuClick = "<Button-3>"
        try:
            from arelle import webserver
            self.hasWebServer = True
        except ImportError:
            self.hasWebServer = False
        # assert that app dir must exist
        self.config = None
        if self.hasFileSystem:
            if not os.path.exists(self.userAppDir):
                os.makedirs(self.userAppDir)
            # load config if it exists
            self.configJsonFile = self.userAppDir + os.sep + "config.json"
            if os.path.exists(self.configJsonFile):
                try:
                    with io.open(self.configJsonFile, 'rt', encoding='utf-8') as f:
                        self.config = json.load(f)
                except Exception as ex:
                    self.config = None # restart with a new config
        if not self.config:
            self.config = {
                'fileHistory': [],
                'windowGeometry': "{0}x{1}+{2}+{3}".format(800, 500, 200, 100),                
            }
            
        # start language translation for domain
        self.setUiLanguage(self.config.get("userInterfaceLangOverride",None), fallbackToDefault=True)
            
        from arelle.WebCache import WebCache
        self.webCache = WebCache(self, self.config.get("proxySettings"))
        self.modelManager = ModelManager.initialize(self)
        
        # start plug in server (requres web cache initialized, but not logger)
        PluginManager.init(self)
 
        # start taxonomy package server (requres web cache initialized, but not logger)
        PackageManager.init(self)
 
        self.startLogging(logFileName, logFileMode, logFileEncoding, logFormat)
        
        # Cntlr.Init after logging started
        for pluginMethod in PluginManager.pluginClassMethods("Cntlr.Init"):
            pluginMethod(self)
예제 #6
0
def xuleCmdUtilityRun(cntlr, options, **kwargs): 
    # Save the controller and options in the module global variable
    global _cntlr
    _cntlr = cntlr
    global _options
    _options = options

    cntlr.addToLog("Xule version: %s" % __version__, 'info')

    # check option combinations
    parser = OptionParser()
    
    if getattr(options, "xule_version", False):
        cntlr.addToLog("Xule version: %s" % __version__, 'xule')
        cntlr.close()

    if getattr(options, "xule_cpu", None) is not None and not getattr(options, 'xule_multi', None):
            parser.error(_("--xule-multi is required with --xule_cpu."))

    if getattr(options, "xule_server", None) is not None and not getattr(options, 'xule_rule_set', None):
            parser.error(_("--xule-rule-set is required with --xule_server."))

    from os import name
    if getattr(options, "xule_multi", False) and name == 'nt':
        parser.error(_("--xule-multi can't be used in Windows"))    

            
    if getattr(options, "xule-numthreads", None) == None:
        setattr(options, "xule-numthreads", 1)   
    
    if getattr(options, 'xule_add_packages', None) is not None and not getattr(options, 'xule_rule_set', None):
        parser.error(_("--xule-rule-set is required with --xule-add-packages.")) 

    if getattr(options, 'xule_remove_packages', None) is not None and not getattr(options, 'xule_rule_set', None):
        parser.error(_("--xule-rule-set is required with --xule-remove-packages.")) 

    if getattr(options, 'xule_show_packages', None) is not None and not getattr(options, 'xule_rule_set', None):
        parser.error(_("--xule-rule-set is required with --xule-show-packages."))    

    if len([x for x in (getattr(options, "xule_update_rule_set_map", False),
                       getattr(options, "xule_replace_rule_set_map", False),
                       getattr(options, "xule_reset_rule_set_map", False)) if x]) > 1:
        parser.error(_("Cannot use --xule-update-rule-set-map or --xule-replace-rule-set-map or --xule-reset-rule-set-map the same time."))
    
    if getattr(options, 'xule_validate', None) is not None and getattr(options, 'xule_rule_set', None) is None:
        parser.error(_("--xule-validate requires a Xule ruleset. Use option --xule-rule-set."))

    if getattr(options, 'xule_filing_list', None) is not None and getattr(options, 'entrypointFile', None) is not None:
        parser.error(_("--xule-filing-list cannot be used with -f"))

    # compile rules
    if getattr(options, "xule_compile", None):
        compile_destination = getattr(options, "xule_rule_set", "xuleRules") 
        xp.parseRules(options.xule_compile.split("|"), compile_destination, getattr(options, "xule_compile_type"))
    
    # add packages
    if getattr(options, "xule_add_packages", None):
        rule_set = xr.XuleRuleSet(cntlr)
        rule_set.open(getattr(options, "xule_rule_set"), open_packages=False, open_files=False)
        packages = options.xule_add_packages.split('|')
        rule_set.manage_packages(packages, 'add')

    # remove packages
    if getattr(options, "xule_remove_packages", None):
        rule_set = xr.XuleRuleSet(cntlr)
        rule_set.open(getattr(options, "xule_rule_set"), open_packages=False, open_files=False)
        packages = options.xule_remove_packages.split('|')
        rule_set.manage_packages(packages, 'del')
    
    # show packages
    if getattr(options, "xule_show_packages", False):
        rule_set = xr.XuleRuleSet(cntlr)
        rule_set.open(getattr(options, "xule_rule_set"), open_packages=False, open_files=False)
        print("Packages in rule set:")
        for package_info in rule_set.get_packages_info():
            print('\t' + package_info.get('name') + ' (' + os.path.basename(package_info.get('URL')) + ')')
        
    # update rule set map
    if getattr(options, 'xule_update_rule_set_map', None):
        xu.update_rule_set_map(cntlr, getattr(options, 'xule_update_rule_set_map'), _xule_rule_set_map_name)
    
    # replace rule set map
    if getattr(options, 'xule_replace_rule_set_map', None):
        xu.update_rule_set_map(cntlr, getattr(options, 'xule_replace_rule_set_map'), _xule_rule_set_map_name, overwrite=True)
    
    # reset rule set map
    if getattr(options, 'xule_reset_rule_set_map', False):
        xu.reset_rule_set_map(cntlr, _xule_rule_set_map_name)

    # display the rule set map
    if getattr(options, 'xule_display_rule_set_map', False):
        displayValidatorRulesetMap(cntlr, 'Xule', _xule_rule_set_map_name)
        
    # validate ruleset
    if getattr(options, 'xule_validate', False):
        rule_set = xr.XuleRuleSet(cntlr)
        rule_set.open(options.xule_rule_set, open_packages=not getattr(options, 'xule_bypass_packages', False))
        xv.XuleValidate(cntlr, rule_set, options.xule_rule_set)
    
    if getattr(options, "xule_server", None):
        from threading import Thread
        
        try:
            rule_set = xr.XuleRuleSet()
            rule_set.open(options.xule_rule_set, False)
        except xr.XuleRuleSetError:
            raise

        # Create global Context
        global_context = XuleGlobalContext(rule_set, cntlr=cntlr, options=options)

        global_context.message_queue.print("Using %d processors" % (global_context.num_processors)) 

        # Start Output message queue
        if getattr(options, "xule_multi", False):
            t = Thread(target=xm.output_message_queue, args=(global_context,))
            t.start()
        
        global_context.message_queue.logging("Building Constant and Rule Groups")
        global_context.all_constants = rule_set.get_grouped_constants()
        global_context.all_rules = rule_set.get_grouped_rules()        

        
        for g in global_context.all_constants:
            global_context.message_queue.logging("Constants: %s - %d" % (g, len(global_context.all_constants[g])))

        for g in global_context.all_rules:
            global_context.message_queue.logging("Rules: %s - %d" % (g, len(global_context.all_rules[g])))

        # evaluate valid constants (no dependency, rules taxonomy)
        global_context.message_queue.logging("Calculating and Storing Constants")
        xm.run_constant_group(global_context, 'c', 'rtc')

                                   
        # Add precalculated information to the cntlr to pass to XuleServer
        setattr(cntlr, "xule_options", options)
        setattr(cntlr, "rule_set", global_context.rule_set)        
        setattr(cntlr, "constant_list", global_context._constants)
        setattr(cntlr, "all_constants", global_context.all_constants)
        setattr(cntlr, "all_rules", global_context.all_rules)        


        global_context.message_queue.logging("Finished Server Initialization")
        
        # stop message_queue
        global_context.message_queue.stop()
        
        if getattr(options, "xule_multi", False):
            t.join()
    else:
        if getattr(options, 'xule_filing_list', None) is not None:
            # process filing list
            if getattr(options, "xule_filing_list", None):
                try:
                    with open(options.xule_filing_list, 'r') as filing_list_file:
                        # Try json
                        try:
                            filing_list = json.load(filing_list_file, object_pairs_hook=collections.OrderedDict)
                        except:
                            # Try a flat list of file names
                            try:
                                # reset the file pointer
                                filing_list_file.seek(0)
                                filing_list = [{"file": file_name} for file_name in filing_list_file]
                            except:
                                cntlr.addToLog(_("Unable to open Filing listing file '%s'." % options.xule_filing_list), 'xule')
                                raise
                except FileNotFoundError:
                    cntlr.addToLog(_("Filing listing file '%s' is not found" % options.xule_filing_list), 'xule')
                    raise

                if isinstance(filing_list, list):
                    for file_info in filing_list:
                        if isinstance(file_info, dict):
                            input_file_name = file_info.get('file')
                            if input_file_name is not None:
                                input_file_name = input_file_name.strip()
                                print("Processing filing", input_file_name)
                                filing_filesource = FileSource.openFileSource(input_file_name, cntlr)
                                modelManager = ModelManager.initialize(cntlr)
                                modelXbrl = modelManager.load(filing_filesource)
                                # Update options
                                new_options = copy.copy(options)
                                delattr(new_options, 'xule_filing_list')
                                for k, v in file_info.items():
                                    if k != 'file' and k.strip().lower().startswith('xule'): # Only change xule options
                                        setattr(new_options, k.strip().lower(), v)
                                if getattr(new_options, 'xule_run'):
                                    xuleCmdXbrlLoaded(cntlr, new_options, modelXbrl)
                                elif getattr(new_options, 'validate'):
                                    for xule_validator in _xule_validators:
                                        runXule(_cntlr, new_options, modelXbrl, xule_validator['map_name'])
                                modelXbrl.close()
        else:
            if options.entrypointFile is None:
                # try running the xule processor - This is when rules are run without an instance document
                xuleCmdXbrlLoaded(cntlr, options, None)

    # Only register xule as a validator if the xule plugin was directly added in the --plugin options.
    if isXuleDirect():
        xuleRegisterValidators('Xule', _xule_rule_set_map_name)
예제 #7
0
def xuleCmdUtilityRun(cntlr, options, **kwargs):
    # Save the controller and options in the module global variable
    global _cntlr
    _cntlr = cntlr
    global _options
    _options = options

    cntlr.addToLog("Xule version: %s" % __version__, 'info')

    # check option combinations
    parser = optparse.OptionParser()

    if getattr(options, "xule_version", False):
        cntlr.addToLog("Xule version: %s" % __version__, 'xule')
        cntlr.close()

    if getattr(options, "xule_cpu",
               None) is not None and not getattr(options, 'xule_multi', None):
        parser.error(_("--xule-multi is required with --xule_cpu."))

    if getattr(options, "xule_server", None) is not None and not getattr(
            options, 'xule_rule_set', None):
        parser.error(_("--xule-rule-set is required with --xule_server."))

    from os import name
    if getattr(options, "xule_multi", False) and name == 'nt':
        parser.error(_("--xule-multi can't be used in Windows"))

    if getattr(options, "xule-numthreads", None) == None:
        setattr(options, "xule-numthreads", 1)

    if getattr(options, 'xule_add_packages', None) is not None and not getattr(
            options, 'xule_rule_set', None):
        parser.error(
            _("--xule-rule-set is required with --xule-add-packages."))

    if getattr(
            options, 'xule_remove_packages',
            None) is not None and not getattr(options, 'xule_rule_set', None):
        parser.error(
            _("--xule-rule-set is required with --xule-remove-packages."))

    if getattr(
            options, 'xule_show_packages',
            None) is not None and not getattr(options, 'xule_rule_set', None):
        parser.error(
            _("--xule-rule-set is required with --xule-show-packages."))

    if len([
            x
            for x in (getattr(options, "xule_update_rule_set_map", False),
                      getattr(options, "xule_replace_rule_set_map", False),
                      getattr(options, "xule_reset_rule_set_map", False)) if x
    ]) > 1:
        parser.error(
            _("Cannot use --xule-update-rule-set-map or --xule-replace-rule-set-map or --xule-reset-rule-set-map the same time."
              ))

    if getattr(options, 'xule_validate', None) is not None and getattr(
            options, 'xule_rule_set', None) is None:
        parser.error(
            _("--xule-validate requires a Xule ruleset. Use option --xule-rule-set."
              ))

    if getattr(options, 'xule_filing_list', None) is not None and getattr(
            options, 'entrypointFile', None) is not None:
        parser.error(_("--xule-filing-list cannot be used with -f"))

    # compile rules
    if getattr(options, "xule_compile", None):
        compile_destination = getattr(options, "xule_rule_set", "xuleRules")
        xp.parseRules(options.xule_compile.split("|"), compile_destination,
                      getattr(options, "xule_compile_type"))

    # add packages
    if getattr(options, "xule_add_packages", None):
        rule_set = xr.XuleRuleSet(cntlr)
        rule_set.open(getattr(options, "xule_rule_set"),
                      open_packages=False,
                      open_files=False)
        packages = options.xule_add_packages.split('|')
        rule_set.manage_packages(packages, 'add')

    # remove packages
    if getattr(options, "xule_remove_packages", None):
        rule_set = xr.XuleRuleSet(cntlr)
        rule_set.open(getattr(options, "xule_rule_set"),
                      open_packages=False,
                      open_files=False)
        packages = options.xule_remove_packages.split('|')
        rule_set.manage_packages(packages, 'del')

    # show packages
    if getattr(options, "xule_show_packages", False):
        rule_set = xr.XuleRuleSet(cntlr)
        rule_set.open(getattr(options, "xule_rule_set"),
                      open_packages=False,
                      open_files=False)
        print("Packages in rule set:")
        for package_info in rule_set.get_packages_info():
            print('\t' + package_info.get('name') + ' (' +
                  os.path.basename(package_info.get('URL')) + ')')

    # update rule set map
    if getattr(options, 'xule_update_rule_set_map', None):
        xu.update_rule_set_map(cntlr,
                               getattr(options, 'xule_update_rule_set_map'),
                               _xule_rule_set_map_name)

    # replace rule set map
    if getattr(options, 'xule_replace_rule_set_map', None):
        xu.update_rule_set_map(cntlr,
                               getattr(options, 'xule_replace_rule_set_map'),
                               _xule_rule_set_map_name,
                               overwrite=True)

    # reset rule set map
    if getattr(options, 'xule_reset_rule_set_map', False):
        xu.reset_rule_set_map(cntlr, _xule_rule_set_map_name)

    # display the rule set map
    if getattr(options, 'xule_display_rule_set_map', False):
        displayValidatorRulesetMap(cntlr, 'Xule', _xule_rule_set_map_name)

    # validate ruleset
    if getattr(options, 'xule_validate', False):
        rule_set = xr.XuleRuleSet(cntlr)
        rule_set.open(
            options.xule_rule_set,
            open_packages=not getattr(options, 'xule_bypass_packages', False))
        xv.XuleValidate(cntlr, rule_set, options.xule_rule_set)

    if getattr(options, "xule_server", None):
        from threading import Thread

        try:
            rule_set = xr.XuleRuleSet()
            rule_set.open(options.xule_rule_set, False)
        except xr.XuleRuleSetError:
            raise

        # Create global Context
        global_context = XuleGlobalContext(rule_set,
                                           cntlr=cntlr,
                                           options=options)

        global_context.message_queue.print("Using %d processors" %
                                           (global_context.num_processors))

        # Start Output message queue
        if getattr(options, "xule_multi", False):
            t = Thread(target=xm.output_message_queue, args=(global_context, ))
            t.start()

        global_context.message_queue.logging(
            "Building Constant and Rule Groups")
        global_context.all_constants = rule_set.get_grouped_constants()
        global_context.all_rules = rule_set.get_grouped_rules()

        for g in global_context.all_constants:
            global_context.message_queue.logging(
                "Constants: %s - %d" %
                (g, len(global_context.all_constants[g])))

        for g in global_context.all_rules:
            global_context.message_queue.logging(
                "Rules: %s - %d" % (g, len(global_context.all_rules[g])))

        # evaluate valid constants (no dependency, rules taxonomy)
        global_context.message_queue.logging(
            "Calculating and Storing Constants")
        xm.run_constant_group(global_context, 'c', 'rtc')

        # Add precalculated information to the cntlr to pass to XuleServer
        setattr(cntlr, "xule_options", options)
        setattr(cntlr, "rule_set", global_context.rule_set)
        setattr(cntlr, "constant_list", global_context._constants)
        setattr(cntlr, "all_constants", global_context.all_constants)
        setattr(cntlr, "all_rules", global_context.all_rules)

        global_context.message_queue.logging("Finished Server Initialization")

        # stop message_queue
        global_context.message_queue.stop()

        if getattr(options, "xule_multi", False):
            t.join()
    else:
        if getattr(options, 'xule_filing_list', None) is not None:
            # process filing list
            if getattr(options, "xule_filing_list", None):
                try:
                    with open(options.xule_filing_list,
                              'r') as filing_list_file:
                        # Try json
                        try:
                            filing_list = json.load(
                                filing_list_file,
                                object_pairs_hook=collections.OrderedDict)
                        except:
                            # Try a flat list of file names
                            try:
                                # reset the file pointer
                                filing_list_file.seek(0)
                                filing_list = [{
                                    "file": file_name
                                } for file_name in filing_list_file]
                            except:
                                cntlr.addToLog(
                                    _("Unable to open Filing listing file '%s'."
                                      % options.xule_filing_list), 'xule')
                                raise
                except FileNotFoundError:
                    cntlr.addToLog(
                        _("Filing listing file '%s' is not found" %
                          options.xule_filing_list), 'xule')
                    raise

                if isinstance(filing_list, list):
                    for file_info in filing_list:
                        if isinstance(file_info, dict):
                            input_file_name = file_info.get('file')
                            if input_file_name is not None:
                                input_file_name = input_file_name.strip()
                                print("Processing filing", input_file_name)
                                filing_filesource = FileSource.openFileSource(
                                    input_file_name, cntlr)
                                modelManager = ModelManager.initialize(cntlr)
                                modelXbrl = modelManager.load(
                                    filing_filesource)
                                # Update options
                                new_options = copy.copy(options)
                                delattr(new_options, 'xule_filing_list')
                                for k, v in file_info.items():
                                    if k != 'file' and k.strip().lower(
                                    ).startswith('xule'
                                                 ):  # Only change xule options
                                        setattr(new_options,
                                                k.strip().lower(), v)
                                if getattr(new_options, 'xule_run'):
                                    xuleCmdXbrlLoaded(cntlr, new_options,
                                                      modelXbrl)
                                elif getattr(new_options, 'validate'):
                                    for xule_validator in _xule_validators:
                                        runXule(_cntlr, new_options, modelXbrl,
                                                xule_validator['map_name'])
                                modelXbrl.close()
        else:
            if options.entrypointFile is None:
                # try running the xule processor - This is when rules are run without an instance document
                xuleCmdXbrlLoaded(cntlr, options, None)

    # Only register xule as a validator if the xule plugin was directly added in the --plugin options.
    if isXuleDirect():
        xuleRegisterValidators('Xule', _xule_rule_set_map_name)
예제 #8
0
def xuleCmdUtilityRun(cntlr, options, **kwargs):
    # Save the controller and options in the module global variable
    global _cntlr
    _cntlr = cntlr
    global _options
    _options = options

    #check option combinations
    parser = OptionParser()

    if getattr(options, "xule_version", False):
        cntlr.addToLog("Xule version: %s" % __version__)
        cntlr.close()

    if getattr(options, "xule_cpu",
               None) is not None and not getattr(options, 'xule_multi', None):
        parser.error(_("--xule-multi is required with --xule_cpu."))

#     if  getattr(options, "xule_run", None) is not None and not getattr(options, 'xule_rule_set', None):
#             parser.error(_("--xule-rule-set is required with --xule-run."))

    if getattr(options, "xule_server", None) is not None and not getattr(
            options, 'xule_rule_set', None):
        parser.error(_("--xule-rule-set is required with --xule_server."))

    if getattr(options, "xule-numthreads", None) == None:
        setattr(options, "xule-numthreads", 1)

    if getattr(options, 'xule_add_packages', None) is not None and not getattr(
            options, 'xule_rule_set', None):
        parser.error(
            _("--xule-rule-set is required with --xule-add-packages."))

    if getattr(
            options, 'xule_remove_packages',
            None) is not None and not getattr(options, 'xule_rule_set', None):
        parser.error(
            _("--xule-rule-set is required with --xule-remove-packages."))

    if getattr(
            options, 'xule_show_packages',
            None) is not None and not getattr(options, 'xule_rule_set', None):
        parser.error(
            _("--xule-rule-set is required with --xule-show-packages."))

    from os import name
    if getattr(options, "xule_multi", False) and name == 'nt':
        parser.error(_("--xule-multi can't be used in Windows"))

    if not getattr(options, "xule_multi", False) and getattr(
            options, "xule_cpu", None) is not None:
        parser.error(
            _("--xule-cpu can only be used with --xule-multi enabled"))

    if len([
            x
            for x in (getattr(options, "xule_update_rule_set_map", False),
                      getattr(options, "xule_replace_rule_set_map", False),
                      getattr(options, "xule_reset_rule_set_map", False)) if x
    ]) > 1:
        parser.error(
            _("Cannot use --xule-update-rule-set-map or --xule-replace-rule-set-map or --xule-reset-rule-set-map the same time."
              ))

    #compile rules
    if getattr(options, "xule_compile", None):
        compile_destination = getattr(options, "xule_rule_set", "xuleRules")
        from .XuleParser import parseRules
        parseRules(options.xule_compile.split("|"), compile_destination)

    #add packages
    if getattr(options, "xule_add_packages", None):
        rule_set = xr.XuleRuleSet(cntlr)
        rule_set.open(getattr(options, "xule_rule_set"),
                      open_packages=False,
                      open_files=False)
        packages = options.xule_add_packages.split('|')
        rule_set.manage_packages(packages, 'add')

    #remove packages
    if getattr(options, "xule_remove_packages", None):
        rule_set = xr.XuleRuleSet(cntlr)
        rule_set.open(getattr(options, "xule_rule_set"),
                      open_packages=False,
                      open_files=False)
        packages = options.xule_remove_packages.split('|')
        rule_set.manage_packages(packages, 'del')

    #show packages
    if getattr(options, "xule_show_packages", False):
        rule_set = xr.XuleRuleSet(cntlr)
        rule_set.open(getattr(options, "xule_rule_set"),
                      open_packages=False,
                      open_files=False)
        print("Packages in rule set:")
        for package_info in rule_set.get_packages_info():
            print('\t' + package_info.get('name') + ' (' +
                  os.path.basename(package_info.get('URL')) + ')')

    #update rule set map
    if getattr(options, 'xule_update_rule_set_map', None):
        xu.update_rule_set_map(cntlr,
                               getattr(options, 'xule_update_rule_set_map'))

    #replace rule set map
    if getattr(options, 'xule_replace_rule_set_map', None):
        xu.update_rule_set_map(cntlr,
                               getattr(options, 'xule_replace_rule_set_map'),
                               overwrite=True)

    #reset rule set map
    if getattr(options, 'xule_reset_rule_set_map', False):
        xu.reset_rule_set_map(cntlr)

    if getattr(options, "xule_server", None):
        from .XuleMultiProcessing import run_constant_group, output_message_queue
        from threading import Thread

        try:
            rule_set = xr.XuleRuleSet()
            rule_set.open(options.xule_rule_set, False)
        except xr.XuleRuleSetError:
            raise

        # Create global Context
        global_context = XuleGlobalContext(
            rule_set,
            cntlr=cntlr,
            options=options  #,
            #multi=getattr(options, "xule_multi", False),
            #async=getattr(options, "xule_async", False),
            #cpunum=getattr(options, "xule_cpu", None))
        )
        #global_context.show_timing = getattr(options, "xule_time", None)
        #global_context.show_debug = getattr(options, "xule_debug", False)
        #global_context.show_debug_table = getattr(options, "xule_debug_table", False)
        #global_context.show_trace = getattr(options, "xule_trace", None)
        #global_context.crash_on_error = getattr(options, "xule_crash", False)

        global_context.message_queue.print("Using %d processors" %
                                           (global_context.num_processors))

        # Start Output message queue
        if getattr(options, "xule_multi", False):
            t = Thread(target=output_message_queue, args=(global_context, ))
            t.start()


#         #load rules taxonomy
#         global_context.message_queue.logging("Loading rules taxonomy")
#         global_context.get_rules_dts()
# #        rules_dts = global_context.get_rules_dts()
# #        from .XuleProcessor import load_networks
# #        load_networks(rules_dts)

        global_context.message_queue.logging(
            "Building Constant and Rule Groups")
        global_context.all_constants = rule_set.get_grouped_constants()
        global_context.all_rules = rule_set.get_grouped_rules()

        for g in global_context.all_constants:
            global_context.message_queue.logging(
                "Constants: %s - %d" %
                (g, len(global_context.all_constants[g])))
        #    for c in global_context.all_constants[g]:
        #        print(" -- %s" % (c))

        for g in global_context.all_rules:
            global_context.message_queue.logging(
                "Rules: %s - %d" % (g, len(global_context.all_rules[g])))
            #for c in global_context.all_rules[g]:
            #    print(" -- %s" % (c))

        # evaluate valid constants (no dependency, rules taxonomy)
        global_context.message_queue.logging(
            "Calculating and Storing Constants")
        run_constant_group(global_context, 'c', 'rtc')

        # Add precalculated information to the cntlr to pass to XuleServer
        setattr(cntlr, "xule_options", options)
        setattr(cntlr, "rule_set", global_context.rule_set)
        setattr(cntlr, "constant_list", global_context._constants)
        setattr(cntlr, "all_constants", global_context.all_constants)
        setattr(cntlr, "all_rules", global_context.all_rules)

        global_context.message_queue.logging("Finished Server Initialization")

        # stop message_queue
        global_context.message_queue.stop()

        if getattr(options, "xule_multi", False):
            t.join()
    else:
        if options.entrypointFile is None:
            #try running the xule processor
            xuleCmdXbrlLoaded(cntlr, options, None)
    #process filing list
    if getattr(options, "xule_filing_list", None):
        try:
            with open(options.xule_filing_list, "r") as filing_list:
                for line in filing_list:

                    filing = line.strip()
                    print("Processing filing", filing)
                    filing_filesource = FileSource.openFileSource(
                        filing, cntlr)
                    modelManager = ModelManager.initialize(cntlr)
                    modelXbrl = modelManager.load(filing_filesource)
                    xuleCmdXbrlLoaded(cntlr, options, modelXbrl)
                    modelXbrl.close()

        except FileNotFoundError:
            print("Filing listing file '%s' is not found" %
                  options.xule_filing_list)
예제 #9
0
def xuleCmdUtilityRun(cntlr, options, **kwargs):  
    #check option combinations
    parser = OptionParser()
    
    if getattr(options, "xule_version", False):
        cntlr.addToLog("Xule version: %s" % __version__)
        cntlr.close()

    if getattr(options, "xule_cpu", None) is not None and not getattr(options, 'xule_multi', None):
            parser.error(_("--xule-multi is required with --xule_cpu."))

    if  getattr(options, "xule_run", None) is not None and not getattr(options, 'xule_rule_set', None):
            parser.error(_("--xule-rule-set is required with --xule-run."))
    
    if getattr(options, "xule_server", None) is not None and not getattr(options, 'xule_rule_set', None):
            parser.error(_("--xule-rule-set is required with --xule_server."))
            
    if getattr(options, "xule-numthreads", None) == None:
        setattr(options, "xule-numthreads", 1)   
    
    if getattr(options, 'xule_add_packages', None) is not None and not getattr(options, 'xule_rule_set', None):
        parser.error(_("--xule-rule-set is required with --xule-add-packages.")) 

    if getattr(options, 'xule_remove_packages', None) is not None and not getattr(options, 'xule_rule_set', None):
        parser.error(_("--xule-rule-set is required with --xule-remove-packages.")) 

    if getattr(options, 'xule_show_packages', None) is not None and not getattr(options, 'xule_rule_set', None):
        parser.error(_("--xule-rule-set is required with --xule-show-packages.")) 

    from os import name
    if getattr(options, "xule_multi", False) and name == 'nt':
            parser.error(_("--xule-multi can't be used in Windows"))    

    if not getattr(options, "xule_multi", False) and getattr(options, "xule_cpu", None) is not None:
            parser.error(_("--xule-cpu can only be used with --xule-multi enabled"))    

    #compile rules
    if getattr(options, "xule_compile", None):
        compile_destination = getattr(options, "xule_rule_set", "xuleRules") 
        from .XuleParser import parseRules
        parseRules(options.xule_compile.split("|"),compile_destination)
    
    #add packages
    if getattr(options, "xule_add_packages", None):
        rule_set = xr.XuleRuleSet(cntlr)
        rule_set.open(getattr(options, "xule_rule_set"), open_packages=False, open_files=False)
        packages = options.xule_add_packages.split('|')
        rule_set.manage_packages(packages, 'add')

    #remove packages
    if getattr(options, "xule_remove_packages", None):
        rule_set = xr.XuleRuleSet(cntlr)
        rule_set.open(getattr(options, "xule_rule_set"), open_packages=False, open_files=False)
        packages = options.xule_remove_packages.split('|')
        rule_set.manage_packages(packages, 'del')
    
    #show packages
    if getattr(options, "xule_show_packages", False):
        rule_set = xr.XuleRuleSet(cntlr)
        rule_set.open(getattr(options, "xule_rule_set"), open_packages=False, open_files=False)
        print("Packages in rule set:")
        for package_info in rule_set.get_packages_info():
            print('\t' + package_info.get('name') + ' (' + os.path.basename(package_info.get('URL')) + ')' )
    
    if getattr(options, "xule_server", None):
        from .XuleMultiProcessing import run_constant_group, output_message_queue
        from threading import Thread
        
        try:
            rule_set = xr.XuleRuleSet()
            rule_set.open(options.xule_rule_set, False)
        except xr.XuleRuleSetError:
            raise

        # Create global Context
        global_context = XuleGlobalContext(rule_set, cntlr=cntlr, options=options#,
                                           #multi=getattr(options, "xule_multi", False), 
                                           #async=getattr(options, "xule_async", False),
                                           #cpunum=getattr(options, "xule_cpu", None))
                                           )
        #global_context.show_timing = getattr(options, "xule_time", None)
        #global_context.show_debug = getattr(options, "xule_debug", False)
        #global_context.show_debug_table = getattr(options, "xule_debug_table", False)
        #global_context.show_trace = getattr(options, "xule_trace", None)
        #global_context.crash_on_error = getattr(options, "xule_crash", False)

        global_context.message_queue.print("Using %d processors" % (global_context.num_processors)) 

        # Start Output message queue
        if getattr(options, "xule_multi", False):
            t = Thread(target=output_message_queue, args=(global_context,))
            t.start()
        
#         #load rules taxonomy
#         global_context.message_queue.logging("Loading rules taxonomy")
#         global_context.get_rules_dts()        
# #        rules_dts = global_context.get_rules_dts()
# #        from .XuleProcessor import load_networks
# #        load_networks(rules_dts)
         
        global_context.message_queue.logging("Building Constant and Rule Groups")
        global_context.all_constants = rule_set.get_grouped_constants()
        global_context.all_rules = rule_set.get_grouped_rules()        

        
        for g in global_context.all_constants:
            global_context.message_queue.logging("Constants: %s - %d" % (g, len(global_context.all_constants[g])))
        #    for c in global_context.all_constants[g]:
        #        print(" -- %s" % (c))

        for g in global_context.all_rules:
            global_context.message_queue.logging("Rules: %s - %d" % (g, len(global_context.all_rules[g])))
            #for c in global_context.all_rules[g]:
            #    print(" -- %s" % (c))


        # evaluate valid constants (no dependency, rules taxonomy)
        global_context.message_queue.logging("Calculating and Storing Constants")
        run_constant_group(global_context, 'c', 'rtc')

                                   
        # Add precalculated information to the cntlr to pass to XuleServer
        setattr(cntlr, "xule_options", options)
        setattr(cntlr, "rule_set", global_context.rule_set)        
        setattr(cntlr, "constant_list", global_context._constants)
        setattr(cntlr, "all_constants", global_context.all_constants)
        setattr(cntlr, "all_rules", global_context.all_rules)        


        global_context.message_queue.logging("Finished Server Initialization")
        
        # stop message_queue
        global_context.message_queue.stop()
        
        if getattr(options, "xule_multi", False):
            t.join()
    else:
        if options.entrypointFile is None:
            #try running the xule processor
            xuleCmdXbrlLoaded(cntlr, options, None)
    #process filing list
    if getattr(options, "xule_filing_list", None):
        try:
            with open(options.xule_filing_list, "r") as filing_list:
                for line in filing_list:

                    filing = line.strip()
                    print("Processing filing", filing)
                    filing_filesource = FileSource.openFileSource(filing, cntlr)            
                    modelManager = ModelManager.initialize(cntlr)
                    modelXbrl = modelManager.load(filing_filesource) 
                    xuleCmdXbrlLoaded(cntlr, options, modelXbrl)
                    modelXbrl.close()

        except FileNotFoundError:
            print("Filing listing file '%s' is not found" % options.xule_filing_list)
예제 #10
0
    def __init__(self,
                 hasGui=False,
                 logFileName=None,
                 logFileMode=None,
                 logFileEncoding=None,
                 logFormat=None):
        self.hasWin32gui = False
        self.hasGui = hasGui
        self.hasFileSystem = True  # no file system on Google App Engine servers
        self.isGAE = False
        self.isCGI = False
        self.systemWordSize = int(round(math.log(sys.maxsize, 2)) +
                                  1)  # e.g., 32 or 64

        _resourcesDir = resourcesDir()
        self.configDir = os.path.join(_resourcesDir, "config")
        self.imagesDir = os.path.join(_resourcesDir, "images")
        self.localeDir = os.path.join(_resourcesDir, "locale")
        self.pluginDir = os.path.join(_resourcesDir, "plugin")
        _mplDir = os.path.join(_resourcesDir, "mpl-data")
        if os.path.exists(
                _mplDir
        ):  # set matplotlibdata for cx_Freeze with local directory
            os.environ["MATPLOTLIBDATA"] = _mplDir

        serverSoftware = os.getenv("SERVER_SOFTWARE", "")
        if serverSoftware.startswith(
                "Google App Engine/") or serverSoftware.startswith(
                    "Development/"):
            self.hasFileSystem = False  # no file system, userAppDir does not exist
            self.isGAE = True
        else:
            gatewayInterface = os.getenv("GATEWAY_INTERFACE", "")
            if gatewayInterface.startswith("CGI/"):
                self.isCGI = True

        configHomeDir = None  # look for path configDir/CONFIG_HOME in argv and environment parameters
        for i, arg in enumerate(
                sys.argv):  # check if config specified in a argv
            if arg.startswith("--xdgConfigHome="):
                configHomeDir = arg[16:]
                break
            elif arg == "--xdgConfigHome" and i + 1 < len(sys.argv):
                configHomeDir = sys.argv[i + 1]
                break
        if not configHomeDir:  # not in argv, may be an environment parameter
            configHomeDir = os.getenv('XDG_CONFIG_HOME')
        if not configHomeDir:  # look for path configDir/CONFIG_HOME
            configHomeDirFile = os.path.join(self.configDir, "XDG_CONFIG_HOME")
            if os.path.exists(configHomeDirFile):
                try:
                    with io.open(configHomeDirFile, 'rt',
                                 encoding='utf-8') as f:
                        configHomeDir = f.read().strip()
                    if configHomeDir and not os.path.isabs(configHomeDir):
                        configHomeDir = os.path.abspath(
                            configHomeDir)  # make into a full path if relative
                except EnvironmentError:
                    configHomeDir = None
        if self.hasFileSystem and configHomeDir and os.path.exists(
                configHomeDir):
            # check if a cache exists in this directory (e.g. from XPE or other tool)
            impliedAppDir = os.path.join(configHomeDir, "arelle")
            if os.path.exists(impliedAppDir):
                self.userAppDir = impliedAppDir
            elif os.path.exists(os.path.join(configHomeDir, "cache")):
                self.userAppDir = configHomeDir  # use the XDG_CONFIG_HOME because cache is already a subdirectory
            else:
                self.userAppDir = impliedAppDir
        if sys.platform == "darwin":
            self.isMac = True
            self.isMSW = False
            if self.hasFileSystem and not configHomeDir:
                self.userAppDir = os.path.expanduser(
                    "~") + "/Library/Application Support/Arelle"
            # note that cache is in ~/Library/Caches/Arelle
            self.contextMenuClick = "<Button-2>"
            self.hasClipboard = hasGui  # clipboard always only if Gui (not command line mode)
            self.updateURL = "http://arelle.org/downloads/8"
        elif sys.platform.startswith("win"):
            self.isMac = False
            self.isMSW = True
            if self.hasFileSystem and not configHomeDir:
                tempDir = tempfile.gettempdir()
                if tempDir.lower().endswith('local\\temp'):
                    impliedAppDir = tempDir[:-10] + 'local'
                else:
                    impliedAppDir = tempDir
                self.userAppDir = os.path.join(impliedAppDir, "Arelle")
            if hasGui:
                try:
                    import win32clipboard
                    self.hasClipboard = True
                except ImportError:
                    self.hasClipboard = False
                try:
                    import win32gui
                    self.hasWin32gui = True  # active state for open file dialogs
                except ImportError:
                    pass
            else:
                self.hasClipboard = False
            self.contextMenuClick = "<Button-3>"
            if "64 bit" in sys.version:
                self.updateURL = "http://arelle.org/downloads/9"
            else:  # 32 bit
                self.updateURL = "http://arelle.org/downloads/10"
        else:  # Unix/Linux
            self.isMac = False
            self.isMSW = False
            if self.hasFileSystem and not configHomeDir:
                self.userAppDir = os.path.join(os.path.expanduser("~/.config"),
                                               "arelle")
            if hasGui:
                try:
                    import gtk
                    self.hasClipboard = True
                except ImportError:
                    self.hasClipboard = False
            else:
                self.hasClipboard = False
            self.contextMenuClick = "<Button-3>"
        try:
            from arelle import webserver
            self.hasWebServer = True
        except ImportError:
            self.hasWebServer = False
        # assert that app dir must exist
        self.config = None
        if self.hasFileSystem:
            if not os.path.exists(self.userAppDir):
                os.makedirs(self.userAppDir)
            # load config if it exists
            self.configJsonFile = self.userAppDir + os.sep + "config.json"
            if os.path.exists(self.configJsonFile):
                try:
                    with io.open(self.configJsonFile, 'rt',
                                 encoding='utf-8') as f:
                        self.config = json.load(f)
                except Exception as ex:
                    self.config = None  # restart with a new config
        if not self.config:
            self.config = {
                'fileHistory': [],
                'windowGeometry': "{0}x{1}+{2}+{3}".format(800, 500, 200, 100),
            }

        # start language translation for domain
        self.setUiLanguage(self.config.get("userInterfaceLangOverride", None),
                           fallbackToDefault=True)

        from arelle.WebCache import WebCache
        self.webCache = WebCache(self, self.config.get("proxySettings"))

        # start plug in server (requres web cache initialized, but not logger)
        PluginManager.init(self, loadPluginConfig=hasGui)

        # requires plug ins initialized
        self.modelManager = ModelManager.initialize(self)

        # start taxonomy package server (requres web cache initialized, but not logger)
        PackageManager.init(self, loadPackagesConfig=hasGui)

        self.startLogging(logFileName, logFileMode, logFileEncoding, logFormat)

        # Cntlr.Init after logging started
        for pluginMethod in PluginManager.pluginClassMethods("Cntlr.Init"):
            pluginMethod(self)
예제 #11
0
파일: Cntlr.py 프로젝트: ojones20/Arelle
    def __init__(self, logFileName=None, logFileMode=None, logFileEncoding=None, logFormat=None):
        self.hasWin32gui = False
        self.hasGui = False
        if sys.platform == "darwin":
            self.isMac = True
            self.isMSW = False
            self.userAppDir = os.path.expanduser("~") + "/Library/Application Support/Arelle"
            self.contextMenuClick = "<Button-2>"
            self.hasClipboard = True
            self.updateURL = "http://arelle.org/downloads/8"
        elif sys.platform.startswith("win"):
            self.isMac = False
            self.isMSW = True
            tempDir = tempfile.gettempdir()
            if tempDir.endswith('local\\temp'):
                impliedAppDir = tempDir[:-10] + 'local'
            else:
                impliedAppDir = tempDir
            self.userAppDir = os.path.join(
                   os.getenv('XDG_CONFIG_HOME', impliedAppDir),
                   "Arelle")
            try:
                import win32clipboard
                self.hasClipboard = True
            except ImportError:
                self.hasClipboard = False
            try:
                import win32gui
                self.hasWin32gui = True # active state for open file dialogs
            except ImportError:
                pass
            self.contextMenuClick = "<Button-3>"
            if "64 bit" in sys.version:
                self.updateURL = "http://arelle.org/downloads/9"
            else: # 32 bit
                self.updateURL = "http://arelle.org/downloads/10"
        else: # Unix/Linux
            self.isMac = False
            self.isMSW = False
            self.userAppDir = os.path.join(
                   os.getenv('XDG_CONFIG_HOME', os.path.expanduser("~/.config")),
                   "arelle")
            try:
                import gtk
                self.hasClipboard = True
            except ImportError:
                self.hasClipboard = False
            self.contextMenuClick = "<Button-3>"
        self.moduleDir = os.path.dirname(__file__)
        # for python 3.2 remove __pycache__
        if self.moduleDir.endswith("__pycache__"):
            self.moduleDir = os.path.dirname(self.moduleDir)
        if self.moduleDir.endswith("python32.zip/arelle"):
            '''
            distZipFile = os.path.dirname(self.moduleDir)
            d = os.path.join(self.userAppDir, "arelle")
            self.configDir = os.path.join(d, "config")
            self.imagesDir = os.path.join(d, "images")
            import zipfile
            distZip = zipfile.ZipFile(distZipFile, mode="r")
            distNames = distZip.namelist()
            distZip.extractall(path=self.userAppDir,
                               members=[f for f in distNames if "/config/" in f or "/images/" in f]
                               )
            distZip.close()
            '''
            resources = os.path.dirname(os.path.dirname(os.path.dirname(self.moduleDir)))
            self.configDir = os.path.join(resources, "config")
            self.imagesDir = os.path.join(resources, "images")
            self.localeDir = os.path.join(resources, "locale")
        elif self.moduleDir.endswith("library.zip\\arelle") or self.moduleDir.endswith("library.zip/arelle"): # cx_Freexe
            resources = os.path.dirname(os.path.dirname(self.moduleDir))
            self.configDir = os.path.join(resources, "config")
            self.imagesDir = os.path.join(resources, "images")
            self.localeDir = os.path.join(resources, "locale")
        else:
            self.configDir = os.path.join(self.moduleDir, "config")
            self.imagesDir = os.path.join(self.moduleDir, "images")
            self.localeDir = os.path.join(self.moduleDir, "locale")
        try:
            from arelle import webserver
            self.hasWebServer = True
        except ImportError:
            self.hasWebServer = False
        # assert that app dir must exist
        if not os.path.exists(self.userAppDir):
            os.makedirs(self.userAppDir)
        # load config if it exists
        self.configJsonFile = self.userAppDir + os.sep + "config.json"
        self.config = None
        if os.path.exists(self.configJsonFile):
            try:
                with io.open(self.configJsonFile, 'rt', encoding='utf-8') as f:
                    self.config = json.load(f)
            except Exception as ex:
                self.config = None # restart with a new config
        if not self.config:
            self.config = {
                'fileHistory': [],
                'windowGeometry': "{0}x{1}+{2}+{3}".format(800, 500, 200, 100),                
            }
            
        # start language translation for domain
        self.setUiLanguage(self.config.get("userInterfaceLangOverride",None), fallbackToDefault=True)
            
        from arelle.WebCache import WebCache
        self.webCache = WebCache(self, self.config.get("proxySettings"))
        self.modelManager = ModelManager.initialize(self)
        
        # start plug in server (requres web cache initialized
        PluginManager.init(self)
 
        self.startLogging(logFileName, logFileMode, logFileEncoding, logFormat)
예제 #12
0
    def __init__(self, logFileName=None, logFileMode=None, logFileEncoding=None, logFormat=None):
        self.hasWin32gui = False
        self.hasGui = False
        if sys.platform == "darwin":
            self.isMac = True
            self.isMSW = False
            self.userAppDir = os.path.expanduser("~") + "/Library/Application Support/Arelle"
            self.contextMenuClick = "<Button-2>"
            self.hasClipboard = True
            self.updateURL = "http://arelle.org/downloads/8"
        elif sys.platform.startswith("win"):
            self.isMac = False
            self.isMSW = True
            tempDir = tempfile.gettempdir()
            if tempDir.endswith('local\\temp'):
                self.userAppDir = tempDir[:-10] + 'local\\Arelle'
            else:
                self.userAppDir = tempDir + os.sep + 'arelle'
            try:
                import win32clipboard
                self.hasClipboard = True
            except ImportError:
                self.hasClipboard = False
            try:
                import win32gui
                self.hasWin32gui = True # active state for open file dialogs
            except ImportError:
                pass
            self.contextMenuClick = "<Button-3>"
            if "64 bit" in sys.version:
                self.updateURL = "http://arelle.org/downloads/9"
            else: # 32 bit
                self.updateURL = "http://arelle.org/downloads/10"
        else: # Unix/Linux
            self.isMac = False
            self.isMSW = False
            self.userAppDir = os.path.join(
                   os.getenv('XDG_CONFIG_HOME', os.path.expanduser("~/.config")),
                   "arelle")
            try:
                import gtk
                self.hasClipboard = True
            except ImportError:
                self.hasClipboard = False
            self.contextMenuClick = "<Button-3>"
        self.moduleDir = os.path.dirname(__file__)
        # for python 3.2 remove __pycache__
        if self.moduleDir.endswith("__pycache__"):
            self.moduleDir = os.path.dirname(self.moduleDir)
        if self.moduleDir.endswith("python32.zip/arelle"):
            '''
            distZipFile = os.path.dirname(self.moduleDir)
            d = os.path.join(self.userAppDir, "arelle")
            self.configDir = os.path.join(d, "config")
            self.imagesDir = os.path.join(d, "images")
            import zipfile
            distZip = zipfile.ZipFile(distZipFile, mode="r")
            distNames = distZip.namelist()
            distZip.extractall(path=self.userAppDir,
                               members=[f for f in distNames if "/config/" in f or "/images/" in f]
                               )
            distZip.close()
            '''
            resources = os.path.dirname(os.path.dirname(os.path.dirname(self.moduleDir)))
            self.configDir = os.path.join(resources, "config")
            self.imagesDir = os.path.join(resources, "images")
            self.localeDir = os.path.join(resources, "locale")
        elif self.moduleDir.endswith("library.zip\\arelle") or self.moduleDir.endswith("library.zip/arelle"): # cx_Freexe
            resources = os.path.dirname(os.path.dirname(self.moduleDir))
            self.configDir = os.path.join(resources, "config")
            self.imagesDir = os.path.join(resources, "images")
            self.localeDir = os.path.join(resources, "locale")
        else:
            self.configDir = os.path.join(self.moduleDir, "config")
            self.imagesDir = os.path.join(self.moduleDir, "images")
            self.localeDir = os.path.join(self.moduleDir, "locale")
        try:
            from arelle import webserver
            self.hasWebServer = True
        except ImportError:
            self.hasWebServer = False
        # assert that app dir must exist
        if not os.path.exists(self.userAppDir):
            os.makedirs(self.userAppDir)
        # load config if it exists
        self.configJsonFile = self.userAppDir + os.sep + "config.json"
        self.config = None
        if os.path.exists(self.configJsonFile):
            try:
                with io.open(self.configJsonFile, 'rt', encoding='utf-8') as f:
                    self.config = json.load(f)
            except Exception as ex:
                self.config = None # restart with a new config
        if not self.config:
            self.config = {
                'fileHistory': [],
                'windowGeometry': "{0}x{1}+{2}+{3}".format(800, 500, 200, 100),                
            }
            
        # start language translation for domain
        try:
            gettext.translation("arelle", 
                                self.localeDir, 
                                getLanguageCodes(self.config.get("userInterfaceLangOverride",None))).install()
        except Exception as msg:
            gettext.install("arelle", 
                            self.localeDir)
            
        from arelle.WebCache import WebCache
        self.webCache = WebCache(self, self.config.get("proxySettings"))
        self.modelManager = ModelManager.initialize(self)
        
        # start plug in server (requres web cache initialized
        PluginManager.init(self)
 
        self.startLogging(logFileName, logFileMode, logFileEncoding, logFormat)
예제 #13
0
파일: Cntlr.py 프로젝트: marado/Arelle
    def __init__(self, logFileName=None, logFileMode=None, logFileEncoding=None, logFormat=None):
        self.hasWin32gui = False
        if sys.platform == "darwin":
            self.isMac = True
            self.isMSW = False
            self.userAppDir = os.path.expanduser("~") + "/Library/Application Support/Arelle"
            self.contextMenuClick = "<Button-2>"
            self.hasClipboard = True
            self.updateURL = "http://arelle.org/downloads/8"
        elif sys.platform.startswith("win"):
            self.isMac = False
            self.isMSW = True
            tempDir = tempfile.gettempdir()
            if tempDir.endswith("local\\temp"):
                self.userAppDir = tempDir[:-10] + "local\\Arelle"
            else:
                self.userAppDir = tempDir + os.sep + "arelle"
            try:
                import win32clipboard

                self.hasClipboard = True
            except ImportError:
                self.hasClipboard = False
            try:
                import win32gui

                self.hasWin32gui = True  # active state for open file dialogs
            except ImportError:
                pass
            self.contextMenuClick = "<Button-3>"
            if "64 bit" in sys.version:
                self.updateURL = "http://arelle.org/downloads/9"
            else:  # 32 bit
                self.updateURL = "http://arelle.org/downloads/10"
        else:  # Unix/Linux
            self.isMac = False
            self.isMSW = False
            self.userAppDir = os.path.join(os.getenv("XDG_CONFIG_HOME", os.path.expanduser("~/.config")), "arelle")
            try:
                import gtk

                self.hasClipboard = True
            except ImportError:
                self.hasClipboard = False
            self.contextMenuClick = "<Button-3>"
        self.moduleDir = os.path.dirname(__file__)
        # for python 3.2 remove __pycache__
        if self.moduleDir.endswith("__pycache__"):
            self.moduleDir = os.path.dirname(self.moduleDir)
        if self.moduleDir.endswith("python32.zip/arelle"):
            """
            distZipFile = os.path.dirname(self.moduleDir)
            d = os.path.join(self.userAppDir, "arelle")
            self.configDir = os.path.join(d, "config")
            self.imagesDir = os.path.join(d, "images")
            import zipfile
            distZip = zipfile.ZipFile(distZipFile, mode="r")
            distNames = distZip.namelist()
            distZip.extractall(path=self.userAppDir,
                               members=[f for f in distNames if "/config/" in f or "/images/" in f]
                               )
            distZip.close()
            """
            resources = os.path.dirname(os.path.dirname(os.path.dirname(self.moduleDir)))
            self.configDir = os.path.join(resources, "config")
            self.imagesDir = os.path.join(resources, "images")
            self.localeDir = os.path.join(resources, "locale")
        elif self.moduleDir.endswith("library.zip\\arelle"):  # cx_Freexe
            resources = os.path.dirname(os.path.dirname(self.moduleDir))
            self.configDir = os.path.join(resources, "config")
            self.imagesDir = os.path.join(resources, "images")
            self.localeDir = os.path.join(resources, "locale")
        else:
            self.configDir = os.path.join(self.moduleDir, "config")
            self.imagesDir = os.path.join(self.moduleDir, "images")
            self.localeDir = os.path.join(self.moduleDir, "locale")
        # assert that app dir must exist
        if not os.path.exists(self.userAppDir):
            os.makedirs(self.userAppDir)
        # load config if it exists
        self.configPickleFile = self.userAppDir + os.sep + "config.pickle"
        self.config = None
        if os.path.exists(self.configPickleFile):
            try:
                with open(self.configPickleFile, "rb") as f:
                    self.config = pickle.load(f)
            except Exception as ex:
                self.config = None  # restart with a new config
        if not self.config:
            self.config = {"fileHistory": [], "windowGeometry": "{0}x{1}+{2}+{3}".format(800, 500, 200, 100)}

        # start language translation for domain
        try:
            gettext.translation("arelle", self.localeDir, getLanguageCodes()).install()
        except Exception as msg:
            gettext.install("arelle", self.localeDir)

        from arelle.WebCache import WebCache

        self.webCache = WebCache(self, self.config.get("proxySettings"))
        self.modelManager = ModelManager.initialize(self)

        if logFileName:  # use default logging
            self.logger = logging.getLogger("arelle")
            if logFileName == "logToPrint":
                self.logHandler = LogToPrintHandler()
            else:
                self.logHandler = logging.FileHandler(
                    filename=logFileName,
                    mode=logFileMode if logFileMode else "w",
                    encoding=logFileEncoding if logFileEncoding else "utf-8",
                )
            self.logHandler.level = logging.DEBUG
            self.logHandler.setFormatter(
                logging.Formatter(
                    logFormat if logFormat else "%(asctime)s [%(messageCode)s] %(message)s - %(file)s %(sourceLine)s \n"
                )
            )
            self.logger.addHandler(self.logHandler)
        else:
            self.logger = None
예제 #14
0
def load_edinet():

    edinet_list = []

    # xbrl 読み込み
    xbrl_files = glob.glob(const.FOLDER_XBRL + '*.xbrl')

    # EdinetcodeDlInfo.csv 読み込み
    edinetcodedata = pd.read_csv(const.FILE_EDINETCODE,
                                 skiprows=1,
                                 encoding='cp932')

    for index, xbrl_file in enumerate(xbrl_files):
        ctrl = Cntlr.Cntlr()
        model_manager = ModelManager.initialize(ctrl)
        model_xbrl = model_manager.load(xbrl_file)

        print(xbrl_file, ":", index + 1, "/", len(xbrl_files))

        # XBRLをDataFrameにセット
        factData = pd.DataFrame(data=[
            (fact.concept.qname.localName, fact.value, fact.isNumeric,
             fact.contextID, fact.decimals) for fact in model_xbrl.facts
        ],
                                columns=[
                                    'element_id', 'value', 'isNumeric',
                                    'contextID', 'decimals'
                                ])

        # 取得したい要素
        datalist = []
        datalist_ren = []
        flg_ren = False
        for key, value in dict_cols.items():
            contextRef = value['contextRef']
            element_ids = value['element_id']
            if len(element_ids) > 0:
                for element_id in element_ids:
                    # data1 = factData[factData['element_id'].str.contains(element_id)]
                    data1 = []
                    data2 = []
                    data3 = []
                    data1 = factData[factData['element_id'] == element_id]
                    if len(data1) == 1: break

                    data2 = data1[data1['contextID'] == (
                        contextRef + '_NonConsolidatedMember')]
                    data3 = data1[data1['contextID'] == contextRef]
                    if len(data2) >= 1 or len(data3) >= 1: break

                data, data_ren, unitRef = '', '', ''
                if len(data1) == 1:
                    data = data1['value'].values[0]
                    data_ren = data
                    print(key + ' : ' + data)
                if len(data2) >= 1 and len(data3) == 0:
                    data = data2['value'].values[0]
                    data_ren = data
                    print(key + ' : ' + data)
                if len(data3) >= 1:
                    data = ''
                    data_ren = data3['value'].values[0]
                    print(key + '(連結) : ' + data_ren)
                if len(data1) == 0:
                    print(key + ' : ' + 'NoData ')
                    data = ''

                datalist.append(data)
                if flg_ren:
                    datalist_ren.append(data_ren)

                if key == 'EDINETコード':
                    company = edinetcodedata[edinetcodedata['EDINETコード'] ==
                                             data]
                    if len(company) == 1:
                        print('提出者業種 : ' + company['提出者業種'].values[0])
                        data = company['提出者業種'].values[0]

                        flg_ren = False
                        if company['連結の有無'].values[0] == '有':
                            print('連結の有無 : ' + company['連結の有無'].values[0])
                            flg_ren = True
                    else:
                        print('提出者業種 : ' + 'NoData ')
                        data = ''
                    datalist.append(data)
                    datalist.append('連結' if flg_ren else '単独')

                    if flg_ren:
                        # 連結用にデータをコピー
                        datalist_ren = datalist[:]

        if flg_ren:
            edinet_list.append(datalist_ren)
        else:
            edinet_list.append(datalist)

    return edinet_list
예제 #15
0
파일: Cntlr.py 프로젝트: 8maki/Arelle
 def __init__(self):
     if sys.platform == "darwin":
         self.userAppDir = os.path.expanduser("~") + "/Library/Application Support/Arelle"
         self.contextMenuClick = "<Button-2>"
         self.hasClipboard = True
         self.updateURL = "http://arelle.org/downloads/5"
     elif sys.platform.startswith("win"):
         tempDir = tempfile.gettempdir()
         if tempDir.endswith('local\\temp'):
             self.userAppDir = tempDir[:-10] + 'local\\Arelle'
         else:
             self.userAppDir = tempDir + os.sep + 'arelle'
         try:
             import win32clipboard
             self.hasClipboard = True
         except ImportError:
             self.hasClipboard = False
         self.contextMenuClick = "<Button-3>"
         if "64 bit" in sys.version:
             self.updateURL = "http://arelle.org/downloads/6"
         else: # 32 bit
             self.updateURL = "http://arelle.org/downloads/7"
     else: # Unix/Linux
         self.userAppDir = os.path.join(
                os.getenv('XDG_CONFIG_HOME', os.path.expanduser("~/.config")),
                "arelle")
         try:
             import gtk
             self.hasClipboard = True
         except ImportError:
             self.hasClipboard = False
         self.contextMenuClick = "<Button-3>"
     self.moduleDir = os.path.dirname(__file__)
     # for python 3.2 remove __pycache__
     if self.moduleDir.endswith("__pycache__"):
         self.moduleDir = os.path.dirname(self.moduleDir)
     if self.moduleDir.endswith("python32.zip/arelle"):
         '''
         distZipFile = os.path.dirname(self.moduleDir)
         d = os.path.join(self.userAppDir, "arelle")
         self.configDir = os.path.join(d, "config")
         self.imagesDir = os.path.join(d, "images")
         import zipfile
         distZip = zipfile.ZipFile(distZipFile, mode="r")
         distNames = distZip.namelist()
         distZip.extractall(path=self.userAppDir,
                            members=[f for f in distNames if "/config/" in f or "/images/" in f]
                            )
         distZip.close()
         '''
         resources = os.path.dirname(os.path.dirname(os.path.dirname(self.moduleDir)))
         self.configDir = os.path.join(resources, "config")
         self.imagesDir = os.path.join(resources, "images")
     elif self.moduleDir.endswith("library.zip\\arelle"): # cx_Freexe
         resources = os.path.dirname(os.path.dirname(self.moduleDir))
         self.configDir = os.path.join(resources, "config")
         self.imagesDir = os.path.join(resources, "images")
     else:
         self.configDir = os.path.join(self.moduleDir, "config")
         self.imagesDir = os.path.join(self.moduleDir, "images")
     # assert that app dir must exist
     if not os.path.exists(self.userAppDir):
         os.makedirs(self.userAppDir)
     # load config if it exists
     self.configPickleFile = self.userAppDir + os.sep + "config.pickle"
     self.config = None
     if os.path.exists(self.configPickleFile):
         try:
             with open(self.configPickleFile, 'rb') as f:
                 self.config = pickle.load(f)
         except Exception as ex:
             self.config = None # restart with a new config
     if not self.config:
         self.config = {
             'fileHistory': [],
             'windowGeometry': "{0}x{1}+{2}+{3}".format(800, 500, 200, 100),                
         }
     from arelle.WebCache import WebCache
     self.webCache = WebCache(self, self.config.get("proxySettings"))
     self.modelManager = ModelManager.initialize(self)
예제 #16
0
 def __init__(self):
     if sys.platform == "darwin":
         self.userAppDir = os.path.expanduser(
             "~") + "/Library/Application Support/Arelle"
         self.contextMenuClick = "<Button-2>"
         self.hasClipboard = True
         self.updateURL = "http://arelle.org/downloads/5"
     elif sys.platform.startswith("win"):
         tempDir = tempfile.gettempdir()
         if tempDir.endswith('local\\temp'):
             self.userAppDir = tempDir[:-10] + 'local\\Arelle'
         else:
             self.userAppDir = tempDir + os.sep + 'arelle'
         try:
             import win32clipboard
             self.hasClipboard = True
         except ImportError:
             self.hasClipboard = False
         self.contextMenuClick = "<Button-3>"
         if "64 bit" in sys.version:
             self.updateURL = "http://arelle.org/downloads/6"
         else:  # 32 bit
             self.updateURL = "http://arelle.org/downloads/7"
     else:  # Unix/Linux
         self.userAppDir = os.path.join(
             os.getenv('XDG_CONFIG_HOME', os.path.expanduser("~/.config")),
             "arelle")
         try:
             import gtk
             self.hasClipboard = True
         except ImportError:
             self.hasClipboard = False
         self.contextMenuClick = "<Button-3>"
     self.moduleDir = os.path.dirname(__file__)
     # for python 3.2 remove __pycache__
     if self.moduleDir.endswith("__pycache__"):
         self.moduleDir = os.path.dirname(self.moduleDir)
     if self.moduleDir.endswith("python32.zip/arelle"):
         '''
         distZipFile = os.path.dirname(self.moduleDir)
         d = os.path.join(self.userAppDir, "arelle")
         self.configDir = os.path.join(d, "config")
         self.imagesDir = os.path.join(d, "images")
         import zipfile
         distZip = zipfile.ZipFile(distZipFile, mode="r")
         distNames = distZip.namelist()
         distZip.extractall(path=self.userAppDir,
                            members=[f for f in distNames if "/config/" in f or "/images/" in f]
                            )
         distZip.close()
         '''
         resources = os.path.dirname(
             os.path.dirname(os.path.dirname(self.moduleDir)))
         self.configDir = os.path.join(resources, "config")
         self.imagesDir = os.path.join(resources, "images")
     elif self.moduleDir.endswith("library.zip\\arelle"):  # cx_Freexe
         resources = os.path.dirname(os.path.dirname(self.moduleDir))
         self.configDir = os.path.join(resources, "config")
         self.imagesDir = os.path.join(resources, "images")
     else:
         self.configDir = os.path.join(self.moduleDir, "config")
         self.imagesDir = os.path.join(self.moduleDir, "images")
     # assert that app dir must exist
     if not os.path.exists(self.userAppDir):
         os.makedirs(self.userAppDir)
     # load config if it exists
     self.configPickleFile = self.userAppDir + os.sep + "config.pickle"
     self.config = None
     if os.path.exists(self.configPickleFile):
         try:
             with open(self.configPickleFile, 'rb') as f:
                 self.config = pickle.load(f)
         except Exception as ex:
             self.config = None  # restart with a new config
     if not self.config:
         self.config = {
             'fileHistory': [],
             'windowGeometry': "{0}x{1}+{2}+{3}".format(800, 500, 200, 100),
         }
     from arelle.WebCache import WebCache
     self.webCache = WebCache(self, self.config.get("proxySettings"))
     self.modelManager = ModelManager.initialize(self)
예제 #17
0
    def __init__(self,
                 hasGui=False,
                 logFileName=None,
                 logFileMode=None,
                 logFileEncoding=None,
                 logFormat=None):
        self.hasWin32gui = False
        self.hasGui = hasGui
        self.hasFileSystem = True  # no file system on Google App Engine servers

        self.moduleDir = os.path.dirname(__file__)
        # for python 3.2 remove __pycache__
        if self.moduleDir.endswith("__pycache__"):
            self.moduleDir = os.path.dirname(self.moduleDir)
        if self.moduleDir.endswith("python32.zip/arelle"):
            '''
            distZipFile = os.path.dirname(self.moduleDir)
            d = os.path.join(self.userAppDir, "arelle")
            self.configDir = os.path.join(d, "config")
            self.imagesDir = os.path.join(d, "images")
            import zipfile
            distZip = zipfile.ZipFile(distZipFile, mode="r")
            distNames = distZip.namelist()
            distZip.extractall(path=self.userAppDir,
                               members=[f for f in distNames if "/config/" in f or "/images/" in f]
                               )
            distZip.close()
            '''
            resources = os.path.dirname(
                os.path.dirname(os.path.dirname(self.moduleDir)))
            self.configDir = os.path.join(resources, "config")
            self.imagesDir = os.path.join(resources, "images")
            self.localeDir = os.path.join(resources, "locale")
            self.pluginDir = os.path.join(resources, "plugin")
        elif self.moduleDir.endswith(
                "library.zip\\arelle") or self.moduleDir.endswith(
                    "library.zip/arelle"):  # cx_Freexe
            resources = os.path.dirname(os.path.dirname(self.moduleDir))
            self.configDir = os.path.join(resources, "config")
            self.imagesDir = os.path.join(resources, "images")
            self.localeDir = os.path.join(resources, "locale")
            self.pluginDir = os.path.join(resources, "plugin")
        else:
            self.configDir = os.path.join(self.moduleDir, "config")
            self.imagesDir = os.path.join(self.moduleDir, "images")
            self.localeDir = os.path.join(self.moduleDir, "locale")
            self.pluginDir = os.path.join(self.moduleDir, "plugin")

        configHomeDir = os.getenv('XDG_CONFIG_HOME')
        if not configHomeDir:  # look for path configDir/CONFIG_HOME
            configHomeDirFile = os.path.join(self.configDir, "XDG_CONFIG_HOME")
            if os.path.exists(configHomeDirFile):
                try:
                    with io.open(configHomeDirFile, 'rt',
                                 encoding='utf-8') as f:
                        configHomeDir = f.read().strip()
                    if configHomeDir and not os.path.isabs(configHomeDir):
                        configHomeDir = os.path.abspath(
                            configHomeDir)  # make into a full path if relative
                except EnvironmentError:
                    configHomeDir = None
        if configHomeDir and os.path.exists(configHomeDir):
            # check if a cache exists in this directory (e.g. from XPE or other tool)
            impliedAppDir = os.path.join(configHomeDir, "arelle")
            if os.path.exists(impliedAppDir):
                self.userAppDir = impliedAppDir
            elif os.path.exists(os.path.join(configHomeDir, "cache")):
                self.userAppDir = configHomeDir  # use the XDG_CONFIG_HOME because cache is already a subdirectory
            else:
                self.userAppDir = impliedAppDir
        if sys.platform == "darwin":
            self.isMac = True
            self.isMSW = False
            self.isGAE = False
            if not configHomeDir:
                self.userAppDir = os.path.expanduser(
                    "~") + "/Library/Application Support/Arelle"
            # note that cache is in ~/Library/Caches/Arelle
            self.contextMenuClick = "<Button-2>"
            self.hasClipboard = hasGui  # clipboard always only if Gui (not command line mode)
            self.updateURL = "http://arelle.org/downloads/8"
        elif sys.platform.startswith("win"):
            self.isMac = False
            self.isMSW = True
            self.isGAE = False
            if not configHomeDir:
                tempDir = tempfile.gettempdir()
                if tempDir.endswith('local\\temp'):
                    impliedAppDir = tempDir[:-10] + 'local'
                else:
                    impliedAppDir = tempDir
                self.userAppDir = os.path.join(impliedAppDir, "Arelle")
            if hasGui:
                try:
                    import win32clipboard
                    self.hasClipboard = True
                except ImportError:
                    self.hasClipboard = False
                try:
                    import win32gui
                    self.hasWin32gui = True  # active state for open file dialogs
                except ImportError:
                    pass
            else:
                self.hasClipboard = False
            self.contextMenuClick = "<Button-3>"
            if "64 bit" in sys.version:
                self.updateURL = "http://arelle.org/downloads/9"
            else:  # 32 bit
                self.updateURL = "http://arelle.org/downloads/10"
        else:  # Unix/Linux
            self.isMac = False
            self.isMSW = False
            serverSoftware = os.getenv("SERVER_SOFTWARE", "")
            if serverSoftware.startswith(
                    "Google App Engine/") or serverSoftware.startswith(
                        "Development/"):
                self.hasFileSystem = False  # no file system, userAppDir does not exist
                self.isGAE = True
            elif not configHomeDir:
                self.userAppDir = os.path.join(os.path.expanduser("~/.config"),
                                               "arelle")
                self.isGAE = False
            if hasGui:
                try:
                    import gtk
                    self.hasClipboard = True
                except ImportError:
                    self.hasClipboard = False
            else:
                self.hasClipboard = False
            self.contextMenuClick = "<Button-3>"
        try:
            from arelle import webserver
            self.hasWebServer = True
        except ImportError:
            self.hasWebServer = False
        # assert that app dir must exist
        self.config = None
        if self.hasFileSystem:
            if not os.path.exists(self.userAppDir):
                os.makedirs(self.userAppDir)
            # load config if it exists
            self.configJsonFile = self.userAppDir + os.sep + "config.json"
            if os.path.exists(self.configJsonFile):
                try:
                    with io.open(self.configJsonFile, 'rt',
                                 encoding='utf-8') as f:
                        self.config = json.load(f)
                except Exception as ex:
                    self.config = None  # restart with a new config
        if not self.config:
            self.config = {
                'fileHistory': [],
                'windowGeometry': "{0}x{1}+{2}+{3}".format(800, 500, 200, 100),
            }

        # start language translation for domain
        self.setUiLanguage(self.config.get("userInterfaceLangOverride", None),
                           fallbackToDefault=True)

        from arelle.WebCache import WebCache
        self.webCache = WebCache(self, self.config.get("proxySettings"))
        self.modelManager = ModelManager.initialize(self)

        # start plug in server (requres web cache initialized, but not logger)
        PluginManager.init(self)

        self.startLogging(logFileName, logFileMode, logFileEncoding, logFormat)

        # Cntlr.Init after logging started
        for pluginMethod in PluginManager.pluginClassMethods("Cntlr.Init"):
            pluginMethod(self)