def get_test_data(args): """ Produces a list of Pytest Params that can be fed into a parameterized pytest function :param args: The args to be parsed by arelle in order to correctly produce the desired result set :type args: list of strings :return: A list of PyTest Params that can be used to run a parameterized pytest function :rtype: list of ::class:: `~pytest.param` """ cntlr = parseAndRun(args) results = [] model_document = cntlr.modelManager.modelXbrl.modelDocument if model_document is not None: if model_document.type == ModelDocument.Type.TESTCASESINDEX: for tc in sorted(model_document.referencesDocument.keys(), key=lambda doc: doc.uri): uri_dir_parts = os.path.dirname(tc.uri).split('/') test_case_dir = '/'.join(uri_dir_parts[-2:]) if hasattr(tc, "testcaseVariations"): for mv in tc.testcaseVariations: param = pytest.param( { 'status': mv.status, 'expected': mv.expected, 'actual': mv.actual }, id='{}/{}'.format(test_case_dir, str(mv.id or mv.name))) results.append(param) cntlr.modelManager.close() PackageManager.close() PluginManager.close() return results
def packageRemove(self): if 0 <= self.selectedPackageIndex < len(self.packagesConfig["packages"]): packageInfo = self.packagesConfig["packages"][self.selectedPackageIndex] self.removePackageInfo(packageInfo["name"], packageInfo["version"]) self.packagesConfigChanged = True PackageManager.rebuildRemappings() self.loadTreeViews()
def packageRemove(self): if 0 <= self.selectedPackageIndex < len(self.packagesConfig["packages"]): packageInfo = self.packagesConfig["packages"][self.selectedPackageIndex] self.removePackageInfo(packageInfo["name"], packageInfo["version"]) self.packagesConfigChanged = True PackageManager.rebuildRemappings(self.cntlr) self.loadTreeViews()
def packageReload(self): if 0 <= self.selectedPackageIndex < len( self.packagesConfig["packages"]): packageInfo = self.packagesConfig["packages"][ self.selectedPackageIndex] url = packageInfo.get("URL") if url: packageInfo = PackageManager.packageInfo( self.cntlr, url, reload=True, packageManifestName=packageInfo.get("manifestName")) if packageInfo: self.addPackageInfo(packageInfo) PackageManager.rebuildRemappings(self.cntlr) self.loadTreeViews() self.cntlr.showStatus(_("{0} reloaded").format( packageInfo.get("name")), clearAfter=5000) else: messagebox.showwarning( _("Package error"), _("File or package cannot be reloaded: \n\n{0}" ).format(url), parent=self)
def addPackageInfo(self, packageInfo): name = packageInfo["name"] version = packageInfo["version"] self.removePackageInfo(name, version) # remove any prior entry for this package self.packageNamesWithNewerFileDates.discard(name) # no longer has an update available self.packagesConfig["packages"].append(packageInfo) PackageManager.rebuildRemappings(self.cntlr) self.packagesConfigChanged = True
def loadTaxonomyPackageMappings(self, errors=[], expectTaxonomyPackage=None): if not self.mappedPaths and (self.taxonomyPackageMetadataFiles or expectTaxonomyPackage): if PackageManager.validateTaxonomyPackage(self.cntlr, self, errors=errors): metadata = self.baseurl + os.sep + self.taxonomyPackageMetadataFiles[0] self.taxonomyPackage = PackageManager.parsePackage(self.cntlr, self, metadata, os.sep.join(os.path.split(metadata)[:-1]) + os.sep, errors=errors) self.mappedPaths = self.taxonomyPackage.get("remappings")
def addPackageInfo(self, packageInfo): name = packageInfo["name"] version = packageInfo["version"] self.removePackageInfo(name, version) # remove any prior entry for this package self.packageNamesWithNewerFileDates.discard(name) # no longer has an update available self.packagesConfig["packages"].append(packageInfo) PackageManager.rebuildRemappings() self.packagesConfigChanged = True
def packageMoveDown(self): if 0 <= self.selectedPackageIndex < len(self.packagesConfig["packages"]) - 1: packages = self.packagesConfig["packages"] packageInfo = packages[self.selectedPackageIndex] del packages[self.selectedPackageIndex] packages.insert(self.selectedPackageIndex + 1, packageInfo) self.packagesConfigChanged = True PackageManager.rebuildRemappings(self.cntlr) self.loadTreeViews()
def loadPackagesFromDir(self, directory): packages = glob.glob(os.path.join(directory, "*.zip")) for p in packages: pi = PackageManager.addPackage(self, p) if pi: self.addToLog("Package added", messageCode="info", file=pi.get("URL")) else: self.addToLog("Failed to load package", messageCode="error", file=p) PackageManager.rebuildRemappings(self)
def packageMoveDown(self): if 0 <= self.selectedPackageIndex < len(self.packagesConfig["packages"]) - 1: packages = self.packagesConfig["packages"] packageInfo = packages[self.selectedPackageIndex] del packages[self.selectedPackageIndex] packages.insert(self.selectedPackageIndex + 1, packageInfo) self.packagesConfigChanged = True PackageManager.rebuildRemappings() self.loadTreeViews()
def load_xbrl_model(file_path, taxonomies_dir=None): controller = Cntlr(logFileName='logToStdErr') if taxonomies_dir is not None: save_taxonomy_config(taxonomies_dir, controller) else: PackageManager.init(controller) model_manager = ModelManager(controller) model_manager.abortOnMajorError = True xbrl_model = model_manager.load(file_path) loadDimensionDefaults(xbrl_model) return xbrl_model
def enableDisableAll(self, doEnable): for iPkg in range(len(self.packagesConfig["packages"])): packageInfo = self.packagesConfig["packages"][iPkg] if doEnable: packageInfo["status"] = "enabled" self.packageEnableButton['text'] = self.DISABLE else: packageInfo["status"] = "disabled" self.packageEnableButton['text'] = self.ENABLE self.packagesConfigChanged = True PackageManager.rebuildRemappings(self.cntlr) self.loadTreeViews()
def test_package_manager_reset(): """ Test that packagesConfig and packagesMappings are cleared when reset is called """ cntlr = Mock() PackageManager.init(cntlr, loadPackagesConfig=False) assert len(PackageManager.packagesMappings) == 0 PackageManager.packagesMappings['mapping'] = 'package' PackageManager.reset() assert len(PackageManager.packagesConfig) == 0 assert len(PackageManager.packagesMappings) == 0 assert PackageManager._cntlr == cntlr
def packageEnable(self): if 0 <= self.selectedPackageIndex < len(self.packagesConfig["packages"]): packageInfo = self.packagesConfig["packages"][self.selectedPackageIndex] if self.packageEnableButton['text'] == self.ENABLE: packageInfo["status"] = "enabled" self.packageEnableButton['text'] = self.DISABLE elif self.packageEnableButton['text'] == self.DISABLE: packageInfo["status"] = "disabled" self.packageEnableButton['text'] = self.ENABLE self.packagesConfigChanged = True PackageManager.rebuildRemappings() self.loadTreeViews()
def packageEnable(self): if 0 <= self.selectedPackageIndex < len(self.packagesConfig["packages"]): packageInfo = self.packagesConfig["packages"][self.selectedPackageIndex] if self.packageEnableButton['text'] == self.ENABLE: packageInfo["status"] = "enabled" self.packageEnableButton['text'] = self.DISABLE elif self.packageEnableButton['text'] == self.DISABLE: packageInfo["status"] = "disabled" self.packageEnableButton['text'] = self.ENABLE self.packagesConfigChanged = True PackageManager.rebuildRemappings(self.cntlr) self.loadTreeViews()
def openFileStream(cntlr, filepath, mode='r', encoding=None): if PackageManager.isMappedUrl(filepath): filepath = PackageManager.mappedUrl(filepath) elif isHttpUrl(filepath) and cntlr and hasattr( cntlr, "modelManager" ): # may be called early in initialization for PluginManager filepath = cntlr.modelManager.disclosureSystem.mappedUrl(filepath) if archiveFilenameParts(filepath): # file is in an archive return openFileSource(filepath, cntlr).file(filepath, binary='b' in mode, encoding=encoding)[0] if isHttpUrl(filepath) and cntlr: _cacheFilepath = cntlr.webCache.getfilename( filepath, normalize=True ) # normalize is separate step in ModelDocument retrieval, combined here if _cacheFilepath is None: raise IOError(_("Unable to open file: {0}.").format(filepath)) filepath = _cacheFilepath # file path may be server (or memcache) or local file system if filepath.startswith(SERVER_WEB_CACHE) and cntlr: filestream = None cacheKey = filepath[len(SERVER_WEB_CACHE) + 1:].replace("\\", "/") if cntlr.isGAE: # check if in memcache cachedBytes = gaeGet(cacheKey) if cachedBytes: filestream = io.BytesIO(cachedBytes) if filestream is None: filestream = io.BytesIO() cntlr.webCache.retrieve( cntlr.webCache.cacheFilepathToUrl(filepath), filestream=filestream) if cntlr.isGAE: gaeSet(cacheKey, filestream.getvalue()) if mode.endswith('t') or encoding: contents = filestream.getvalue() filestream.close() filestream = FileNamedStringIO( filepath, contents.decode(encoding or 'utf-8')) return filestream # local file system elif encoding is None and 'b' not in mode: openedFileStream = io.open(filepath, mode='rb') hdrBytes = openedFileStream.read(512) encoding = XmlUtil.encoding(hdrBytes, default=None) openedFileStream.close() return io.open(filepath, mode=mode, encoding=encoding) else: # local file system return io.open(filepath, mode=mode, encoding=encoding)
def test_package_manager_init_first_pass(): """ Test that packagesConfig is correctly setup during init on fresh pass """ cntlr = Mock() PackageManager.init(cntlr, loadPackagesConfig=False) assert len(PackageManager.packagesConfig) == 2 assert 'packages' in PackageManager.packagesConfig assert isinstance(PackageManager.packagesConfig.get('packages'), list) assert len(PackageManager.packagesConfig.get('packages')) == 0 assert 'remappings' in PackageManager.packagesConfig assert isinstance(PackageManager.packagesConfig.get('remappings'), dict) assert len(PackageManager.packagesConfig.get('remappings')) == 0 assert PackageManager._cntlr == cntlr
def close(self, saveConfig=False): """Closes the controller and its logger, optionally saving the user preferences configuration :param saveConfig: save the user preferences configuration :type saveConfig: bool """ PluginManager.save(self) PackageManager.save(self) if saveConfig: self.saveConfig() if self.logger is not None: try: self.logHandler.close() except Exception: # fails on some earlier pythons (3.1) pass
def load(self, filesource, nextaction=None, taxonomyPackages=None, **kwargs): """Load an entry point modelDocument object(s), which in turn load documents they discover (for the case of instance, taxonomies, and versioning reports), but defer loading instances for test case and RSS feeds. The modelXbrl that is loaded is 'stacked', by this class, so that any modelXbrl operations such as validate, and close, operate on the most recently loaded modelXbrl, and compareDTSes operates on the two most recently loaded modelXbrl's. :param filesource: may be a FileSource object, with the entry point selected, or string file name (or web URL). :type filesource: FileSource or str :param nextAction: status line text string, if any, to show upon completion :type nextAction: str :param taxonomyPackages: array of URLs of taxonomy packages required for load operation """ if taxonomyPackages: resetPackageMappings = False for pkgUrl in taxonomyPackages: if PackageManager.addPackage(self.cntlr, pkgUrl): resetPackageMappings = True if resetPackageMappings: PackageManager.rebuildRemappings(self.cntlr) try: if filesource.url.startswith( "urn:uuid:"): # request for an open modelXbrl for modelXbrl in self.loadedModelXbrls: if not modelXbrl.isClosed and modelXbrl.uuid == filesource.url: return modelXbrl raise IOError( _("Open file handle is not open: {0}").format( filesource.url)) except AttributeError: pass # filesource may be a string, which has no url attribute self.filesource = filesource modelXbrl = None # loaded modelXbrl for customLoader in pluginClassMethods("ModelManager.Load"): modelXbrl = customLoader(self, filesource) if modelXbrl is not None: break # custom loader did the loading if modelXbrl is None: # use default xbrl loader modelXbrl = ModelXbrl.load(self, filesource, nextaction, **kwargs) self.modelXbrl = modelXbrl self.loadedModelXbrls.append(self.modelXbrl) return self.modelXbrl
def loadTaxonomyPackageMappings(self): if self.taxonomyPackageMetadataFiles: metadata = self.url + os.sep + self.taxonomyPackageMetadataFiles[0] taxonomyPackage = PackageManager.parsePackage( self.cntlr, self, metadata, os.sep.join(os.path.split(metadata)[:-1]) + os.sep) self.mappedPaths = taxonomyPackage["remappings"]
def func_json_data(xule_context, *args): """Read a json file/url. Arguments: file_url (string or url) Returns a dictionary/list of the json data. """ file_url = args[0] if file_url.type not in ('string', 'uri'): raise XuleProcessingError(_("The file url argument of the json-dta() function must be a string or uri, found '{}'.".format(file_url.value)), xule_context) from arelle import PackageManager mapped_file_url = PackageManager.mappedUrl(file_url.value) # Using the FileSource object in arelle. This will open the file and handle taxonomy package mappings. from arelle import FileSource file_source = FileSource.openFileSource(file_url.value, xule_context.global_context.cntlr) file = file_source.file(file_url.value, binary=True) # file is tuple of one item as a BytesIO stream. Since this is in bytes, it needs to be converted to text via a decoder. # Assuming the file is in utf-8. data_source = [x.decode('utf-8') for x in file[0].readlines()] try: json_source = json.loads(''.join(data_source)) #except JSONDecodeError: except ValueError: raise XuleProcessingError(_("The file '{}' is not a valid JSON file.".format(file_url.value)), xule_context) x = xv.system_collection_to_xule(json_source, xule_context) return xv.system_collection_to_xule(json_source, xule_context)
def findLocally(self): initialdir = self.cntlr.pluginDir # default plugin directory if not self.cntlr.isMac: # can't navigate within app easily, always start in default directory initialdir = self.cntlr.config.setdefault("packageOpenDir", initialdir) filename = self.cntlr.uiFileDialog( "open", parent=self, title=_("Choose taxonomy package file"), initialdir=initialdir, filetypes=[(_("Taxonomy package files (*.zip)"), "*.zip"), (_("PWD Manifest (taxonomyPackage.xml)"), "taxonomyPackage.xml"), (_("pre-PWD Manifest (*.taxonomyPackage.xml)"), "*.taxonomyPackage.xml"), (_("pre-PWD Oasis Catalog (*catalog.xml)"), "*catalog.xml")], defaultextension=".zip") if filename: # check if a package is selected (any file in a directory containing an __init__.py self.cntlr.config["packageOpenDir"] = os.path.dirname(filename) packageInfo = PackageManager.packageInfo( self.cntlr, filename, packageManifestName=self.manifestNamePattern) self.loadFoundPackageInfo(packageInfo, filename)
def findOnWeb(self): url = DialogURL.askURL(self) if url: # url is the in-cache or local file packageInfo = PackageManager.packageInfo( self.cntlr, url, packageManifestName=self.manifestNamePattern) self.cntlr.showStatus("") # clear web loading status self.loadFoundPackageInfo(packageInfo, url)
def packageReload(self): if 0 <= self.selectedPackageIndex < len(self.packagesConfig["packages"]): packageInfo = self.packagesConfig["packages"][self.selectedPackageIndex] url = packageInfo.get("URL") if url: packageInfo = PackageManager.packageInfo(url, reload=True, packageManifestName=packageInfo.get("manifestName")) if packageInfo: self.addPackageInfo(packageInfo) PackageManager.rebuildRemappings() self.loadTreeViews() self.cntlr.showStatus(_("{0} reloaded").format(packageInfo.get("name")), clearAfter=5000) else: messagebox.showwarning(_("Package error"), _("File or package cannot be reloaded: \n\n{0}") .format(url), parent=self)
def backgroundCheckForUpdates(cntlr): cntlr.showStatus(_("Checking for updates to packages")) # clear web loading status packageNamesWithNewerFileDates = PackageManager.packageNamesWithNewerFileDates() if packageNamesWithNewerFileDates: cntlr.showStatus(_("Updates are available for these packages: {0}") .format(', '.join(packageNamesWithNewerFileDates)), clearAfter=5000) else: cntlr.showStatus(_("No updates found for packages."), clearAfter=5000) time.sleep(0.1) # Mac locks up without this, may be needed for empty ui queue? cntlr.uiThreadQueue.put((DialogPackageManager, [cntlr, packageNamesWithNewerFileDates]))
def openFileStream(cntlr, filepath, mode='r', encoding=None): if PackageManager.isMappedUrl(filepath): filepath = PackageManager.mappedUrl(filepath) else: filepath = cntlr.modelManager.disclosureSystem.mappedUrl(filepath) if archiveFilenameParts(filepath): # file is in an archive return openFileSource(filepath, cntlr).file(filepath, binary='b' in mode, encoding=encoding)[0] if isHttpUrl(filepath) and cntlr: _cacheFilepath = cntlr.webCache.getfilename(filepath) if _cacheFilepath is None: raise IOError(_("Unable to open file: {0}.").format(filepath)) filepath = _cacheFilepath # file path may be server (or memcache) or local file system if filepath.startswith(SERVER_WEB_CACHE) and cntlr: filestream = None cacheKey = filepath[len(SERVER_WEB_CACHE) + 1:].replace("\\","/") if cntlr.isGAE: # check if in memcache cachedBytes = gaeGet(cacheKey) if cachedBytes: filestream = io.BytesIO(cachedBytes) if filestream is None: filestream = io.BytesIO() cntlr.webCache.retrieve(cntlr.webCache.cacheFilepathToUrl(filepath), filestream=filestream) if cntlr.isGAE: gaeSet(cacheKey, filestream.getvalue()) if mode.endswith('t') or encoding: contents = filestream.getvalue() filestream.close() filestream = FileNamedStringIO(filepath, contents.decode(encoding or 'utf-8')) return filestream # local file system elif encoding is None and 'b' not in mode: openedFileStream = io.open(filepath, mode='rb') hdrBytes = openedFileStream.read(512) encoding = XmlUtil.encoding(hdrBytes, default=None) openedFileStream.close() return io.open(filepath, mode=mode, encoding=encoding) else: # local file system return io.open(filepath, mode=mode, encoding=encoding)
def save_taxonomy_config(taxonomies_dir, controller=None): if controller is None: controller = Cntlr(logFileName='logToStdErr') PackageManager.init(controller) for taxonomy_zip in os.listdir(taxonomies_dir): if taxonomy_zip.endswith('.zip'): taxonomy_zip_path = os.path.join(taxonomies_dir, taxonomy_zip) PackageManager.addPackage(controller, taxonomy_zip_path) PackageManager.save(controller)
def findLocally(self): initialdir = self.cntlr.pluginDir # default plugin directory if not self.cntlr.isMac: # can't navigate within app easily, always start in default directory initialdir = self.cntlr.config.setdefault("packageOpenDir", initialdir) filename = self.cntlr.uiFileDialog("open", owner=self, title=_("Choose taxonomy package file"), initialdir=initialdir, filetypes=[(_("Taxonomy package files (*.zip)"), "*.zip"), (_("Manifest (*.taxonomyPackage.xml)"), "*.taxonomyPackage.xml")], defaultextension=".zip") if filename: # check if a package is selected (any file in a directory containing an __init__.py self.cntlr.config["packageOpenDir"] = os.path.dirname(filename) packageInfo = PackageManager.packageInfo(filename) self.loadFoundPackageInfo(packageInfo, filename)
def open_package_file(self, file_name): """Open a taxonomy package in the rule set :param file_name: the name of the package file in the rule set :type file_name: str :return: The package information. This is the return from Arelle when activating the package """ package_info = PackageManager.addPackage(self._cntlr, file_name) if package_info: # print("Activation of package {0} successful.".format(package_info.get("name"))) self._cntlr.addToLog(_("Activation of package {0} successful.").format(package_info.get("name")), messageCode="info", file=package_info.get("URL")) else: # print("Unable to load package \"{}\". ".format(file_name)) self._cntlr.addToLog(_("Unable to load package \"%(name)s\". "), messageCode="arelle:packageLoadingError", level=logging.ERROR) return package_info
def get_packages_info(self): """Get a list of taxonomy packages in the rule set :returns: List of package information. The package information is returned when activating the package in Arelle :rtype: list """ results = [] temp_dir = tempfile.TemporaryDirectory() #Using arelle file source object. This will handle files from the web. file_object = self._get_rule_set_file_object() try: with zipfile.ZipFile(file_object, 'r') as zf: for package_file_name in zf.namelist(): if package_file_name.startswith('packages/'): package_file = zf.extract(package_file_name, temp_dir.name) package_info = PackageManager.addPackage(self._cntlr, package_file) results.append(package_info) finally: file_object.close() return results
def func_json_data(xule_context, *args): """Read a json file/url. Arguments: file_url (string or url) Returns a dictionary/list of the json data. """ file_url = args[0] if file_url.type not in ('string', 'uri'): raise XuleProcessingError( _("The file url argument of the json-dta() function must be a string or uri, found '{}'." .format(file_url.value)), xule_context) from arelle import PackageManager mapped_file_url = PackageManager.mappedUrl(file_url.value) # Using the FileSource object in arelle. This will open the file and handle taxonomy package mappings. from arelle import FileSource file_source = FileSource.openFileSource(file_url.value, xule_context.global_context.cntlr) file = file_source.file(file_url.value, binary=True) # file is tuple of one item as a BytesIO stream. Since this is in bytes, it needs to be converted to text via a decoder. # Assuming the file is in utf-8. data_source = [x.decode('utf-8') for x in file[0].readlines()] try: json_source = json.loads(''.join(data_source)) #except JSONDecodeError: except ValueError: raise XuleProcessingError( _("The file '{}' is not a valid JSON file.".format( file_url.value)), xule_context) x = xv.system_collection_to_xule(json_source, xule_context) return xv.system_collection_to_xule(json_source, xule_context)
def func_csv_data(xule_context, *args): """Read a csv file/url. Arguments: file_url (string or url) has_header (boolean) - determines if the first line of the csv file has headers type list (list) - list of xule types in the order of the columns of the csv file. This is optional. If not provided, then all the data will be treated as stirngs. as_dictionary (boolean) - return the row as a dictionary instead of a list. This is optional. """ if len(args) < 2: raise XuleProcessingError( _("The csv-data() function requires at least 2 arguments (file url, has headers), found {} arguments." .format(len(args))), xule_context) if len(args) > 4: raise XuleProcessingError( _("The csv-data() function takes no more than 3 arguments (file url, has headers, column types, as dictionary), found {} arguments." .format(len(args))), xule_context) file_url = args[0] has_headers = args[1] if file_url.type not in ('string', 'uri'): raise XuleProcessingError( _("The file url argument (1st argument) of the csv-dta() function must be a string or uri, found '{}'." .format(file_url.value)), xule_context) if has_headers.type != 'bool': raise XuleProcessingError( _("The has headers argument (2nd argument) of the csv-data() function muset be a boolean, found '{}'." .format(has_headers.type)), xule_context) if len(args) >= 3: column_types = args[2] if column_types.type == 'none': ordered_cols = None elif column_types.type == 'list': ordered_cols = list() for col in column_types.value: if col.type != 'string': raise XuleProcessingError( _("The type list argument (3rd argument) of the csv-data() function must be a list of strings, found '{}'." .format(col.type)), xule_context) ordered_cols.append(col.value) else: raise XuleProcessingError( _("The type list argument (3rd argument) of the csv-data() fucntion must be list, found '{}'." .format(column_types.type)), xule_context) else: ordered_cols = None if len(args) == 4: if args[3].type != 'bool': raise XuleProcessingError( _("The as dictionary argument (4th argument) of the csv-data() function must be a boolean, found '{}'." .format(args[3].type)), xule_context) if args[3].value: return_row_type = 'dictionary' else: return_row_type = 'list' else: return_row_type = 'list' if return_row_type == 'dictionary' and not has_headers.value: raise XuleProcessingError( _("When the csv-data() function is returning the rows as dictionaries (4th argument), the has headers argument (2nd argument) must be true." ), xule_context) result = list() result_shadow = list() from arelle import PackageManager mapped_file_url = PackageManager.mappedUrl(file_url.value) # Using the FileSource object in arelle. This will open the file and handle taxonomy package mappings. from arelle import FileSource file_source = FileSource.openFileSource(file_url.value, xule_context.global_context.cntlr) file = file_source.file(file_url.value, binary=True) # file is tuple of one item as a BytesIO stream. Since this is in bytes, it needs to be converted to text via a decoder. # Assuming the file is in utf-8. data_source = [x.decode('utf-8') for x in file[0].readlines()] import csv reader = csv.reader(data_source) first_line = True row_num = 0 for line in reader: row_num += 1 if first_line and has_headers.value: first_line = False #skip the headers line if return_row_type == 'dictionary': # Need to get the names from the first row column_names = [x for x in line] if len(column_names) != len(set(column_names)): raise XuleProcessingError( _("There are duplicate column names in the csv file. This is not allowed when return rows as dictionaries. File: {}" .format(file_url.value)), xule_context) continue if return_row_type == 'list': result_line = list() result_line_shadow = list() else: #dictionary result_line = dict() result_line_shadow = dict() for col_num, item in enumerate(line): if ordered_cols is not None and col_num >= len(ordered_cols): raise XuleProcessingError( _("The nubmer of columns on row {} is greater than the number of column types provided in the third argument of the csv-data() function. File: {}" .format(row_num, file_url.value)), xule_context) item_value = convert_file_data_item( item, ordered_cols[col_num] if ordered_cols is not None else None, xule_context) if return_row_type == 'list': result_line.append(item_value) result_line_shadow.append(item_value.value) else: #dictonary if col_num >= len(column_names): raise xule_context( _("The number of columns on row {} is greater than the number of headers in the csv file. File: {}" .format( row_num, mappedUrl if mapped_file_url == file_url.value else file_url.value + ' --> ' + mapped_file_url)), xule_context) result_line[xv.XuleValue(xule_context, column_names[col_num], 'string')] = item_value result_line_shadow[column_names[col_num]] = item_value.value if return_row_type == 'list': result.append( xv.XuleValue(xule_context, tuple(result_line), 'list', shadow_collection=tuple(result_line_shadow))) result_shadow.append(tuple(result_line_shadow)) else: #dictionary result.append( xv.XuleValue(xule_context, frozenset(result_line.items()), 'dictionary', shadow_collection=frozenset( result_line_shadow.items()))) result_shadow.append(frozenset(result_line_shadow.items())) return xv.XuleValue(xule_context, tuple(result), 'list', shadow_collection=tuple(result_shadow))
def loadPackageUrl(self, url): if url: # url is the in-cache or local file packageInfo = PackageManager.packageInfo( self.cntlr, url, packageManifestName=self.manifestNamePattern) self.cntlr.showStatus("") # clear web loading status self.loadFoundPackageInfo(packageInfo, url)
def validateTestcase(self, testcase): self.modelXbrl.info("info", "Testcase", modelDocument=testcase) self.modelXbrl.viewModelObject(testcase.objectId()) if hasattr(testcase, "testcaseVariations"): for modelTestcaseVariation in testcase.testcaseVariations: # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) # is this a versioning report? resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance resultIsTaxonomyPackage = modelTestcaseVariation.resultIsTaxonomyPackage formulaOutputInstance = None inputDTSes = defaultdict(list) baseForElement = testcase.baseForElement(modelTestcaseVariation) # try to load instance document self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s - %(description)s"), modelObject=modelTestcaseVariation, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, expected=modelTestcaseVariation.expected, description=modelTestcaseVariation.description) errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY parameters = modelTestcaseVariation.parameters.copy() for readMeFirstUri in modelTestcaseVariation.readMeFirstUris: if isinstance(readMeFirstUri,tuple): # dtsName is for formula instances, but is from/to dts if versioning dtsName, readMeFirstUri = readMeFirstUri elif resultIsVersioningReport: if inputDTSes: dtsName = "to" else: dtsName = "from" else: dtsName = None if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document if dtsName in inputDTSes: dtsName = inputDTSes[dtsName] else: modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager, Type.DTSENTRIES, self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement), isEntry=True, errorCaptureLevel=errorCaptureLevel) DTSdoc = modelXbrl.modelDocument DTSdoc.inDTS = True doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement) if doc is not None: DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement) #fake import doc.inDTS = True elif resultIsTaxonomyPackage: from arelle import PackageManager, PrototypeInstanceObject dtsName = readMeFirstUri modelXbrl = PrototypeInstanceObject.XbrlPrototype(self.modelXbrl.modelManager, readMeFirstUri) PackageManager.packageInfo(self.modelXbrl.modelManager.cntlr, readMeFirstUri, reload=True, errors=modelXbrl.errors) else: # not a multi-schemaRef versioning report if self.useFileSource.isArchive: modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, readMeFirstUri, _("validating"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) else: # need own file source, may need instance discovery filesource = FileSource.FileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr) if filesource and not filesource.selection and filesource.isArchive: for _archiveFile in filesource.dir: # find instance document in archive filesource.select(_archiveFile) if ModelDocument.Type.identify(filesource, filesource.url) in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL): break # use this selection modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, filesource, _("validating"), base=baseForElement, errorCaptureLevel=errorCaptureLevel) modelXbrl.isTestcaseVariation = True if modelXbrl.modelDocument is None: modelXbrl.error("arelle:notLoaded", _("Testcase %(id)s %(name)s document not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) self.determineNotLoadedTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif resultIsVersioningReport or resultIsTaxonomyPackage: inputDTSes[dtsName] = modelXbrl elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl) self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif testcase.type == Type.REGISTRYTESTCASE: self.instValidator.validate(modelXbrl) # required to set up dimensions, etc self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id, modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest) self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) self.instValidator.close() modelXbrl.close() else: inputDTSes[dtsName].append(modelXbrl) # validate except for formulas _hasFormulae = modelXbrl.hasFormulae modelXbrl.hasFormulae = False try: for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"): pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation) self.instValidator.validate(modelXbrl, parameters) for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"): pluginXbrlMethod(self.modelXbrl, modelXbrl) except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase variation validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) modelXbrl.hasFormulae = _hasFormulae if resultIsVersioningReport and modelXbrl.modelDocument: versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl( modelTestcaseVariation.versioningReportUri, baseForElement) if os.path.exists(versReportFile): #validate existing modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report")) if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport) self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors) modelVersReport.close() elif len(inputDTSes) == 2: ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes( versReportFile, inputDTSes["from"], inputDTSes["to"]) modelTestcaseVariation.status = "generated" else: modelXbrl.error("arelle:notLoaded", _("Testcase %(id)s %(name)s DTSes not loaded, unable to generate versioning report: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) modelTestcaseVariation.status = "failed" for inputDTS in inputDTSes.values(): inputDTS.close() del inputDTSes # dereference elif resultIsTaxonomyPackage: self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif inputDTSes: # validate schema, linkbase, or instance modelXbrl = inputDTSes[None][0] for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters if dtsName: # named instance parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes) elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable: RenderingEvaluator.init(modelXbrl) if modelXbrl.hasFormulae: try: # validate only formulae self.instValidator.parameters = parameters ValidateFormula.validate(self.instValidator) except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset: for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"): pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri) infoset = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultInfosetUri, _("loading result infoset"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) if infoset.modelDocument is None: modelXbrl.error("arelle:notLoaded", _("Testcase %(id)s %(name)s result infoset not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstance)) modelTestcaseVariation.status = "result infoset not loadable" else: # check infoset ValidateInfoset.validate(self.instValidator, modelXbrl, infoset) infoset.close() if modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset: # diff (or generate) table infoset resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement) if not any(alternativeValidation(modelXbrl, resultTableUri) for alternativeValidation in pluginClassMethods("Validate.TableInfoset")): ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files self.instValidator.close() extraErrors = [] for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"): pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors) self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual): # if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost) self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters) self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors) if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance formulaOutputInstance = modelXbrl.formulaOutputInstance modelXbrl.formulaOutputInstance = None # prevent it from being closed now self.instValidator.close() compareIxResultInstance = getattr(modelXbrl, "extractedInlineInstance", False) and modelTestcaseVariation.resultXbrlInstanceUri if compareIxResultInstance: formulaOutputInstance = modelXbrl # compare modelXbrl to generated output instance errMsgPrefix = "ix" else: # delete input instances before formula output comparision for inputDTSlist in inputDTSes.values(): for inputDTS in inputDTSlist: inputDTS.close() del inputDTSes # dereference errMsgPrefix = "formula" if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument: expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultXbrlInstanceUri, _("loading expected result XBRL instance"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) if expectedInstance.modelDocument is None: self.modelXbrl.error("{}:expectedResultNotLoaded".format(errMsgPrefix), _("Testcase %(id)s %(name)s expected result instance not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstanceUri), messageCodes=("formula:expectedResultNotLoaded","ix:expectedResultNotLoaded")) modelTestcaseVariation.status = "result not loadable" else: # compare facts if len(expectedInstance.facts) != len(formulaOutputInstance.facts): formulaOutputInstance.error("{}:resultFactCounts".format(errMsgPrefix), _("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"), modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts), expectedFacts=len(expectedInstance.facts), messageCodes=("formula:resultFactCounts","ix:resultFactCounts")) else: formulaOutputFootnotesRelSet = ModelRelationshipSet(formulaOutputInstance, "XBRL-footnotes") expectedFootnotesRelSet = ModelRelationshipSet(expectedInstance, "XBRL-footnotes") def factFootnotes(fact, footnotesRelSet): footnotes = [] footnoteRels = footnotesRelSet.fromModelObject(fact) if footnoteRels: # most process rels in same order between two instances, use labels to sort for i, footnoteRel in enumerate(sorted(footnoteRels, key=lambda r: (r.fromLabel,r.toLabel))): modelObject = footnoteRel.toModelObject if isinstance(modelObject, ModelResource): xml = modelObject.viewText().strip() footnotes.append("Footnote {}: {}".format( i+1, # compare footnote with HTML serialized xml, #re.sub(r'\s+', ' ', collapseWhitespace(modelObject.stringValue)) )) elif isinstance(modelObject, ModelFact): footnotes.append("Footnoted fact {}: {} context: {} value: {}".format( i+1, modelObject.qname, modelObject.contextID, collapseWhitespace(modelObject.value))) return footnotes for expectedInstanceFact in expectedInstance.facts: unmatchedFactsStack = [] formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True) if formulaOutputFact is None: if unmatchedFactsStack: # get missing nested tuple fact, if possible missingFact = unmatchedFactsStack[-1] else: missingFact = expectedInstanceFact formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix), _("Output missing expected fact %(fact)s"), modelXbrl=missingFact, fact=missingFact.qname, messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing")) else: # compare footnotes expectedInstanceFactFootnotes = factFootnotes(expectedInstanceFact, expectedFootnotesRelSet) formulaOutputFactFootnotes = factFootnotes(formulaOutputFact, formulaOutputFootnotesRelSet) if expectedInstanceFactFootnotes != formulaOutputFactFootnotes: formulaOutputInstance.error("{}:expectedFactFootnoteDifference".format(errMsgPrefix), _("Output expected fact %(fact)s expected footnotes %(footnotes1)s produced footnotes %(footnotes2)s"), modelXbrl=(formulaOutputFact,expectedInstanceFact), fact=expectedInstanceFact.qname, footnotes1=expectedInstanceFactFootnotes, footnotes2=formulaOutputFactFootnotes, messageCodes=("formula:expectedFactFootnoteDifference","ix:expectedFactFootnoteDifference")) # for debugging uncomment next line to save generated instance document # formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml") expectedInstance.close() del expectedInstance # dereference self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors) formulaOutputInstance.close() del formulaOutputInstance if compareIxResultInstance: for inputDTSlist in inputDTSes.values(): for inputDTS in inputDTSlist: inputDTS.close() del inputDTSes # dereference # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def open(self, reloadCache=False): if not self.isOpen: if (self.isZip or self.isTarGz or self.isEis or self.isXfd or self.isRss or self.isInstalledTaxonomyPackage) and self.cntlr: self.basefile = self.cntlr.webCache.getfilename( self.url, reload=reloadCache) else: self.basefile = self.url self.baseurl = self.url # url gets changed by selection if not self.basefile: return # an error should have been logged if self.isZip: try: self.fs = zipfile.ZipFile(openFileStream( self.cntlr, self.basefile, 'rb'), mode="r") self.isOpen = True except EnvironmentError as err: self.logError(err) pass elif self.isTarGz: try: self.fs = tarfile.open(self.basefile, "r:gz") self.isOpen = True except EnvironmentError as err: self.logError(err) pass elif self.isEis: # check first line of file buf = b'' try: file = open(self.basefile, 'rb') more = True while more: l = file.read(8) if len(l) < 8: break if len(buf) == 0 and l.startswith( b"<?xml "): # not compressed buf = l + file.read() # not compressed break compressedBytes = file.read( struct.unpack(">L", l[0:4])[0]) if len(compressedBytes) <= 0: break buf += zlib.decompress(compressedBytes) file.close() except EnvironmentError as err: self.logError(err) pass #uncomment to save for debugging #with open("c:/temp/test.xml", "wb") as f: # f.write(buf) if buf.startswith(b"<?xml "): try: # must strip encoding str = buf.decode(XmlUtil.encoding(buf)) endEncoding = str.index("?>", 0, 128) if endEncoding > 0: str = str[endEncoding + 2:] file = io.StringIO(initial_value=str) parser = etree.XMLParser(recover=True, huge_tree=True) self.eisDocument = etree.parse(file, parser=parser) file.close() self.isOpen = True except EnvironmentError as err: self.logError(err) return # provide error message later except etree.LxmlError as err: self.logError(err) return # provide error message later elif self.isXfd: # check first line of file file = open(self.basefile, 'rb') firstline = file.readline() if firstline.startswith( b"application/x-xfdl;content-encoding=\"asc-gzip\""): # file has been gzipped base64input = file.read(-1) file.close() file = None fb = base64.b64decode(base64input) ungzippedBytes = b"" totalLenUncompr = 0 i = 0 while i < len(fb): lenCompr = fb[i + 0] * 256 + fb[i + 1] lenUncomp = fb[i + 2] * 256 + fb[i + 3] lenRead = 0 totalLenUncompr += lenUncomp gzchunk = (bytes((31, 139, 8, 0)) + fb[i:i + lenCompr]) try: with gzip.GzipFile( fileobj=io.BytesIO(gzchunk)) as gf: while True: readSize = min(16384, lenUncomp - lenRead) readBytes = gf.read(size=readSize) lenRead += len(readBytes) ungzippedBytes += readBytes if len(readBytes) == 0 or (lenUncomp - lenRead) <= 0: break except IOError as err: pass # provide error message later i += lenCompr + 4 #for learning the content of xfd file, uncomment this: #with open("c:\\temp\\test.xml", "wb") as fh: # fh.write(ungzippedBytes) file = io.StringIO( initial_value=ungzippedBytes.decode("utf-8")) else: # position to start of file file.seek(0, io.SEEK_SET) try: self.xfdDocument = etree.parse(file) file.close() self.isOpen = True except EnvironmentError as err: self.logError(err) return # provide error message later except etree.LxmlError as err: self.logError(err) return # provide error message later elif self.isRss: try: self.rssDocument = etree.parse(self.basefile) self.isOpen = True except EnvironmentError as err: self.logError(err) return # provide error message later except etree.LxmlError as err: self.logError(err) return # provide error message later elif self.isInstalledTaxonomyPackage: self.isOpen = True # load mappings try: metadataFiles = self.taxonomyPackageMetadataFiles if len(metadataFiles) != 1: raise IOError( _("Taxonomy package must contain one and only one metadata file: {0}." ).format(', '.join(metadataFiles))) # HF: this won't work, see DialogOpenArchive for correct code # not sure if it is used taxonomyPackage = PackageManager.parsePackage( self.cntlr, self.url) fileSourceDir = os.path.dirname(self.baseurl) + os.sep self.mappedPaths = \ dict((prefix, remapping if isHttpUrl(remapping) else (fileSourceDir + remapping.replace("/", os.sep))) for prefix, remapping in taxonomyPackage["remappings"].items()) except EnvironmentError as err: self.logError(err) return # provide error message later
def __init__(self, hasGui=False, logFileName=None, logFileMode=None, logFileEncoding=None, logFormat=None): self.hasWin32gui = False self.hasGui = hasGui self.hasFileSystem = True # no file system on Google App Engine servers self.isGAE = False self.systemWordSize = int(round(math.log(sys.maxsize, 2)) + 1) # e.g., 32 or 64 self.moduleDir = os.path.dirname(__file__) # for python 3.2 remove __pycache__ if self.moduleDir.endswith("__pycache__"): self.moduleDir = os.path.dirname(self.moduleDir) if self.moduleDir.endswith("python32.zip/arelle"): ''' distZipFile = os.path.dirname(self.moduleDir) d = os.path.join(self.userAppDir, "arelle") self.configDir = os.path.join(d, "config") self.imagesDir = os.path.join(d, "images") import zipfile distZip = zipfile.ZipFile(distZipFile, mode="r") distNames = distZip.namelist() distZip.extractall(path=self.userAppDir, members=[f for f in distNames if "/config/" in f or "/images/" in f] ) distZip.close() ''' resources = os.path.dirname(os.path.dirname(os.path.dirname(self.moduleDir))) self.configDir = os.path.join(resources, "config") self.imagesDir = os.path.join(resources, "images") self.localeDir = os.path.join(resources, "locale") self.pluginDir = os.path.join(resources, "plugin") elif self.moduleDir.endswith("library.zip\\arelle") or self.moduleDir.endswith("library.zip/arelle"): # cx_Freexe resources = os.path.dirname(os.path.dirname(self.moduleDir)) self.configDir = os.path.join(resources, "config") self.imagesDir = os.path.join(resources, "images") self.localeDir = os.path.join(resources, "locale") self.pluginDir = os.path.join(resources, "plugin") else: self.configDir = os.path.join(self.moduleDir, "config") self.imagesDir = os.path.join(self.moduleDir, "images") self.localeDir = os.path.join(self.moduleDir, "locale") self.pluginDir = os.path.join(self.moduleDir, "plugin") serverSoftware = os.getenv("SERVER_SOFTWARE", "") if serverSoftware.startswith("Google App Engine/") or serverSoftware.startswith("Development/"): self.hasFileSystem = False # no file system, userAppDir does not exist self.isGAE = True configHomeDir = None # look for path configDir/CONFIG_HOME in argv and environment parameters for i, arg in enumerate(sys.argv): # check if config specified in a argv if arg.startswith("--xdgConfigHome="): configHomeDir = arg[16:] break elif arg == "--xdgConfigHome" and i + 1 < len(sys.argv): configHomeDir = sys.argv[i + 1] break if not configHomeDir: # not in argv, may be an environment parameter configHomeDir = os.getenv('XDG_CONFIG_HOME') if not configHomeDir: # look for path configDir/CONFIG_HOME configHomeDirFile = os.path.join(self.configDir, "XDG_CONFIG_HOME") if os.path.exists(configHomeDirFile): try: with io.open(configHomeDirFile, 'rt', encoding='utf-8') as f: configHomeDir = f.read().strip() if configHomeDir and not os.path.isabs(configHomeDir): configHomeDir = os.path.abspath(configHomeDir) # make into a full path if relative except EnvironmentError: configHomeDir = None if self.hasFileSystem and configHomeDir and os.path.exists(configHomeDir): # check if a cache exists in this directory (e.g. from XPE or other tool) impliedAppDir = os.path.join(configHomeDir, "arelle") if os.path.exists(impliedAppDir): self.userAppDir = impliedAppDir elif os.path.exists(os.path.join(configHomeDir, "cache")): self.userAppDir = configHomeDir # use the XDG_CONFIG_HOME because cache is already a subdirectory else: self.userAppDir = impliedAppDir if sys.platform == "darwin": self.isMac = True self.isMSW = False if self.hasFileSystem and not configHomeDir: self.userAppDir = os.path.expanduser("~") + "/Library/Application Support/Arelle" # note that cache is in ~/Library/Caches/Arelle self.contextMenuClick = "<Button-2>" self.hasClipboard = hasGui # clipboard always only if Gui (not command line mode) self.updateURL = "http://arelle.org/downloads/8" elif sys.platform.startswith("win"): self.isMac = False self.isMSW = True if self.hasFileSystem and not configHomeDir: tempDir = tempfile.gettempdir() if tempDir.endswith('local\\temp'): impliedAppDir = tempDir[:-10] + 'local' else: impliedAppDir = tempDir self.userAppDir = os.path.join( impliedAppDir, "Arelle") if hasGui: try: import win32clipboard self.hasClipboard = True except ImportError: self.hasClipboard = False try: import win32gui self.hasWin32gui = True # active state for open file dialogs except ImportError: pass else: self.hasClipboard = False self.contextMenuClick = "<Button-3>" if "64 bit" in sys.version: self.updateURL = "http://arelle.org/downloads/9" else: # 32 bit self.updateURL = "http://arelle.org/downloads/10" else: # Unix/Linux self.isMac = False self.isMSW = False if self.hasFileSystem and not configHomeDir: self.userAppDir = os.path.join( os.path.expanduser("~/.config"), "arelle") if hasGui: try: import gtk self.hasClipboard = True except ImportError: self.hasClipboard = False else: self.hasClipboard = False self.contextMenuClick = "<Button-3>" try: from arelle import webserver self.hasWebServer = True except ImportError: self.hasWebServer = False # assert that app dir must exist self.config = None if self.hasFileSystem: if not os.path.exists(self.userAppDir): os.makedirs(self.userAppDir) # load config if it exists self.configJsonFile = self.userAppDir + os.sep + "config.json" if os.path.exists(self.configJsonFile): try: with io.open(self.configJsonFile, 'rt', encoding='utf-8') as f: self.config = json.load(f) except Exception as ex: self.config = None # restart with a new config if not self.config: self.config = { 'fileHistory': [], 'windowGeometry': "{0}x{1}+{2}+{3}".format(800, 500, 200, 100), } # start language translation for domain self.setUiLanguage(self.config.get("userInterfaceLangOverride",None), fallbackToDefault=True) from arelle.WebCache import WebCache self.webCache = WebCache(self, self.config.get("proxySettings")) self.modelManager = ModelManager.initialize(self) # start plug in server (requres web cache initialized, but not logger) PluginManager.init(self) # start taxonomy package server (requres web cache initialized, but not logger) PackageManager.init(self) self.startLogging(logFileName, logFileMode, logFileEncoding, logFormat) # Cntlr.Init after logging started for pluginMethod in PluginManager.pluginClassMethods("Cntlr.Init"): pluginMethod(self)
def validateTestcase(self, testcase): self.modelXbrl.info("info", "Testcase", modelDocument=testcase) self.modelXbrl.viewModelObject(testcase.objectId()) if testcase.type in (Type.TESTCASESINDEX, Type.REGISTRY): for doc in sorted(testcase.referencesDocument.keys(), key=lambda doc: doc.uri): self.validateTestcase(doc) # testcases doc's are sorted by their uri (file names), e.g., for formula elif hasattr(testcase, "testcaseVariations"): for modelTestcaseVariation in testcaseVariationsByTarget(testcase.testcaseVariations): # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) # is this a versioning report? resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance resultIsTaxonomyPackage = modelTestcaseVariation.resultIsTaxonomyPackage formulaOutputInstance = None inputDTSes = defaultdict(list) baseForElement = testcase.baseForElement(modelTestcaseVariation) # try to load instance document self.modelXbrl.info("info", _("Variation %(id)s%(name)s%(target)s: %(expected)s - %(description)s"), modelObject=modelTestcaseVariation, id=modelTestcaseVariation.id, name=(" {}".format(modelTestcaseVariation.name) if modelTestcaseVariation.name else ""), target=(" target {}".format(modelTestcaseVariation.ixdsTarget) if modelTestcaseVariation.ixdsTarget else ""), expected=modelTestcaseVariation.expected, description=modelTestcaseVariation.description) if self.modelXbrl.modelManager.formulaOptions.testcaseResultsCaptureWarnings: errorCaptureLevel = logging._checkLevel("WARNING") else: errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY parameters = modelTestcaseVariation.parameters.copy() for readMeFirstUri in modelTestcaseVariation.readMeFirstUris: if isinstance(readMeFirstUri,tuple): # dtsName is for formula instances, but is from/to dts if versioning dtsName, readMeFirstUri = readMeFirstUri elif resultIsVersioningReport: if inputDTSes: dtsName = "to" else: dtsName = "from" else: dtsName = None if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document if dtsName in inputDTSes: dtsName = inputDTSes[dtsName] else: modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager, Type.DTSENTRIES, self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement), isEntry=True, errorCaptureLevel=errorCaptureLevel) DTSdoc = modelXbrl.modelDocument DTSdoc.inDTS = True doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement) if doc is not None: DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement) #fake import doc.inDTS = True elif resultIsTaxonomyPackage: from arelle import PackageManager, PrototypeInstanceObject dtsName = readMeFirstUri modelXbrl = PrototypeInstanceObject.XbrlPrototype(self.modelXbrl.modelManager, readMeFirstUri) PackageManager.packageInfo(self.modelXbrl.modelManager.cntlr, readMeFirstUri, reload=True, errors=modelXbrl.errors) else: # not a multi-schemaRef versioning report if self.useFileSource.isArchive: modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, readMeFirstUri, _("validating"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel, ixdsTarget=modelTestcaseVariation.ixdsTarget) else: # need own file source, may need instance discovery filesource = FileSource.openFileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr, base=baseForElement) if filesource and not filesource.selection and filesource.isArchive: try: if filesource.isTaxonomyPackage: _rptPkgIxdsOptions = {} for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ReportPackageIxdsOptions"): pluginXbrlMethod(self, _rptPkgIxdsOptions) filesource.loadTaxonomyPackageMappings() for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ReportPackageIxds"): filesource.select(pluginXbrlMethod(filesource, **_rptPkgIxdsOptions)) else: from arelle.CntlrCmdLine import filesourceEntrypointFiles entrypoints = filesourceEntrypointFiles(filesource) if entrypoints: # resolve an IXDS in entrypoints for pluginXbrlMethod in pluginClassMethods("ModelTestcaseVariation.ArchiveIxds"): pluginXbrlMethod(self, filesource,entrypoints) filesource.select(entrypoints[0].get("file", None) ) except Exception as err: self.modelXbrl.error("exception:" + type(err).__name__, _("Testcase variation validation exception: %(error)s, entry URL: %(instance)s"), modelXbrl=self.modelXbrl, instance=readMeFirstUri, error=err) continue # don't try to load this entry URL modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, filesource, _("validating"), base=baseForElement, errorCaptureLevel=errorCaptureLevel, ixdsTarget=modelTestcaseVariation.ixdsTarget) modelXbrl.isTestcaseVariation = True if modelXbrl.modelDocument is None: modelXbrl.info("arelle:notLoaded", _("Variation %(id)s %(name)s readMeFirst document not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) self.determineNotLoadedTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif resultIsVersioningReport or resultIsTaxonomyPackage: inputDTSes[dtsName] = modelXbrl elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl) self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif testcase.type == Type.REGISTRYTESTCASE: self.instValidator.validate(modelXbrl) # required to set up dimensions, etc self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id, modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest) self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) self.instValidator.close() modelXbrl.close() else: inputDTSes[dtsName].append(modelXbrl) # validate except for formulas _hasFormulae = modelXbrl.hasFormulae modelXbrl.hasFormulae = False try: for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"): pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation) self.instValidator.validate(modelXbrl, parameters) for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"): pluginXbrlMethod(self.modelXbrl, modelXbrl) except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase variation validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=(type(err) is not AssertionError)) modelXbrl.hasFormulae = _hasFormulae if resultIsVersioningReport and modelXbrl.modelDocument: versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl( modelTestcaseVariation.versioningReportUri, baseForElement) if os.path.exists(versReportFile): #validate existing modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report")) if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport) self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors) modelVersReport.close() elif len(inputDTSes) == 2: ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes( versReportFile, inputDTSes["from"], inputDTSes["to"]) modelTestcaseVariation.status = "generated" else: modelXbrl.error("arelle:notLoaded", _("Variation %(id)s %(name)s input DTSes not loaded, unable to generate versioning report: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) modelTestcaseVariation.status = "failed" for inputDTS in inputDTSes.values(): inputDTS.close() del inputDTSes # dereference elif resultIsTaxonomyPackage: self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif inputDTSes: # validate schema, linkbase, or instance modelXbrl = inputDTSes[None][0] expectedDataFiles = set(modelXbrl.modelManager.cntlr.webCache.normalizeUrl(uri, baseForElement) for d in modelTestcaseVariation.dataUris.values() for uri in d if not UrlUtil.isAbsolute(uri)) foundDataFiles = set() variationBase = os.path.dirname(baseForElement) for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters if dtsName: # named instance parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes) elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula for _inputDTS in inputDTS: for docUrl, doc in _inputDTS.urlDocs.items(): if docUrl.startswith(variationBase) and not doc.type == Type.INLINEXBRLDOCUMENTSET: if getattr(doc,"loadedFromXbrlFormula", False): # may have been sourced from xf file if docUrl.replace("-formula.xml", ".xf") in expectedDataFiles: docUrl = docUrl.replace("-formula.xml", ".xf") foundDataFiles.add(docUrl) if expectedDataFiles - foundDataFiles: modelXbrl.info("arelle:testcaseDataNotUsed", _("Variation %(id)s %(name)s data files not used: %(missingDataFiles)s"), modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id, missingDataFiles=", ".join(sorted(os.path.basename(f) for f in expectedDataFiles - foundDataFiles))) if foundDataFiles - expectedDataFiles: modelXbrl.info("arelle:testcaseDataUnexpected", _("Variation %(id)s %(name)s files not in variation data: %(unexpectedDataFiles)s"), modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id, unexpectedDataFiles=", ".join(sorted(os.path.basename(f) for f in foundDataFiles - expectedDataFiles))) if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable: try: RenderingEvaluator.init(modelXbrl) except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase RenderingEvaluator.init exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) modelXbrlHasFormulae = modelXbrl.hasFormulae if modelXbrlHasFormulae and self.modelXbrl.modelManager.formulaOptions.formulaAction != "none": try: # validate only formulae self.instValidator.parameters = parameters ValidateFormula.validate(self.instValidator) except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=(type(err) is not AssertionError)) if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset: for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"): pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri) infoset = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultInfosetUri, _("loading result infoset"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) if infoset.modelDocument is None: modelXbrl.error("arelle:notLoaded", _("Variation %(id)s %(name)s result infoset not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstance)) modelTestcaseVariation.status = "result infoset not loadable" else: # check infoset ValidateInfoset.validate(self.instValidator, modelXbrl, infoset) infoset.close() if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset: # diff (or generate) table infoset resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement) if not any(alternativeValidation(modelXbrl, resultTableUri) for alternativeValidation in pluginClassMethods("Validate.TableInfoset")): try: ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase table linkbase validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) self.instValidator.close() extraErrors = [] for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"): pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors) self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual): # if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost) self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters) self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors) if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance formulaOutputInstance = modelXbrl.formulaOutputInstance modelXbrl.formulaOutputInstance = None # prevent it from being closed now self.instValidator.close() compareIxResultInstance = (modelXbrl.modelDocument.type in (Type.INLINEXBRL, Type.INLINEXBRLDOCUMENTSET) and modelTestcaseVariation.resultXbrlInstanceUri is not None) if compareIxResultInstance: formulaOutputInstance = modelXbrl # compare modelXbrl to generated output instance errMsgPrefix = "ix" else: # delete input instances before formula output comparision for inputDTSlist in inputDTSes.values(): for inputDTS in inputDTSlist: inputDTS.close() del inputDTSes # dereference errMsgPrefix = "formula" if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument: _matchExpectedResultIDs = not modelXbrlHasFormulae # formula restuls have inconsistent IDs expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultXbrlInstanceUri, _("loading expected result XBRL instance"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) if expectedInstance.modelDocument is None: self.modelXbrl.error("{}:expectedResultNotLoaded".format(errMsgPrefix), _("Testcase \"%(name)s\" %(id)s expected result instance not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstanceUri), messageCodes=("formula:expectedResultNotLoaded","ix:expectedResultNotLoaded")) modelTestcaseVariation.status = "result not loadable" else: # compare facts for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.ExpectedInstance.Loaded"): pluginXbrlMethod(expectedInstance, formulaOutputInstance) if len(expectedInstance.facts) != len(formulaOutputInstance.facts): formulaOutputInstance.error("{}:resultFactCounts".format(errMsgPrefix), _("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"), modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts), expectedFacts=len(expectedInstance.facts), messageCodes=("formula:resultFactCounts","ix:resultFactCounts")) else: formulaOutputFootnotesRelSet = ModelRelationshipSet(formulaOutputInstance, "XBRL-footnotes") expectedFootnotesRelSet = ModelRelationshipSet(expectedInstance, "XBRL-footnotes") def factFootnotes(fact, footnotesRelSet): footnotes = {} footnoteRels = footnotesRelSet.fromModelObject(fact) if footnoteRels: # most process rels in same order between two instances, use labels to sort for i, footnoteRel in enumerate(sorted(footnoteRels, key=lambda r: (r.fromLabel,r.toLabel))): modelObject = footnoteRel.toModelObject if isinstance(modelObject, ModelResource): xml = collapseWhitespace(modelObject.viewText().strip()) footnotes["Footnote {}".format(i+1)] = xml #re.sub(r'\s+', ' ', collapseWhitespace(modelObject.stringValue)) elif isinstance(modelObject, ModelFact): footnotes["Footnoted fact {}".format(i+1)] = \ "{} context: {} value: {}".format( modelObject.qname, modelObject.contextID, collapseWhitespace(modelObject.value)) return footnotes for expectedInstanceFact in expectedInstance.facts: unmatchedFactsStack = [] formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=_matchExpectedResultIDs, matchLang=False) #formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=True, matchLang=True) if formulaOutputFact is None: if unmatchedFactsStack: # get missing nested tuple fact, if possible missingFact = unmatchedFactsStack[-1] else: missingFact = expectedInstanceFact # is it possible to show value mismatches? expectedFacts = formulaOutputInstance.factsByQname.get(missingFact.qname) if len(expectedFacts) == 1: formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix), _("Output missing expected fact %(fact)s, extracted value \"%(value1)s\", expected value \"%(value2)s\""), modelXbrl=missingFact, fact=missingFact.qname, value1=missingFact.xValue, value2=next(iter(expectedFacts)).xValue, messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing")) else: formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix), _("Output missing expected fact %(fact)s"), modelXbrl=missingFact, fact=missingFact.qname, messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing")) else: # compare footnotes expectedInstanceFactFootnotes = factFootnotes(expectedInstanceFact, expectedFootnotesRelSet) formulaOutputFactFootnotes = factFootnotes(formulaOutputFact, formulaOutputFootnotesRelSet) if (len(expectedInstanceFactFootnotes) != len(formulaOutputFactFootnotes) or set(expectedInstanceFactFootnotes.values()) != set(formulaOutputFactFootnotes.values())): formulaOutputInstance.error("{}:expectedFactFootnoteDifference".format(errMsgPrefix), _("Output expected fact %(fact)s expected footnotes %(footnotes1)s produced footnotes %(footnotes2)s"), modelXbrl=(formulaOutputFact,expectedInstanceFact), fact=expectedInstanceFact.qname, footnotes1=sorted(expectedInstanceFactFootnotes.items()), footnotes2=sorted(formulaOutputFactFootnotes.items()), messageCodes=("formula:expectedFactFootnoteDifference","ix:expectedFactFootnoteDifference")) # for debugging uncomment next line to save generated instance document # formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml") expectedInstance.close() del expectedInstance # dereference self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors) formulaOutputInstance.close() del formulaOutputInstance if compareIxResultInstance: for inputDTSlist in inputDTSes.values(): for inputDTS in inputDTSlist: inputDTS.close() del inputDTSes # dereference # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) _statusCounts = OrderedDict((("pass",0),("fail",0))) for tv in getattr(testcase, "testcaseVariations", ()): _statusCounts[tv.status] = _statusCounts.get(tv.status, 0) + 1 self.modelXbrl.info("arelle:testCaseResults", ", ".join("{}={}".format(k,c) for k, c in _statusCounts.items() if k)) self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def func_csv_data(xule_context, *args): """Read a csv file/url. Arguments: file_url (string or url) has_header (boolean) - determines if the first line of the csv file has headers type list (list) - list of xule types in the order of the columns of the csv file. This is optional. If not provided, then all the data will be treated as stirngs. as_dictionary (boolean) - return the row as a dictionary instead of a list. This is optional. """ file_url = args[0] has_headers = args[1] if len(args) < 2: raise XuleProcessingError(_("The csv-data() function requires at least 2 arguments (file url, has headers), found {} arguments.".format(len(args))), xule_context) if len(args) > 4: raise XuleProcessingError(_("The csv-data() function takes no more than 3 arguments (file url, has headers, column types, as dictionary), found {} arguments.".format(len(args))), xule_context) if file_url.type not in ('string', 'uri'): raise XuleProcessingError(_("The file url argument (1st argument) of the csv-dta() function must be a string or uri, found '{}'.".format(file_url.value)), xule_contet) if has_headers.type != 'bool': raise XuleProcessingError(_("The has headers argument (2nd argument) of the csv-data() function muset be a boolean, found '{}'.".format(has_headers.type)), xule_context) if len(args) >= 3: column_types = args[2] if column_types.type == 'none': ordered_cols = None elif column_types.type == 'list': ordered_cols = list() for col in column_types.value: if col.type != 'string': raise XuleProcessingError(_("The type list argument (3rd argument) of the csv-data() function must be a list of strings, found '{}'.".format(col.type)), xule_context) ordered_cols.append(col.value) else: raise XuleProcessingError(_("The type list argument (3rd argument) of the csv-data() fucntion must be list, found '{}'.".format(column_types.type)), xule_context) else: ordered_cols = None if len(args) == 4: if args[3].type != 'bool': raise XuleProcessingError(_("The as dictionary argument (4th argument) of the csv-data() function must be a boolean, found '{}'.".format(args[3].type)), xule_context) if args[3].value: return_row_type = 'dictionary' else: return_row_type = 'list' else: return_row_type = 'list' if return_row_type == 'dictionary' and not has_headers.value: raise XuleProcessingError(_("When the csv-data() function is returning the rows as dictionaries (4th argument), the has headers argument (2nd argument) must be true."), xule_context) result = list() result_shadow = list() from arelle import PackageManager mapped_file_url = PackageManager.mappedUrl(file_url.value) # Using the FileSource object in arelle. This will open the file and handle taxonomy package mappings. from arelle import FileSource file_source = FileSource.openFileSource(file_url.value, xule_context.global_context.cntlr) file = file_source.file(file_url.value, binary=True) # file is tuple of one item as a BytesIO stream. Since this is in bytes, it needs to be converted to text via a decoder. # Assuming the file is in utf-8. data_source = [x.decode('utf-8') for x in file[0].readlines()] # if mapped_file_url.startswith('http://') or mapped_file_url.startswith('https://'): # # if mapped_file_url.startswith('https://') and getattr(xule_context.global_context.options, 'noCertificateCheck', False): # try: # import ssl # context = ssl.create_default_context() # context.check_hostname = False # context.verify_mode = ssl.CERT_NONE # except ImportError: # context=None # else: # context = None # try: # data_source = urllib.request.urlopen(mapped_file_url, context=context).read().decode('utf-8').splitlines() # except urllib.error.HTTPError as he: # raise XuleProcessingError(_("Trying to open url '{}', got HTTP {} - {}, error".format(mapped_file_url, he.code, he.reason)), xule_context) # else: # try: # with open(mapped_file_url, 'r', newline='') as data_file: # data_source = data_file.readlines() # except FileNotFoundError: # raise XuleProcessingError(_("Trying to open file '{}', but file is not found.".format(mapped_file_url)), xule_context) import csv reader = csv.reader(data_source) first_line = True row_num = 0 for line in reader: row_num += 1 if first_line and has_headers.value: first_line = False #skip the headers line if return_row_type == 'dictionary': # Need to get the names from the first row column_names = [x for x in line] if len(column_names) != len(set(column_names)): raise XuleProcessingError(_("There are duplicate column names in the csv file. This is not allowed when return rows as dictionaries. File: {}".format(file_url.value)), xule_context) continue if return_row_type == 'list': result_line = list() result_line_shadow = list() else: #dictionary result_line = dict() result_line_shadow = dict() for col_num, item in enumerate(line): if ordered_cols is not None and col_num >= len(ordered_cols): raise XuleProcessingError(_("The nubmer of columns on row {} is greater than the number of column types provided in the third argument of the csv-data() function. File: {}".format(row_num, file_url.value)), xule_context) item_value = convert_file_data_item(item, ordered_cols[col_num] if ordered_cols is not None else None, xule_context) if return_row_type == 'list': result_line.append(item_value) result_line_shadow.append(item_value.value) else: #dictonary if col_num >= len(column_names): raise xule_context(_("The number of columns on row {} is greater than the number of headers in the csv file. File: {}".format(row_num, mappedUrl if mapped_file_url == file_url.value else file_url.value + ' --> ' + mapped_file_url)), xule_context) result_line[xv.XuleValue(xule_context, column_names[col_num], 'string')] = item_value result_line_shadow[column_names[col_num]] = item_value.value if return_row_type == 'list': result.append(xv.XuleValue(xule_context, tuple(result_line), 'list', shadow_collection=tuple(result_line_shadow))) result_shadow.append(result_line_shadow) else: #dictionary result.append(xv.XuleValue(xule_context, frozenset(result_line.items()), 'dictionary', shadow_collection=frozenset(result_line_shadow.items()))) result_shadow.append(frozenset(result_line_shadow.items())) return xv.XuleValue(xule_context, tuple(result), 'list', shadow_collection=tuple(result_shadow))
def validateTestcase(self, testcase): self.modelXbrl.info("info", "Testcase", modelDocument=testcase) self.modelXbrl.viewModelObject(testcase.objectId()) if testcase.type in (Type.TESTCASESINDEX, Type.REGISTRY): for doc in sorted(testcase.referencesDocument.keys(), key=lambda doc: doc.uri): self.validateTestcase(doc) # testcases doc's are sorted by their uri (file names), e.g., for formula elif hasattr(testcase, "testcaseVariations"): for modelTestcaseVariation in testcase.testcaseVariations: # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) # is this a versioning report? resultIsVersioningReport = modelTestcaseVariation.resultIsVersioningReport resultIsXbrlInstance = modelTestcaseVariation.resultIsXbrlInstance resultIsTaxonomyPackage = modelTestcaseVariation.resultIsTaxonomyPackage formulaOutputInstance = None inputDTSes = defaultdict(list) baseForElement = testcase.baseForElement(modelTestcaseVariation) # try to load instance document self.modelXbrl.info("info", _("Variation %(id)s %(name)s: %(expected)s - %(description)s"), modelObject=modelTestcaseVariation, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, expected=modelTestcaseVariation.expected, description=modelTestcaseVariation.description) if self.modelXbrl.modelManager.formulaOptions.testcaseResultsCaptureWarnings: errorCaptureLevel = logging._checkLevel("WARNING") else: errorCaptureLevel = modelTestcaseVariation.severityLevel # default is INCONSISTENCY parameters = modelTestcaseVariation.parameters.copy() for readMeFirstUri in modelTestcaseVariation.readMeFirstUris: if isinstance(readMeFirstUri,tuple): # dtsName is for formula instances, but is from/to dts if versioning dtsName, readMeFirstUri = readMeFirstUri elif resultIsVersioningReport: if inputDTSes: dtsName = "to" else: dtsName = "from" else: dtsName = None if resultIsVersioningReport and dtsName: # build multi-schemaRef containing document if dtsName in inputDTSes: dtsName = inputDTSes[dtsName] else: modelXbrl = ModelXbrl.create(self.modelXbrl.modelManager, Type.DTSENTRIES, self.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(readMeFirstUri[:-4] + ".dts", baseForElement), isEntry=True, errorCaptureLevel=errorCaptureLevel) DTSdoc = modelXbrl.modelDocument DTSdoc.inDTS = True doc = modelDocumentLoad(modelXbrl, readMeFirstUri, base=baseForElement) if doc is not None: DTSdoc.referencesDocument[doc] = ModelDocumentReference("import", DTSdoc.xmlRootElement) #fake import doc.inDTS = True elif resultIsTaxonomyPackage: from arelle import PackageManager, PrototypeInstanceObject dtsName = readMeFirstUri modelXbrl = PrototypeInstanceObject.XbrlPrototype(self.modelXbrl.modelManager, readMeFirstUri) PackageManager.packageInfo(self.modelXbrl.modelManager.cntlr, readMeFirstUri, reload=True, errors=modelXbrl.errors) else: # not a multi-schemaRef versioning report if self.useFileSource.isArchive: modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, readMeFirstUri, _("validating"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) else: # need own file source, may need instance discovery filesource = FileSource.FileSource(readMeFirstUri, self.modelXbrl.modelManager.cntlr) if filesource and not filesource.selection and filesource.isArchive: for _archiveFile in filesource.dir or (): # find instance document in archive filesource.select(_archiveFile) if ModelDocument.Type.identify(filesource, filesource.url) in (ModelDocument.Type.INSTANCE, ModelDocument.Type.INLINEXBRL): break # use this selection modelXbrl = ModelXbrl.load(self.modelXbrl.modelManager, filesource, _("validating"), base=baseForElement, errorCaptureLevel=errorCaptureLevel) modelXbrl.isTestcaseVariation = True if modelXbrl.modelDocument is None: modelXbrl.error("arelle:notLoaded", _("Variation %(id)s %(name)s readMeFirst document not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) self.determineNotLoadedTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif resultIsVersioningReport or resultIsTaxonomyPackage: inputDTSes[dtsName] = modelXbrl elif modelXbrl.modelDocument.type == Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelXbrl) self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif testcase.type == Type.REGISTRYTESTCASE: self.instValidator.validate(modelXbrl) # required to set up dimensions, etc self.instValidator.executeCallTest(modelXbrl, modelTestcaseVariation.id, modelTestcaseVariation.cfcnCall, modelTestcaseVariation.cfcnTest) self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) self.instValidator.close() modelXbrl.close() else: inputDTSes[dtsName].append(modelXbrl) # validate except for formulas _hasFormulae = modelXbrl.hasFormulae modelXbrl.hasFormulae = False try: for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Loaded"): pluginXbrlMethod(self.modelXbrl, modelXbrl, modelTestcaseVariation) self.instValidator.validate(modelXbrl, parameters) for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Xbrl.Validated"): pluginXbrlMethod(self.modelXbrl, modelXbrl) except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase variation validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) modelXbrl.hasFormulae = _hasFormulae if resultIsVersioningReport and modelXbrl.modelDocument: versReportFile = modelXbrl.modelManager.cntlr.webCache.normalizeUrl( modelTestcaseVariation.versioningReportUri, baseForElement) if os.path.exists(versReportFile): #validate existing modelVersReport = ModelXbrl.load(self.modelXbrl.modelManager, versReportFile, _("validating existing version report")) if modelVersReport and modelVersReport.modelDocument and modelVersReport.modelDocument.type == Type.VERSIONINGREPORT: ValidateVersReport.ValidateVersReport(self.modelXbrl).validate(modelVersReport) self.determineTestStatus(modelTestcaseVariation, modelVersReport.errors) modelVersReport.close() elif len(inputDTSes) == 2: ModelVersReport.ModelVersReport(self.modelXbrl).diffDTSes( versReportFile, inputDTSes["from"], inputDTSes["to"]) modelTestcaseVariation.status = "generated" else: modelXbrl.error("arelle:notLoaded", _("Variation %(id)s %(name)s input DTSes not loaded, unable to generate versioning report: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(readMeFirstUri)) modelTestcaseVariation.status = "failed" for inputDTS in inputDTSes.values(): inputDTS.close() del inputDTSes # dereference elif resultIsTaxonomyPackage: self.determineTestStatus(modelTestcaseVariation, modelXbrl.errors) modelXbrl.close() elif inputDTSes: # validate schema, linkbase, or instance modelXbrl = inputDTSes[None][0] expectedDataFiles = set(modelXbrl.modelManager.cntlr.webCache.normalizeUrl(uri, baseForElement) for d in modelTestcaseVariation.dataUris.values() for uri in d if not UrlUtil.isAbsolute(uri)) foundDataFiles = set() variationBase = os.path.dirname(baseForElement) for dtsName, inputDTS in inputDTSes.items(): # input instances are also parameters if dtsName: # named instance parameters[dtsName] = (None, inputDTS) #inputDTS is a list of modelXbrl's (instance DTSes) elif len(inputDTS) > 1: # standard-input-instance with multiple instance documents parameters[XbrlConst.qnStandardInputInstance] = (None, inputDTS) # allow error detection in validateFormula for _inputDTS in inputDTS: for docUrl, doc in _inputDTS.urlDocs.items(): if docUrl.startswith(variationBase) and not doc.type == Type.INLINEXBRLDOCUMENTSET: if getattr(doc,"loadedFromXbrlFormula", False): # may have been sourced from xf file if docUrl.replace("-formula.xml", ".xf") in expectedDataFiles: docUrl = docUrl.replace("-formula.xml", ".xf") foundDataFiles.add(docUrl) if expectedDataFiles - foundDataFiles: modelXbrl.info("arelle:testcaseDataNotUsed", _("Variation %(id)s %(name)s data files not used: %(missingDataFiles)s"), modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id, missingDataFiles=", ".join(sorted(os.path.basename(f) for f in expectedDataFiles - foundDataFiles))) if foundDataFiles - expectedDataFiles: modelXbrl.info("arelle:testcaseDataUnexpected", _("Variation %(id)s %(name)s files not in variation data: %(unexpectedDataFiles)s"), modelObject=modelTestcaseVariation, name=modelTestcaseVariation.name, id=modelTestcaseVariation.id, unexpectedDataFiles=", ".join(sorted(os.path.basename(f) for f in foundDataFiles - expectedDataFiles))) if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable: try: RenderingEvaluator.init(modelXbrl) except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase RenderingEvaluator.init exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) modelXbrlHasFormulae = modelXbrl.hasFormulae if modelXbrlHasFormulae: try: # validate only formulae self.instValidator.parameters = parameters ValidateFormula.validate(self.instValidator) except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase formula variation validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) if modelTestcaseVariation.resultIsInfoset and self.modelXbrl.modelManager.validateInfoset: for pluginXbrlMethod in pluginClassMethods("Validate.Infoset"): pluginXbrlMethod(modelXbrl, modelTestcaseVariation.resultInfosetUri) infoset = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultInfosetUri, _("loading result infoset"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) if infoset.modelDocument is None: modelXbrl.error("arelle:notLoaded", _("Variation %(id)s %(name)s result infoset not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstance)) modelTestcaseVariation.status = "result infoset not loadable" else: # check infoset ValidateInfoset.validate(self.instValidator, modelXbrl, infoset) infoset.close() if modelXbrl.hasTableRendering or modelTestcaseVariation.resultIsTable: # and self.modelXbrl.modelManager.validateInfoset: # diff (or generate) table infoset resultTableUri = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(modelTestcaseVariation.resultTableUri, baseForElement) if not any(alternativeValidation(modelXbrl, resultTableUri) for alternativeValidation in pluginClassMethods("Validate.TableInfoset")): try: ViewFileRenderedGrid.viewRenderedGrid(modelXbrl, resultTableUri, diffToFile=True) # false to save infoset files except Exception as err: modelXbrl.error("exception:" + type(err).__name__, _("Testcase table linkbase validation exception: %(error)s, instance: %(instance)s"), modelXbrl=modelXbrl, instance=modelXbrl.modelDocument.basename, error=err, exc_info=True) self.instValidator.close() extraErrors = [] for pluginXbrlMethod in pluginClassMethods("TestcaseVariation.Validated"): pluginXbrlMethod(self.modelXbrl, modelXbrl, extraErrors) self.determineTestStatus(modelTestcaseVariation, [e for inputDTSlist in inputDTSes.values() for inputDTS in inputDTSlist for e in inputDTS.errors] + extraErrors) # include infoset errors in status if modelXbrl.formulaOutputInstance and self.noErrorCodes(modelTestcaseVariation.actual): # if an output instance is created, and no string error codes, ignoring dict of assertion results, validate it modelXbrl.formulaOutputInstance.hasFormulae = False # block formulae on output instance (so assertion of input is not lost) self.instValidator.validate(modelXbrl.formulaOutputInstance, modelTestcaseVariation.parameters) self.determineTestStatus(modelTestcaseVariation, modelXbrl.formulaOutputInstance.errors) if self.noErrorCodes(modelTestcaseVariation.actual): # if still 'clean' pass it forward for comparison to expected result instance formulaOutputInstance = modelXbrl.formulaOutputInstance modelXbrl.formulaOutputInstance = None # prevent it from being closed now self.instValidator.close() compareIxResultInstance = (modelXbrl.modelDocument.type in (Type.INLINEXBRL, Type.INLINEXBRLDOCUMENTSET) and modelTestcaseVariation.resultXbrlInstanceUri is not None) if compareIxResultInstance: formulaOutputInstance = modelXbrl # compare modelXbrl to generated output instance errMsgPrefix = "ix" else: # delete input instances before formula output comparision for inputDTSlist in inputDTSes.values(): for inputDTS in inputDTSlist: inputDTS.close() del inputDTSes # dereference errMsgPrefix = "formula" if resultIsXbrlInstance and formulaOutputInstance and formulaOutputInstance.modelDocument: _matchExpectedResultIDs = not modelXbrlHasFormulae # formula restuls have inconsistent IDs expectedInstance = ModelXbrl.load(self.modelXbrl.modelManager, modelTestcaseVariation.resultXbrlInstanceUri, _("loading expected result XBRL instance"), base=baseForElement, useFileSource=self.useFileSource, errorCaptureLevel=errorCaptureLevel) if expectedInstance.modelDocument is None: self.modelXbrl.error("{}:expectedResultNotLoaded".format(errMsgPrefix), _("Testcase \"%(name)s\" %(id)s expected result instance not loaded: %(file)s"), modelXbrl=testcase, id=modelTestcaseVariation.id, name=modelTestcaseVariation.name, file=os.path.basename(modelTestcaseVariation.resultXbrlInstanceUri), messageCodes=("formula:expectedResultNotLoaded","ix:expectedResultNotLoaded")) modelTestcaseVariation.status = "result not loadable" else: # compare facts if len(expectedInstance.facts) != len(formulaOutputInstance.facts): formulaOutputInstance.error("{}:resultFactCounts".format(errMsgPrefix), _("Formula output %(countFacts)s facts, expected %(expectedFacts)s facts"), modelXbrl=modelXbrl, countFacts=len(formulaOutputInstance.facts), expectedFacts=len(expectedInstance.facts), messageCodes=("formula:resultFactCounts","ix:resultFactCounts")) else: formulaOutputFootnotesRelSet = ModelRelationshipSet(formulaOutputInstance, "XBRL-footnotes") expectedFootnotesRelSet = ModelRelationshipSet(expectedInstance, "XBRL-footnotes") def factFootnotes(fact, footnotesRelSet): footnotes = {} footnoteRels = footnotesRelSet.fromModelObject(fact) if footnoteRels: # most process rels in same order between two instances, use labels to sort for i, footnoteRel in enumerate(sorted(footnoteRels, key=lambda r: (r.fromLabel,r.toLabel))): modelObject = footnoteRel.toModelObject if isinstance(modelObject, ModelResource): xml = modelObject.viewText().strip() footnotes["Footnote {}".format(i+1)] = xml #re.sub(r'\s+', ' ', collapseWhitespace(modelObject.stringValue)) elif isinstance(modelObject, ModelFact): footnotes["Footnoted fact {}".format(i+1)] = \ "{} context: {} value: {}".format( modelObject.qname, modelObject.contextID, collapseWhitespace(modelObject.value)) return footnotes for expectedInstanceFact in expectedInstance.facts: unmatchedFactsStack = [] formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=_matchExpectedResultIDs, matchLang=False) #formulaOutputFact = formulaOutputInstance.matchFact(expectedInstanceFact, unmatchedFactsStack, deemP0inf=True, matchId=True, matchLang=True) if formulaOutputFact is None: if unmatchedFactsStack: # get missing nested tuple fact, if possible missingFact = unmatchedFactsStack[-1] else: missingFact = expectedInstanceFact formulaOutputInstance.error("{}:expectedFactMissing".format(errMsgPrefix), _("Output missing expected fact %(fact)s"), modelXbrl=missingFact, fact=missingFact.qname, messageCodes=("formula:expectedFactMissing","ix:expectedFactMissing")) else: # compare footnotes expectedInstanceFactFootnotes = factFootnotes(expectedInstanceFact, expectedFootnotesRelSet) formulaOutputFactFootnotes = factFootnotes(formulaOutputFact, formulaOutputFootnotesRelSet) if (len(expectedInstanceFactFootnotes) != len(formulaOutputFactFootnotes) or set(expectedInstanceFactFootnotes.values()) != set(formulaOutputFactFootnotes.values())): formulaOutputInstance.error("{}:expectedFactFootnoteDifference".format(errMsgPrefix), _("Output expected fact %(fact)s expected footnotes %(footnotes1)s produced footnotes %(footnotes2)s"), modelXbrl=(formulaOutputFact,expectedInstanceFact), fact=expectedInstanceFact.qname, footnotes1=sorted(expectedInstanceFactFootnotes.items()), footnotes2=sorted(formulaOutputFactFootnotes.items()), messageCodes=("formula:expectedFactFootnoteDifference","ix:expectedFactFootnoteDifference")) # for debugging uncomment next line to save generated instance document # formulaOutputInstance.saveInstance(r"c:\temp\test-out-inst.xml") expectedInstance.close() del expectedInstance # dereference self.determineTestStatus(modelTestcaseVariation, formulaOutputInstance.errors) formulaOutputInstance.close() del formulaOutputInstance if compareIxResultInstance: for inputDTSlist in inputDTSes.values(): for inputDTS in inputDTSlist: inputDTS.close() del inputDTSes # dereference # update ui thread via modelManager (running in background here) self.modelXbrl.modelManager.viewModelObject(self.modelXbrl, modelTestcaseVariation.objectId()) _statusCounts = OrderedDict((("pass",0),("fail",0))) for tv in getattr(testcase, "testcaseVariations", ()): _statusCounts[tv.status] = _statusCounts.get(tv.status, 0) + 1 self.modelXbrl.info("arelle:testCaseResults", ", ".join("{}={}".format(k,c) for k, c in _statusCounts.items() if k)) self.modelXbrl.modelManager.showStatus(_("ready"), 2000)
def open(self): if not self.isOpen: if (self.isZip or self.isTarGz or self.isEis or self.isXfd or self.isRss or self.isInstalledTaxonomyPackage) and self.cntlr: self.basefile = self.cntlr.webCache.getfilename(self.url) else: self.basefile = self.url self.baseurl = self.url # url gets changed by selection if not self.basefile: return # an error should have been logged if self.isZip: try: self.fs = zipfile.ZipFile(openFileStream(self.cntlr, self.basefile, 'rb'), mode="r") self.isOpen = True except EnvironmentError as err: self.logError(err) pass elif self.isTarGz: try: self.fs = tarfile.open(self.basefile, "r:gz") self.isOpen = True except EnvironmentError as err: self.logError(err) pass elif self.isEis: # check first line of file buf = b'' try: file = open(self.basefile, 'rb') more = True while more: l = file.read(8) if len(l) < 8: break if len(buf) == 0 and l.startswith(b"<?xml "): # not compressed buf = l + file.read() # not compressed break compressedBytes = file.read( struct.unpack(">L", l[0:4])[0]) if len(compressedBytes) <= 0: break buf += zlib.decompress(compressedBytes) file.close() except EnvironmentError as err: self.logError(err) pass #uncomment to save for debugging #with open("c:/temp/test.xml", "wb") as f: # f.write(buf) if buf.startswith(b"<?xml "): try: # must strip encoding str = buf.decode(XmlUtil.encoding(buf)) endEncoding = str.index("?>", 0, 128) if endEncoding > 0: str = str[endEncoding+2:] file = io.StringIO(initial_value=str) parser = etree.XMLParser(recover=True, huge_tree=True) self.eisDocument = etree.parse(file, parser=parser) file.close() self.isOpen = True except EnvironmentError as err: self.logError(err) return # provide error message later except etree.LxmlError as err: self.logError(err) return # provide error message later elif self.isXfd: # check first line of file file = open(self.basefile, 'rb') firstline = file.readline() if firstline.startswith(b"application/x-xfdl;content-encoding=\"asc-gzip\""): # file has been gzipped base64input = file.read(-1) file.close(); file = None; fb = base64.b64decode(base64input) ungzippedBytes = b"" totalLenUncompr = 0 i = 0 while i < len(fb): lenCompr = fb[i + 0] * 256 + fb[i + 1] lenUncomp = fb[i + 2] * 256 + fb[i + 3] lenRead = 0 totalLenUncompr += lenUncomp gzchunk = (bytes((31,139,8,0)) + fb[i:i+lenCompr]) try: with gzip.GzipFile(fileobj=io.BytesIO(gzchunk)) as gf: while True: readSize = min(16384, lenUncomp - lenRead) readBytes = gf.read(size=readSize) lenRead += len(readBytes) ungzippedBytes += readBytes if len(readBytes) == 0 or (lenUncomp - lenRead) <= 0: break except IOError as err: pass # provide error message later i += lenCompr + 4 #for learning the content of xfd file, uncomment this: #with open("c:\\temp\\test.xml", "wb") as fh: # fh.write(ungzippedBytes) file = io.StringIO(initial_value=ungzippedBytes.decode("utf-8")) else: # position to start of file file.seek(0,io.SEEK_SET) try: self.xfdDocument = etree.parse(file) file.close() self.isOpen = True except EnvironmentError as err: self.logError(err) return # provide error message later except etree.LxmlError as err: self.logError(err) return # provide error message later elif self.isRss: try: self.rssDocument = etree.parse(self.basefile) self.isOpen = True except EnvironmentError as err: self.logError(err) return # provide error message later except etree.LxmlError as err: self.logError(err) return # provide error message later elif self.isInstalledTaxonomyPackage: self.isOpen = True # load mappings try: metadataFiles = self.taxonomyPackageMetadataFiles if len(metadataFiles) != 1: raise IOError(_("Taxonomy package must contain one and only one metadata file: {0}.") .format(', '.join(metadataFiles))) # HF: this won't work, see DialogOpenArchive for correct code # not sure if it is used taxonomyPackage = PackageManager.parsePackage(self.cntlr, self.url) fileSourceDir = os.path.dirname(self.baseurl) + os.sep self.mappedPaths = \ dict((prefix, remapping if isHttpUrl(remapping) else (fileSourceDir + remapping.replace("/", os.sep))) for prefix, remapping in taxonomyPackage["remappings"].items()) except EnvironmentError as err: self.logError(err) return # provide error message later
def run(self, options, sourceZipStream=None): """Process command line arguments or web service request, such as to load and validate an XBRL document, or start web server. When a web server has been requested, this method may be called multiple times, once for each web service (REST) request that requires processing. Otherwise (when called for a command line request) this method is called only once for the command line arguments request. :param options: OptionParser options from parse_args of main argv arguments (when called from command line) or corresponding arguments from web service (REST) request. :type options: optparse.Values """ if options.showOptions: # debug options for optName, optValue in sorted(options.__dict__.items(), key=lambda optItem: optItem[0]): self.addToLog("Option {0}={1}".format(optName, optValue), messageCode="info") self.addToLog("sys.argv {0}".format(sys.argv), messageCode="info") if options.uiLang: # set current UI Lang (but not config setting) self.setUiLanguage(options.uiLang) if options.proxy: if options.proxy != "show": proxySettings = proxyTuple(options.proxy) self.webCache.resetProxies(proxySettings) self.config["proxySettings"] = proxySettings self.saveConfig() self.addToLog(_("Proxy configuration has been set."), messageCode="info") useOsProxy, urlAddr, urlPort, user, password = self.config.get("proxySettings", proxyTuple("none")) if useOsProxy: self.addToLog(_("Proxy configured to use {0}.").format( _('Microsoft Windows Internet Settings') if sys.platform.startswith("win") else (_('Mac OS X System Configuration') if sys.platform in ("darwin", "macos") else _('environment variables'))), messageCode="info") elif urlAddr: self.addToLog(_("Proxy setting: http://{0}{1}{2}{3}{4}").format( user if user else "", ":****" if password else "", "@" if (user or password) else "", urlAddr, ":{0}".format(urlPort) if urlPort else ""), messageCode="info") else: self.addToLog(_("Proxy is disabled."), messageCode="info") if options.plugins: from arelle import PluginManager resetPlugins = False savePluginChanges = True showPluginModules = False for pluginCmd in options.plugins.split('|'): cmd = pluginCmd.strip() if cmd == "show": showPluginModules = True elif cmd == "temp": savePluginChanges = False elif cmd.startswith("+"): moduleInfo = PluginManager.addPluginModule(cmd[1:]) if moduleInfo: self.addToLog(_("Addition of plug-in {0} successful.").format(moduleInfo.get("name")), messageCode="info", file=moduleInfo.get("moduleURL")) resetPlugins = True else: self.addToLog(_("Unable to load plug-in."), messageCode="info", file=cmd[1:]) elif cmd.startswith("~"): if PluginManager.reloadPluginModule(cmd[1:]): self.addToLog(_("Reload of plug-in successful."), messageCode="info", file=cmd[1:]) resetPlugins = True else: self.addToLog(_("Unable to reload plug-in."), messageCode="info", file=cmd[1:]) elif cmd.startswith("-"): if PluginManager.removePluginModule(cmd[1:]): self.addToLog(_("Deletion of plug-in successful."), messageCode="info", file=cmd[1:]) resetPlugins = True else: self.addToLog(_("Unable to delete plug-in."), messageCode="info", file=cmd[1:]) else: # assume it is a module or package savePluginChanges = False moduleInfo = PluginManager.addPluginModule(cmd) if moduleInfo: self.addToLog(_("Activation of plug-in {0} successful.").format(moduleInfo.get("name")), messageCode="info", file=moduleInfo.get("moduleURL")) resetPlugins = True else: self.addToLog(_("Unable to load {0} as a plug-in or {0} is not recognized as a command. ").format(cmd), messageCode="info", file=cmd) if resetPlugins: PluginManager.reset() if savePluginChanges: PluginManager.save(self) if showPluginModules: self.addToLog(_("Plug-in modules:"), messageCode="info") for i, moduleItem in enumerate(sorted(PluginManager.pluginConfig.get("modules", {}).items())): moduleInfo = moduleItem[1] self.addToLog(_("Plug-in: {0}; author: {1}; version: {2}; status: {3}; date: {4}; description: {5}; license {6}.").format( moduleItem[0], moduleInfo.get("author"), moduleInfo.get("version"), moduleInfo.get("status"), moduleInfo.get("fileDate"), moduleInfo.get("description"), moduleInfo.get("license")), messageCode="info", file=moduleInfo.get("moduleURL")) if options.packages: from arelle import PackageManager savePackagesChanges = True showPackages = False for packageCmd in options.packages.split('|'): cmd = packageCmd.strip() if cmd == "show": showPackages = True elif cmd == "temp": savePackagesChanges = False elif cmd.startswith("+"): packageInfo = PackageManager.addPackage(cmd[1:]) if packageInfo: self.addToLog(_("Addition of package {0} successful.").format(packageInfo.get("name")), messageCode="info", file=packageInfo.get("URL")) else: self.addToLog(_("Unable to load plug-in."), messageCode="info", file=cmd[1:]) elif cmd.startswith("~"): if PackageManager.reloadPackageModule(cmd[1:]): self.addToLog(_("Reload of package successful."), messageCode="info", file=cmd[1:]) else: self.addToLog(_("Unable to reload package."), messageCode="info", file=cmd[1:]) elif cmd.startswith("-"): if PackageManager.removePackageModule(cmd[1:]): self.addToLog(_("Deletion of package successful."), messageCode="info", file=cmd[1:]) else: self.addToLog(_("Unable to delete package."), messageCode="info", file=cmd[1:]) else: # assume it is a module or package savePackagesChanges = False packageInfo = PackageManager.addPackage(cmd) if packageInfo: self.addToLog(_("Activation of package {0} successful.").format(packageInfo.get("name")), messageCode="info", file=packageInfo.get("URL")) resetPlugins = True else: self.addToLog(_("Unable to load {0} as a package or {0} is not recognized as a command. ").format(cmd), messageCode="info", file=cmd) if savePackagesChanges: PackageManager.save(self) if showPackages: self.addToLog(_("Taxonomy packages:"), messageCode="info") for packageInfo in PackageManager.orderedPackagesConfig()["packages"]: self.addToLog(_("Package: {0}; version: {1}; status: {2}; date: {3}; description: {4}.").format( packageInfo.get("name"), packageInfo.get("version"), packageInfo.get("status"), packageInfo.get("fileDate"), packageInfo.get("description")), messageCode="info", file=packageInfo.get("URL")) # run utility command line options that don't depend on entrypoint Files hasUtilityPlugin = False for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Utility.Run"): hasUtilityPlugin = True pluginXbrlMethod(self, options) # if no entrypointFile is applicable, quit now if options.proxy or options.plugins or hasUtilityPlugin: if not options.entrypointFile: return True # success self.username = options.username self.password = options.password self.entrypointFile = options.entrypointFile if self.entrypointFile: filesource = FileSource.openFileSource(self.entrypointFile, self, sourceZipStream) else: filesource = None if options.validateEFM: if options.disclosureSystemName: self.addToLog(_("both --efm and --disclosureSystem validation are requested, proceeding with --efm only"), messageCode="info", file=self.entrypointFile) self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select("efm") elif options.disclosureSystemName: self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select(options.disclosureSystemName) elif options.validateHMRC: self.modelManager.validateDisclosureSystem = True self.modelManager.disclosureSystem.select("hmrc") else: self.modelManager.disclosureSystem.select(None) # just load ordinary mappings self.modelManager.validateDisclosureSystem = False if options.utrUrl: # override disclosureSystem utrUrl self.modelManager.disclosureSystem.utrUrl = options.utrUrl # can be set now because the utr is first loaded at validation time # disclosure system sets logging filters, override disclosure filters, if specified by command line if options.logLevelFilter: self.setLogLevelFilter(options.logLevelFilter) if options.logCodeFilter: self.setLogCodeFilter(options.logCodeFilter) if options.calcDecimals: if options.calcPrecision: self.addToLog(_("both --calcDecimals and --calcPrecision validation are requested, proceeding with --calcDecimals only"), messageCode="info", file=self.entrypointFile) self.modelManager.validateInferDecimals = True self.modelManager.validateCalcLB = True elif options.calcPrecision: self.modelManager.validateInferDecimals = False self.modelManager.validateCalcLB = True if options.utrValidate: self.modelManager.validateUtr = True if options.infosetValidate: self.modelManager.validateInfoset = True if options.abortOnMajorError: self.modelManager.abortOnMajorError = True if options.collectProfileStats: self.modelManager.collectProfileStats = True if options.internetConnectivity == "offline": self.webCache.workOffline = True elif options.internetConnectivity == "online": self.webCache.workOffline = False if options.internetTimeout is not None: self.webCache.timeout = (options.internetTimeout or None) # use None if zero specified to disable timeout fo = FormulaOptions() if options.parameters: parameterSeparator = (options.parameterSeparator or ',') fo.parameterValues = dict(((qname(key, noPrefixIsNoNamespace=True),(None,value)) for param in options.parameters.split(parameterSeparator) for key,sep,value in (param.partition('='),) ) ) if options.formulaParamExprResult: fo.traceParameterExpressionResult = True if options.formulaParamInputValue: fo.traceParameterInputValue = True if options.formulaCallExprSource: fo.traceCallExpressionSource = True if options.formulaCallExprCode: fo.traceCallExpressionCode = True if options.formulaCallExprEval: fo.traceCallExpressionEvaluation = True if options.formulaCallExprResult: fo.traceCallExpressionResult = True if options.formulaVarSetExprEval: fo.traceVariableSetExpressionEvaluation = True if options.formulaVarSetExprResult: fo.traceVariableSetExpressionResult = True if options.formulaAsserResultCounts: fo.traceAssertionResultCounts = True if options.formulaFormulaRules: fo.traceFormulaRules = True if options.formulaVarsOrder: fo.traceVariablesOrder = True if options.formulaVarExpressionSource: fo.traceVariableExpressionSource = True if options.formulaVarExpressionCode: fo.traceVariableExpressionCode = True if options.formulaVarExpressionEvaluation: fo.traceVariableExpressionEvaluation = True if options.formulaVarExpressionResult: fo.traceVariableExpressionResult = True if options.timeVariableSetEvaluation: fo.timeVariableSetEvaluation = True if options.formulaVarFilterWinnowing: fo.traceVariableFilterWinnowing = True if options.formulaVarFiltersResult: fo.traceVariableFiltersResult = True if options.formulaVarFiltersResult: fo.traceVariableFiltersResult = True self.modelManager.formulaOptions = fo timeNow = XmlUtil.dateunionValue(datetime.datetime.now()) firstStartedAt = startedAt = time.time() modelDiffReport = None success = True modelXbrl = None try: if filesource: modelXbrl = self.modelManager.load(filesource, _("views loading")) except ModelDocument.LoadingException: pass except Exception as err: self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format( err, traceback.format_tb(sys.exc_info()[2]))) success = False # loading errors, don't attempt to utilize loaded DTS if modelXbrl and modelXbrl.modelDocument: loadTime = time.time() - startedAt modelXbrl.profileStat(_("load"), loadTime) self.addToLog(format_string(self.modelManager.locale, _("loaded in %.2f secs at %s"), (loadTime, timeNow)), messageCode="info", file=self.entrypointFile) if options.importFiles: for importFile in options.importFiles.split("|"): fileName = importFile.strip() if sourceZipStream is not None and not (fileName.startswith('http://') or os.path.isabs(fileName)): fileName = os.path.dirname(modelXbrl.uri) + os.sep + fileName # make relative to sourceZipStream ModelDocument.load(modelXbrl, fileName) loadTime = time.time() - startedAt self.addToLog(format_string(self.modelManager.locale, _("import in %.2f secs at %s"), (loadTime, timeNow)), messageCode="info", file=importFile) modelXbrl.profileStat(_("import"), loadTime) if modelXbrl.errors: success = False # loading errors, don't attempt to utilize loaded DTS if modelXbrl.modelDocument.type in ModelDocument.Type.TESTCASETYPES: for pluginXbrlMethod in pluginClassMethods("Testcases.Start"): pluginXbrlMethod(self, options, modelXbrl) else: # not a test case, probably instance or DTS for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Loaded"): pluginXbrlMethod(self, options, modelXbrl) else: success = False if success and options.diffFile and options.versReportFile: try: diffFilesource = FileSource.FileSource(options.diffFile,self) startedAt = time.time() modelXbrl2 = self.modelManager.load(diffFilesource, _("views loading")) if modelXbrl2.errors: if not options.keepOpen: modelXbrl2.close() success = False else: loadTime = time.time() - startedAt modelXbrl.profileStat(_("load"), loadTime) self.addToLog(format_string(self.modelManager.locale, _("diff comparison DTS loaded in %.2f secs"), loadTime), messageCode="info", file=self.entrypointFile) startedAt = time.time() modelDiffReport = self.modelManager.compareDTSes(options.versReportFile) diffTime = time.time() - startedAt modelXbrl.profileStat(_("diff"), diffTime) self.addToLog(format_string(self.modelManager.locale, _("compared in %.2f secs"), diffTime), messageCode="info", file=self.entrypointFile) except ModelDocument.LoadingException: success = False except Exception as err: success = False self.addToLog(_("[Exception] Failed to doad diff file: \n{0} \n{1}").format( err, traceback.format_tb(sys.exc_info()[2]))) if success: try: modelXbrl = self.modelManager.modelXbrl hasFormulae = modelXbrl.hasFormulae if options.validate: startedAt = time.time() if options.formulaAction: # don't automatically run formulas modelXbrl.hasFormulae = False self.modelManager.validate() if options.formulaAction: # restore setting modelXbrl.hasFormulae = hasFormulae self.addToLog(format_string(self.modelManager.locale, _("validated in %.2f secs"), time.time() - startedAt), messageCode="info", file=self.entrypointFile) if options.formulaAction in ("validate", "run"): # do nothing here if "none" from arelle import ValidateXbrlDimensions, ValidateFormula startedAt = time.time() if not options.validate: ValidateXbrlDimensions.loadDimensionDefaults(modelXbrl) # setup fresh parameters from formula optoins modelXbrl.parameters = fo.typedParameters() ValidateFormula.validate(modelXbrl, compileOnly=(options.formulaAction != "run")) self.addToLog(format_string(self.modelManager.locale, _("formula validation and execution in %.2f secs") if options.formulaAction == "run" else _("formula validation only in %.2f secs"), time.time() - startedAt), messageCode="info", file=self.entrypointFile) if options.testReport: ViewFileTests.viewTests(self.modelManager.modelXbrl, options.testReport, options.testReportCols) if options.rssReport: ViewFileRssFeed.viewRssFeed(self.modelManager.modelXbrl, options.rssReport, options.rssReportCols) if options.DTSFile: ViewFileDTS.viewDTS(modelXbrl, options.DTSFile) if options.factsFile: ViewFileFactList.viewFacts(modelXbrl, options.factsFile, labelrole=options.labelRole, lang=options.labelLang, cols=options.factListCols) if options.factTableFile: ViewFileFactTable.viewFacts(modelXbrl, options.factTableFile, labelrole=options.labelRole, lang=options.labelLang) if options.conceptsFile: ViewFileConcepts.viewConcepts(modelXbrl, options.conceptsFile, labelrole=options.labelRole, lang=options.labelLang) if options.preFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.preFile, "Presentation Linkbase", "http://www.xbrl.org/2003/arcrole/parent-child", labelrole=options.labelRole, lang=options.labelLang) if options.calFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.calFile, "Calculation Linkbase", "http://www.xbrl.org/2003/arcrole/summation-item", labelrole=options.labelRole, lang=options.labelLang) if options.dimFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.dimFile, "Dimensions", "XBRL-dimensions", labelrole=options.labelRole, lang=options.labelLang) if options.formulaeFile: ViewFileFormulae.viewFormulae(modelXbrl, options.formulaeFile, "Formulae", lang=options.labelLang) if options.viewArcrole and options.viewFile: ViewFileRelationshipSet.viewRelationshipSet(modelXbrl, options.viewFile, os.path.basename(options.viewArcrole), options.viewArcrole, labelrole=options.labelRole, lang=options.labelLang) if options.roleTypesFile: ViewFileRoleTypes.viewRoleTypes(modelXbrl, options.roleTypesFile, "Role Types", isArcrole=False, lang=options.labelLang) if options.arcroleTypesFile: ViewFileRoleTypes.viewRoleTypes(modelXbrl, options.arcroleTypesFile, "Arcrole Types", isArcrole=True, lang=options.labelLang) for pluginXbrlMethod in pluginClassMethods("CntlrCmdLine.Xbrl.Run"): pluginXbrlMethod(self, options, modelXbrl) except (IOError, EnvironmentError) as err: self.addToLog(_("[IOError] Failed to save output:\n {0}").format(err)) success = False except Exception as err: self.addToLog(_("[Exception] Failed to complete request: \n{0} \n{1}").format( err, traceback.format_tb(sys.exc_info()[2]))) success = False if modelXbrl: modelXbrl.profileStat(_("total"), time.time() - firstStartedAt) if options.collectProfileStats and modelXbrl: modelXbrl.logProfileStats() if not options.keepOpen: if modelDiffReport: self.modelManager.close(modelDiffReport) elif modelXbrl: self.modelManager.close(modelXbrl) self.username = self.password = None #dereference password return success
def loadTaxonomyPackageMappings(self): if self.taxonomyPackageMetadataFiles: metadata = self.url + os.sep + self.taxonomyPackageMetadataFiles[0] taxonomyPackage = PackageManager.parsePackage(self.cntlr, self, metadata, os.sep.join(os.path.split(metadata)[:-1]) + os.sep) self.mappedPaths = taxonomyPackage["remappings"]
def loadPackageUrl(self, url): if url: # url is the in-cache or local file packageInfo = PackageManager.packageInfo(self.cntlr, url, packageManifestName=self.manifestNamePattern) self.cntlr.showStatus("") # clear web loading status self.loadFoundPackageInfo(packageInfo, url)
def findOnWeb(self): url = DialogURL.askURL(self) if url: # url is the in-cache or local file packageInfo = PackageManager.packageInfo(url, packageManifestName=self.manifestNamePattern) self.cntlr.showStatus("") # clear web loading status self.loadFoundPackageInfo(packageInfo, url)
def __init__(self, hasGui=False, logFileName=None, logFileMode=None, logFileEncoding=None, logFormat=None): self.hasWin32gui = False self.hasGui = hasGui self.hasFileSystem = True # no file system on Google App Engine servers self.isGAE = False self.isCGI = False self.systemWordSize = int(round(math.log(sys.maxsize, 2)) + 1) # e.g., 32 or 64 _resourcesDir = resourcesDir() self.configDir = os.path.join(_resourcesDir, "config") self.imagesDir = os.path.join(_resourcesDir, "images") self.localeDir = os.path.join(_resourcesDir, "locale") self.pluginDir = os.path.join(_resourcesDir, "plugin") _mplDir = os.path.join(_resourcesDir, "mpl-data") if os.path.exists( _mplDir ): # set matplotlibdata for cx_Freeze with local directory os.environ["MATPLOTLIBDATA"] = _mplDir serverSoftware = os.getenv("SERVER_SOFTWARE", "") if serverSoftware.startswith( "Google App Engine/") or serverSoftware.startswith( "Development/"): self.hasFileSystem = False # no file system, userAppDir does not exist self.isGAE = True else: gatewayInterface = os.getenv("GATEWAY_INTERFACE", "") if gatewayInterface.startswith("CGI/"): self.isCGI = True configHomeDir = None # look for path configDir/CONFIG_HOME in argv and environment parameters for i, arg in enumerate( sys.argv): # check if config specified in a argv if arg.startswith("--xdgConfigHome="): configHomeDir = arg[16:] break elif arg == "--xdgConfigHome" and i + 1 < len(sys.argv): configHomeDir = sys.argv[i + 1] break if not configHomeDir: # not in argv, may be an environment parameter configHomeDir = os.getenv('XDG_CONFIG_HOME') if not configHomeDir: # look for path configDir/CONFIG_HOME configHomeDirFile = os.path.join(self.configDir, "XDG_CONFIG_HOME") if os.path.exists(configHomeDirFile): try: with io.open(configHomeDirFile, 'rt', encoding='utf-8') as f: configHomeDir = f.read().strip() if configHomeDir and not os.path.isabs(configHomeDir): configHomeDir = os.path.abspath( configHomeDir) # make into a full path if relative except EnvironmentError: configHomeDir = None if self.hasFileSystem and configHomeDir and os.path.exists( configHomeDir): # check if a cache exists in this directory (e.g. from XPE or other tool) impliedAppDir = os.path.join(configHomeDir, "arelle") if os.path.exists(impliedAppDir): self.userAppDir = impliedAppDir elif os.path.exists(os.path.join(configHomeDir, "cache")): self.userAppDir = configHomeDir # use the XDG_CONFIG_HOME because cache is already a subdirectory else: self.userAppDir = impliedAppDir if sys.platform == "darwin": self.isMac = True self.isMSW = False if self.hasFileSystem and not configHomeDir: self.userAppDir = os.path.expanduser( "~") + "/Library/Application Support/Arelle" # note that cache is in ~/Library/Caches/Arelle self.contextMenuClick = "<Button-2>" self.hasClipboard = hasGui # clipboard always only if Gui (not command line mode) self.updateURL = "http://arelle.org/downloads/8" elif sys.platform.startswith("win"): self.isMac = False self.isMSW = True if self.hasFileSystem and not configHomeDir: tempDir = tempfile.gettempdir() if tempDir.lower().endswith('local\\temp'): impliedAppDir = tempDir[:-10] + 'local' else: impliedAppDir = tempDir self.userAppDir = os.path.join(impliedAppDir, "Arelle") if hasGui: try: import win32clipboard self.hasClipboard = True except ImportError: self.hasClipboard = False try: import win32gui self.hasWin32gui = True # active state for open file dialogs except ImportError: pass else: self.hasClipboard = False self.contextMenuClick = "<Button-3>" if "64 bit" in sys.version: self.updateURL = "http://arelle.org/downloads/9" else: # 32 bit self.updateURL = "http://arelle.org/downloads/10" else: # Unix/Linux self.isMac = False self.isMSW = False if self.hasFileSystem and not configHomeDir: self.userAppDir = os.path.join(os.path.expanduser("~/.config"), "arelle") if hasGui: try: import gtk self.hasClipboard = True except ImportError: self.hasClipboard = False else: self.hasClipboard = False self.contextMenuClick = "<Button-3>" try: from arelle import webserver self.hasWebServer = True except ImportError: self.hasWebServer = False # assert that app dir must exist self.config = None if self.hasFileSystem: if not os.path.exists(self.userAppDir): os.makedirs(self.userAppDir) # load config if it exists self.configJsonFile = self.userAppDir + os.sep + "config.json" if os.path.exists(self.configJsonFile): try: with io.open(self.configJsonFile, 'rt', encoding='utf-8') as f: self.config = json.load(f) except Exception as ex: self.config = None # restart with a new config if not self.config: self.config = { 'fileHistory': [], 'windowGeometry': "{0}x{1}+{2}+{3}".format(800, 500, 200, 100), } # start language translation for domain self.setUiLanguage(self.config.get("userInterfaceLangOverride", None), fallbackToDefault=True) from arelle.WebCache import WebCache self.webCache = WebCache(self, self.config.get("proxySettings")) # start plug in server (requres web cache initialized, but not logger) PluginManager.init(self, loadPluginConfig=hasGui) # requires plug ins initialized self.modelManager = ModelManager.initialize(self) # start taxonomy package server (requres web cache initialized, but not logger) PackageManager.init(self, loadPackagesConfig=hasGui) self.startLogging(logFileName, logFileMode, logFileEncoding, logFormat) # Cntlr.Init after logging started for pluginMethod in PluginManager.pluginClassMethods("Cntlr.Init"): pluginMethod(self)
return # provide error message later except etree.LxmlError, err: self.logError(err) return # provide error message later elif self.isInstalledTaxonomyPackage: self.isOpen = True # load mappings try: metadataFiles = self.taxonomyPackageMetadataFiles if len(metadataFiles) != 1: raise IOError(_(u"Taxonomy package must contain one and only one metadata file: {0}.") .format(u', '.join(metadataFiles))) # HF: this won't work, see DialogOpenArchive for correct code # not sure if it is used taxonomyPackage = PackageManager.parsePackage(self.cntlr, self.url) fileSourceDir = os.path.dirname(self.baseurl) + os.sep self.mappedPaths = \ dict((prefix, remapping if isHttpUrl(remapping) else (fileSourceDir + remapping.replace(u"/", os.sep))) for prefix, remapping in taxonomyPackage[u"remappings"].items()) except EnvironmentError, err: self.logError(err) return # provide error message later def openZipStream(self, sourceZipStream): if not self.isOpen: self.basefile = self.url self.baseurl = self.url # url gets changed by selection self.fs = zipfile.ZipFile(sourceZipStream, mode=u"r")