def runOpenInlineDocumentSetMenuCommand(cntlr, runInBackground=False, saveTargetFiling=False): filenames = cntlr.uiFileDialog( "open", multiple=True, title=_("arelle - Multi-open inline XBRL file(s)"), initialdir=cntlr.config.setdefault("fileOpenDir", "."), filetypes=[(_("XBRL files"), "*.*")], defaultextension=".xbrl") if os.sep == "\\": filenames = [f.replace("/", "\\") for f in filenames] if not filenames: filename = "" elif len(filenames) == 1 and (filenames[0].endswith(".zip") or filenames[0].endswith(".tar.gz")): # get archive file names from arelle.FileSource import openFileSource filesource = openFileSource(filenames[0], cntlr) if filesource.isArchive: from arelle import DialogOpenArchive archiveEntries = DialogOpenArchive.askArchiveFile(cntlr, filesource, multiselect=True) if archiveEntries: ixdsFirstFile = archiveEntries[0] _archiveFilenameParts = archiveFilenameParts(ixdsFirstFile) if _archiveFilenameParts is not None: ixdsDir = _archiveFilenameParts[ 0] # it's a zip or package, use zip file name as head of ixds else: ixdsDir = os.path.dirname(ixdsFirstFile) docsetSurrogatePath = os.path.join(ixdsDir, IXDS_SURROGATE) filename = docsetSurrogatePath + IXDS_DOC_SEPARATOR.join( archiveEntries) else: filename = None filesource.close() elif len(filenames) >= MINIMUM_IXDS_DOC_COUNT: ixdsFirstFile = filenames[0] _archiveFilenameParts = archiveFilenameParts(ixdsFirstFile) if _archiveFilenameParts is not None: ixdsDir = _archiveFilenameParts[ 0] # it's a zip or package, use zip file name as head of ixds else: ixdsDir = os.path.dirname(ixdsFirstFile) docsetSurrogatePath = os.path.join(ixdsDir, IXDS_SURROGATE) filename = docsetSurrogatePath + IXDS_DOC_SEPARATOR.join(filenames) else: filename = filenames[0] if filename is not None: cntlr.fileOpenFile(filename)
def runOpenInlineDocumentSetMenuCommand(cntlr, filenames, runInBackground=False, saveTargetFiling=False): if os.sep == "\\": filenames = [f.replace("/", "\\") for f in filenames] if not filenames: filename = "" elif len(filenames) == 1 and any(filenames[0].endswith(s) for s in archiveFilenameSuffixes): # get archive file names from arelle.FileSource import openFileSource filesource = openFileSource(filenames[0], cntlr) if filesource.isArchive: from arelle import DialogOpenArchive archiveEntries = DialogOpenArchive.askArchiveFile(cntlr, filesource, multiselect=True) if archiveEntries: ixdsFirstFile = archiveEntries[0] _archiveFilenameParts = archiveFilenameParts(ixdsFirstFile) if _archiveFilenameParts is not None: ixdsDir = _archiveFilenameParts[ 0] # it's a zip or package, use zip file name as head of ixds else: ixdsDir = os.path.dirname(ixdsFirstFile) docsetSurrogatePath = os.path.join(ixdsDir, IXDS_SURROGATE) filename = docsetSurrogatePath + IXDS_DOC_SEPARATOR.join( archiveEntries) else: filename = None filesource.close() elif len(filenames) >= MINIMUM_IXDS_DOC_COUNT: ixdsFirstFile = filenames[0] _archiveFilenameParts = archiveFilenameParts(ixdsFirstFile) if _archiveFilenameParts is not None: ixdsDir = _archiveFilenameParts[ 0] # it's a zip or package, use zip file name as head of ixds else: ixdsDir = os.path.dirname(ixdsFirstFile) docsetSurrogatePath = os.path.join(ixdsDir, IXDS_SURROGATE) filename = docsetSurrogatePath + IXDS_DOC_SEPARATOR.join(filenames) else: filename = filenames[0] if filename is not None: cntlr.fileOpenFile(filename)
def launchLocalViewer(cntlr, modelXbrl): from arelle import LocalViewer try: viewerBuilder = IXBRLViewerBuilder(cntlr.modelManager.modelXbrl) iv = viewerBuilder.createViewer(scriptUrl="/ixbrlviewer.js", showValidations=False) # first check if source file was in an archive (e.g., taxonomy package) _archiveFilenameParts = archiveFilenameParts( modelXbrl.modelDocument.filepath) if _archiveFilenameParts is not None: outDir = os.path.dirname( _archiveFilenameParts[0]) # it's a zip or package else: outDir = modelXbrl.modelDocument.filepathdir _localhost = localViewer.init(cntlr, outDir) # for IXDS, outPath must be a directory name, suffix is applied in saving files if len(iv.files) > 1: # save files in a separate directory from source files _localhost += "/" + VIEWER_BASENAME_SUFFIX outDir = os.path.join(outDir, VIEWER_BASENAME_SUFFIX) os.makedirs(outDir, exist_ok=True) iv.save( outDir ) # no changes to html inline files so inter-file refereences still can work htmlFile = iv.files[0].filename else: iv.save(outDir, outBasenameSuffix=VIEWER_BASENAME_SUFFIX) htmlFile = "{0[0]}{1}{0[1]}".format( os.path.splitext(modelXbrl.modelDocument.basename), VIEWER_BASENAME_SUFFIX) import webbrowser webbrowser.open(url="{}/{}".format(_localhost, htmlFile)) except Exception as ex: modelXbrl.error("viewer:exception", "Exception %(exception)s \sTraceback %(traceback)s", modelObject=modelXbrl, exception=ex, traceback=traceback.format_tb(sys.exc_info()[2]))
def getfilename(self, url, base=None, reload=False, checkModifiedTime=False, normalize=False, filenameOnly=False): if url is None: return url if base is not None or normalize: url = self.normalizeUrl(url, base) urlScheme, schemeSep, urlSchemeSpecificPart = url.partition("://") if schemeSep and urlScheme in ("http", "https"): # is this a mapped archive file contents? _archiveFileNameParts = archiveFilenameParts(url) if _archiveFileNameParts: _archiveFilename = self.getfilename( _archiveFileNameParts[0], reload=reload, checkModifiedTime=checkModifiedTime) if _archiveFilename: return os.path.join(_archiveFilename, _archiveFileNameParts[1]) return None # form cache file name (substituting _ for any illegal file characters) filepath = self.urlToCacheFilepath(url) if self.cacheDir == SERVER_WEB_CACHE: # server web-cached files are downloaded when opening to prevent excessive memcache api calls return filepath # quotedUrl has scheme-specific-part quoted except for parameter separators quotedUrl = urlScheme + schemeSep + quote(urlSchemeSpecificPart, '/?=&') # handle default directory requests if filepath.endswith("/"): filepath += DIRECTORY_INDEX_FILE if os.sep == '\\': filepath = filepath.replace('/', '\\') if self.workOffline or filenameOnly: return filepath filepathtmp = filepath + ".tmp" fileExt = os.path.splitext(filepath)[1] timeNow = time.time() timeNowStr = time.strftime('%Y-%m-%dT%H:%M:%S UTC', time.gmtime(timeNow)) retrievingDueToRecheckInterval = False if not reload and os.path.exists(filepath): if url in self.cachedUrlCheckTimes and not checkModifiedTime: cachedTime = calendar.timegm( time.strptime(self.cachedUrlCheckTimes[url], '%Y-%m-%dT%H:%M:%S UTC')) else: cachedTime = 0 if timeNow - cachedTime > self.maxAgeSeconds: # weekly check if newer file exists newerOnWeb = False try: # no provision here for proxy authentication!!! remoteFileTime = lastModifiedTime( self.getheaders(quotedUrl)) if remoteFileTime and remoteFileTime > os.path.getmtime( filepath): newerOnWeb = True except: pass # for now, forget about authentication here if not newerOnWeb: # update ctime by copying file and return old file self.cachedUrlCheckTimes[url] = timeNowStr self.cachedUrlCheckTimesModified = True return filepath retrievingDueToRecheckInterval = True else: return filepath filedir = os.path.dirname(filepath) if not os.path.exists(filedir): os.makedirs(filedir) # Retrieve over HTTP and cache, using rename to avoid collisions # self.modelManager.addToLog('web caching: {0}'.format(url)) # download to a temporary name so it is not left readable corrupted if download fails retryCount = 5 while retryCount > 0: try: self.progressUrl = url savedfile, headers, initialBytes = self.retrieve( #savedfile, headers = self.opener.retrieve( quotedUrl, filename=filepathtmp, reporthook=self.reportProgress) # check if this is a real file or a wifi or web logon screen if fileExt in {".xsd", ".xml", ".xbrl"}: if b"<html" in initialBytes: if retrievingDueToRecheckInterval: return self.internetRecheckFailedRecovery( filepath, url, "file contents appear to be an html logon request", timeNowStr) response = None # found possible logon request if self.cntlr.hasGui: response = self.cntlr.internet_logon( url, quotedUrl, _("Unexpected HTML in {0}").format(url), _("Is this a logon page? If so, click 'yes', else click 'no' if it is the expected XBRL content, or 'cancel' to abort retrieval: \n\n{0}" ).format(initialBytes[:1500])) if response == "retry": retryCount -= 1 continue elif response != "no": self.cntlr.addToLog( _("Web file appears to be an html logon request, not retrieved: %(URL)s \nContents: \n%(contents)s" ), messageCode="webCache:invalidRetrieval", messageArgs={ "URL": url, "contents": initialBytes }, level=logging.ERROR) return None retryCount = 0 except (ContentTooShortError, IncompleteRead) as err: if retrievingDueToRecheckInterval: return self.internetRecheckFailedRecovery( filepath, url, err, timeNowStr) if retryCount > 1: self.cntlr.addToLog( _("%(error)s \nunsuccessful retrieval of %(URL)s \n%(retryCount)s retries remaining" ), messageCode="webCache:retryingOperation", messageArgs={ "error": err, "URL": url, "retryCount": retryCount }, level=logging.ERROR) retryCount -= 1 continue self.cntlr.addToLog( _("%(error)s \nretrieving %(URL)s"), messageCode="webCache:contentTooShortError", messageArgs={ "URL": url, "error": err }, level=logging.ERROR) if os.path.exists(filepathtmp): os.remove(filepathtmp) return None # handle file is bad except (HTTPError, URLError) as err: try: tryWebAuthentication = False if isinstance(err, HTTPError) and err.code == 401: tryWebAuthentication = True if 'www-authenticate' in err.hdrs: match = re.match( '[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"', err.hdrs['www-authenticate']) if match: scheme, realm = match.groups() if scheme.lower() == 'basic': host = os.path.dirname(quotedUrl) userPwd = self.cntlr.internet_user_password( host, realm) if isinstance(userPwd, (tuple, list)): self.http_auth_handler.add_password( realm=realm, uri=host, user=userPwd[0], passwd=userPwd[1]) retryCount -= 1 continue self.cntlr.addToLog( _("'%(scheme)s' www-authentication for realm '%(realm)s' is required to access %(URL)s\n%(error)s" ), messageCode= "webCache:unsupportedWWWAuthentication", messageArgs={ "scheme": scheme, "realm": realm, "URL": url, "error": err }, level=logging.ERROR) elif isinstance(err, HTTPError) and err.code == 407: tryWebAuthentication = True if 'proxy-authenticate' in err.hdrs: match = re.match( '[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"', err.hdrs['proxy-authenticate']) if match: scheme, realm = match.groups() host = self.proxy_handler.proxies.get( 'http') if scheme.lower() == 'basic': userPwd = self.cntlr.internet_user_password( host, realm) if isinstance(userPwd, (tuple, list)): self.proxy_auth_handler.add_password( realm=realm, uri=host, user=userPwd[0], passwd=userPwd[1]) retryCount -= 1 continue self.cntlr.addToLog( _("'%(scheme)s' proxy-authentication for realm '%(realm)s' is required to access %(URL)s\n%(error)s" ), messageCode= "webCache:unsupportedProxyAuthentication", messageArgs={ "scheme": scheme, "realm": realm, "URL": url, "error": err }, level=logging.ERROR) if retrievingDueToRecheckInterval: return self.internetRecheckFailedRecovery( filepath, url, err, timeNowStr) if tryWebAuthentication: # may be a web login authentication request response = None # found possible logon request if self.cntlr.hasGui: response = self.cntlr.internet_logon( url, quotedUrl, _("HTTP {0} authentication request"). format(err.code), _("Unexpected HTML in {0}").format(url), _("Is browser-based possible? If so, click 'yes', or 'cancel' to abort retrieval: \n\n{0}" ).format(url)) if response == "retry": retryCount -= 1 continue elif response != "no": self.cntlr.addToLog( _("Web file HTTP 401 (authentication required) response, not retrieved: %(URL)s" ), messageCode= "webCache:authenticationRequired", messageArgs={"URL": url}, level=logging.ERROR) return None except AttributeError: pass if retrievingDueToRecheckInterval: return self.internetRecheckFailedRecovery( filepath, url, err, timeNowStr) self.cntlr.addToLog( _("%(error)s \nretrieving %(URL)s"), messageCode="webCache:retrievalError", messageArgs={ "error": err.reason if hasattr(err, "reason") else err, "URL": url }, level=logging.ERROR) return None except Exception as err: if retryCount > 1: self.cntlr.addToLog( _("%(error)s \nunsuccessful retrieval of %(URL)s \n%(retryCount)s retries remaining" ), messageCode="webCache:retryingOperation", messageArgs={ "error": err, "URL": url, "retryCount": retryCount }, level=logging.ERROR) retryCount -= 1 continue if retrievingDueToRecheckInterval: return self.internetRecheckFailedRecovery( filepath, url, err, timeNowStr) if self.cntlr.hasGui: self.cntlr.addToLog( _("%(error)s \nunsuccessful retrieval of %(URL)s \nswitching to work offline" ), messageCode="webCache:attemptingOfflineOperation", messageArgs={ "error": err, "URL": url }, level=logging.ERROR) # try working offline self.workOffline = True return filepath else: # don't switch offline unexpectedly in scripted (batch) operation self.cntlr.addToLog( _("%(error)s \nunsuccessful retrieval of %(URL)s"), messageCode="webCache:unsuccessfulRetrieval", messageArgs={ "error": err, "URL": url }, level=logging.ERROR) if os.path.exists(filepathtmp): os.remove(filepathtmp) return None # rename temporarily named downloaded file to desired name if os.path.exists(filepath): try: if os.path.isfile(filepath) or os.path.islink( filepath): os.remove(filepath) elif os.path.isdir(filepath): shutil.rmtree(filepath) except Exception as err: self.cntlr.addToLog( _("%(error)s \nUnsuccessful removal of prior file %(filepath)s \nPlease remove with file manager." ), messageCode="webCache:cachedPriorFileLocked", messageArgs={ "error": err, "filepath": filepath }, level=logging.ERROR) try: os.rename(filepathtmp, filepath) if self._logDownloads: self.cntlr.addToLog(_("Downloaded %(URL)s"), messageCode="webCache:download", messageArgs={ "URL": url, "filepath": filepath }, level=logging.INFO) except Exception as err: self.cntlr.addToLog( _("%(error)s \nUnsuccessful renaming of downloaded file to active file %(filepath)s \nPlease remove with file manager." ), messageCode="webCache:cacheDownloadRenamingError", messageArgs={ "error": err, "filepath": filepath }, level=logging.ERROR) webFileTime = lastModifiedTime(headers) if webFileTime: # set mtime to web mtime os.utime(filepath, (webFileTime, webFileTime)) self.cachedUrlCheckTimes[url] = timeNowStr self.cachedUrlCheckTimesModified = True return filepath if url.startswith("file://"): url = url[7:] elif url.startswith("file:\\"): url = url[6:] if os.sep == '\\': url = url.replace('/', '\\') return url