def runOpenInlineDocumentSetMenuCommand(cntlr,
runInBackground=False,
saveTargetFiling=False):
filenames = cntlr.uiFileDialog(
"open",
multiple=True,
title=_("arelle - Multi-open inline XBRL file(s)"),
initialdir=cntlr.config.setdefault("fileOpenDir", "."),
filetypes=[(_("XBRL files"), "*.*")],
defaultextension=".xbrl")
if os.sep == "\\":
filenames = [f.replace("/", "\\") for f in filenames]
if not filenames:
filename = ""
elif len(filenames) == 1 and (filenames[0].endswith(".zip")
or filenames[0].endswith(".tar.gz")):
# get archive file names
from arelle.FileSource import openFileSource
filesource = openFileSource(filenames[0], cntlr)
if filesource.isArchive:
from arelle import DialogOpenArchive
archiveEntries = DialogOpenArchive.askArchiveFile(cntlr,
filesource,
multiselect=True)
if archiveEntries:
ixdsFirstFile = archiveEntries[0]
_archiveFilenameParts = archiveFilenameParts(ixdsFirstFile)
if _archiveFilenameParts is not None:
ixdsDir = _archiveFilenameParts[
0] # it's a zip or package, use zip file name as head of ixds
else:
ixdsDir = os.path.dirname(ixdsFirstFile)
docsetSurrogatePath = os.path.join(ixdsDir, IXDS_SURROGATE)
filename = docsetSurrogatePath + IXDS_DOC_SEPARATOR.join(
archiveEntries)
else:
filename = None
filesource.close()
elif len(filenames) >= MINIMUM_IXDS_DOC_COUNT:
ixdsFirstFile = filenames[0]
_archiveFilenameParts = archiveFilenameParts(ixdsFirstFile)
if _archiveFilenameParts is not None:
ixdsDir = _archiveFilenameParts[
0] # it's a zip or package, use zip file name as head of ixds
else:
ixdsDir = os.path.dirname(ixdsFirstFile)
docsetSurrogatePath = os.path.join(ixdsDir, IXDS_SURROGATE)
filename = docsetSurrogatePath + IXDS_DOC_SEPARATOR.join(filenames)
else:
filename = filenames[0]
if filename is not None:
cntlr.fileOpenFile(filename)
def runOpenInlineDocumentSetMenuCommand(cntlr,
filenames,
runInBackground=False,
saveTargetFiling=False):
if os.sep == "\\":
filenames = [f.replace("/", "\\") for f in filenames]
if not filenames:
filename = ""
elif len(filenames) == 1 and any(filenames[0].endswith(s)
for s in archiveFilenameSuffixes):
# get archive file names
from arelle.FileSource import openFileSource
filesource = openFileSource(filenames[0], cntlr)
if filesource.isArchive:
from arelle import DialogOpenArchive
archiveEntries = DialogOpenArchive.askArchiveFile(cntlr,
filesource,
multiselect=True)
if archiveEntries:
ixdsFirstFile = archiveEntries[0]
_archiveFilenameParts = archiveFilenameParts(ixdsFirstFile)
if _archiveFilenameParts is not None:
ixdsDir = _archiveFilenameParts[
0] # it's a zip or package, use zip file name as head of ixds
else:
ixdsDir = os.path.dirname(ixdsFirstFile)
docsetSurrogatePath = os.path.join(ixdsDir, IXDS_SURROGATE)
filename = docsetSurrogatePath + IXDS_DOC_SEPARATOR.join(
archiveEntries)
else:
filename = None
filesource.close()
elif len(filenames) >= MINIMUM_IXDS_DOC_COUNT:
ixdsFirstFile = filenames[0]
_archiveFilenameParts = archiveFilenameParts(ixdsFirstFile)
if _archiveFilenameParts is not None:
ixdsDir = _archiveFilenameParts[
0] # it's a zip or package, use zip file name as head of ixds
else:
ixdsDir = os.path.dirname(ixdsFirstFile)
docsetSurrogatePath = os.path.join(ixdsDir, IXDS_SURROGATE)
filename = docsetSurrogatePath + IXDS_DOC_SEPARATOR.join(filenames)
else:
filename = filenames[0]
if filename is not None:
cntlr.fileOpenFile(filename)
def launchLocalViewer(cntlr, modelXbrl):
from arelle import LocalViewer
try:
viewerBuilder = IXBRLViewerBuilder(cntlr.modelManager.modelXbrl)
iv = viewerBuilder.createViewer(scriptUrl="/ixbrlviewer.js",
showValidations=False)
# first check if source file was in an archive (e.g., taxonomy package)
_archiveFilenameParts = archiveFilenameParts(
modelXbrl.modelDocument.filepath)
if _archiveFilenameParts is not None:
outDir = os.path.dirname(
_archiveFilenameParts[0]) # it's a zip or package
else:
outDir = modelXbrl.modelDocument.filepathdir
_localhost = localViewer.init(cntlr, outDir)
# for IXDS, outPath must be a directory name, suffix is applied in saving files
if len(iv.files) > 1:
# save files in a separate directory from source files
_localhost += "/" + VIEWER_BASENAME_SUFFIX
outDir = os.path.join(outDir, VIEWER_BASENAME_SUFFIX)
os.makedirs(outDir, exist_ok=True)
iv.save(
outDir
) # no changes to html inline files so inter-file refereences still can work
htmlFile = iv.files[0].filename
else:
iv.save(outDir, outBasenameSuffix=VIEWER_BASENAME_SUFFIX)
htmlFile = "{0[0]}{1}{0[1]}".format(
os.path.splitext(modelXbrl.modelDocument.basename),
VIEWER_BASENAME_SUFFIX)
import webbrowser
webbrowser.open(url="{}/{}".format(_localhost, htmlFile))
except Exception as ex:
modelXbrl.error("viewer:exception",
"Exception %(exception)s \sTraceback %(traceback)s",
modelObject=modelXbrl,
exception=ex,
traceback=traceback.format_tb(sys.exc_info()[2]))
def getfilename(self,
url,
base=None,
reload=False,
checkModifiedTime=False,
normalize=False,
filenameOnly=False):
if url is None:
return url
if base is not None or normalize:
url = self.normalizeUrl(url, base)
urlScheme, schemeSep, urlSchemeSpecificPart = url.partition("://")
if schemeSep and urlScheme in ("http", "https"):
# is this a mapped archive file contents?
_archiveFileNameParts = archiveFilenameParts(url)
if _archiveFileNameParts:
_archiveFilename = self.getfilename(
_archiveFileNameParts[0],
reload=reload,
checkModifiedTime=checkModifiedTime)
if _archiveFilename:
return os.path.join(_archiveFilename,
_archiveFileNameParts[1])
return None
# form cache file name (substituting _ for any illegal file characters)
filepath = self.urlToCacheFilepath(url)
if self.cacheDir == SERVER_WEB_CACHE:
# server web-cached files are downloaded when opening to prevent excessive memcache api calls
return filepath
# quotedUrl has scheme-specific-part quoted except for parameter separators
quotedUrl = urlScheme + schemeSep + quote(urlSchemeSpecificPart,
'/?=&')
# handle default directory requests
if filepath.endswith("/"):
filepath += DIRECTORY_INDEX_FILE
if os.sep == '\\':
filepath = filepath.replace('/', '\\')
if self.workOffline or filenameOnly:
return filepath
filepathtmp = filepath + ".tmp"
fileExt = os.path.splitext(filepath)[1]
timeNow = time.time()
timeNowStr = time.strftime('%Y-%m-%dT%H:%M:%S UTC',
time.gmtime(timeNow))
retrievingDueToRecheckInterval = False
if not reload and os.path.exists(filepath):
if url in self.cachedUrlCheckTimes and not checkModifiedTime:
cachedTime = calendar.timegm(
time.strptime(self.cachedUrlCheckTimes[url],
'%Y-%m-%dT%H:%M:%S UTC'))
else:
cachedTime = 0
if timeNow - cachedTime > self.maxAgeSeconds:
# weekly check if newer file exists
newerOnWeb = False
try: # no provision here for proxy authentication!!!
remoteFileTime = lastModifiedTime(
self.getheaders(quotedUrl))
if remoteFileTime and remoteFileTime > os.path.getmtime(
filepath):
newerOnWeb = True
except:
pass # for now, forget about authentication here
if not newerOnWeb:
# update ctime by copying file and return old file
self.cachedUrlCheckTimes[url] = timeNowStr
self.cachedUrlCheckTimesModified = True
return filepath
retrievingDueToRecheckInterval = True
else:
return filepath
filedir = os.path.dirname(filepath)
if not os.path.exists(filedir):
os.makedirs(filedir)
# Retrieve over HTTP and cache, using rename to avoid collisions
# self.modelManager.addToLog('web caching: {0}'.format(url))
# download to a temporary name so it is not left readable corrupted if download fails
retryCount = 5
while retryCount > 0:
try:
self.progressUrl = url
savedfile, headers, initialBytes = self.retrieve(
#savedfile, headers = self.opener.retrieve(
quotedUrl,
filename=filepathtmp,
reporthook=self.reportProgress)
# check if this is a real file or a wifi or web logon screen
if fileExt in {".xsd", ".xml", ".xbrl"}:
if b"<html" in initialBytes:
if retrievingDueToRecheckInterval:
return self.internetRecheckFailedRecovery(
filepath, url,
"file contents appear to be an html logon request",
timeNowStr)
response = None # found possible logon request
if self.cntlr.hasGui:
response = self.cntlr.internet_logon(
url, quotedUrl,
_("Unexpected HTML in {0}").format(url),
_("Is this a logon page? If so, click 'yes', else click 'no' if it is the expected XBRL content, or 'cancel' to abort retrieval: \n\n{0}"
).format(initialBytes[:1500]))
if response == "retry":
retryCount -= 1
continue
elif response != "no":
self.cntlr.addToLog(
_("Web file appears to be an html logon request, not retrieved: %(URL)s \nContents: \n%(contents)s"
),
messageCode="webCache:invalidRetrieval",
messageArgs={
"URL": url,
"contents": initialBytes
},
level=logging.ERROR)
return None
retryCount = 0
except (ContentTooShortError, IncompleteRead) as err:
if retrievingDueToRecheckInterval:
return self.internetRecheckFailedRecovery(
filepath, url, err, timeNowStr)
if retryCount > 1:
self.cntlr.addToLog(
_("%(error)s \nunsuccessful retrieval of %(URL)s \n%(retryCount)s retries remaining"
),
messageCode="webCache:retryingOperation",
messageArgs={
"error": err,
"URL": url,
"retryCount": retryCount
},
level=logging.ERROR)
retryCount -= 1
continue
self.cntlr.addToLog(
_("%(error)s \nretrieving %(URL)s"),
messageCode="webCache:contentTooShortError",
messageArgs={
"URL": url,
"error": err
},
level=logging.ERROR)
if os.path.exists(filepathtmp):
os.remove(filepathtmp)
return None
# handle file is bad
except (HTTPError, URLError) as err:
try:
tryWebAuthentication = False
if isinstance(err, HTTPError) and err.code == 401:
tryWebAuthentication = True
if 'www-authenticate' in err.hdrs:
match = re.match(
'[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"',
err.hdrs['www-authenticate'])
if match:
scheme, realm = match.groups()
if scheme.lower() == 'basic':
host = os.path.dirname(quotedUrl)
userPwd = self.cntlr.internet_user_password(
host, realm)
if isinstance(userPwd, (tuple, list)):
self.http_auth_handler.add_password(
realm=realm,
uri=host,
user=userPwd[0],
passwd=userPwd[1])
retryCount -= 1
continue
self.cntlr.addToLog(
_("'%(scheme)s' www-authentication for realm '%(realm)s' is required to access %(URL)s\n%(error)s"
),
messageCode=
"webCache:unsupportedWWWAuthentication",
messageArgs={
"scheme": scheme,
"realm": realm,
"URL": url,
"error": err
},
level=logging.ERROR)
elif isinstance(err, HTTPError) and err.code == 407:
tryWebAuthentication = True
if 'proxy-authenticate' in err.hdrs:
match = re.match(
'[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"',
err.hdrs['proxy-authenticate'])
if match:
scheme, realm = match.groups()
host = self.proxy_handler.proxies.get(
'http')
if scheme.lower() == 'basic':
userPwd = self.cntlr.internet_user_password(
host, realm)
if isinstance(userPwd, (tuple, list)):
self.proxy_auth_handler.add_password(
realm=realm,
uri=host,
user=userPwd[0],
passwd=userPwd[1])
retryCount -= 1
continue
self.cntlr.addToLog(
_("'%(scheme)s' proxy-authentication for realm '%(realm)s' is required to access %(URL)s\n%(error)s"
),
messageCode=
"webCache:unsupportedProxyAuthentication",
messageArgs={
"scheme": scheme,
"realm": realm,
"URL": url,
"error": err
},
level=logging.ERROR)
if retrievingDueToRecheckInterval:
return self.internetRecheckFailedRecovery(
filepath, url, err, timeNowStr)
if tryWebAuthentication:
# may be a web login authentication request
response = None # found possible logon request
if self.cntlr.hasGui:
response = self.cntlr.internet_logon(
url, quotedUrl,
_("HTTP {0} authentication request").
format(err.code),
_("Unexpected HTML in {0}").format(url),
_("Is browser-based possible? If so, click 'yes', or 'cancel' to abort retrieval: \n\n{0}"
).format(url))
if response == "retry":
retryCount -= 1
continue
elif response != "no":
self.cntlr.addToLog(
_("Web file HTTP 401 (authentication required) response, not retrieved: %(URL)s"
),
messageCode=
"webCache:authenticationRequired",
messageArgs={"URL": url},
level=logging.ERROR)
return None
except AttributeError:
pass
if retrievingDueToRecheckInterval:
return self.internetRecheckFailedRecovery(
filepath, url, err, timeNowStr)
self.cntlr.addToLog(
_("%(error)s \nretrieving %(URL)s"),
messageCode="webCache:retrievalError",
messageArgs={
"error":
err.reason if hasattr(err, "reason") else err,
"URL": url
},
level=logging.ERROR)
return None
except Exception as err:
if retryCount > 1:
self.cntlr.addToLog(
_("%(error)s \nunsuccessful retrieval of %(URL)s \n%(retryCount)s retries remaining"
),
messageCode="webCache:retryingOperation",
messageArgs={
"error": err,
"URL": url,
"retryCount": retryCount
},
level=logging.ERROR)
retryCount -= 1
continue
if retrievingDueToRecheckInterval:
return self.internetRecheckFailedRecovery(
filepath, url, err, timeNowStr)
if self.cntlr.hasGui:
self.cntlr.addToLog(
_("%(error)s \nunsuccessful retrieval of %(URL)s \nswitching to work offline"
),
messageCode="webCache:attemptingOfflineOperation",
messageArgs={
"error": err,
"URL": url
},
level=logging.ERROR)
# try working offline
self.workOffline = True
return filepath
else: # don't switch offline unexpectedly in scripted (batch) operation
self.cntlr.addToLog(
_("%(error)s \nunsuccessful retrieval of %(URL)s"),
messageCode="webCache:unsuccessfulRetrieval",
messageArgs={
"error": err,
"URL": url
},
level=logging.ERROR)
if os.path.exists(filepathtmp):
os.remove(filepathtmp)
return None
# rename temporarily named downloaded file to desired name
if os.path.exists(filepath):
try:
if os.path.isfile(filepath) or os.path.islink(
filepath):
os.remove(filepath)
elif os.path.isdir(filepath):
shutil.rmtree(filepath)
except Exception as err:
self.cntlr.addToLog(
_("%(error)s \nUnsuccessful removal of prior file %(filepath)s \nPlease remove with file manager."
),
messageCode="webCache:cachedPriorFileLocked",
messageArgs={
"error": err,
"filepath": filepath
},
level=logging.ERROR)
try:
os.rename(filepathtmp, filepath)
if self._logDownloads:
self.cntlr.addToLog(_("Downloaded %(URL)s"),
messageCode="webCache:download",
messageArgs={
"URL": url,
"filepath": filepath
},
level=logging.INFO)
except Exception as err:
self.cntlr.addToLog(
_("%(error)s \nUnsuccessful renaming of downloaded file to active file %(filepath)s \nPlease remove with file manager."
),
messageCode="webCache:cacheDownloadRenamingError",
messageArgs={
"error": err,
"filepath": filepath
},
level=logging.ERROR)
webFileTime = lastModifiedTime(headers)
if webFileTime: # set mtime to web mtime
os.utime(filepath, (webFileTime, webFileTime))
self.cachedUrlCheckTimes[url] = timeNowStr
self.cachedUrlCheckTimesModified = True
return filepath
if url.startswith("file://"): url = url[7:]
elif url.startswith("file:\\"): url = url[6:]
if os.sep == '\\':
url = url.replace('/', '\\')
return url
