Exemplo n.º 1
0
 def normalizeUrl(self, url, base=None):
     if url and not (isHttpUrl(url) or os.path.isabs(url)):
         if base is not None and not isHttpUrl(base) and u'%' in url:
             url = unquote(url)
         if base:
             if isHttpUrl(base):
                 scheme, sep, path = base.partition(u"://")
                 normedPath = scheme + sep + posixpath.normpath(os.path.dirname(path) + u"/" + url)
             else:
                 if u'%' in base:
                     base = unquote(base)
                 normedPath = os.path.normpath(os.path.join(os.path.dirname(base),url))
         else: # includes base == '' (for forcing relative path)
             normedPath = url
         if normedPath.startswith(u"file://"): normedPath = normedPath[7:]
         elif normedPath.startswith(u"file:\\"): normedPath = normedPath[6:]
         
         # no base, not normalized, must be relative to current working directory
         if base is None and not os.path.isabs(url): 
             normedPath = os.path.abspath(normedPath)
     else:
         normedPath = url
     
     if normedPath:
         if isHttpUrl(normedPath):
             scheme, sep, pathpart = normedPath.partition(u"://")
             pathpart = pathpart.replace(u'\\',u'/')
             endingSep = u'/' if pathpart[-1] == u'/' else u''  # normpath drops ending directory separator
             return scheme + u"://" + posixpath.normpath(pathpart) + endingSep
         normedPath = os.path.normpath(normedPath)
         if normedPath.startswith(self.cacheDir):
             normedPath = self.cacheFilepathToUrl(normedPath)
     return normedPath
Exemplo n.º 2
0
 def ok(self, event=None):
     selection = self.treeView.selection()
     if len(selection) > 0:
         if self.openType in (ARCHIVE, DISCLOSURE_SYSTEM):
             filename = self.filenames[int(selection[0][4:])]
             if isinstance(filename,tuple):
                 if self.isRss:
                     filename = filename[4]
                 else:
                     filename = filename[0]
             if not filename.endswith("/"):
                 self.filesource.select(filename)
                 self.accepted = True
                 self.close()
         elif self.openType == ENTRY_POINTS:
             epName = selection[0]
             #index 0 is the remapped Url, as opposed to the canonical one used for display
             urlOrFile = self.nameToUrls[epName][0]
             
             # load file source remappings
             self.filesource.mappedPaths = \
                 dict((prefix, 
                       remapping if isHttpUrl(remapping)
                       else (self.filesource.baseurl + os.sep + self.metadataFilePrefix +remapping.replace("/", os.sep)))
                       for prefix, remapping in self.remappings.items())
 
             if not urlOrFile.endswith("/"):
                 # check if it's an absolute URL rather than a path into the archive
                 if isHttpUrl(urlOrFile):
                     self.filesource.select(urlOrFile)  # absolute path selection
                 else:
                     # assume it's a path inside the archive:
                     self.filesource.select(self.metadataFilePrefix + urlOrFile)
                 self.accepted = True
                 self.close()
Exemplo n.º 3
0
 def ok(self, event=None):
     selection = self.treeView.selection()
     if len(selection) > 0:
         if hasattr(self, "taxonomyPackage"):
             # load file source remappings
             self.filesource.mappedPaths = self.taxonomyPackage["remappings"]
         filename = None
         if self.openType in (ARCHIVE, DISCLOSURE_SYSTEM):
             filename = self.filenames[int(selection[0][4:])]
             if isinstance(filename,tuple):
                 if self.isRss:
                     filename = filename[4]
                 else:
                     filename = filename[0]
         elif self.openType == ENTRY_POINTS:
             epName = selection[0]
             #index 0 is the remapped Url, as opposed to the canonical one used for display
             # Greg Acsone reports [0] does not work for Corep 1.6 pkgs, need [1], old style packages
             filenames = []
             for _url, _type in self.packageContainedInstances: # check if selection was an inline instance
                 if _type == epName:
                     filenames.append(_url)
             if not filenames: # else if it's a named taxonomy entry point
                 for url in self.taxonomyPackage["entryPoints"][epName]:
                     filename = url[0]
                     if not filename.endswith("/"):
                         # check if it's an absolute URL rather than a path into the archive
                         if not isHttpUrl(filename) and self.metadataFilePrefix != self.taxonomyPkgMetaInf:
                             # assume it's a path inside the archive:
                             filename = self.metadataFilePrefix + filename
                     filenames.append(filename)
             if filenames:
                 self.filesource.select(filenames)
                 self.accepted = True
                 self.close()
             return
         elif self.openType in (PLUGIN, PACKAGE):
             filename = self.filenames[int(selection[0][4:])][2]
         if filename is not None and not filename.endswith("/"):
             if hasattr(self, "taxonomyPackage"):
                 # attempt to unmap the filename to original file
                 # will be mapped again in loading, but this allows schemaLocation to be unmapped
                 for prefix, remapping in self.taxonomyPackage["remappings"].items():
                     if isHttpUrl(remapping):
                         remapStart = remapping
                     else:
                         remapStart = self.metadataFilePrefix + remapping
                     if filename.startswith(remapStart):
                         # set unmmapped file
                         filename = prefix + filename[len(remapStart):]
                         break
             if self.openType in (PLUGIN, PACKAGE):
                 self.filesource.selection = filename
             else:
                 self.filesource.select(filename)
             self.accepted = True
             self.close()
Exemplo n.º 4
0
 def ok(self, event=None):
     selection = self.treeView.selection()
     if len(selection) > 0:
         if hasattr(self, "taxonomyPackage"):
             # load file source remappings
             self.filesource.mappedPaths = self.taxonomyPackage[
                 "remappings"]
         filename = None
         if self.openType in (ARCHIVE, DISCLOSURE_SYSTEM):
             filename = self.filenames[int(selection[0][4:])]
             if isinstance(filename, tuple):
                 if self.isRss:
                     filename = filename[4]
                 else:
                     filename = filename[0]
         elif self.openType == ENTRY_POINTS:
             epName = selection[0]
             #index 0 is the remapped Url, as opposed to the canonical one used for display
             # Greg Acsone reports [0] does not work for Corep 1.6 pkgs, need [1], old style packages
             filenames = []
             for url in self.taxonomyPackage["entryPoints"][epName]:
                 filename = url[0]
                 if not filename.endswith("/"):
                     # check if it's an absolute URL rather than a path into the archive
                     if not isHttpUrl(
                             filename
                     ) and self.metadataFilePrefix != self.taxonomyPkgMetaInf:
                         # assume it's a path inside the archive:
                         filename = self.metadataFilePrefix + filename
                 filenames.append(filename)
             if filenames:
                 self.filesource.select(filenames)
                 self.accepted = True
                 self.close()
             return
         elif self.openType == PLUGIN:
             filename = self.filenames[int(selection[0][4:])][2]
         if filename is not None and not filename.endswith("/"):
             if hasattr(self, "taxonomyPackage"):
                 # attempt to unmap the filename to original file
                 # will be mapped again in loading, but this allows schemaLocation to be unmapped
                 for prefix, remapping in self.taxonomyPackage[
                         "remappings"].items():
                     if isHttpUrl(remapping):
                         remapStart = remapping
                     else:
                         remapStart = self.metadataFilePrefix + remapping
                     if filename.startswith(remapStart):
                         # set unmmapped file
                         filename = prefix + filename[len(remapStart):]
                         break
             if self.openType == PLUGIN:
                 self.filesource.selection = filename
             else:
                 self.filesource.select(filename)
             self.accepted = True
             self.close()
Exemplo n.º 5
0
def openFileStream(cntlr, filepath, mode='r', encoding=None):

    if PackageManager.isMappedUrl(filepath):
        filepath = PackageManager.mappedUrl(filepath)
    elif isHttpUrl(filepath) and cntlr and hasattr(
            cntlr, "modelManager"
    ):  # may be called early in initialization for PluginManager
        filepath = cntlr.modelManager.disclosureSystem.mappedUrl(filepath)
    if archiveFilenameParts(filepath):  # file is in an archive
        return openFileSource(filepath, cntlr).file(filepath,
                                                    binary='b' in mode,
                                                    encoding=encoding)[0]
    if isHttpUrl(filepath) and cntlr:
        _cacheFilepath = cntlr.webCache.getfilename(
            filepath, normalize=True
        )  # normalize is separate step in ModelDocument retrieval, combined here
        if _cacheFilepath is None:
            raise IOError(_("Unable to open file: {0}.").format(filepath))
        filepath = _cacheFilepath
    # file path may be server (or memcache) or local file system
    if filepath.startswith(SERVER_WEB_CACHE) and cntlr:
        filestream = None
        cacheKey = filepath[len(SERVER_WEB_CACHE) + 1:].replace("\\", "/")
        if cntlr.isGAE:  # check if in memcache
            cachedBytes = gaeGet(cacheKey)
            if cachedBytes:
                filestream = io.BytesIO(cachedBytes)
        if filestream is None:
            filestream = io.BytesIO()
            cntlr.webCache.retrieve(
                cntlr.webCache.cacheFilepathToUrl(filepath),
                filestream=filestream)
            if cntlr.isGAE:
                gaeSet(cacheKey, filestream.getvalue())
        if mode.endswith('t') or encoding:
            contents = filestream.getvalue()
            filestream.close()
            filestream = FileNamedStringIO(
                filepath, contents.decode(encoding or 'utf-8'))
        return filestream
    # local file system
    elif encoding is None and 'b' not in mode:
        openedFileStream = io.open(filepath, mode='rb')
        hdrBytes = openedFileStream.read(512)
        encoding = XmlUtil.encoding(hdrBytes, default=None)
        openedFileStream.close()
        return io.open(filepath, mode=mode, encoding=encoding)
    else:
        # local file system
        return io.open(filepath, mode=mode, encoding=encoding)
Exemplo n.º 6
0
 def basedUrl(self, selection):
     if isHttpUrl(selection) or os.path.isabs(selection):
         return selection
     elif self.baseIsHttp or os.sep == '/':
         return self.baseurl + "/" + selection
     else:  # MSFT os.sep == '\\'
         return self.baseurl + os.sep + selection.replace("/", os.sep)
Exemplo n.º 7
0
 def __init__(self, url, cntlr=None, checkIfXmlIsEis=False):
     self.url = str(url)  # allow either string or FileNamedStringIO
     self.baseIsHttp = isHttpUrl(self.url)
     self.cntlr = cntlr
     self.type = self.url.lower()[-4:]
     self.isZip = self.type == ".zip"
     self.isEis = self.type == ".eis"
     self.isXfd = (self.type == ".xfd" or self.type == ".frm")
     self.isRss = (self.type == ".rss" or self.url.endswith(".rss.xml"))
     self.isOpen = False
     self.fs = None
     self.selection = None
     self.filesDir = None
     self.referencedFileSources = {}  # archive file name, fileSource object
     self.mappedPaths = {}  # remappings of path segments may be loaded by taxonomyPackage manifest
     
     # for SEC xml files, check if it's an EIS anyway
     if (not (self.isZip or self.isEis or self.isXfd or self.isRss) and
         self.type == ".xml" and
         checkIfXmlIsEis):
         try:
             file = open(self.cntlr.webCache.getfilename(self.url), 'r', errors='replace')
             l = file.read(128)
             file.close()
             if re.match(r"\s*(<[?]xml[^?]+[?]>)?\s*<cor[a-z]*:edgarSubmission", l):
                 self.isEis = True
         except EnvironmentError as err:
             if self.cntlr:
                 self.cntlr.addToLog(_("[{0}] {1}").format(type(err).__name__, err))
             pass
Exemplo n.º 8
0
def openFileStream(cntlr, filepath, mode='r', encoding=None):
    if isHttpUrl(filepath) and cntlr:
        filepath = cntlr.webCache.getfilename(filepath)
    # file path may be server (or memcache) or local file system
    if filepath.startswith(SERVER_WEB_CACHE) and cntlr:
        filestream = None
        cacheKey = filepath[len(SERVER_WEB_CACHE) + 1:].replace("\\", "/")
        if cntlr.isGAE:  # check if in memcache
            cachedBytes = gaeGet(cacheKey)
            if cachedBytes:
                filestream = io.BytesIO(cachedBytes)
        if filestream is None:
            filestream = io.BytesIO()
            cntlr.webCache.retrieve(
                cntlr.webCache.cacheFilepathToUrl(filepath),
                filestream=filestream)
            if cntlr.isGAE:
                gaeSet(cacheKey, filestream.getvalue())
        if mode.endswith('t') or encoding:
            contents = filestream.getvalue()
            filestream.close()
            filestream = FileNamedStringIO(
                filepath, contents.decode(encoding or 'utf-8'))
        return filestream
    # local file system
    elif encoding is None and 'b' not in mode:
        openedFileStream = io.open(filepath, mode='rb')
        hdrBytes = openedFileStream.read(512)
        encoding = XmlUtil.encoding(hdrBytes, default=None)
        openedFileStream.close()
        return io.open(filepath, mode=mode, encoding=encoding)
    else:
        # local file system
        return io.open(filepath, mode=mode, encoding=encoding)
Exemplo n.º 9
0
def package(dts):
    if dts.fileSource.isArchive:
        return
    import os
    from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED 
    from arelle.UrlUtil import isHttpUrl
    try:
        import zlib
        compression = ZIP_DEFLATED
    except ImportError:
        compression = ZIP_STORED
        dts.info("info:packageDTS",
                 _("Python's zlib module is not available, output is not compressed."),
                 modelObject=dts)
    entryFilename = dts.fileSource.url
    pkgFilename = entryFilename + ".zip"
    with ZipFile(pkgFilename, 'w', compression) as zipFile:
        numFiles = 0
        for fileUri in sorted(dts.urlDocs.keys()):
            if not isHttpUrl(fileUri):
                numFiles += 1
                # this has to be a relative path because the hrefs will break
                zipFile.write(fileUri, os.path.basename(fileUri))
    dts.info("info:packageDTS",
             _("DTS of %(entryFile)s has %(numberOfFiles)s files packaged into %(packageOutputFile)s."),
             modelObject=dts,
             entryFile=entryFilename, numberOfFiles=numFiles, packageOutputFile=pkgFilename)
Exemplo n.º 10
0
def openFileStream(cntlr, filepath, mode='r', encoding=None):
    if isHttpUrl(filepath) and cntlr:
        _cacheFilepath = cntlr.webCache.getfilename(filepath)
        if _cacheFilepath is None:
            raise IOError(_("Unable to open file: {0}.").format(filepath))
        filepath = _cacheFilepath
    # file path may be server (or memcache) or local file system
    if filepath.startswith(SERVER_WEB_CACHE) and cntlr:
        filestream = None
        cacheKey = filepath[len(SERVER_WEB_CACHE) + 1:].replace("\\","/")
        if cntlr.isGAE: # check if in memcache
            cachedBytes = gaeGet(cacheKey)
            if cachedBytes:
                filestream = io.BytesIO(cachedBytes)
        if filestream is None:
            filestream = io.BytesIO()
            cntlr.webCache.retrieve(cntlr.webCache.cacheFilepathToUrl(filepath),
                                    filestream=filestream)
            if cntlr.isGAE:
                gaeSet(cacheKey, filestream.getvalue())
        if mode.endswith('t') or encoding:
            contents = filestream.getvalue()
            filestream.close()
            filestream = FileNamedStringIO(filepath, contents.decode(encoding or 'utf-8'))
        return filestream
    # local file system
    elif encoding is None and 'b' not in mode:
        openedFileStream = io.open(filepath, mode='rb')
        hdrBytes = openedFileStream.read(512)
        encoding = XmlUtil.encoding(hdrBytes, default=None)
        openedFileStream.close()
        return io.open(filepath, mode=mode, encoding=encoding)
    else:
        # local file system
        return io.open(filepath, mode=mode, encoding=encoding)
Exemplo n.º 11
0
 def addLocallyReferencedFile(elt):
     if elt.tag in ("a", "img", "{http://www.w3.org/1999/xhtml}a", "{http://www.w3.org/1999/xhtml}img"):
         for attrTag, attrValue in elt.items():
             if attrTag in ("href", "src") and not isHttpUrl(attrValue) and not os.path.isabs(attrValue):
                 file = os.path.join(sourceDir,attrValue)
                 if os.path.exists(file):
                     self.reportedFiles.add(os.path.join(sourceDir,attrValue))
Exemplo n.º 12
0
 def basedUrl(self, selection):
     if isHttpUrl(selection) or os.path.isabs(selection):
         return selection
     elif self.baseIsHttp or os.sep == '/':
         return self.baseurl + "/" + selection
     else: # MSFT os.sep == '\\'
         return self.baseurl + os.sep + selection.replace("/", os.sep)
Exemplo n.º 13
0
def openFileStream(cntlr, filepath, mode='r', encoding=None):
    if isHttpUrl(filepath) and cntlr:
        filepath = cntlr.webCache.getfilename(filepath)
    # file path may be server (or memcache) or local file system
    if filepath.startswith(SERVER_WEB_CACHE) and cntlr and not cntlr.localOnly:
        filestream = None
        cacheKey = filepath[len(SERVER_WEB_CACHE) + 1:].replace("\\","/")
        if cntlr.isGAE: # check if in memcache
            cachedBytes = gaeGet(cacheKey)
            if cachedBytes:
                filestream = io.BytesIO(cachedBytes)
        if filestream is None:
            filestream = io.BytesIO()
            cntlr.webCache.retrieve(cntlr.webCache.cacheFilepathToUrl(filepath),
                                    filestream=filestream)
            if cntlr.isGAE:
                gaeSet(cacheKey, filestream.getvalue())
        if mode.endswith('t') or encoding:
            contents = filestream.getvalue()
            filestream.close()
            filestream = FileNamedStringIO(filepath, contents.decode(encoding or 'utf-8'))
        return filestream
    # local file system
    elif encoding is None and 'b' not in mode:
        openedFileStream = io.open(filepath, mode='rb')
        hdrBytes = openedFileStream.read(512)
        encoding = XmlUtil.encoding(hdrBytes, default=None)
        openedFileStream.close()
        return io.open(filepath, mode=mode, encoding=encoding)
    else:
        # local file system
        if cntlr and cntlr.isGAE and cntlr.localOnly and filepath[0] == '/':
            # can't do an absolute path here, so presume it is a relative to the path of the app source
            filepath = '.%s' % filepath
        return io.open(filepath, mode=mode, encoding=encoding)
Exemplo n.º 14
0
 def createFacts(facts, parent):
     for fact in facts:
         if fact.isItem:
             attrs = {"contextRef": fact.contextID}
             if fact.id:
                 attrs["id"] = fact.id
             if fact.isNumeric:
                 attrs["unitRef"] = fact.unitID
                 if fact.get("decimals"):
                     attrs["decimals"] = fact.get("decimals")
                 if fact.get("precision"):
                     attrs["precision"] = fact.get("precision")
             if fact.isNil:
                 attrs[XbrlConst.qnXsiNil] = "true"
                 text = None
             else:
                 text = fact.xValue if fact.xValid else fact.textValue
             newFact = targetInstance.createFact(fact.qname, attributes=attrs, text=text, parent=parent)
             newFactForOldObjId[fact.objectIndex] = newFact
             if filingFiles and fact.concept is not None and fact.concept.isTextBlock:
                 # check for img and other filing references
                 for xmltext in [text] + CDATApattern.findall(text):
                     try:
                         for elt in XML("<body>\n{0}\n</body>\n".format(xmltext)):
                             if elt.tag in ("a", "img") and not isHttpUrl(attrValue) and not os.path.isabs(attrvalue):
                                 for attrTag, attrValue in elt.items():
                                     if attrTag in ("href", "src"):
                                         filingFiles.add(attrValue)
                     except (XMLSyntaxError, UnicodeDecodeError):
                         pass
         elif fact.isTuple:
             newTuple = targetInstance.createFact(fact.qname, parent=parent)
             newFactForOldObjId[fact.objectIndex] = newTuple
             createFacts(fact.modelTupleFacts, newTuple)
Exemplo n.º 15
0
def openFileStream(cntlr, filepath, mode='r', encoding=None):
    if isHttpUrl(filepath) and cntlr:
        filepath = cntlr.webCache.getfilename(filepath)
    # file path may be server (or memcache) or local file system
    if filepath.startswith(SERVER_WEB_CACHE) and cntlr:
        filestream = None
        cacheKey = filepath[len(SERVER_WEB_CACHE) + 1:].replace("\\","/")
        if cntlr.isGAE: # check if in memcache
            cachedBytes = gaeGet(cacheKey)
            if cachedBytes:
                filestream = io.BytesIO(cachedBytes)
        if filestream is None:
            filestream = io.BytesIO()
            cntlr.webCache.retrieve(cntlr.webCache.cacheFilepathToUrl(filepath),
                                    filestream=filestream)
            if cntlr.isGAE:
                gaeSet(cacheKey, filestream.getvalue())
        if mode.endswith('t') or encoding:
            contents = filestream.getvalue()
            filestream.close()
            filestream = io.StringIO(contents.decode(encoding or 'utf-8'))
        return filestream
    else:
        # local file system
        return io.open(filepath, mode=mode, encoding=encoding)
Exemplo n.º 16
0
    def __init__(self, url, cntlr=None, checkIfXmlIsEis=False):
        self.url = str(url)  # allow either string or FileNamedStringIO
        self.baseIsHttp = isHttpUrl(self.url)
        self.cntlr = cntlr
        self.type = self.url.lower()[-7:]
        self.isTarGz = self.type == ".tar.gz"
        if not self.isTarGz:
            self.type = self.type[3:]
        self.isZip = self.type == ".zip"
        self.isEis = self.type == ".eis"
        self.isXfd = self.type == ".xfd" or self.type == ".frm"
        self.isRss = self.type == ".rss" or self.url.endswith(".rss.xml")
        self.isInstalledTaxonomyPackage = False
        self.isOpen = False
        self.fs = None
        self.selection = None
        self.filesDir = None
        self.referencedFileSources = {}  # archive file name, fileSource object
        self.mappedPaths = None  # remappings of path segments may be loaded by taxonomyPackage manifest

        # for SEC xml files, check if it's an EIS anyway
        if not (self.isZip or self.isEis or self.isXfd or self.isRss) and self.type == ".xml":
            if os.path.split(self.url)[-1] in TAXONOMY_PACKAGE_FILE_NAMES:
                self.isInstalledTaxonomyPackage = True
            elif checkIfXmlIsEis:
                try:
                    file = open(self.cntlr.webCache.getfilename(self.url), "r", errors="replace")
                    l = file.read(128)
                    file.close()
                    if re.match(r"\s*(<[?]xml[^?]+[?]>)?\s*<(cor[a-z]*:)?edgarSubmission", l):
                        self.isEis = True
                except EnvironmentError as err:
                    if self.cntlr:
                        self.cntlr.addToLog(_("[{0}] {1}").format(type(err).__name__, err))
                    pass
Exemplo n.º 17
0
def package(dts):
    if dts.fileSource.isArchive:
        return
    import os
    from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED
    from arelle.UrlUtil import isHttpUrl
    try:
        import zlib
        compression = ZIP_DEFLATED
    except ImportError:
        compression = ZIP_STORED
        dts.info(
            "info:packageDTS",
            _("Python's zlib module is not available, output is not compressed."
              ),
            modelObject=dts)
    entryFilename = dts.fileSource.url
    pkgFilename = entryFilename + ".zip"
    with ZipFile(pkgFilename, 'w', compression) as zipFile:
        numFiles = 0
        for fileUri in sorted(dts.urlDocs.keys()):
            if not isHttpUrl(fileUri):
                numFiles += 1
                # this has to be a relative path because the hrefs will break
                zipFile.write(fileUri, os.path.basename(fileUri))
    dts.info(
        "info:packageDTS",
        _("DTS of %(entryFile)s has %(numberOfFiles)s files packaged into %(packageOutputFile)s."
          ),
        modelObject=dts,
        entryFile=entryFilename,
        numberOfFiles=numFiles,
        packageOutputFile=pkgFilename)
Exemplo n.º 18
0
 def ok(self, event=None):
     selection = self.treeView.selection()
     if len(selection) > 0:
         if hasattr(self, "taxonomyPackage"):
             # load file source remappings
             self.filesource.mappedPaths = self.taxonomyPackage["remappings"]
         filename = None
         if self.openType in (ARCHIVE, DISCLOSURE_SYSTEM):
             filename = self.filenames[int(selection[0][4:])]
             if isinstance(filename,tuple):
                 if self.isRss:
                     filename = filename[4]
                 else:
                     filename = filename[0]
         elif self.openType == ENTRY_POINTS:
             epName = selection[0]
             self.filesource.reportName = epName
             #index 0 is the remapped Url, as opposed to the canonical one used for display
             # Greg Acsone reports [0] does not work for Corep 1.6 pkgs, need [1], old style packages
             filename = self.taxonomyPackage["nameToUrls"][epName][1]
             if not filename.endswith("/"):
                 # check if it's an absolute URL rather than a path into the archive
                 if not isHttpUrl(filename) and self.metadataFilePrefix != self.taxonomyPkgMetaInf:
                     # assume it's a path inside the archive:
                     filename = self.metadataFilePrefix + filename
         if filename is not None and not filename.endswith("/"):
             if hasattr(self, "taxonomyPackage"):
                 # attempt to unmap the filename to original file
                 # will be mapped again in loading, but this allows schemaLocation to be unmapped
                 for prefix, remapping in self.taxonomyPackage["remappings"].items():
                     if isHttpUrl(remapping):
                         remapStart = remapping
                     elif self.filesource.isArchive:
                         remapStart = remapping[len(self.filesource.baseurl):]
                         if remapStart.startswith('\\'):
                             remapStart = remapStart[1:]
                         if remapStart.endswith('\\'):
                             remapStart = remapStart[:-1]+'/'
                     else:
                         remapStart = self.metadataFilePrefix + remapping
                     if filename.startswith(remapStart):
                         # set unmmapped file
                         filename = prefix + filename[len(remapStart):]
                         break
             self.filesource.select(filename)
             self.accepted = True
             self.close()
Exemplo n.º 19
0
 def select(self, selection):
     self.selection = selection
     if isHttpUrl(selection) or os.path.isabs(selection):
         self.url = selection
     elif self.baseIsHttp or os.sep == '/':
         self.url = self.baseurl + "/" + selection
     else:  # MSFT os.sep == '\\'
         self.url = self.baseurl + os.sep + selection.replace("/", os.sep)
Exemplo n.º 20
0
 def getAttachmentFilename(self, url):  # get the filename attachment from the header
     if url and isHttpUrl(url):
         try:
             fp = self.opener.open(url, timeout=self.timeout)
             return cgi.parse_header(fp.headers.get("Content-Disposition"))[1]["filename"]
         except Exception:
             pass
     return None
Exemplo n.º 21
0
 def select(self, selection):
     self.selection = selection
     if isHttpUrl(selection) or os.path.isabs(selection):
         self.url = selection
     elif self.baseIsHttp or os.sep == '/':
         self.url = self.baseurl + "/" + selection
     else: # MSFT os.sep == '\\'
         self.url = self.baseurl + os.sep + selection.replace("/", os.sep)
Exemplo n.º 22
0
def packageInfo(URL, reload=False):
    # TODO several directories, eg User Application Data
    packageFilename = _cntlr.webCache.getfilename(URL, reload=reload, normalize=True)
    if packageFilename:
        filesource = None
        try:
            global openFileSource
            if openFileSource is None:
                from arelle.FileSource import openFileSource
            filesource = openFileSource(packageFilename, _cntlr)
            if filesource.isZip:
                metadataFiles = filesource.taxonomyPackageMetadataFiles
                if len(metadataFiles) != 1:
                    raise IOError(
                        _("Taxonomy package contained more than one metadata file: {0}.").format(
                            ", ".join(metadataFiles)
                        )
                    )
                metadataFile = metadataFiles[0]
                metadata = filesource.file(filesource.url + os.sep + metadataFile)[0]
                metadataFilePrefix = os.sep.join(os.path.split(metadataFile)[:-1])
                if metadataFilePrefix:
                    metadataFilePrefix += os.sep
                metadataFilePrefix = filesource.baseurl + os.sep + metadataFilePrefix
            elif os.path.basename(filesource.url) == ".taxonomyPackage.xml":  # individual manifest file
                metadataFile = metadata = filesource.url
                metadataFilePrefix = os.sep.join(os.path.split(metadataFile)[:-1])
                if metadataFilePrefix:
                    metadataFilePrefix += os.sep
            else:
                raise IOError(
                    _("File must be a taxonomy package (zip file) or manifest (.taxonomyPackage.xml): {0}.").format(
                        metadataFile
                    )
                )
            parsedPackage = parsePackage(_cntlr, metadata)
            package = {
                "name": parsedPackage["name"],
                "status": "enabled",
                "version": parsedPackage["version"],
                "fileDate": time.strftime("%Y-%m-%dT%H:%M:%S UTC", time.gmtime(os.path.getmtime(packageFilename))),
                "URL": URL,
                "description": parsedPackage["description"],
                "remappings": dict(
                    (
                        prefix,
                        remapping if isHttpUrl(remapping) else (metadataFilePrefix + remapping.replace("/", os.sep)),
                    )
                    for prefix, remapping in parsedPackage["remappings"].items()
                ),
            }
            filesource.close()
            return package
        except EnvironmentError:
            pass
        if filesource:
            filesource.close()
    return None
Exemplo n.º 23
0
    def ok(self, event=None):
        selection = self.treeView.selection()
        if len(selection) > 0:
            if hasattr(self, "taxonomyPackage"):
                # load file source remappings
                self.filesource.mappedPaths = \
                    dict((prefix,
                          remapping if isHttpUrl(remapping)
                          else (self.filesource.baseurl + os.sep + self.metadataFilePrefix +remapping.replace("/", os.sep)))
                          for prefix, remapping in self.taxonomyPackage["remappings"].items())
            filename = None
            if self.openType in (ARCHIVE, DISCLOSURE_SYSTEM):
                filename = self.filenames[int(selection[0][4:])]
                if isinstance(filename, tuple):
                    if self.isRss:
                        filename = filename[4]
                    else:
                        filename = filename[0]
            elif self.openType == ENTRY_POINTS:
                epName = selection[0]
                #index 0 is the remapped Url, as opposed to the canonical one used for display
                filename = self.taxonomyPackage["nameToUrls"][epName][0]

                if not filename.endswith("/"):
                    # check if it's an absolute URL rather than a path into the archive
                    if not isHttpUrl(filename):
                        # assume it's a path inside the archive:
                        filename = self.metadataFilePrefix + filename
            if filename is not None and not filename.endswith("/"):
                if hasattr(self, "taxonomyPackage"):
                    # attempt to unmap the filename to original file
                    # will be mapped again in loading, but this allows schemaLocation to be unmapped
                    for prefix, remapping in self.taxonomyPackage[
                            "remappings"].items():
                        if isHttpUrl(remapping):
                            remapStart = remapping
                        else:
                            remapStart = self.metadataFilePrefix + remapping
                        if filename.startswith(remapStart):
                            # set unmmapped file
                            filename = prefix + filename[len(remapStart):]
                            break
                self.filesource.select(filename)
                self.accepted = True
                self.close()
Exemplo n.º 24
0
def archiveFilenameParts(filename, checkIfXmlIsEis=False):
    # check if path has an archive file plus appended in-archive content reference
    for archiveSep in archivePathSeparators:
        if filename and archiveSep in filename and (not archiveSep.startswith(".xml") or checkIfXmlIsEis):
            filenameParts = filename.partition(archiveSep)
            fileDir = filenameParts[0] + archiveSep[:-1]
            if isHttpUrl(fileDir) or os.path.isfile(fileDir):  # if local, be sure it is not a directory name
                return (fileDir, filenameParts[2])
    return None
Exemplo n.º 25
0
 def disallowedHrefOfNamespace(self, href, namespaceUri):
     if namespaceUri in self.standardTaxonomiesDict:
         if href in self.standardTaxonomiesDict[namespaceUri]:
             return False
     if namespaceUri in self.standardLocalHrefs and not isHttpUrl(href):
         normalizedHref = href.replace("\\", "/")
         if any(normalizedHref.endswith(localHref) for localHref in self.standardLocalHrefs[namespaceUri]):
             return False
     return False
Exemplo n.º 26
0
 def ok(self, event=None):
     selection = self.treeView.selection()
     if len(selection) > 0:
         if hasattr(self, "taxonomyPackage"):
             # load file source remappings
             self.filesource.mappedPaths = \
                 dict((prefix, 
                       remapping if isHttpUrl(remapping)
                       else (self.filesource.baseurl + os.sep + self.metadataFilePrefix +remapping.replace("/", os.sep)))
                       for prefix, remapping in self.taxonomyPackage["remappings"].items())
         filename = None
         if self.openType in (ARCHIVE, DISCLOSURE_SYSTEM):
             filename = self.filenames[int(selection[0][4:])]
             if isinstance(filename,tuple):
                 if self.isRss:
                     filename = filename[4]
                 else:
                     filename = filename[0]
         elif self.openType == ENTRY_POINTS:
             epName = selection[0]
             #index 0 is the remapped Url, as opposed to the canonical one used for display
             filename = self.taxonomyPackage["nameToUrls"][epName][0]
                 
             if not filename.endswith("/"):
                 # check if it's an absolute URL rather than a path into the archive
                 if not isHttpUrl(filename):
                     # assume it's a path inside the archive:
                     filename = self.metadataFilePrefix + filename
         if filename is not None and not filename.endswith("/"):
             if hasattr(self, "taxonomyPackage"):
                 # attempt to unmap the filename to original file
                 # will be mapped again in loading, but this allows schemaLocation to be unmapped
                 for prefix, remapping in self.taxonomyPackage["remappings"].items():
                     if isHttpUrl(remapping):
                         remapStart = remapping
                     else:
                         remapStart = self.metadataFilePrefix + remapping
                     if filename.startswith(remapStart):
                         # set unmmapped file
                         filename = prefix + filename[len(remapStart):]
                         break
             self.filesource.select(filename)
             self.accepted = True
             self.close()
Exemplo n.º 27
0
 def getheaders(self, url):
     if url and isHttpUrl(url):
         try:
             fp = self.opener.open(url, timeout=self.timeout)
             headers = fp.info()
             fp.close()
             return headers
         except Exception:
             pass
     return {}
Exemplo n.º 28
0
 def addLocallyReferencedFile(elt):
     if elt.tag in ("a", "img", "{http://www.w3.org/1999/xhtml}a", "{http://www.w3.org/1999/xhtml}img"):
         for attrTag, attrValue in elt.items():
             if attrTag in ("href", "src") and not isHttpUrl(attrValue) and not os.path.isabs(attrValue):
                 attrValue = attrValue.partition('#')[0] # remove anchor
                 if attrValue: # ignore anchor references to base document
                     attrValue = os.path.normpath(attrValue) # change url path separators to host separators
                     file = os.path.join(sourceDir,attrValue)
                     if modelXbrl.fileSource.isInArchive(file, checkExistence=True) or os.path.exists(file):
                         self.reportedFiles.add(attrValue) # add file name within source directory
Exemplo n.º 29
0
 def geturl(self, url):  # get the url that the argument url redirects or resolves to
     if url and isHttpUrl(url):
         try:
             fp = self.opener.open(url, timeout=self.timeout)
             actualurl = fp.geturl()
             fp.close()
             return actualurl
         except Exception:
             pass
     return None
Exemplo n.º 30
0
 def addLocallyReferencedFile(elt,filingFiles):
     if elt.tag in ("a", "img"):
         for attrTag, attrValue in elt.items():
             if attrTag in ("href", "src") and not isHttpUrl(attrValue) and not os.path.isabs(attrValue):
                 attrValue = attrValue.partition('#')[0] # remove anchor
                 if attrValue: # ignore anchor references to base document
                     attrValue = os.path.normpath(attrValue) # change url path separators to host separators
                     file = os.path.join(sourceDir,attrValue)
                     if modelXbrl.fileSource.isInArchive(file, checkExistence=True) or os.path.exists(file):
                         filingFiles.add(file)
Exemplo n.º 31
0
 def geturl(self, url):  # get the url that the argument url redirects or resolves to
     if url and isHttpUrl(url):
         try:
             fp = self.opener.open(url, timeout=self.timeout)
             actualurl = fp.geturl()
             fp.close()
             return actualurl
         except Exception:
             pass
     return None
Exemplo n.º 32
0
 def getheaders(self, url):
     if url and isHttpUrl(url):
         try:
             fp = self.opener.open(url, timeout=self.timeout)
             headers = fp.info()
             fp.close()
             return headers
         except Exception:
             pass
     return {}
Exemplo n.º 33
0
 def addLocallyReferencedFile(elt,filingFiles):
     if elt.tag in ("a", "img"):
         for attrTag, attrValue in elt.items():
             if attrTag in ("href", "src") and not isHttpUrl(attrValue) and not os.path.isabs(attrValue):
                 attrValue = attrValue.partition('#')[0] # remove anchor
                 if attrValue: # ignore anchor references to base document
                     attrValue = os.path.normpath(attrValue) # change url path separators to host separators
                     file = os.path.join(sourceDir,attrValue)
                     if modelXbrl.fileSource.isInArchive(file, checkExistence=True) or os.path.exists(file):
                         filingFiles.add(file)
Exemplo n.º 34
0
 def disallowedHrefOfNamespace(self, href, namespaceUri):
     if namespaceUri in self.standardTaxonomiesDict:
         if href in self.standardTaxonomiesDict[namespaceUri]:
             return False
     if namespaceUri in self.standardLocalHrefs and not isHttpUrl(href):
         normalizedHref = href.replace("\\","/")
         if any(normalizedHref.endswith(localHref)
                for localHref in self.standardLocalHrefs[namespaceUri]):
             return False
     return False
Exemplo n.º 35
0
 def addLocallyReferencedFile(elt):
     if elt.tag in ("a", "img", "{http://www.w3.org/1999/xhtml}a",
                    "{http://www.w3.org/1999/xhtml}img"):
         for attrTag, attrValue in elt.items():
             if attrTag in ("href", "src") and not isHttpUrl(
                     attrValue) and not os.path.isabs(attrValue):
                 file = os.path.join(sourceDir, attrValue)
                 if os.path.exists(file):
                     self.reportedFiles.add(
                         os.path.join(sourceDir, attrValue))
Exemplo n.º 36
0
 def getAttachmentFilename(
         self, url):  # get the filename attachment from the header
     if url and isHttpUrl(url):
         try:
             fp = self.opener.open(url, timeout=self.timeout)
             return cgi.parse_header(
                 fp.headers.get("Content-Disposition"))[1]["filename"]
         except Exception:
             pass
     return None
Exemplo n.º 37
0
def inlineXbrlDocumentSetLoader(modelXbrl, normalizedUri, filepath, isEntry=False, namespace=None, **kwargs):
    if isEntry:
        try:
            if "entrypoint" in kwargs:
                _target = kwargs["entrypoint"]["ixdsTarget"]
            elif "ixdsTarget" in kwargs: # passed from validate (multio test cases)
                _target = kwargs["ixdsTarget"]
            else:
                _target = modelXbrl.modelManager.formulaOptions.parameterValues.get("ixdsTarget")[1]
        except (KeyError, AttributeError, IndexError, TypeError):
            _target = None
        modelXbrl.ixdsTarget = _target or None # None if an empty string specified
    if IXDS_SURROGATE in normalizedUri:
        # create surrogate entry object for inline document set which references ix documents
        xml = ["<instances>\n"]
        modelXbrl.ixdsDocUrls = []
        schemeFixup = isHttpUrl(normalizedUri) # schemes after separator have // normalized to single /
        msUNCfixup = modelXbrl.modelManager.cntlr.isMSW and normalizedUri.startswith("\\\\") # path starts with double backslash \\
        if schemeFixup:
            defectiveScheme = normalizedUri.partition("://")[0] + ":/"
            fixupPosition = len(defectiveScheme)
        for i, url in enumerate(normalizedUri.split(IXDS_DOC_SEPARATOR)):
            if schemeFixup and url.startswith(defectiveScheme) and url[len(defectiveScheme)] != "/":
                url = url[:fixupPosition] + "/" + url[fixupPosition:]
            if i == 0:
                docsetUrl = url
            else:
                if msUNCfixup and not url.startswith("\\\\") and url.startswith("\\"):
                    url = "\\" + url
                xml.append("<instance>{}</instance>\n".format(url))
                modelXbrl.ixdsDocUrls.append(url)
        xml.append("</instances>\n")
        ixdocset = create(modelXbrl, Type.INLINEXBRLDOCUMENTSET, docsetUrl, isEntry=True, initialXml="".join(xml))
        ixdocset.type = Type.INLINEXBRLDOCUMENTSET
        ixdocset.targetDocumentSchemaRefs = set()  # union all the instance schemaRefs
        _firstdoc = True
        for elt in ixdocset.xmlRootElement.iter(tag="instance"):
            # load ix document
            ixdoc = load(modelXbrl, elt.text, referringElement=elt)
            if ixdoc is not None and ixdoc.type == Type.INLINEXBRL:
                # set reference to ix document in document set surrogate object
                referencedDocument = ModelDocumentReference("inlineDocument", elt)
                ixdocset.referencesDocument[ixdoc] = referencedDocument
                for referencedDoc in ixdoc.referencesDocument.keys():
                    if referencedDoc.type == Type.SCHEMA:
                        ixdocset.targetDocumentSchemaRefs.add(ixdoc.relativeUri(referencedDoc.uri))
                ixdocset.ixNS = ixdoc.ixNS # set docset ixNS
                if _firstdoc:
                    _firstdoc = False
                    ixdocset.targetDocumentPreferredFilename = os.path.splitext(ixdoc.uri)[0] + ".xbrl"
                ixdoc.inDTS = True # behaves like an entry
        if hasattr(modelXbrl, "ixdsHtmlElements"): # has any inline root elements
            inlineIxdsDiscover(modelXbrl, ixdocset) # compile cross-document IXDS references
            return ixdocset
    return None
Exemplo n.º 38
0
def saveFile(cntlr, filepath, contents, encoding=None):
    if isHttpUrl(filepath):
        filepath = cntlr.webCache.getfilename(filepath)
    # file path may be server (or memcache) or local file system
    if filepath.startswith(SERVER_WEB_CACHE):
        cacheKey = filepath[len(SERVER_WEB_CACHE) + 1:].replace("\\","/")
        if cntlr.isGAE: # check if in memcache
            gaeSet(cacheKey, contents.encode(encoding or 'utf-8'))
    else:
        with io.open(filepath, 'wt', encoding=(encoding or 'utf-8')) as f:
            f.write(contents)
Exemplo n.º 39
0
def archiveFilenameParts(filename, checkIfXmlIsEis=False):
    # check if path has an archive file plus appended in-archive content reference
    for archiveSep in archivePathSeparators:
        if (filename and archiveSep in filename
                and (not archiveSep.startswith(".xml") or checkIfXmlIsEis)):
            filenameParts = filename.partition(archiveSep)
            fileDir = filenameParts[0] + archiveSep[:-1]
            if (isHttpUrl(fileDir) or os.path.isfile(fileDir)
                ):  # if local, be sure it is not a directory name
                return (fileDir, filenameParts[2])
    return None
Exemplo n.º 40
0
def saveFile(cntlr, filepath, contents, encoding=None):
    if isHttpUrl(filepath):
        filepath = cntlr.webCache.getfilename(filepath)
    # file path may be server (or memcache) or local file system
    if filepath.startswith(SERVER_WEB_CACHE):
        cacheKey = filepath[len(SERVER_WEB_CACHE) + 1:].replace("\\", "/")
        if cntlr.isGAE:  # check if in memcache
            gaeSet(cacheKey, contents.encode(encoding or 'utf-8'))
    else:
        with io.open(filepath, 'wt', encoding=(encoding or 'utf-8')) as f:
            f.write(contents)
Exemplo n.º 41
0
def openFileStream(cntlr, filepath, mode='r', encoding=None):
    if PackageManager.isMappedUrl(filepath):
        filepath = PackageManager.mappedUrl(filepath)
    elif isHttpUrl(filepath) and cntlr and hasattr(cntlr, "modelManager"): # may be called early in initialization for PluginManager
        filepath = cntlr.modelManager.disclosureSystem.mappedUrl(filepath)
    if archiveFilenameParts(filepath): # file is in an archive
        return openFileSource(filepath, cntlr).file(filepath, binary='b' in mode, encoding=encoding)[0]
    if isHttpUrl(filepath) and cntlr:
        _cacheFilepath = cntlr.webCache.getfilename(filepath, normalize=True) # normalize is separate step in ModelDocument retrieval, combined here
        if _cacheFilepath is None:
            raise IOError(_("Unable to open file: {0}.").format(filepath))
        filepath = _cacheFilepath
    # file path may be server (or memcache) or local file system
    if filepath.startswith(SERVER_WEB_CACHE) and cntlr:
        filestream = None
        cacheKey = filepath[len(SERVER_WEB_CACHE) + 1:].replace("\\","/")
        if cntlr.isGAE: # check if in memcache
            cachedBytes = gaeGet(cacheKey)
            if cachedBytes:
                filestream = io.BytesIO(cachedBytes)
        if filestream is None:
            filestream = io.BytesIO()
            cntlr.webCache.retrieve(cntlr.webCache.cacheFilepathToUrl(filepath),
                                    filestream=filestream)
            if cntlr.isGAE:
                gaeSet(cacheKey, filestream.getvalue())
        if mode.endswith('t') or encoding:
            contents = filestream.getvalue()
            filestream.close()
            filestream = FileNamedStringIO(filepath, contents.decode(encoding or 'utf-8'))
        return filestream
    # local file system
    elif encoding is None and 'b' not in mode:
        openedFileStream = io.open(filepath, mode='rb')
        hdrBytes = openedFileStream.read(512)
        encoding = XmlUtil.encoding(hdrBytes, default=None)
        openedFileStream.close()
        return io.open(filepath, mode=mode, encoding=encoding)
    else:
        # local file system
        return io.open(filepath, mode=mode, encoding=encoding)
Exemplo n.º 42
0
def isStandardUri(val, uri):
    try:
        return val._isStandardUri[uri]
    except KeyError:
        isStd = (uri in val.disclosureSystem.standardTaxonomiesDict or
                 (not isHttpUrl(uri) and 
                  # try 2011-12-23 RH: if works, remove the localHrefs
                  # any(u.endswith(e) for u in (uri.replace("\\","/"),) for e in disclosureSystem.standardLocalHrefs)
                  "/basis/sbr/" in uri.replace("\\","/")
                  ))
        val._isStandardUri[uri] = isStd
        return isStd
Exemplo n.º 43
0
def isStandardUri(val, uri):
    try:
        return val._isStandardUri[uri]
    except KeyError:
        isStd = (
            uri in val.disclosureSystem.standardTaxonomiesDict or
            (not isHttpUrl(uri) and
             # try 2011-12-23 RH: if works, remove the localHrefs
             # any(u.endswith(e) for u in (uri.replace("\\","/"),) for e in disclosureSystem.standardLocalHrefs)
             "/basis/sbr/" in uri.replace("\\", "/")))
        val._isStandardUri[uri] = isStd
        return isStd
Exemplo n.º 44
0
    def normalizeUrl(self, url, base=None):
        if url:
            if url.startswith("file://"): url = url[7:]
            elif url.startswith("file:\\"): url = url[6:]
        if url and not (isHttpUrl(url) or os.path.isabs(url)):
            if base is not None and not isHttpUrl(base) and '%' in url:
                url = unquote(url)
            if base:
                if isHttpUrl(base):
                    scheme, sep, path = base.partition("://")
                    normedPath = scheme + sep + posixpath.normpath(
                        os.path.dirname(path) + "/" + url)
                else:
                    if '%' in base:
                        base = unquote(base)
                    normedPath = os.path.normpath(
                        os.path.join(os.path.dirname(base), url))
            else:  # includes base == '' (for forcing relative path)
                normedPath = url
            if normedPath.startswith("file://"): normedPath = normedPath[7:]
            elif normedPath.startswith("file:\\"): normedPath = normedPath[6:]

            # no base, not normalized, must be relative to current working directory
            if base is None and not os.path.isabs(url):
                normedPath = os.path.abspath(normedPath)
        else:
            normedPath = url

        if normedPath:
            if isHttpUrl(normedPath):
                scheme, sep, pathpart = normedPath.partition("://")
                pathpart = pathpart.replace('\\', '/')
                endingSep = '/' if pathpart[
                    -1] == '/' else ''  # normpath drops ending directory separator
                return scheme + "://" + posixpath.normpath(
                    pathpart) + endingSep
            normedPath = os.path.normpath(normedPath)
            if normedPath.startswith(self.cacheDir):
                normedPath = self.cacheFilepathToUrl(normedPath)
        return normedPath
Exemplo n.º 45
0
def packageInfo(URL, reload=False):
    #TODO several directories, eg User Application Data
    packageFilename = _cntlr.webCache.getfilename(URL, reload=reload, normalize=True)
    if packageFilename:
        from arelle.FileSource import TAXONOMY_PACKAGE_FILE_NAMES
        filesource = None
        try:
            global openFileSource
            if openFileSource is None:
                from arelle.FileSource import openFileSource
            filesource = openFileSource(packageFilename, _cntlr)
            if filesource.isZip:
                metadataFiles = filesource.taxonomyPackageMetadataFiles
                ''' allow multiple
                if len(metadataFiles) != 1:
                    raise IOError(_("Taxonomy package contained more than one metadata file: {0}.")
                                  .format(', '.join(metadataFiles)))
                '''
                metadataFile = metadataFiles[0]
                metadata = filesource.file(filesource.url + os.sep + metadataFile)[0]
                metadataFilePrefix = os.sep.join(os.path.split(metadataFile)[:-1])
                if metadataFilePrefix:
                    metadataFilePrefix += os.sep
                metadataFilePrefix = filesource.baseurl + os.sep +  metadataFilePrefix
            elif os.path.basename(filesource.url) in TAXONOMY_PACKAGE_FILE_NAMES: # individual manifest file
                metadataFile = metadata = filesource.url
                metadataFilePrefix = os.sep.join(os.path.split(metadataFile)[:-1])
                if metadataFilePrefix:
                    metadataFilePrefix += os.sep
            else:
                raise IOError(_("File must be a taxonomy package (zip file), catalog file, or manifest (): {0}.")
                              .format(metadataFile, ', '.join(TAXONOMY_PACKAGE_FILE_NAMES)))
            parsedPackage = parsePackage(_cntlr, metadata)
            package = {'name': parsedPackage['name'],
                       'status': 'enabled',
                       'version': parsedPackage['version'],
                       'fileDate': time.strftime('%Y-%m-%dT%H:%M:%S UTC', time.gmtime(os.path.getmtime(packageFilename))),
                       'URL': URL,
                       'description': parsedPackage['description'],
                       'remappings': dict(
                            (prefix, 
                             remapping if isHttpUrl(remapping)
                             else (metadataFilePrefix +remapping.replace("/", os.sep)))
                            for prefix, remapping in parsedPackage["remappings"].items()),
                       }
            filesource.close()
            return package
        except EnvironmentError:
            pass
        if filesource:
            filesource.close()
    return None
Exemplo n.º 46
0
def packageInfo(URL, reload=False):
    #TODO several directories, eg User Application Data
    packageFilename = _cntlr.webCache.getfilename(URL, reload=reload, normalize=True)
    if packageFilename:
        from arelle.FileSource import TAXONOMY_PACKAGE_FILE_NAMES
        filesource = None
        try:
            global openFileSource
            if openFileSource is None:
                from arelle.FileSource import openFileSource
            filesource = openFileSource(packageFilename, _cntlr)
            if filesource.isZip:
                metadataFiles = filesource.taxonomyPackageMetadataFiles
                ''' allow multiple
                if len(metadataFiles) != 1:
                    raise IOError(_("Taxonomy package contained more than one metadata file: {0}.")
                                  .format(', '.join(metadataFiles)))
                '''
                metadataFile = metadataFiles[0]
                metadata = filesource.file(filesource.url + os.sep + metadataFile)[0]
                metadataFilePrefix = os.sep.join(os.path.split(metadataFile)[:-1])
                if metadataFilePrefix:
                    metadataFilePrefix += os.sep
                metadataFilePrefix = filesource.baseurl + os.sep +  metadataFilePrefix
            elif os.path.basename(filesource.url) in TAXONOMY_PACKAGE_FILE_NAMES: # individual manifest file
                metadataFile = metadata = filesource.url
                metadataFilePrefix = os.sep.join(os.path.split(metadataFile)[:-1])
                if metadataFilePrefix:
                    metadataFilePrefix += os.sep
            else:
                raise IOError(_("File must be a taxonomy package (zip file), catalog file, or manifest (): {0}.")
                              .format(packageFilename, ', '.join(TAXONOMY_PACKAGE_FILE_NAMES)))
            parsedPackage = parsePackage(_cntlr, metadata)
            package = {'name': parsedPackage['name'],
                       'status': 'enabled',
                       'version': parsedPackage['version'],
                       'fileDate': time.strftime('%Y-%m-%dT%H:%M:%S UTC', time.gmtime(os.path.getmtime(packageFilename))),
                       'URL': URL,
                       'description': parsedPackage['description'],
                       'remappings': dict(
                            (prefix, 
                             remapping if isHttpUrl(remapping)
                             else (metadataFilePrefix +remapping.replace("/", os.sep)))
                            for prefix, remapping in parsedPackage["remappings"].items()),
                       }
            filesource.close()
            return package
        except EnvironmentError:
            pass
        if filesource:
            filesource.close()
    return None
Exemplo n.º 47
0
def saveFile(cntlr, filepath, contents, encoding=None, mode='wt'):
    if isHttpUrl(filepath):
        _cacheFilepath = cntlr.webCache.getfilename(filepath)
        if _cacheFilepath is None:
            raise IOError(_("Unable to open file: {0}.").format(filepath))
        filepath = _cacheFilepath
    # file path may be server (or memcache) or local file system
    if filepath.startswith(SERVER_WEB_CACHE):
        cacheKey = filepath[len(SERVER_WEB_CACHE) + 1:].replace("\\", "/")
        if cntlr.isGAE:  # check if in memcache
            gaeSet(cacheKey, contents.encode(encoding or 'utf-8'))
    else:
        with io.open(filepath, mode, encoding=(encoding or 'utf-8')) as f:
            f.write(contents)
Exemplo n.º 48
0
 def addReferencedFile(docElt, elt):
     if elt.tag in ("a", "img", "{http://www.w3.org/1999/xhtml}a", "{http://www.w3.org/1999/xhtml}img"):
         for attrTag, attrValue in elt.items():
             if attrTag in ("href", "src") and (
                     not localFilesOnly or 
                     (not isHttpUrl(attrValue) and not os.path.isabs(attrValue))):
                 attrValue = attrValue.partition('#')[0] # remove anchor
                 if attrValue: # ignore anchor references to base document
                     base = docElt.modelDocument.baseForElement(docElt)
                     normalizedUri = docElt.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(attrValue, base)
                     if not docElt.modelXbrl.fileSource.isInArchive(normalizedUri):
                         normalizedUri = docElt.modelXbrl.modelManager.cntlr.webCache.getfilename(normalizedUri)
                     if modelXbrl.fileSource.isInArchive(normalizedUri, checkExistence=True) or os.path.exists(normalizedUri):
                         referencedFiles.add(attrValue) # add file name within source directory
Exemplo n.º 49
0
 def addReferencedFile(docElt, elt):
     if elt.tag in ("a", "img", "{http://www.w3.org/1999/xhtml}a", "{http://www.w3.org/1999/xhtml}img"):
         for attrTag, attrValue in elt.items():
             if attrTag in ("href", "src") and (
                     not localFilesOnly or 
                     (not isHttpUrl(attrValue) and not os.path.isabs(attrValue))):
                 attrValue = attrValue.partition('#')[0] # remove anchor
                 if attrValue: # ignore anchor references to base document
                     base = docElt.modelDocument.baseForElement(docElt)
                     normalizedUri = docElt.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(attrValue, base)
                     if not docElt.modelXbrl.fileSource.isInArchive(normalizedUri):
                         normalizedUri = docElt.modelXbrl.modelManager.cntlr.webCache.getfilename(normalizedUri)
                     if modelXbrl.fileSource.isInArchive(normalizedUri, checkExistence=True) or os.path.exists(normalizedUri):
                         referencedFiles.add(attrValue) # add file name within source directory
Exemplo n.º 50
0
def saveFile(cntlr, filepath, contents, encoding=None, mode='wt'):
    if isHttpUrl(filepath):
        _cacheFilepath = cntlr.webCache.getfilename(filepath)
        if _cacheFilepath is None:
            raise IOError(_("Unable to open file: {0}.").format(filepath))
        filepath = _cacheFilepath
    # file path may be server (or memcache) or local file system
    if filepath.startswith(SERVER_WEB_CACHE):
        cacheKey = filepath[len(SERVER_WEB_CACHE) + 1:].replace("\\","/")
        if cntlr.isGAE: # check if in memcache
            gaeSet(cacheKey, contents.encode(encoding or 'utf-8'))
    else:
        with io.open(filepath, mode, encoding=(encoding or 'utf-8')) as f:
            f.write(contents)
Exemplo n.º 51
0
    def __init__(self, url, cntlr=None, checkIfXmlIsEis=False):
        global pluginClassMethods
        if pluginClassMethods is None:  # dynamic import
            from arelle.PluginManager import pluginClassMethods
        self.url = str(url)  # allow either string or FileNamedStringIO
        self.baseIsHttp = isHttpUrl(self.url)
        self.cntlr = cntlr
        self.type = self.url.lower()[-7:]
        self.isTarGz = self.type == ".tar.gz"
        if not self.isTarGz:
            self.type = self.type[3:]
        self.isZip = self.type == ".zip"
        self.isZipBackslashed = False  # windows style backslashed paths
        self.isEis = self.type == ".eis"
        self.isXfd = (self.type == ".xfd" or self.type == ".frm")
        self.isRss = (self.type == ".rss" or self.url.endswith(".rss.xml"))
        self.isInstalledTaxonomyPackage = False
        self.isOpen = False
        self.fs = None
        self.selection = None
        self.filesDir = None
        self.referencedFileSources = {}  # archive file name, fileSource object
        self.mappedPaths = None  # remappings of path segments may be loaded by taxonomyPackage manifest

        # for SEC xml files, check if it's an EIS anyway
        if (not (self.isZip or self.isEis or self.isXfd or self.isRss)
                and self.type == ".xml"):
            if os.path.split(self.url)[-1] in TAXONOMY_PACKAGE_FILE_NAMES:
                self.isInstalledTaxonomyPackage = True
            elif checkIfXmlIsEis:
                try:
                    file = open(self.cntlr.webCache.getfilename(self.url),
                                'r',
                                errors='replace')
                    l = file.read(
                        256)  # may have comments before first element
                    file.close()
                    if re.match(
                            r"\s*(<[?]xml[^?]+[?]>)?\s*(<!--.*-->\s*)*<(cor[a-z]*:|sdf:)?edgarSubmission",
                            l):
                        self.isEis = True
                except EnvironmentError as err:
                    if self.cntlr:
                        self.cntlr.addToLog(
                            _("[{0}] {1}").format(type(err).__name__, err))
                    pass
Exemplo n.º 52
0
 def __init__(self, url, cntlr=None, checkIfXmlIsEis=False):
     global pluginClassMethods
     if pluginClassMethods is None: # dynamic import
         from arelle.PluginManager import pluginClassMethods
     self.url = str(url)  # allow either string or FileNamedStringIO
     self.baseIsHttp = isHttpUrl(self.url)
     self.cntlr = cntlr
     self.type = self.url.lower()[-7:]
     self.isTarGz = self.type == ".tar.gz"
     if not self.isTarGz:
         self.type = self.type[3:]
     self.isZip = self.type == ".zip"
     self.isZipBackslashed = False # windows style backslashed paths
     self.isEis = self.type == ".eis"
     self.isXfd = (self.type == ".xfd" or self.type == ".frm")
     self.isRss = (self.type == ".rss" or self.url.endswith(".rss.xml"))
     self.isInstalledTaxonomyPackage = False
     self.isOpen = False
     self.fs = None
     self.selection = None
     self.filesDir = None
     self.referencedFileSources = {}  # archive file name, fileSource object
     self.mappedPaths = None  # remappings of path segments may be loaded by taxonomyPackage manifest
     
     # for SEC xml files, check if it's an EIS anyway
     if (not (self.isZip or self.isEis or self.isXfd or self.isRss) and
         self.type == ".xml"):
         if os.path.split(self.url)[-1] in TAXONOMY_PACKAGE_FILE_NAMES:
             self.isInstalledTaxonomyPackage = True
         elif checkIfXmlIsEis:
             try:
                 file = open(self.cntlr.webCache.getfilename(self.url), 'r', errors='replace')
                 l = file.read(256) # may have comments before first element
                 file.close()
                 if re.match(r"\s*(<[?]xml[^?]+[?]>)?\s*(<!--.*-->\s*)*<(cor[a-z]*:|sdf:)?edgarSubmission", l):
                     self.isEis = True
             except EnvironmentError as err:
                 if self.cntlr:
                     self.cntlr.addToLog(_("[{0}] {1}").format(type(err).__name__, err))
                 pass
Exemplo n.º 53
0
 def createFacts(facts, parent):
     for fact in facts:
         if fact.isItem:
             attrs = {"contextRef": fact.contextID}
             if fact.id:
                 attrs["id"] = fact.id
             if fact.isNumeric:
                 attrs["unitRef"] = fact.unitID
                 if fact.get("decimals"):
                     attrs["decimals"] = fact.get("decimals")
                 if fact.get("precision"):
                     attrs["precision"] = fact.get("precision")
             if fact.isNil:
                 attrs[XbrlConst.qnXsiNil] = "true"
                 text = None
             else:
                 text = fact.xValue if fact.xValid else fact.textValue
             newFact = targetInstance.createFact(fact.qname,
                                                 attributes=attrs,
                                                 text=text,
                                                 parent=parent)
             newFactForOldObjId[fact.objectIndex] = newFact
             if filingFiles and fact.concept is not None and fact.concept.isTextBlock:
                 # check for img and other filing references
                 for xmltext in [text] + CDATApattern.findall(text):
                     try:
                         for elt in XML(
                                 "<body>\n{0}\n</body>\n".format(xmltext)):
                             if elt.tag in ("a", "img") and not isHttpUrl(
                                     attrValue) and not os.path.isabs(
                                         attrvalue):
                                 for attrTag, attrValue in elt.items():
                                     if attrTag in ("href", "src"):
                                         filingFiles.add(attrValue)
                     except (XMLSyntaxError, UnicodeDecodeError):
                         pass
         elif fact.isTuple:
             newTuple = targetInstance.createFact(fact.qname, parent=parent)
             newFactForOldObjId[fact.objectIndex] = newTuple
             createFacts(fact.modelTupleFacts, newTuple)
Exemplo n.º 54
0
 def __init__(self, url, cntlr=None, checkIfXmlIsEis=False):
     self.url = str(url)  # allow either string or FileNamedStringIO
     self.baseIsHttp = isHttpUrl(self.url)
     self.cntlr = cntlr
     self.type = self.url.lower()[-7:]
     self.isTarGz = self.type == ".tar.gz"
     if not self.isTarGz:
         self.type = self.type[3:]
     self.isZip = self.type == ".zip"
     self.isEis = self.type == ".eis"
     self.isXfd = (self.type == ".xfd" or self.type == ".frm")
     self.isRss = (self.type == ".rss" or self.url.endswith(".rss.xml"))
     self.isInstalledTaxonomyPackage = False
     self.isOpen = False
     self.fs = None
     self.selection = None
     self.filesDir = None
     self.referencedFileSources = {}  # archive file name, fileSource object
     self.mappedPaths = None  # remappings of path segments may be loaded by taxonomyPackage manifest
     self.reportName = None # used when selecting an entry from a taxonomy package
     
     # for SEC xml files, check if it's an EIS anyway
     if (not (self.isZip or self.isEis or self.isXfd or self.isRss) and
         self.type == ".xml"):
         if os.path.split(self.url)[-1] in TAXONOMY_PACKAGE_FILE_NAMES:
             self.isInstalledTaxonomyPackage = True
         elif checkIfXmlIsEis:
             try:
                 file = open(self.cntlr.webCache.getfilename(self.url), 'r', errors='replace')
                 l = file.read(128)
                 file.close()
                 if re.match(r"\s*(<[?]xml[^?]+[?]>)?\s*<cor[a-z]*:edgarSubmission", l):
                     self.isEis = True
             except EnvironmentError as err:
                 if self.cntlr:
                     self.cntlr.addToLog(_("[{0}] {1}").format(type(err).__name__, err))
                 pass
Exemplo n.º 55
0
    def open(self, reloadCache=False):
        if not self.isOpen:
            if (self.isZip or self.isTarGz or self.isEis or self.isXfd
                    or self.isRss
                    or self.isInstalledTaxonomyPackage) and self.cntlr:
                self.basefile = self.cntlr.webCache.getfilename(
                    self.url, reload=reloadCache)
            else:
                self.basefile = self.url
            self.baseurl = self.url  # url gets changed by selection
            if not self.basefile:
                return  # an error should have been logged
            if self.isZip:
                try:
                    self.fs = zipfile.ZipFile(openFileStream(
                        self.cntlr, self.basefile, 'rb'),
                                              mode="r")
                    self.isOpen = True
                except EnvironmentError as err:
                    self.logError(err)
                    pass
            elif self.isTarGz:
                try:
                    self.fs = tarfile.open(self.basefile, "r:gz")
                    self.isOpen = True
                except EnvironmentError as err:
                    self.logError(err)
                    pass
            elif self.isEis:
                # check first line of file
                buf = b''
                try:
                    file = open(self.basefile, 'rb')
                    more = True
                    while more:
                        l = file.read(8)
                        if len(l) < 8:
                            break
                        if len(buf) == 0 and l.startswith(
                                b"<?xml "):  # not compressed
                            buf = l + file.read()  # not compressed
                            break
                        compressedBytes = file.read(
                            struct.unpack(">L", l[0:4])[0])
                        if len(compressedBytes) <= 0:
                            break
                        buf += zlib.decompress(compressedBytes)
                    file.close()
                except EnvironmentError as err:
                    self.logError(err)
                    pass
                #uncomment to save for debugging
                #with open("c:/temp/test.xml", "wb") as f:
                #    f.write(buf)

                if buf.startswith(b"<?xml "):
                    try:
                        # must strip encoding
                        str = buf.decode(XmlUtil.encoding(buf))
                        endEncoding = str.index("?>", 0, 128)
                        if endEncoding > 0:
                            str = str[endEncoding + 2:]
                        file = io.StringIO(initial_value=str)
                        parser = etree.XMLParser(recover=True, huge_tree=True)
                        self.eisDocument = etree.parse(file, parser=parser)
                        file.close()
                        self.isOpen = True
                    except EnvironmentError as err:
                        self.logError(err)
                        return  # provide error message later
                    except etree.LxmlError as err:
                        self.logError(err)
                        return  # provide error message later

            elif self.isXfd:
                # check first line of file
                file = open(self.basefile, 'rb')
                firstline = file.readline()
                if firstline.startswith(
                        b"application/x-xfdl;content-encoding=\"asc-gzip\""):
                    # file has been gzipped
                    base64input = file.read(-1)
                    file.close()
                    file = None

                    fb = base64.b64decode(base64input)
                    ungzippedBytes = b""
                    totalLenUncompr = 0
                    i = 0
                    while i < len(fb):
                        lenCompr = fb[i + 0] * 256 + fb[i + 1]
                        lenUncomp = fb[i + 2] * 256 + fb[i + 3]
                        lenRead = 0
                        totalLenUncompr += lenUncomp

                        gzchunk = (bytes((31, 139, 8, 0)) + fb[i:i + lenCompr])
                        try:
                            with gzip.GzipFile(
                                    fileobj=io.BytesIO(gzchunk)) as gf:
                                while True:
                                    readSize = min(16384, lenUncomp - lenRead)
                                    readBytes = gf.read(size=readSize)
                                    lenRead += len(readBytes)
                                    ungzippedBytes += readBytes
                                    if len(readBytes) == 0 or (lenUncomp -
                                                               lenRead) <= 0:
                                        break
                        except IOError as err:
                            pass  # provide error message later

                        i += lenCompr + 4
                    #for learning the content of xfd file, uncomment this:
                    #with open("c:\\temp\\test.xml", "wb") as fh:
                    #    fh.write(ungzippedBytes)
                    file = io.StringIO(
                        initial_value=ungzippedBytes.decode("utf-8"))
                else:
                    # position to start of file
                    file.seek(0, io.SEEK_SET)

                try:
                    self.xfdDocument = etree.parse(file)
                    file.close()
                    self.isOpen = True
                except EnvironmentError as err:
                    self.logError(err)
                    return  # provide error message later
                except etree.LxmlError as err:
                    self.logError(err)
                    return  # provide error message later

            elif self.isRss:
                try:
                    self.rssDocument = etree.parse(self.basefile)
                    self.isOpen = True
                except EnvironmentError as err:
                    self.logError(err)
                    return  # provide error message later
                except etree.LxmlError as err:
                    self.logError(err)
                    return  # provide error message later

            elif self.isInstalledTaxonomyPackage:
                self.isOpen = True
                # load mappings
                try:
                    metadataFiles = self.taxonomyPackageMetadataFiles
                    if len(metadataFiles) != 1:
                        raise IOError(
                            _("Taxonomy package must contain one and only one metadata file: {0}."
                              ).format(', '.join(metadataFiles)))
                    # HF: this won't work, see DialogOpenArchive for correct code
                    # not sure if it is used
                    taxonomyPackage = PackageManager.parsePackage(
                        self.cntlr, self.url)
                    fileSourceDir = os.path.dirname(self.baseurl) + os.sep
                    self.mappedPaths = \
                        dict((prefix,
                              remapping if isHttpUrl(remapping)
                              else (fileSourceDir + remapping.replace("/", os.sep)))
                              for prefix, remapping in taxonomyPackage["remappings"].items())
                except EnvironmentError as err:
                    self.logError(err)
                    return  # provide error message later
Exemplo n.º 56
0
def validateXbrlFinally(val, *args, **kwargs):
    if not (val.validateESEFplugin):
        return

    _xhtmlNs = "{{{}}}".format(xhtml)
    _xhtmlNsLen = len(_xhtmlNs)
    modelXbrl = val.modelXbrl
    modelDocument = modelXbrl.modelDocument

    _statusMsg = _("validating {0} filing rules").format(
        val.disclosureSystem.name)
    modelXbrl.profileActivity()
    modelXbrl.modelManager.showStatus(_statusMsg)

    reportXmlLang = None
    firstRootmostXmlLangDepth = 9999999

    _ifrsNs = None
    for targetNs in modelXbrl.namespaceDocs.keys():
        if ifrsNsPattern.match(targetNs):
            _ifrsNs = targetNs
    if not _ifrsNs:
        modelXbrl.error("ESEF.RTS.ifrsRequired",
                        _("RTS on ESEF requires IFRS taxonomy."),
                        modelObject=modelXbrl)
        return

    esefPrimaryStatementPlaceholders = set(
        qname(_ifrsNs, n) for n in esefPrimaryStatementPlaceholderNames)
    esefMandatoryElements2020 = set(
        qname(_ifrsNs, n) for n in esefMandatoryElementNames2020)

    if modelDocument.type == ModelDocument.Type.INSTANCE:
        modelXbrl.error("ESEF.I.1.instanceShallBeInlineXBRL",
                        _("RTS on ESEF requires inline XBRL instances."),
                        modelObject=modelXbrl)

    checkFilingDimensions(
        val)  # sets up val.primaryItems and val.domainMembers
    val.hasExtensionSchema = val.hasExtensionPre = val.hasExtensionCal = val.hasExtensionDef = val.hasExtensionLbl = False
    checkFilingDTS(val, modelXbrl.modelDocument, [])
    modelXbrl.profileActivity("... filer DTS checks", minTimeToShow=1.0)

    if not (val.hasExtensionSchema and val.hasExtensionPre
            and val.hasExtensionCal and val.hasExtensionDef
            and val.hasExtensionLbl):
        missingFiles = []
        if not val.hasExtensionSchema: missingFiles.append("schema file")
        if not val.hasExtensionPre:
            missingFiles.append("presentation linkbase")
        if not val.hasExtensionCal: missingFiles.append("calculation linkbase")
        if not val.hasExtensionDef: missingFiles.append("definition linkbase")
        if not val.hasExtensionLbl: missingFiles.append("label linkbase")
        modelXbrl.warning(
            "ESEF.3.1.1.extensionTaxonomyWrongFilesStructure",
            _("Extension taxonomies MUST consist of at least a schema file and presentation, calculation, definition and label linkbases"
              ": missing %(missingFiles)s"),
            modelObject=modelXbrl,
            missingFiles=", ".join(missingFiles))

    #if modelDocument.type == ModelDocument.Type.INLINEXBRLDOCUMENTSET:
    #    # reports only under reports, none elsewhere
    #    modelXbrl.fileSource.dir

    if modelDocument.type in (ModelDocument.Type.INLINEXBRL,
                              ModelDocument.Type.INLINEXBRLDOCUMENTSET,
                              ModelDocument.Type.INSTANCE):
        footnotesRelationshipSet = modelXbrl.relationshipSet("XBRL-footnotes")
        orphanedFootnotes = set()
        noLangFootnotes = set()
        factLangFootnotes = defaultdict(set)
        footnoteRoleErrors = set()
        transformRegistryErrors = set()

        def checkFootnote(elt, text):
            if text:  # non-empty footnote must be linked to a fact if not empty
                if not any(
                        isinstance(rel.fromModelObject, ModelFact) for rel in
                        footnotesRelationshipSet.toModelObject(elt)):
                    orphanedFootnotes.add(elt)
            lang = elt.xmlLang
            if not lang:
                noLangFootnotes.add(elt)
            else:
                for rel in footnotesRelationshipSet.toModelObject(elt):
                    if rel.fromModelObject is not None:
                        factLangFootnotes[rel.fromModelObject].add(lang)
            if elt.role != XbrlConst.footnote or not all(
                    rel.arcrole == XbrlConst.factFootnote
                    and rel.linkrole == XbrlConst.defaultLinkRole
                    for rel in footnotesRelationshipSet.toModelObject(elt)):
                footnoteRoleErrors.add(elt)

        # check file name of each inline document (which might be below a top-level IXDS)
        for doc in modelXbrl.urlDocs.values():
            if doc.type == ModelDocument.Type.INLINEXBRL:
                _baseName, _baseExt = os.path.splitext(doc.basename)
                if _baseExt not in (".xhtml", ".html"):
                    modelXbrl.warning(
                        "ESEF.RTS.Art.3.fileNameExtension",
                        _("FileName SHOULD have the extension .xhtml or .html: %(fileName)s"
                          ),
                        modelObject=doc,
                        fileName=doc.basename)
                docinfo = doc.xmlRootElement.getroottree().docinfo
                if " html" in docinfo.doctype:
                    modelXbrl.warning(
                        "ESEF.RTS.Art.3.htmlDoctype",
                        _("Doctype SHOULD NOT be html: %(fileName)s"),
                        modelObject=doc,
                        fileName=doc.basename)

        if modelDocument.type in (ModelDocument.Type.INLINEXBRL,
                                  ModelDocument.Type.INLINEXBRLDOCUMENTSET):
            hiddenEltIds = {}
            presentedHiddenEltIds = defaultdict(list)
            eligibleForTransformHiddenFacts = []
            requiredToDisplayFacts = []
            requiredToDisplayFactIds = {}
            firstIxdsDoc = True
            for ixdsHtmlRootElt in modelXbrl.ixdsHtmlElements:  # ix root elements for all ix docs in IXDS
                ixNStag = ixdsHtmlRootElt.modelDocument.ixNStag
                ixTags = set(ixNStag + ln
                             for ln in ("nonNumeric", "nonFraction",
                                        "references", "relationship"))
                ixTextTags = set(ixNStag + ln
                                 for ln in ("nonFraction", "continuation",
                                            "footnote"))
                ixExcludeTag = ixNStag + "exclude"
                ixTupleTag = ixNStag + "tuple"
                ixFractionTag = ixNStag + "fraction"
                for elt, depth in etreeIterWithDepth(ixdsHtmlRootElt):
                    eltTag = elt.tag
                    if isinstance(
                            elt,
                        (_ElementTree, _Comment, _ProcessingInstruction)):
                        continue  # comment or other non-parsed element
                    else:
                        eltTag = elt.tag
                        if eltTag.startswith(_xhtmlNs):
                            eltTag = eltTag[_xhtmlNsLen:]
                            if firstIxdsDoc and (not reportXmlLang or depth <
                                                 firstRootmostXmlLangDepth):
                                xmlLang = elt.get(
                                    "{http://www.w3.org/XML/1998/namespace}lang"
                                )
                                if xmlLang:
                                    reportXmlLang = xmlLang
                                    firstRootmostXmlLangDepth = depth
                        if ((eltTag in ("object", "script"))
                                or (eltTag == "a"
                                    and "javascript:" in elt.get("href", ""))
                                or (eltTag == "img"
                                    and "javascript:" in elt.get("src", ""))):
                            modelXbrl.error(
                                "ESEF.2.5.1.executableCodePresent",
                                _("Inline XBRL documents MUST NOT contain executable code: %(element)s"
                                  ),
                                modelObject=elt,
                                element=eltTag)
                        elif eltTag == "img":
                            src = elt.get("src", "").strip()
                            hasParentIxTextTag = False  # check if image is in an ix text-bearing element
                            _ancestorElt = elt
                            while (_ancestorElt is not None):
                                if _ancestorElt.tag == ixExcludeTag:  # excluded from any parent text-bearing ix element
                                    break
                                if _ancestorElt.tag in ixTextTags:
                                    hasParentIxTextTag = True
                                    break
                                _ancestorElt = _ancestorElt.getparent()
                            if scheme(src) in ("http", "https", "ftp"):
                                modelXbrl.error(
                                    "ESEF.3.5.1.inlinXbrlContainsExternalReferences",
                                    _("Inline XBRL instance documents MUST NOT contain any reference pointing to resources outside the reporting package: %(element)s"
                                      ),
                                    modelObject=elt,
                                    element=eltTag)
                            elif not src.startswith("data:image"):
                                if hasParentIxTextTag:
                                    modelXbrl.error(
                                        "ESEF.2.5.1.imageInIXbrlElementNotEmbedded",
                                        _("Images appearing within an inline XBRL element MUST be embedded regardless of their size."
                                          ),
                                        modelObject=elt)
                                else:
                                    # presume it to be an image file, check image contents
                                    try:
                                        base = elt.modelDocument.baseForElement(
                                            elt)
                                        normalizedUri = elt.modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
                                            src, base)
                                        if not elt.modelXbrl.fileSource.isInArchive(
                                                normalizedUri):
                                            normalizedUri = elt.modelXbrl.modelManager.cntlr.webCache.getfilename(
                                                normalizedUri)
                                        imglen = 0
                                        with elt.modelXbrl.fileSource.file(
                                                normalizedUri,
                                                binary=True)[0] as fh:
                                            imglen += len(fh.read())
                                        if imglen < browserMaxBase64ImageLength:
                                            modelXbrl.error(
                                                "ESEF.2.5.1.embeddedImageNotUsingBase64Encoding",
                                                _("Images MUST be included in the XHTML document as a base64 encoded string unless their size exceeds support of browsers (%(maxImageSize)s): %(file)s."
                                                  ),
                                                modelObject=elt,
                                                maxImageSize=
                                                browserMaxBase64ImageLength,
                                                file=os.path.basename(
                                                    normalizedUri))
                                    except IOError as err:
                                        modelXbrl.error(
                                            "ESEF.2.5.1.imageFileCannotBeLoaded",
                                            _("Image file which isn't openable '%(src)s', error: %(error)s"
                                              ),
                                            modelObject=elt,
                                            src=src,
                                            error=err)
                            elif not any(
                                    src.startswith(m)
                                    for m in allowedImgMimeTypes):
                                modelXbrl.error(
                                    "ESEF.2.5.1.embeddedImageNotUsingBase64Encoding",
                                    _("Images MUST be included in the XHTML document as a base64 encoded string, encoding disallowed: %(src)s."
                                      ),
                                    modelObject=elt,
                                    src=attrValue[:128])

                        elif eltTag == "a":
                            href = elt.get("href", "").strip()
                            if scheme(href) in ("http", "https", "ftp"):
                                modelXbrl.error(
                                    "ESEF.3.5.1.inlinXbrlContainsExternalReferences",
                                    _("Inline XBRL instance documents MUST NOT contain any reference pointing to resources outside the reporting package: %(element)s"
                                      ),
                                    modelObject=elt,
                                    element=eltTag)
                        elif eltTag == "base" or elt.tag == "{http://www.w3.org/XML/1998/namespace}base":
                            modelXbrl.error(
                                "ESEF.2.4.2.htmlOrXmlBaseUsed",
                                _("The HTML <base> elements and xml:base attributes MUST NOT be used in the Inline XBRL document."
                                  ),
                                modelObject=elt,
                                element=eltTag)
                        elif eltTag == "link" and elt.get(
                                "type") == "text/css":
                            if len(modelXbrl.ixdsHtmlElements) > 1:
                                f = elt.get("href")
                                if not f or isHttpUrl(f) or os.path.isabs(f):
                                    modelXbrl.warning(
                                        "ESEF.2.5.4.externalCssReportPackage",
                                        _("The CSS file should be physically stored within the report package: %{file}s."
                                          ),
                                        modelObject=elt,
                                        file=f)
                            else:
                                modelXbrl.error(
                                    "ESEF.2.5.4.externalCssFileForSingleIXbrlDocument",
                                    _("Where an Inline XBRL document set contains a single document, the CSS MUST be embedded within the document."
                                      ),
                                    modelObject=elt,
                                    element=eltTag)
                        elif eltTag == "style" and elt.get(
                                "type") == "text/css":
                            if len(modelXbrl.ixdsHtmlElements) > 1:
                                modelXbrl.warning(
                                    "ESEF.2.5.4.embeddedCssForMultiHtmlIXbrlDocumentSets",
                                    _("Where an Inline XBRL document set contains multiple documents, the CSS SHOULD be defined in a separate file."
                                      ),
                                    modelObject=elt,
                                    element=eltTag)

                    if eltTag in ixTags and elt.get("target"):
                        modelXbrl.error(
                            "ESEF.2.5.3.targetAttributeUsed",
                            _("Target attribute MUST not be used: element %(localName)s, target attribute %(target)s."
                              ),
                            modelObject=elt,
                            localName=elt.elementQname,
                            target=elt.get("target"))
                    if eltTag == ixTupleTag:
                        modelXbrl.error(
                            "ESEF.2.4.1.tupleElementUsed",
                            _("The ix:tuple element MUST not be used in the Inline XBRL document: %(qname)s."
                              ),
                            modelObject=elt,
                            qname=elt.qname)
                    if eltTag == ixFractionTag:
                        modelXbrl.error(
                            "ESEF.2.4.1.fractionElementUsed",
                            _("The ix:fraction element MUST not be used in the Inline XBRL document."
                              ),
                            modelObject=elt)
                    if elt.get("{http://www.w3.org/XML/1998/namespace}base"
                               ) is not None:
                        modelXbrl.error(
                            "ESEF.2.4.1.xmlBaseUsed",
                            _("xml:base attributes MUST NOT be used in the Inline XBRL document: element %(localName)s, base attribute %(base)s."
                              ),
                            modelObject=elt,
                            localName=elt.elementQname,
                            base=elt.get(
                                "{http://www.w3.org/XML/1998/namespace}base"))
                    if isinstance(elt, ModelInlineFootnote):
                        checkFootnote(elt, elt.value)
                    elif isinstance(
                            elt, ModelResource
                    ) and elt.qname == XbrlConst.qnLinkFootnote:
                        checkFootnote(elt, elt.value)
                    elif isinstance(elt, ModelInlineFact):
                        if elt.format is not None and elt.format.namespaceURI not in IXT_NAMESPACES:
                            transformRegistryErrors.add(elt)
                for ixHiddenElt in ixdsHtmlRootElt.iterdescendants(
                        tag=ixNStag + "hidden"):
                    for tag in (ixNStag + "nonNumeric",
                                ixNStag + "nonFraction"):
                        for ixElt in ixHiddenElt.iterdescendants(tag=tag):
                            if (
                                    getattr(ixElt, "xValid",
                                            0) >= VALID  # may not be validated
                            ):  # add future "and" conditions on elements which can be in hidden
                                if (ixElt.concept.baseXsdType
                                        not in untransformableTypes
                                        and not ixElt.isNil):
                                    eligibleForTransformHiddenFacts.append(
                                        ixElt)
                                elif ixElt.id is None:
                                    requiredToDisplayFacts.append(ixElt)
                            if ixElt.id:
                                hiddenEltIds[ixElt.id] = ixElt
                firstIxdsDoc = False
            if eligibleForTransformHiddenFacts:
                modelXbrl.warning(
                    "ESEF.2.4.1.transformableElementIncludedInHiddenSection",
                    _("The ix:hidden section of Inline XBRL document MUST not include elements eligible for transformation. "
                      "%(countEligible)s fact(s) were eligible for transformation: %(elements)s"
                      ),
                    modelObject=eligibleForTransformHiddenFacts,
                    countEligible=len(eligibleForTransformHiddenFacts),
                    elements=", ".join(
                        sorted(
                            set(
                                str(f.qname)
                                for f in eligibleForTransformHiddenFacts))))
            for ixdsHtmlRootElt in modelXbrl.ixdsHtmlElements:
                for ixElt in ixdsHtmlRootElt.getroottree().iterfind(
                        "//{http://www.w3.org/1999/xhtml}*[@style]"):
                    hiddenFactRefMatch = styleIxHiddenPattern.match(
                        ixElt.get("style", ""))
                    if hiddenFactRefMatch:
                        hiddenFactRef = hiddenFactRefMatch.group(2)
                        if hiddenFactRef not in hiddenEltIds:
                            modelXbrl.error(
                                "ESEF.2.4.1.esefIxHiddenStyleNotLinkingFactInHiddenSection",
                                _("\"-esef-ix-hidden\" style identifies @id, %(id)s of a fact that is not in ix:hidden section."
                                  ),
                                modelObject=ixElt,
                                id=hiddenFactRef)
                        else:
                            presentedHiddenEltIds[hiddenFactRef].append(ixElt)
            for hiddenEltId, ixElt in hiddenEltIds.items():
                if (hiddenEltId not in presentedHiddenEltIds
                        and getattr(ixElt, "xValid", 0) >= VALID
                        and  # may not be validated
                    (ixElt.concept.baseXsdType in untransformableTypes
                     or ixElt.isNil)):
                    requiredToDisplayFacts.append(ixElt)
            if requiredToDisplayFacts:
                modelXbrl.warning(
                    "ESEF.2.4.1.factInHiddenSectionNotInReport",
                    _("The ix:hidden section contains %(countUnreferenced)s fact(s) whose @id is not applied on any \"-esef-ix- hidden\" style: %(elements)s"
                      ),
                    modelObject=requiredToDisplayFacts,
                    countUnreferenced=len(requiredToDisplayFacts),
                    elements=", ".join(
                        sorted(
                            set(str(f.qname)
                                for f in requiredToDisplayFacts))))
            del eligibleForTransformHiddenFacts, hiddenEltIds, presentedHiddenEltIds, requiredToDisplayFacts
        elif modelDocument.type == ModelDocument.Type.INSTANCE:
            for elt in modelDocument.xmlRootElement.iter():
                if elt.qname == XbrlConst.qnLinkFootnote:  # for now assume no private elements extend link:footnote
                    checkFootnote(elt, elt.stringValue)

        contextsWithDisallowedOCEs = []
        contextsWithDisallowedOCEcontent = []
        contextsWithPeriodTime = []
        contextsWithPeriodTimeZone = []
        contextIdentifiers = defaultdict(list)
        nonStandardTypedDimensions = defaultdict(set)
        for context in modelXbrl.contexts.values():
            for elt in context.iterdescendants(
                    "{http://www.xbrl.org/2003/instance}startDate",
                    "{http://www.xbrl.org/2003/instance}endDate",
                    "{http://www.xbrl.org/2003/instance}instant"):
                m = datetimePattern.match(elt.stringValue)
                if m:
                    if m.group(1):
                        contextsWithPeriodTime.append(context)
                    if m.group(3):
                        contextsWithPeriodTimeZone.append(context)
            for elt in context.iterdescendants(
                    "{http://www.xbrl.org/2003/instance}segment"):
                contextsWithDisallowedOCEs.append(context)
                break
            for elt in context.iterdescendants(
                    "{http://www.xbrl.org/2003/instance}scenario"):
                if isinstance(elt, ModelObject):
                    if any(True for child in elt.iterchildren()
                           if isinstance(child, ModelObject)
                           and child.tag not in (
                               "{http://xbrl.org/2006/xbrldi}explicitMember",
                               "{http://xbrl.org/2006/xbrldi}typedMember")):
                        contextsWithDisallowedOCEcontent.append(context)
            # check periods here
            contextIdentifiers[context.entityIdentifier].append(context)

        if contextsWithDisallowedOCEs:
            modelXbrl.error(
                "ESEF.2.1.3.segmentUsed",
                _("xbrli:segment container MUST NOT be used in contexts: %(contextIds)s"
                  ),
                modelObject=contextsWithDisallowedOCEs,
                contextIds=", ".join(c.id for c in contextsWithDisallowedOCEs))
        if contextsWithDisallowedOCEcontent:
            modelXbrl.error(
                "ESEF.2.1.3.scenarioContainsNonDimensionalContent",
                _("xbrli:scenario in contexts MUST NOT contain any other content than defined in XBRL Dimensions specification: %(contextIds)s"
                  ),
                modelObject=contextsWithDisallowedOCEcontent,
                contextIds=", ".join(
                    c.id for c in contextsWithDisallowedOCEcontent))
        if len(contextIdentifiers) > 1:
            modelXbrl.error(
                "ESEF.2.1.4.multipleIdentifiers",
                _("All entity identifiers in contexts MUST have identical content: %(contextIdentifiers)s"
                  ),
                modelObject=modelXbrl,
                contextIds=", ".join(i[1] for i in contextIdentifiers))
        for (contextScheme,
             contextIdentifier), contextElts in contextIdentifiers.items():
            if contextScheme != iso17442:
                modelXbrl.warning(
                    "ESEF.2.1.1.nonLEIContextScheme",
                    _("The scheme attribute of the xbrli:identifier element should have \"%(leiScheme)s\" as its content: %(contextScheme)s"
                      ),
                    modelObject=contextElts,
                    contextScheme=contextScheme,
                    leiScheme=iso17442)
            else:
                leiValidity = LeiUtil.checkLei(contextIdentifier)
                if leiValidity == LeiUtil.LEI_INVALID_LEXICAL:
                    modelXbrl.warning(
                        "ESEF.2.1.1.invalidIdentifierFormat",
                        _("The LEI context identifier has an invalid format: %(identifier)s"
                          ),
                        modelObject=contextElts,
                        identifier=contextIdentifier)
                elif leiValidity == LeiUtil.LEI_INVALID_CHECKSUM:
                    modelXbrl.warning(
                        "ESEF.2.1.1.invalidIdentifier",
                        _("The LEI context identifier has checksum error: %(identifier)s"
                          ),
                        modelObject=contextElts,
                        identifier=contextIdentifier)
        if contextsWithPeriodTime:
            modelXbrl.warning(
                "ESEF.2.1.2.periodWithTimeContent",
                _("Context period startDate, endDate and instant elements should be in whole days without time: %(contextIds)s"
                  ),
                modelObject=contextsWithPeriodTime,
                contextIds=", ".join(c.id for c in contextsWithPeriodTime))
        if contextsWithPeriodTimeZone:
            modelXbrl.warning(
                "ESEF.2.1.2.periodWithTimeZone",
                _("Context period startDate, endDate and instant elements should be in whole days without a timezone: %(contextIds)s"
                  ),
                modelObject=contextsWithPeriodTimeZone,
                contextIds=", ".join(c.id for c in contextsWithPeriodTimeZone))

        # identify unique contexts and units
        mapContext = {}
        mapUnit = {}
        uniqueContextHashes = {}
        for context in modelXbrl.contexts.values():
            h = context.contextDimAwareHash
            if h in uniqueContextHashes:
                if context.isEqualTo(uniqueContextHashes[h]):
                    mapContext[context] = uniqueContextHashes[h]
            else:
                uniqueContextHashes[h] = context
        del uniqueContextHashes
        uniqueUnitHashes = {}
        utrValidator = ValidateUtr(modelXbrl)
        utrUnitIds = set(
            u.unitId
            for unitItemType in utrValidator.utrItemTypeEntries.values()
            for u in unitItemType.values())
        for unit in modelXbrl.units.values():
            h = unit.hash
            if h in uniqueUnitHashes:
                if unit.isEqualTo(uniqueUnitHashes[h]):
                    mapUnit[unit] = uniqueUnitHashes[h]
            else:
                uniqueUnitHashes[h] = unit
            # check if any custom measure is in UTR
            for measureTerm in unit.measures:
                for measure in measureTerm:
                    ns = measure.namespaceURI
                    if ns != XbrlConst.iso4217 and not ns.startswith(
                            "http://www.xbrl.org/"):
                        if measure.localName in utrUnitIds:
                            modelXbrl.error(
                                "ESEF.RTS.III.1.G1-7-1.customUnitInUtr",
                                _("Custom measure SHOULD NOT duplicate a UnitID of UTR: %(measure)s"
                                  ),
                                modelObject=unit,
                                measure=measure)
        del uniqueUnitHashes

        reportedMandatory = set()
        precisionFacts = set()
        numFactsByConceptContextUnit = defaultdict(list)
        textFactsByConceptContext = defaultdict(list)
        footnotesRelationshipSet = modelXbrl.relationshipSet(
            XbrlConst.factFootnote, XbrlConst.defaultLinkRole)
        noLangFacts = []
        textFactsMissingReportLang = []
        conceptsUsed = set()

        for qn, facts in modelXbrl.factsByQname.items():
            if qn in mandatory:
                reportedMandatory.add(qn)
            for f in facts:
                if f.precision is not None:
                    precisionFacts.add(f)
                if f.isNumeric:
                    numFactsByConceptContextUnit[(f.qname,
                                                  mapContext.get(
                                                      f.context, f.context),
                                                  mapUnit.get(
                                                      f.unit,
                                                      f.unit))].append(f)
                    if f.concept is not None and not f.isNil and f.xValid >= VALID and f.xValue > 1 and f.concept.type is not None and (
                            f.concept.type.qname == PERCENT_TYPE
                            or f.concept.type.isDerivedFrom(PERCENT_TYPE)):
                        modelXbrl.warning(
                            "ESEF.2.2.2.percentGreaterThan100",
                            _("A percent fact should have value <= 100: %(element)s in context %(context)s value %(value)s"
                              ),
                            modelObject=f,
                            element=f.qname,
                            context=f.context.id,
                            value=f.xValue)
                elif f.concept is not None and f.concept.type is not None:
                    if f.concept.type.isOimTextFactType:
                        if not f.xmlLang:
                            noLangFacts.append(f)
                        elif f.context is not None:
                            textFactsByConceptContext[(
                                f.qname, mapContext.get(f.context,
                                                        f.context))].append(f)
                conceptsUsed.add(f.concept)
                if f.context is not None:
                    for dim in f.context.qnameDims.values():
                        conceptsUsed.add(dim.dimension)
                        if dim.isExplicit:
                            conceptsUsed.add(dim.member)
                        elif dim.isTyped:
                            conceptsUsed.add(dim.typedMember)

        if noLangFacts:
            modelXbrl.error(
                "ESEF.2.5.2.undefinedLanguageForTextFact",
                _("Each tagged text fact MUST have the 'xml:lang' attribute assigned or inherited."
                  ),
                modelObject=noLangFacts)

        # missing report lang text facts
        if reportXmlLang:
            for fList in textFactsByConceptContext.values():
                if not any(f.xmlLang == reportXmlLang for f in fList):
                    modelXbrl.error(
                        "ESEF.2.5.2.taggedTextFactOnlyInLanguagesOtherThanLanguageOfAReport",
                        _("Each tagged text fact MUST have the 'xml:lang' provided in at least the language of the report: %(element)s"
                          ),
                        modelObject=fList,
                        element=fList[0].qname)

        # 2.2.4 test
        for fList in numFactsByConceptContextUnit.values():
            if len(fList) > 1:
                f0 = fList[0]
                if any(f.isNil for f in fList):
                    _inConsistent = not all(f.isNil for f in fList)
                elif all(
                        inferredDecimals(f) == inferredDecimals(f0)
                        for f in fList[1:]):  # same decimals
                    v0 = rangeValue(f0.value)
                    _inConsistent = not all(
                        rangeValue(f.value) == v0 for f in fList[1:])
                else:  # not all have same decimals
                    aMax, bMin = rangeValue(f0.value, inferredDecimals(f0))
                    for f in fList[1:]:
                        a, b = rangeValue(f.value, inferredDecimals(f))
                        if a > aMax: aMax = a
                        if b < bMin: bMin = b
                    _inConsistent = (bMin < aMax)
                if _inConsistent:
                    modelXbrl.error(
                        ("ESEF.2.2.4.inconsistentDuplicateNumericFactInInlineXbrlDocument"
                         ),
                        "Inconsistent duplicate numeric facts MUST NOT appear in the content of an inline XBRL document. %(fact)s that was used more than once in contexts equivalent to %(contextID)s: values %(values)s.  ",
                        modelObject=fList,
                        fact=f0.qname,
                        contextID=f0.contextID,
                        values=", ".join(
                            strTruncate(f.value, 128) for f in fList))

        if precisionFacts:
            modelXbrl.warning(
                "ESEF.2.2.1.precisionAttributeUsed",
                _("The accuracy of numeric facts SHOULD be defined with the 'decimals' attribute rather than the 'precision' attribute: %(elements)s."
                  ),
                modelObject=precisionFacts,
                elements=", ".join(sorted(
                    str(e.qname) for e in precisionFacts)))

        missingElements = (mandatory - reportedMandatory)
        if missingElements:
            modelXbrl.error(
                "ESEF.???.missingRequiredElements",
                _("Required elements missing from document: %(elements)s."),
                modelObject=modelXbrl,
                elements=", ".join(sorted(str(qn) for qn in missingElements)))

        if transformRegistryErrors:
            modelXbrl.warning(
                "ESEF.2.2.3.transformRegistry",
                _("ESMA recommends applying the latest available version of the Transformation Rules Registry marked with 'Recommendation' status for these elements: %(elements)s."
                  ),
                modelObject=transformRegistryErrors,
                elements=", ".join(
                    sorted(
                        str(fact.qname) for fact in transformRegistryErrors)))

        if orphanedFootnotes:
            modelXbrl.error(
                "ESEF.2.3.1.unusedFootnote",
                _("Non-empty footnotes must be connected to fact(s)."),
                modelObject=orphanedFootnotes)

        if noLangFootnotes:
            modelXbrl.error(
                "ESEF.2.3.1.undefinedLanguageForFootnote",
                _("Each footnote MUST have the 'xml:lang' attribute whose value corresponds to the language of the text in the content of the respective footnote."
                  ),
                modelObject=noLangFootnotes)
        nonDefLangFtFacts = set(f for f, langs in factLangFootnotes.items()
                                if reportXmlLang not in langs)
        if nonDefLangFtFacts:
            modelXbrl.error(
                "ESEF.2.3.1.footnoteOnlyInLanguagesOtherThanLanguageOfAReport",
                _("Each fact MUST have at least one footnote with 'xml:lang' attribute whose value corresponds to the language of the text in the content of the respective footnote: %(qnames)s."
                  ),
                modelObject=nonDefLangFtFacts,
                qnames=", ".join(
                    sorted(str(f.qname) for f in nonDefLangFtFacts)))
        del nonDefLangFtFacts
        if footnoteRoleErrors:
            modelXbrl.error(
                "ESEF.2.3.1.nonStandardRoleForFootnote",
                _("The xlink:role attribute of a link:footnote and link:footnoteLink element as well as xlink:arcrole attribute of a link:footnoteArc MUST be defined in the XBRL Specification 2.1."
                  ),
                modelObject=footnoteRoleErrors)

        nonStdFootnoteElts = list()
        for modelLink in modelXbrl.baseSets[("XBRL-footnotes", None, None,
                                             None)]:
            for elt in modelLink.iterchildren():
                if isinstance(
                        elt, (_ElementTree, _Comment, _ProcessingInstruction)):
                    continue  # comment or other non-parsed element
                if elt.qname not in FOOTNOTE_LINK_CHILDREN:
                    nonStdFootnoteElts.append(elt)

        if nonStdFootnoteElts:
            modelXbrl.error(
                "ESEF.2.3.2.nonStandardElementInFootnote",
                _("A link:footnoteLink element MUST have no children other than link:loc, link:footnote, and link:footnoteArc."
                  ),
                modelObject=nonStdFootnoteElts)

        for qn in modelXbrl.qnameDimensionDefaults.values():
            conceptsUsed.add(modelXbrl.qnameConcepts.get(qn))

        # unused elements in linkbases
        for arcroles, err in (
            ((parentChild, ),
             "elementsNotUsedForTaggingAppliedInPresentationLinkbase"),
            ((summationItem, ),
             "elementsNotUsedForTaggingAppliedInCalculationLinkbase"),
            ((dimensionDomain, domainMember),
             "elementsNotUsedForTaggingAppliedInDefinitionLinkbase")):
            unreportedLbElts = set()
            for arcrole in arcroles:
                for rel in modelXbrl.relationshipSet(
                        arcrole).modelRelationships:
                    fr = rel.fromModelObject
                    to = rel.toModelObject
                    if arcrole in (parentChild, summationItem):
                        if fr is not None and not fr.isAbstract and fr not in conceptsUsed and isExtension(
                                val, rel):
                            unreportedLbElts.add(fr)
                        if to is not None and not to.isAbstract and to not in conceptsUsed and isExtension(
                                val, rel):
                            unreportedLbElts.add(to)
                    elif arcrole == dimensionDomain:  # dimension, always abstract
                        if fr is not None and fr not in conceptsUsed and isExtension(
                                val, rel):
                            unreportedLbElts.add(fr)
                        if to is not None and rel.isUsable and to not in conceptsUsed and isExtension(
                                val, rel):
                            unreportedLbElts.add(to)
                    elif arcrole == domainMember:
                        if to is not None and not fr.isAbstract and rel.isUsable and to not in conceptsUsed and isExtension(
                                val, rel):
                            unreportedLbElts.add(to)
            if unreportedLbElts:
                modelXbrl.error(
                    "ESEF.3.4.6." + err,
                    _("All usable concepts in extension taxonomy relationships MUST be applied by tagged facts: %(elements)s."
                      ),
                    modelObject=unreportedLbElts,
                    elements=", ".join(
                        sorted((str(c.qname) for c in unreportedLbElts))))

        # 3.4.4 check for presentation preferred labels
        missingConceptLabels = defaultdict(set)  # by role
        pfsConceptsRootInPreLB = set()

        def checkLabels(parent, relSet, labelrole, visited):
            if not parent.label(
                    labelrole, lang=reportXmlLang, fallbackToQname=False):
                if parent.name != "NotesAccountingPoliciesAndMandatoryTags":  # TEMPORARY TBD remove
                    missingConceptLabels[labelrole].add(parent)
            visited.add(parent)
            conceptRels = defaultdict(
                list)  # counts for concepts without preferred label role
            for rel in relSet.fromModelObject(parent):
                child = rel.toModelObject
                if child is not None:
                    labelrole = rel.preferredLabel
                    if not labelrole:
                        conceptRels[child].append(rel)
                    if child not in visited:
                        checkLabels(child, relSet, labelrole, visited)
            for concept, rels in conceptRels.items():
                if len(rels) > 1:
                    modelXbrl.warning(
                        "ESEF.3.4.4.missingPreferredLabelRole",
                        _("Preferred label role SHOULD be used when concept is duplicated in same presentation tree location: %(qname)s."
                          ),
                        modelObject=rels + [concept],
                        qname=concept.qname)
            visited.remove(parent)

        for ELR in modelXbrl.relationshipSet(parentChild).linkRoleUris:
            relSet = modelXbrl.relationshipSet(parentChild, ELR)
            for rootConcept in relSet.rootConcepts:
                checkLabels(rootConcept, relSet, None, set())
                # check for PFS element which isn't an orphan
                if rootConcept.qname in esefPrimaryStatementPlaceholders and relSet.fromModelObject(
                        rootConcept):
                    pfsConceptsRootInPreLB.add(rootConcept)
        for labelrole, concepts in missingConceptLabels.items():
            modelXbrl.warning(
                "ESEF.3.4.5.missingLabelForRoleInReportLanguage",
                _("Label for %(role)s role SHOULD be available in report language for concepts: %(qnames)s."
                  ),
                modelObject=concepts,
                qnames=", ".join(str(c.qname) for c in concepts),
                role=os.path.basename(labelrole) if labelrole else "standard")
        if not pfsConceptsRootInPreLB:
            # no PFS statements were recognized
            modelXbrl.error(
                "ESEF.RTS.Annex.II.Par.1.Par.7.missingPrimaryFinancialStatement",
                _("A primary financial statement placeholder element MUST be a root of a presentation linkbase tree."
                  ),
                modelObject=modelXbrl)
        # dereference
        del missingConceptLabels, pfsConceptsRootInPreLB

        # mandatory factc RTS Annex II
        missingMandatoryElements = esefMandatoryElements2020 - modelXbrl.factsByQname.keys(
        )
        if missingMandatoryElements:
            modelXbrl.error(
                "ESEF.RTS.Annex.II.Par.2.missingMandatoryMarkups",
                _("Mandatory elements to be marked up are missing: %(qnames)s."
                  ),
                modelObject=missingMandatoryElements,
                qnames=", ".join(
                    sorted(str(qn) for qn in missingMandatoryElements)))

        # duplicated core taxonomy elements
        for name, concepts in modelXbrl.nameConcepts.items():
            if len(concepts) > 1:
                i = None  # ifrs Concept
                for c in concepts:
                    if c.qname.namespaceURI == _ifrsNs:
                        i = c
                        break
                if i is not None:
                    for c in concepts:
                        if c != i and c.balance == i.balance and c.periodType == i.periodType:
                            modelXbrl.error(
                                "ESEF.RTS.Annex.IV.Par.4.2.extensionElementDuplicatesCoreElement",
                                _("Extension elements must not duplicate the existing elements from the core taxonomy and be identifiable %(qname)s."
                                  ),
                                modelObject=(c, i),
                                qname=c.qname)

    modelXbrl.profileActivity(_statusMsg, minTimeToShow=0.0)
    modelXbrl.modelManager.showStatus(None)
Exemplo n.º 57
0
def packageInfo(URL, reload=False, packageManifestName=None):
    #TODO several directories, eg User Application Data
    packageFilename = _cntlr.webCache.getfilename(URL,
                                                  reload=reload,
                                                  normalize=True)
    if packageFilename:
        from arelle.FileSource import TAXONOMY_PACKAGE_FILE_NAMES
        filesource = None
        try:
            global openFileSource
            if openFileSource is None:
                from arelle.FileSource import openFileSource
            filesource = openFileSource(packageFilename, _cntlr)
            # allow multiple manifests [[metadata, prefix]...] for multiple catalogs
            packages = []
            if filesource.isZip:
                if packageManifestName:
                    packageFiles = [
                        fileName for fileName in filesource.dir
                        if fnmatch(fileName, packageManifestName)
                    ]
                else:
                    packageFiles = filesource.taxonomyPackageMetadataFiles
                if len(packageFiles) < 1:
                    raise IOError(
                        _("Taxonomy package contained no metadata file: {0}.").
                        format(', '.join(packageFiles)))
                for packageFile in packageFiles:
                    packageFileUrl = filesource.file(filesource.url + os.sep +
                                                     packageFile)[0]
                    packageFilePrefix = os.sep.join(
                        os.path.split(packageFile)[:-1])
                    if packageFilePrefix:
                        packageFilePrefix += os.sep
                    packageFilePrefix = filesource.baseurl + os.sep + packageFilePrefix
                    packages.append([packageFileUrl, packageFilePrefix])
            elif os.path.basename(
                    filesource.url
            ) in TAXONOMY_PACKAGE_FILE_NAMES:  # individual manifest file
                packageFile = packageFileUrl = filesource.url
                packageFilePrefix = os.sep.join(
                    os.path.split(packageFile)[:-1])
                if packageFilePrefix:
                    packageFilePrefix += os.sep
                packages.append([packageFileUrl, packageFilePrefix])
            else:
                raise IOError(
                    _("File must be a taxonomy package (zip file), catalog file, or manifest (): {0}."
                      ).format(packageFilename,
                               ', '.join(TAXONOMY_PACKAGE_FILE_NAMES)))
            remappings = {}
            packageNames = []
            descriptions = []
            for packageFileUrl, packageFilePrefix in packages:
                parsedPackage = parsePackage(_cntlr, packageFileUrl)
                packageNames.append(parsedPackage['name'])
                if parsedPackage.get('description'):
                    descriptions.append(parsedPackage['description'])
                for prefix, remapping in parsedPackage["remappings"].items():
                    remappings[prefix] = (remapping
                                          if isHttpUrl(remapping) else
                                          (packageFilePrefix +
                                           remapping.replace("/", os.sep)))
            package = {
                'name':
                ", ".join(packageNames),
                'status':
                'enabled',
                'version':
                parsedPackage['version'],
                'fileDate':
                time.strftime('%Y-%m-%dT%H:%M:%S UTC',
                              time.gmtime(os.path.getmtime(packageFilename))),
                'URL':
                URL,
                'manifestName':
                packageManifestName,
                'description':
                "; ".join(descriptions),
                'remappings':
                remappings,
            }
            filesource.close()
            return package
        except EnvironmentError:
            pass
        if filesource:
            filesource.close()
    return None
Exemplo n.º 58
0
 def ok(self, event=None):
     selection = self.treeView.selection()
     if len(selection) > 0:
         filename = None
         if self.openType in (ARCHIVE, DISCLOSURE_SYSTEM):
             if self.multiselect:
                 filenames = []
             for _selection in selection:
                 filename = self.filenames[int(_selection[4:])]
                 if isinstance(filename, tuple):
                     if self.isRss:
                         filename = filename[4]
                     else:
                         filename = filename[0]
                 if self.multiselect:
                     filenames.append(filename)
                 else:
                     break
             if self.multiselect and filenames:
                 self.filesource.select(filenames)  # array of file names
                 self.accepted = True
                 self.close()
         elif self.openType == ENTRY_POINTS:
             epName = selection[0]
             #index 0 is the remapped Url, as opposed to the canonical one used for display
             # Greg Acsone reports [0] does not work for Corep 1.6 pkgs, need [1], old style packages
             filenames = []
             for _url, _type in self.packageContainedInstances:  # check if selection was an inline instance
                 if _type in selection:
                     if _url in self.packageContainedIXDSes:  # taxonomy package
                         filenames.extend(self.packageContainedIXDSes[_url])
                     else:  # single instance
                         filenames.append(_url)
             if not filenames:  # else if it's a named taxonomy entry point of an installed package
                 for url in self.taxonomyPackage["entryPoints"][epName]:
                     filename = url[1]  # use unmapped file name
                     if not filename.endswith("/"):
                         # check if it's an absolute URL rather than a path into the archive
                         if not isHttpUrl(
                                 filename
                         ) and self.metadataFilePrefix != self.taxonomyPkgMetaInf:
                             # assume it's a path inside the archive:
                             filename = self.metadataFilePrefix + filename
                     filenames.append(filename)
             if filenames:
                 self.filesource.select(filenames)
                 self.accepted = True
                 self.close()
             return
         elif self.openType in (PLUGIN, PACKAGE):
             filename = self.filenames[int(selection[0][4:])][2]
         if filename is not None and not self.multiselect and not filename.endswith(
                 "/"):
             if hasattr(self, "taxonomyPackage"):
                 # attempt to unmap the filename to original file
                 # will be mapped again in loading, but this allows schemaLocation to be unmapped
                 for prefix, remapping in self.taxonomyPackage[
                         "remappings"].items():
                     if isHttpUrl(remapping):
                         remapStart = remapping
                     else:
                         remapStart = self.metadataFilePrefix + remapping
                     if filename.startswith(remapStart):
                         # set unmmapped file
                         filename = prefix + filename[len(remapStart):]
                         break
                     if (self.metadataFilePrefix.endswith("/META-INF/")
                             and isHttpUrl(prefix) and filename.startswith(
                                 self.metadataFilePrefix[:-10])
                             and filename.startswith(
                                 remapping[len(self.filesource.url) + 1:])):
                         # recover unmapped file name for chosen in-archive relative file
                         filename = prefix + filename[
                             len(remapping) - len(self.filesource.url) - 1:]
             if self.openType in (PLUGIN, PACKAGE):
                 self.filesource.selection = filename
             else:
                 self.filesource.select(filename)
             self.accepted = True
             self.close()
Exemplo n.º 59
0
def saveTargetDocument(modelXbrl,
                       targetDocumentFilename,
                       targetDocumentSchemaRefs,
                       outputZip=None,
                       filingFiles=None,
                       *args,
                       **kwargs):
    targetUrl = modelXbrl.modelManager.cntlr.webCache.normalizeUrl(
        targetDocumentFilename, modelXbrl.modelDocument.filepath)
    targetUrlParts = targetUrl.rpartition(".")
    targetUrl = targetUrlParts[0] + "_extracted." + targetUrlParts[2]
    modelXbrl.modelManager.showStatus(
        _("Extracting instance ") + os.path.basename(targetUrl))
    targetInstance = ModelXbrl.create(modelXbrl.modelManager,
                                      newDocumentType=Type.INSTANCE,
                                      url=targetUrl,
                                      schemaRefs=targetDocumentSchemaRefs,
                                      isEntry=True)
    ValidateXbrlDimensions.loadDimensionDefaults(
        targetInstance)  # need dimension defaults
    # roleRef and arcroleRef (of each inline document)
    for sourceRefs in (modelXbrl.targetRoleRefs, modelXbrl.targetArcroleRefs):
        for roleRefElt in sourceRefs.values():
            addChild(targetInstance.modelDocument.xmlRootElement,
                     roleRefElt.qname,
                     attributes=roleRefElt.items())

    # contexts
    for context in modelXbrl.contexts.values():
        newCntx = targetInstance.createContext(
            context.entityIdentifier[0],
            context.entityIdentifier[1],
            'instant' if context.isInstantPeriod else
            'duration' if context.isStartEndPeriod else 'forever',
            context.startDatetime,
            context.endDatetime,
            None,
            context.qnameDims, [], [],
            id=context.id)
    for unit in modelXbrl.units.values():
        measures = unit.measures
        newUnit = targetInstance.createUnit(measures[0],
                                            measures[1],
                                            id=unit.id)

    modelXbrl.modelManager.showStatus(_("Creating and validating facts"))
    newFactForOldObjId = {}

    def createFacts(facts, parent):
        for fact in facts:
            if fact.isItem:
                attrs = {"contextRef": fact.contextID}
                if fact.id:
                    attrs["id"] = fact.id
                if fact.isNumeric:
                    attrs["unitRef"] = fact.unitID
                    if fact.get("decimals"):
                        attrs["decimals"] = fact.get("decimals")
                    if fact.get("precision"):
                        attrs["precision"] = fact.get("precision")
                if fact.isNil:
                    attrs[XbrlConst.qnXsiNil] = "true"
                    text = None
                else:
                    text = fact.xValue if fact.xValid else fact.textValue
                newFact = targetInstance.createFact(fact.qname,
                                                    attributes=attrs,
                                                    text=text,
                                                    parent=parent)
                newFactForOldObjId[fact.objectIndex] = newFact
                if filingFiles and fact.concept is not None and fact.concept.isTextBlock:
                    # check for img and other filing references
                    for xmltext in [text] + CDATApattern.findall(text):
                        try:
                            for elt in XML(
                                    "<body>\n{0}\n</body>\n".format(xmltext)):
                                if elt.tag in ("a", "img") and not isHttpUrl(
                                        attrValue) and not os.path.isabs(
                                            attrvalue):
                                    for attrTag, attrValue in elt.items():
                                        if attrTag in ("href", "src"):
                                            filingFiles.add(attrValue)
                        except (XMLSyntaxError, UnicodeDecodeError):
                            pass
            elif fact.isTuple:
                newTuple = targetInstance.createFact(fact.qname, parent=parent)
                newFactForOldObjId[fact.objectIndex] = newTuple
                createFacts(fact.modelTupleFacts, newTuple)

    createFacts(modelXbrl.facts, None)
    # footnote links
    footnoteIdCount = {}
    modelXbrl.modelManager.showStatus(
        _("Creating and validating footnotes & relationships"))
    HREF = "{http://www.w3.org/1999/xlink}href"
    footnoteLinks = defaultdict(list)
    for linkKey, linkPrototypes in modelXbrl.baseSets.items():
        arcrole, linkrole, linkqname, arcqname = linkKey
        if (linkrole and linkqname and arcqname and  # fully specified roles
                arcrole != "XBRL-footnotes" and any(
                    lP.modelDocument.type == Type.INLINEXBRL
                    for lP in linkPrototypes)):
            for linkPrototype in linkPrototypes:
                if linkPrototype not in footnoteLinks[linkrole]:
                    footnoteLinks[linkrole].append(linkPrototype)
    for linkrole in sorted(footnoteLinks.keys()):
        for linkPrototype in footnoteLinks[linkrole]:
            newLink = addChild(targetInstance.modelDocument.xmlRootElement,
                               linkPrototype.qname,
                               attributes=linkPrototype.attributes)
            for linkChild in linkPrototype:
                attributes = linkChild.attributes
                if isinstance(linkChild, LocPrototype):
                    if HREF not in linkChild.attributes:
                        linkChild.attributes[HREF] = \
                        "#" + elementFragmentIdentifier(newFactForOldObjId[linkChild.dereference().objectIndex])
                    addChild(newLink, linkChild.qname, attributes=attributes)
                elif isinstance(linkChild, ArcPrototype):
                    addChild(newLink, linkChild.qname, attributes=attributes)
                elif isinstance(linkChild, ModelInlineFootnote):
                    idUseCount = footnoteIdCount.get(linkChild.footnoteID,
                                                     0) + 1
                    if idUseCount > 1:  # if footnote with id in other links bump the id number
                        attributes = linkChild.attributes.copy()
                        attributes["id"] = "{}_{}".format(
                            attributes["id"], idUseCount)
                    footnoteIdCount[linkChild.footnoteID] = idUseCount
                    newChild = addChild(newLink,
                                        linkChild.qname,
                                        attributes=attributes)
                    copyIxFootnoteHtml(
                        linkChild,
                        newChild,
                        targetModelDocument=targetInstance.modelDocument,
                        withText=True)
                    if filingFiles and linkChild.textValue:
                        footnoteHtml = XML("<body/>")
                        copyIxFootnoteHtml(linkChild, footnoteHtml)
                        for elt in footnoteHtml.iter():
                            if elt.tag in ("a", "img"):
                                for attrTag, attrValue in elt.items():
                                    if attrTag in (
                                            "href", "src") and not isHttpUrl(
                                                attrValue
                                            ) and not os.path.isabs(attrvalue):
                                        filingFiles.add(attrValue)

    targetInstance.saveInstance(overrideFilepath=targetUrl,
                                outputZip=outputZip)
    if getattr(modelXbrl, "isTestcaseVariation", False):
        modelXbrl.extractedInlineInstance = True  # for validation comparison
    modelXbrl.modelManager.showStatus(_("Saved extracted instance"), 5000)