コード例 #1
0
ファイル: FileSource.py プロジェクト: vivekmahajan380/Arelle
 def loadTaxonomyPackageMappings(self):
     if self.taxonomyPackageMetadataFiles:
         metadata = self.url + os.sep + self.taxonomyPackageMetadataFiles[0]
         taxonomyPackage = PackageManager.parsePackage(
             self.cntlr, self, metadata,
             os.sep.join(os.path.split(metadata)[:-1]) + os.sep)
         self.mappedPaths = taxonomyPackage["remappings"]
コード例 #2
0
 def loadTaxonomyPackageMappings(self, errors=[], expectTaxonomyPackage=None):
     if not self.mappedPaths and (self.taxonomyPackageMetadataFiles or expectTaxonomyPackage):
         if PackageManager.validateTaxonomyPackage(self.cntlr, self, errors=errors):
             metadata = self.baseurl + os.sep + self.taxonomyPackageMetadataFiles[0]
             self.taxonomyPackage = PackageManager.parsePackage(self.cntlr, self, metadata,
                                                                os.sep.join(os.path.split(metadata)[:-1]) + os.sep,
                                                                errors=errors)
             self.mappedPaths = self.taxonomyPackage.get("remappings")
コード例 #3
0
ファイル: FileSource.py プロジェクト: namitkewat/Arelle
    def open(self):
        if not self.isOpen:
            if (self.isZip or self.isTarGz or self.isEis or self.isXfd or self.isRss or self.isInstalledTaxonomyPackage) and self.cntlr:
                self.basefile = self.cntlr.webCache.getfilename(self.url)
            else:
                self.basefile = self.url
            self.baseurl = self.url # url gets changed by selection
            if not self.basefile:
                return  # an error should have been logged
            if self.isZip:
                try:
                    self.fs = zipfile.ZipFile(openFileStream(self.cntlr, self.basefile, 'rb'), mode="r")
                    self.isOpen = True
                except EnvironmentError as err:
                    self.logError(err)
                    pass
            elif self.isTarGz:
                try:
                    self.fs = tarfile.open(self.basefile, "r:gz")
                    self.isOpen = True
                except EnvironmentError as err:
                    self.logError(err)
                    pass
            elif self.isEis:
                # check first line of file
                buf = b''
                try:
                    file = open(self.basefile, 'rb')
                    more = True
                    while more:
                        l = file.read(8)
                        if len(l) < 8:
                            break
                        if len(buf) == 0 and l.startswith(b"<?xml "): # not compressed
                            buf = l + file.read()  # not compressed
                            break
                        compressedBytes = file.read( struct.unpack(">L", l[0:4])[0])
                        if len(compressedBytes) <= 0:
                            break
                        buf += zlib.decompress(compressedBytes)
                    file.close()
                except EnvironmentError as err:
                    self.logError(err)
                    pass
                #uncomment to save for debugging
                #with open("c:/temp/test.xml", "wb") as f:
                #    f.write(buf)
                
                if buf.startswith(b"<?xml "):
                    try:
                        # must strip encoding
                        str = buf.decode(XmlUtil.encoding(buf))
                        endEncoding = str.index("?>", 0, 128)
                        if endEncoding > 0:
                            str = str[endEncoding+2:]
                        file = io.StringIO(initial_value=str)
                        parser = etree.XMLParser(recover=True, huge_tree=True)
                        self.eisDocument = etree.parse(file, parser=parser)
                        file.close()
                        self.isOpen = True
                    except EnvironmentError as err:
                        self.logError(err)
                        return # provide error message later
                    except etree.LxmlError as err:
                        self.logError(err)
                        return # provide error message later
                
            elif self.isXfd:
                # check first line of file
                file = open(self.basefile, 'rb')
                firstline = file.readline()
                if firstline.startswith(b"application/x-xfdl;content-encoding=\"asc-gzip\""):
                    # file has been gzipped
                    base64input = file.read(-1)
                    file.close();
                    file = None;
    
                    fb = base64.b64decode(base64input)
                    ungzippedBytes = b""
                    totalLenUncompr = 0
                    i = 0
                    while i < len(fb):
                        lenCompr = fb[i + 0] * 256 + fb[i + 1]
                        lenUncomp = fb[i + 2] * 256 + fb[i + 3]
                        lenRead = 0
                        totalLenUncompr += lenUncomp

                        gzchunk = (bytes((31,139,8,0)) + fb[i:i+lenCompr])
                        try:
                            with gzip.GzipFile(fileobj=io.BytesIO(gzchunk)) as gf:
                                while True:
                                    readSize = min(16384, lenUncomp - lenRead)
                                    readBytes = gf.read(size=readSize)
                                    lenRead += len(readBytes)
                                    ungzippedBytes += readBytes
                                    if len(readBytes) == 0 or (lenUncomp - lenRead) <= 0:
                                        break
                        except IOError as err:
                            pass # provide error message later

                        i += lenCompr + 4
                    #for learning the content of xfd file, uncomment this:
                    #with open("c:\\temp\\test.xml", "wb") as fh:
                    #    fh.write(ungzippedBytes)
                    file = io.StringIO(initial_value=ungzippedBytes.decode("utf-8"))
                else:
                    # position to start of file
                    file.seek(0,io.SEEK_SET)
                    
                try:
                    self.xfdDocument = etree.parse(file)
                    file.close()
                    self.isOpen = True
                except EnvironmentError as err:
                    self.logError(err)
                    return # provide error message later
                except etree.LxmlError as err:
                    self.logError(err)
                    return # provide error message later
                
            elif self.isRss:
                try:
                    self.rssDocument = etree.parse(self.basefile)
                    self.isOpen = True
                except EnvironmentError as err:
                    self.logError(err)
                    return # provide error message later
                except etree.LxmlError as err:
                    self.logError(err)
                    return # provide error message later
            
            elif self.isInstalledTaxonomyPackage:
                self.isOpen = True
                # load mappings
                try:
                    metadataFiles = self.taxonomyPackageMetadataFiles
                    if len(metadataFiles) != 1:
                        raise IOError(_("Taxonomy package must contain one and only one metadata file: {0}.")
                                      .format(', '.join(metadataFiles)))
                    # HF: this won't work, see DialogOpenArchive for correct code
                    # not sure if it is used
                    taxonomyPackage = PackageManager.parsePackage(self.cntlr, self.url)
                    fileSourceDir = os.path.dirname(self.baseurl) + os.sep
                    self.mappedPaths = \
                        dict((prefix, 
                              remapping if isHttpUrl(remapping)
                              else (fileSourceDir + remapping.replace("/", os.sep)))
                              for prefix, remapping in taxonomyPackage["remappings"].items())
                except EnvironmentError as err:
                    self.logError(err)
                    return # provide error message later
コード例 #4
0
ファイル: FileSource.py プロジェクト: camirisk/Arelle
    def open(self, reloadCache=False):
        if not self.isOpen:
            if (self.isZip or self.isTarGz or self.isEis or self.isXfd
                    or self.isRss
                    or self.isInstalledTaxonomyPackage) and self.cntlr:
                self.basefile = self.cntlr.webCache.getfilename(
                    self.url, reload=reloadCache)
            else:
                self.basefile = self.url
            self.baseurl = self.url  # url gets changed by selection
            if not self.basefile:
                return  # an error should have been logged
            if self.isZip:
                try:
                    self.fs = zipfile.ZipFile(openFileStream(
                        self.cntlr, self.basefile, 'rb'),
                                              mode="r")
                    self.isOpen = True
                except EnvironmentError as err:
                    self.logError(err)
                    pass
            elif self.isTarGz:
                try:
                    self.fs = tarfile.open(self.basefile, "r:gz")
                    self.isOpen = True
                except EnvironmentError as err:
                    self.logError(err)
                    pass
            elif self.isEis:
                # check first line of file
                buf = b''
                try:
                    file = open(self.basefile, 'rb')
                    more = True
                    while more:
                        l = file.read(8)
                        if len(l) < 8:
                            break
                        if len(buf) == 0 and l.startswith(
                                b"<?xml "):  # not compressed
                            buf = l + file.read()  # not compressed
                            break
                        compressedBytes = file.read(
                            struct.unpack(">L", l[0:4])[0])
                        if len(compressedBytes) <= 0:
                            break
                        buf += zlib.decompress(compressedBytes)
                    file.close()
                except EnvironmentError as err:
                    self.logError(err)
                    pass
                #uncomment to save for debugging
                #with open("c:/temp/test.xml", "wb") as f:
                #    f.write(buf)

                if buf.startswith(b"<?xml "):
                    try:
                        # must strip encoding
                        str = buf.decode(XmlUtil.encoding(buf))
                        endEncoding = str.index("?>", 0, 128)
                        if endEncoding > 0:
                            str = str[endEncoding + 2:]
                        file = io.StringIO(initial_value=str)
                        parser = etree.XMLParser(recover=True, huge_tree=True)
                        self.eisDocument = etree.parse(file, parser=parser)
                        file.close()
                        self.isOpen = True
                    except EnvironmentError as err:
                        self.logError(err)
                        return  # provide error message later
                    except etree.LxmlError as err:
                        self.logError(err)
                        return  # provide error message later

            elif self.isXfd:
                # check first line of file
                file = open(self.basefile, 'rb')
                firstline = file.readline()
                if firstline.startswith(
                        b"application/x-xfdl;content-encoding=\"asc-gzip\""):
                    # file has been gzipped
                    base64input = file.read(-1)
                    file.close()
                    file = None

                    fb = base64.b64decode(base64input)
                    ungzippedBytes = b""
                    totalLenUncompr = 0
                    i = 0
                    while i < len(fb):
                        lenCompr = fb[i + 0] * 256 + fb[i + 1]
                        lenUncomp = fb[i + 2] * 256 + fb[i + 3]
                        lenRead = 0
                        totalLenUncompr += lenUncomp

                        gzchunk = (bytes((31, 139, 8, 0)) + fb[i:i + lenCompr])
                        try:
                            with gzip.GzipFile(
                                    fileobj=io.BytesIO(gzchunk)) as gf:
                                while True:
                                    readSize = min(16384, lenUncomp - lenRead)
                                    readBytes = gf.read(size=readSize)
                                    lenRead += len(readBytes)
                                    ungzippedBytes += readBytes
                                    if len(readBytes) == 0 or (lenUncomp -
                                                               lenRead) <= 0:
                                        break
                        except IOError as err:
                            pass  # provide error message later

                        i += lenCompr + 4
                    #for learning the content of xfd file, uncomment this:
                    #with open("c:\\temp\\test.xml", "wb") as fh:
                    #    fh.write(ungzippedBytes)
                    file = io.StringIO(
                        initial_value=ungzippedBytes.decode("utf-8"))
                else:
                    # position to start of file
                    file.seek(0, io.SEEK_SET)

                try:
                    self.xfdDocument = etree.parse(file)
                    file.close()
                    self.isOpen = True
                except EnvironmentError as err:
                    self.logError(err)
                    return  # provide error message later
                except etree.LxmlError as err:
                    self.logError(err)
                    return  # provide error message later

            elif self.isRss:
                try:
                    self.rssDocument = etree.parse(self.basefile)
                    self.isOpen = True
                except EnvironmentError as err:
                    self.logError(err)
                    return  # provide error message later
                except etree.LxmlError as err:
                    self.logError(err)
                    return  # provide error message later

            elif self.isInstalledTaxonomyPackage:
                self.isOpen = True
                # load mappings
                try:
                    metadataFiles = self.taxonomyPackageMetadataFiles
                    if len(metadataFiles) != 1:
                        raise IOError(
                            _("Taxonomy package must contain one and only one metadata file: {0}."
                              ).format(', '.join(metadataFiles)))
                    # HF: this won't work, see DialogOpenArchive for correct code
                    # not sure if it is used
                    taxonomyPackage = PackageManager.parsePackage(
                        self.cntlr, self.url)
                    fileSourceDir = os.path.dirname(self.baseurl) + os.sep
                    self.mappedPaths = \
                        dict((prefix,
                              remapping if isHttpUrl(remapping)
                              else (fileSourceDir + remapping.replace("/", os.sep)))
                              for prefix, remapping in taxonomyPackage["remappings"].items())
                except EnvironmentError as err:
                    self.logError(err)
                    return  # provide error message later
コード例 #5
0
ファイル: FileSource.py プロジェクト: sternshus/not_arelle2.7
                    return # provide error message later
                except etree.LxmlError, err:
                    self.logError(err)
                    return # provide error message later
            
            elif self.isInstalledTaxonomyPackage:
                self.isOpen = True
                # load mappings
                try:
                    metadataFiles = self.taxonomyPackageMetadataFiles
                    if len(metadataFiles) != 1:
                        raise IOError(_(u"Taxonomy package must contain one and only one metadata file: {0}.")
                                      .format(u', '.join(metadataFiles)))
                    # HF: this won't work, see DialogOpenArchive for correct code
                    # not sure if it is used
                    taxonomyPackage = PackageManager.parsePackage(self.cntlr, self.url)
                    fileSourceDir = os.path.dirname(self.baseurl) + os.sep
                    self.mappedPaths = \
                        dict((prefix, 
                              remapping if isHttpUrl(remapping)
                              else (fileSourceDir + remapping.replace(u"/", os.sep)))
                              for prefix, remapping in taxonomyPackage[u"remappings"].items())
                except EnvironmentError, err:
                    self.logError(err)
                    return # provide error message later

    def openZipStream(self, sourceZipStream):
        if not self.isOpen:
            self.basefile = self.url
            self.baseurl = self.url # url gets changed by selection
            self.fs = zipfile.ZipFile(sourceZipStream, mode=u"r")
コード例 #6
0
ファイル: FileSource.py プロジェクト: Arelle/Arelle
 def loadTaxonomyPackageMappings(self):
     if self.taxonomyPackageMetadataFiles:
         metadata = self.url + os.sep + self.taxonomyPackageMetadataFiles[0]
         taxonomyPackage = PackageManager.parsePackage(self.cntlr, self, metadata,
                                                       os.sep.join(os.path.split(metadata)[:-1]) + os.sep)
         self.mappedPaths = taxonomyPackage["remappings"]