def loadTaxonomyPackageMappings(self): if self.taxonomyPackageMetadataFiles: metadata = self.url + os.sep + self.taxonomyPackageMetadataFiles[0] taxonomyPackage = PackageManager.parsePackage( self.cntlr, self, metadata, os.sep.join(os.path.split(metadata)[:-1]) + os.sep) self.mappedPaths = taxonomyPackage["remappings"]
def loadTaxonomyPackageMappings(self, errors=[], expectTaxonomyPackage=None): if not self.mappedPaths and (self.taxonomyPackageMetadataFiles or expectTaxonomyPackage): if PackageManager.validateTaxonomyPackage(self.cntlr, self, errors=errors): metadata = self.baseurl + os.sep + self.taxonomyPackageMetadataFiles[0] self.taxonomyPackage = PackageManager.parsePackage(self.cntlr, self, metadata, os.sep.join(os.path.split(metadata)[:-1]) + os.sep, errors=errors) self.mappedPaths = self.taxonomyPackage.get("remappings")
def open(self): if not self.isOpen: if (self.isZip or self.isTarGz or self.isEis or self.isXfd or self.isRss or self.isInstalledTaxonomyPackage) and self.cntlr: self.basefile = self.cntlr.webCache.getfilename(self.url) else: self.basefile = self.url self.baseurl = self.url # url gets changed by selection if not self.basefile: return # an error should have been logged if self.isZip: try: self.fs = zipfile.ZipFile(openFileStream(self.cntlr, self.basefile, 'rb'), mode="r") self.isOpen = True except EnvironmentError as err: self.logError(err) pass elif self.isTarGz: try: self.fs = tarfile.open(self.basefile, "r:gz") self.isOpen = True except EnvironmentError as err: self.logError(err) pass elif self.isEis: # check first line of file buf = b'' try: file = open(self.basefile, 'rb') more = True while more: l = file.read(8) if len(l) < 8: break if len(buf) == 0 and l.startswith(b"<?xml "): # not compressed buf = l + file.read() # not compressed break compressedBytes = file.read( struct.unpack(">L", l[0:4])[0]) if len(compressedBytes) <= 0: break buf += zlib.decompress(compressedBytes) file.close() except EnvironmentError as err: self.logError(err) pass #uncomment to save for debugging #with open("c:/temp/test.xml", "wb") as f: # f.write(buf) if buf.startswith(b"<?xml "): try: # must strip encoding str = buf.decode(XmlUtil.encoding(buf)) endEncoding = str.index("?>", 0, 128) if endEncoding > 0: str = str[endEncoding+2:] file = io.StringIO(initial_value=str) parser = etree.XMLParser(recover=True, huge_tree=True) self.eisDocument = etree.parse(file, parser=parser) file.close() self.isOpen = True except EnvironmentError as err: self.logError(err) return # provide error message later except etree.LxmlError as err: self.logError(err) return # provide error message later elif self.isXfd: # check first line of file file = open(self.basefile, 'rb') firstline = file.readline() if firstline.startswith(b"application/x-xfdl;content-encoding=\"asc-gzip\""): # file has been gzipped base64input = file.read(-1) file.close(); file = None; fb = base64.b64decode(base64input) ungzippedBytes = b"" totalLenUncompr = 0 i = 0 while i < len(fb): lenCompr = fb[i + 0] * 256 + fb[i + 1] lenUncomp = fb[i + 2] * 256 + fb[i + 3] lenRead = 0 totalLenUncompr += lenUncomp gzchunk = (bytes((31,139,8,0)) + fb[i:i+lenCompr]) try: with gzip.GzipFile(fileobj=io.BytesIO(gzchunk)) as gf: while True: readSize = min(16384, lenUncomp - lenRead) readBytes = gf.read(size=readSize) lenRead += len(readBytes) ungzippedBytes += readBytes if len(readBytes) == 0 or (lenUncomp - lenRead) <= 0: break except IOError as err: pass # provide error message later i += lenCompr + 4 #for learning the content of xfd file, uncomment this: #with open("c:\\temp\\test.xml", "wb") as fh: # fh.write(ungzippedBytes) file = io.StringIO(initial_value=ungzippedBytes.decode("utf-8")) else: # position to start of file file.seek(0,io.SEEK_SET) try: self.xfdDocument = etree.parse(file) file.close() self.isOpen = True except EnvironmentError as err: self.logError(err) return # provide error message later except etree.LxmlError as err: self.logError(err) return # provide error message later elif self.isRss: try: self.rssDocument = etree.parse(self.basefile) self.isOpen = True except EnvironmentError as err: self.logError(err) return # provide error message later except etree.LxmlError as err: self.logError(err) return # provide error message later elif self.isInstalledTaxonomyPackage: self.isOpen = True # load mappings try: metadataFiles = self.taxonomyPackageMetadataFiles if len(metadataFiles) != 1: raise IOError(_("Taxonomy package must contain one and only one metadata file: {0}.") .format(', '.join(metadataFiles))) # HF: this won't work, see DialogOpenArchive for correct code # not sure if it is used taxonomyPackage = PackageManager.parsePackage(self.cntlr, self.url) fileSourceDir = os.path.dirname(self.baseurl) + os.sep self.mappedPaths = \ dict((prefix, remapping if isHttpUrl(remapping) else (fileSourceDir + remapping.replace("/", os.sep))) for prefix, remapping in taxonomyPackage["remappings"].items()) except EnvironmentError as err: self.logError(err) return # provide error message later
def open(self, reloadCache=False): if not self.isOpen: if (self.isZip or self.isTarGz or self.isEis or self.isXfd or self.isRss or self.isInstalledTaxonomyPackage) and self.cntlr: self.basefile = self.cntlr.webCache.getfilename( self.url, reload=reloadCache) else: self.basefile = self.url self.baseurl = self.url # url gets changed by selection if not self.basefile: return # an error should have been logged if self.isZip: try: self.fs = zipfile.ZipFile(openFileStream( self.cntlr, self.basefile, 'rb'), mode="r") self.isOpen = True except EnvironmentError as err: self.logError(err) pass elif self.isTarGz: try: self.fs = tarfile.open(self.basefile, "r:gz") self.isOpen = True except EnvironmentError as err: self.logError(err) pass elif self.isEis: # check first line of file buf = b'' try: file = open(self.basefile, 'rb') more = True while more: l = file.read(8) if len(l) < 8: break if len(buf) == 0 and l.startswith( b"<?xml "): # not compressed buf = l + file.read() # not compressed break compressedBytes = file.read( struct.unpack(">L", l[0:4])[0]) if len(compressedBytes) <= 0: break buf += zlib.decompress(compressedBytes) file.close() except EnvironmentError as err: self.logError(err) pass #uncomment to save for debugging #with open("c:/temp/test.xml", "wb") as f: # f.write(buf) if buf.startswith(b"<?xml "): try: # must strip encoding str = buf.decode(XmlUtil.encoding(buf)) endEncoding = str.index("?>", 0, 128) if endEncoding > 0: str = str[endEncoding + 2:] file = io.StringIO(initial_value=str) parser = etree.XMLParser(recover=True, huge_tree=True) self.eisDocument = etree.parse(file, parser=parser) file.close() self.isOpen = True except EnvironmentError as err: self.logError(err) return # provide error message later except etree.LxmlError as err: self.logError(err) return # provide error message later elif self.isXfd: # check first line of file file = open(self.basefile, 'rb') firstline = file.readline() if firstline.startswith( b"application/x-xfdl;content-encoding=\"asc-gzip\""): # file has been gzipped base64input = file.read(-1) file.close() file = None fb = base64.b64decode(base64input) ungzippedBytes = b"" totalLenUncompr = 0 i = 0 while i < len(fb): lenCompr = fb[i + 0] * 256 + fb[i + 1] lenUncomp = fb[i + 2] * 256 + fb[i + 3] lenRead = 0 totalLenUncompr += lenUncomp gzchunk = (bytes((31, 139, 8, 0)) + fb[i:i + lenCompr]) try: with gzip.GzipFile( fileobj=io.BytesIO(gzchunk)) as gf: while True: readSize = min(16384, lenUncomp - lenRead) readBytes = gf.read(size=readSize) lenRead += len(readBytes) ungzippedBytes += readBytes if len(readBytes) == 0 or (lenUncomp - lenRead) <= 0: break except IOError as err: pass # provide error message later i += lenCompr + 4 #for learning the content of xfd file, uncomment this: #with open("c:\\temp\\test.xml", "wb") as fh: # fh.write(ungzippedBytes) file = io.StringIO( initial_value=ungzippedBytes.decode("utf-8")) else: # position to start of file file.seek(0, io.SEEK_SET) try: self.xfdDocument = etree.parse(file) file.close() self.isOpen = True except EnvironmentError as err: self.logError(err) return # provide error message later except etree.LxmlError as err: self.logError(err) return # provide error message later elif self.isRss: try: self.rssDocument = etree.parse(self.basefile) self.isOpen = True except EnvironmentError as err: self.logError(err) return # provide error message later except etree.LxmlError as err: self.logError(err) return # provide error message later elif self.isInstalledTaxonomyPackage: self.isOpen = True # load mappings try: metadataFiles = self.taxonomyPackageMetadataFiles if len(metadataFiles) != 1: raise IOError( _("Taxonomy package must contain one and only one metadata file: {0}." ).format(', '.join(metadataFiles))) # HF: this won't work, see DialogOpenArchive for correct code # not sure if it is used taxonomyPackage = PackageManager.parsePackage( self.cntlr, self.url) fileSourceDir = os.path.dirname(self.baseurl) + os.sep self.mappedPaths = \ dict((prefix, remapping if isHttpUrl(remapping) else (fileSourceDir + remapping.replace("/", os.sep))) for prefix, remapping in taxonomyPackage["remappings"].items()) except EnvironmentError as err: self.logError(err) return # provide error message later
return # provide error message later except etree.LxmlError, err: self.logError(err) return # provide error message later elif self.isInstalledTaxonomyPackage: self.isOpen = True # load mappings try: metadataFiles = self.taxonomyPackageMetadataFiles if len(metadataFiles) != 1: raise IOError(_(u"Taxonomy package must contain one and only one metadata file: {0}.") .format(u', '.join(metadataFiles))) # HF: this won't work, see DialogOpenArchive for correct code # not sure if it is used taxonomyPackage = PackageManager.parsePackage(self.cntlr, self.url) fileSourceDir = os.path.dirname(self.baseurl) + os.sep self.mappedPaths = \ dict((prefix, remapping if isHttpUrl(remapping) else (fileSourceDir + remapping.replace(u"/", os.sep))) for prefix, remapping in taxonomyPackage[u"remappings"].items()) except EnvironmentError, err: self.logError(err) return # provide error message later def openZipStream(self, sourceZipStream): if not self.isOpen: self.basefile = self.url self.baseurl = self.url # url gets changed by selection self.fs = zipfile.ZipFile(sourceZipStream, mode=u"r")
def loadTaxonomyPackageMappings(self): if self.taxonomyPackageMetadataFiles: metadata = self.url + os.sep + self.taxonomyPackageMetadataFiles[0] taxonomyPackage = PackageManager.parsePackage(self.cntlr, self, metadata, os.sep.join(os.path.split(metadata)[:-1]) + os.sep) self.mappedPaths = taxonomyPackage["remappings"]