def __init__(self, pathname, ignoreUsageXml=False): self.pathname = pathname self.filenames = [] self.fileSize = 0 self.nested = [] self.nestedSize = 0 xusage = None if not ignoreUsageXml: # Look for a usage.xml file in this directory. If we find # one, we read it for the file size and then stop here, as # an optimization. usageFilename = Filename(pathname, 'usage.xml') doc = TiXmlDocument(usageFilename.toOsSpecific()) if doc.LoadFile(): xusage = doc.FirstChildElement('usage') if xusage: diskSpace = xusage.Attribute('disk_space') try: diskSpace = int(diskSpace or '') except ValueError: diskSpace = None if diskSpace is not None: self.fileSize = diskSpace return files = vfs.scanDirectory(self.pathname) if files is None: files = [] for vfile in files: if hasattr(vfile, 'getMount'): if not isinstance(vfile.getMount(), VirtualFileMountSystem): # Not a real file; ignore it. continue if vfile.isDirectory(): # A nested directory. subdir = ScanDirectoryNode(vfile.getFilename(), ignoreUsageXml=ignoreUsageXml) self.nested.append(subdir) self.nestedSize += subdir.getTotalSize() elif vfile.isRegularFile(): # A nested file. self.filenames.append(vfile.getFilename()) self.fileSize += vfile.getFileSize() else: # Some other wacky file thing. self.filenames.append(vfile.getFilename()) if xusage: # Now update the usage.xml file with the newly-determined # disk space. xusage.SetAttribute('disk_space', str(self.getTotalSize())) tfile = Filename.temporary(str(pathname), '.xml') if doc.SaveFile(tfile.toOsSpecific()): tfile.renameTo(usageFilename)
def __init__(self, pathname, ignoreUsageXml = False): self.pathname = pathname self.filenames = [] self.fileSize = 0 self.nested = [] self.nestedSize = 0 xusage = None if not ignoreUsageXml: # Look for a usage.xml file in this directory. If we find # one, we read it for the file size and then stop here, as # an optimization. usageFilename = Filename(pathname, 'usage.xml') doc = TiXmlDocument(usageFilename.toOsSpecific()) if doc.LoadFile(): xusage = doc.FirstChildElement('usage') if xusage: diskSpace = xusage.Attribute('disk_space') try: diskSpace = int(diskSpace or '') except ValueError: diskSpace = None if diskSpace is not None: self.fileSize = diskSpace return files = vfs.scanDirectory(self.pathname) if files is None: files = [] for vfile in files: if hasattr(vfile, 'getMount'): if not isinstance(vfile.getMount(), VirtualFileMountSystem): # Not a real file; ignore it. continue if vfile.isDirectory(): # A nested directory. subdir = ScanDirectoryNode(vfile.getFilename(), ignoreUsageXml = ignoreUsageXml) self.nested.append(subdir) self.nestedSize += subdir.getTotalSize() elif vfile.isRegularFile(): # A nested file. self.filenames.append(vfile.getFilename()) self.fileSize += vfile.getFileSize() else: # Some other wacky file thing. self.filenames.append(vfile.getFilename()) if xusage: # Now update the usage.xml file with the newly-determined # disk space. xusage.SetAttribute('disk_space', str(self.getTotalSize())) tfile = Filename.temporary(str(pathname), '.xml') if doc.SaveFile(tfile.toOsSpecific()): tfile.renameTo(usageFilename)
def ramdir(): """Fixture yielding a fresh ramdisk directory.""" from panda3d.core import VirtualFileMountRamdisk, Filename vfs = VirtualFileSystem.get_global_ptr() mount = VirtualFileMountRamdisk() dir = Filename.temporary("/virtual", "ram.") assert vfs.mount(mount, dir, 0) yield dir vfs.unmount(mount)
def __applyPatch(self, step, patchfile): """ Applies the indicated patching in-place to the current uncompressed archive. The patchfile is removed after the operation. Yields one of stepComplete, stepFailed, restartDownload, or stepContinue. """ self.updated = True origPathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename) patchPathname = Filename(self.getPackageDir(), patchfile.file.filename) result = Filename.temporary('', 'patch_') self.notify.info("Patching %s with %s" % (origPathname, patchPathname)) p = core.Patchfile() # The C++ class ret = p.initiate(patchPathname, origPathname, result) if ret == EUSuccess: ret = p.run() while ret == EUOk: step.bytesDone = step.bytesNeeded * p.getProgress() self.__updateStepProgress(step) if taskMgr.destroyed: # If the task manager has been destroyed, we must # be shutting down. Get out of here. self.notify.warning( "Task Manager destroyed, aborting patch %s" % (origPathname)) yield self.stepFailed return yield self.stepContinue ret = p.run() del p patchPathname.unlink() if ret < 0: self.notify.warning("Patching of %s failed." % (origPathname)) result.unlink() yield self.stepFailed return if not result.renameTo(origPathname): self.notify.warning("Couldn't rename %s to %s" % (result, origPathname)) yield self.stepFailed return yield self.stepComplete return
def freshenFile(self, host, fileSpec, localPathname): """ Ensures that the localPathname is the most current version of the file defined by fileSpec, as offered by host. If not, it downloads a new version on-the-spot. Returns true on success, false on failure. """ if fileSpec.quickVerify(pathname = localPathname): # It's good, keep it. return True assert self.http # It's stale, get a new one. doc = None if self.superMirrorUrl: # Use the "super mirror" first. url = core.URLSpec(self.superMirrorUrl + fileSpec.filename) self.notify.info("Freshening %s" % (url)) doc = self.http.getDocument(url) if not doc or not doc.isValid(): # Failing the super mirror, contact the actual host. url = core.URLSpec(host.hostUrlPrefix + fileSpec.filename) self.notify.info("Freshening %s" % (url)) doc = self.http.getDocument(url) if not doc.isValid(): return False file = Filename.temporary('', 'p3d_') if not doc.downloadToFile(file): # Failed to download. file.unlink() return False # Successfully downloaded! localPathname.makeDir() if not file.renameTo(localPathname): # Couldn't move it into place. file.unlink() return False if not fileSpec.fullVerify(pathname = localPathname, notify = self.notify): # No good after download. self.notify.info("%s is still no good after downloading." % (url)) return False return True
def freshenFile(self, host, fileSpec, localPathname): """ Ensures that the localPathname is the most current version of the file defined by fileSpec, as offered by host. If not, it downloads a new version on-the-spot. Returns true on success, false on failure. """ if fileSpec.quickVerify(pathname=localPathname): # It's good, keep it. return True assert self.http # It's stale, get a new one. doc = None if self.superMirrorUrl: # Use the "super mirror" first. url = core.URLSpec(self.superMirrorUrl + fileSpec.filename) self.notify.info("Freshening %s" % (url)) doc = self.http.getDocument(url) if not doc or not doc.isValid(): # Failing the super mirror, contact the actual host. url = core.URLSpec(host.hostUrlPrefix + fileSpec.filename) self.notify.info("Freshening %s" % (url)) doc = self.http.getDocument(url) if not doc.isValid(): return False file = Filename.temporary('', 'p3d_') if not doc.downloadToFile(file): # Failed to download. file.unlink() return False # Successfully downloaded! localPathname.makeDir() if not file.renameTo(localPathname): # Couldn't move it into place. file.unlink() return False if not fileSpec.fullVerify(pathname=localPathname, notify=self.notify): # No good after download. self.notify.info("%s is still no good after downloading." % (url)) return False return True
def _import_extension_module(self, fullname): """ Loads the binary shared object as a Python module, and returns it. """ vfile = vfs.getFile(self.filename, False) # We can only import an extension module if it already exists on # disk. This means if it's a truly virtual file that has no # on-disk equivalent, we have to write it to a temporary file # first. if hasattr(vfile, 'getMount') and \ isinstance(vfile.getMount(), VirtualFileMountSystem): # It's a real file. filename = self.filename elif self.filename.exists(): # It's a virtual file, but it's shadowing a real file in # the same directory. Assume they're the same, and load # the real one. filename = self.filename else: # It's a virtual file with no real-world existence. Dump # it to disk. TODO: clean up this filename. filename = Filename.temporary( '', self.filename.getBasenameWoExtension(), '.' + self.filename.getExtension(), type=Filename.TDso) filename.setExtension(self.filename.getExtension()) filename.setBinary() sin = vfile.openReadFile(True) sout = OFileStream() if not filename.openWrite(sout): raise IOError if not copyStream(sin, sout): raise IOError vfile.closeReadFile(sin) del sout module = imp.load_module(fullname, None, filename.toOsSpecific(), self.desc) module.__file__ = self.filename.toOsSpecific() return module
def __applyPatch(self, step, patchfile): """ Applies the indicated patching in-place to the current uncompressed archive. The patchfile is removed after the operation. Yields one of stepComplete, stepFailed, restartDownload, or stepContinue. """ self.updated = True origPathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename) patchPathname = Filename(self.getPackageDir(), patchfile.file.filename) result = Filename.temporary('', 'patch_') self.notify.info("Patching %s with %s" % (origPathname, patchPathname)) p = core.Patchfile() # The C++ class ret = p.initiate(patchPathname, origPathname, result) if ret == EUSuccess: ret = p.run() while ret == EUOk: step.bytesDone = step.bytesNeeded * p.getProgress() self.__updateStepProgress(step) if taskMgr.destroyed: # If the task manager has been destroyed, we must # be shutting down. Get out of here. self.notify.warning("Task Manager destroyed, aborting patch %s" % (origPathname)) yield self.stepFailed; return yield self.stepContinue ret = p.run() del p patchPathname.unlink() if ret < 0: self.notify.warning("Patching of %s failed." % (origPathname)) result.unlink() yield self.stepFailed; return if not result.renameTo(origPathname): self.notify.warning("Couldn't rename %s to %s" % (result, origPathname)) yield self.stepFailed; return yield self.stepComplete; return
def _import_extension_module(self, fullname): """ Loads the binary shared object as a Python module, and returns it. """ vfile = vfs.getFile(self.filename, False) # We can only import an extension module if it already exists on # disk. This means if it's a truly virtual file that has no # on-disk equivalent, we have to write it to a temporary file # first. if hasattr(vfile, 'getMount') and \ isinstance(vfile.getMount(), VirtualFileMountSystem): # It's a real file. filename = self.filename elif self.filename.exists(): # It's a virtual file, but it's shadowing a real file in # the same directory. Assume they're the same, and load # the real one. filename = self.filename else: # It's a virtual file with no real-world existence. Dump # it to disk. TODO: clean up this filename. filename = Filename.temporary('', self.filename.getBasenameWoExtension(), '.' + self.filename.getExtension(), type = Filename.TDso) filename.setExtension(self.filename.getExtension()) filename.setBinary() sin = vfile.openReadFile(True) sout = OFileStream() if not filename.openWrite(sout): raise IOError if not copyStream(sin, sout): raise IOError vfile.closeReadFile(sin) del sout module = imp.load_module(fullname, None, filename.toOsSpecific(), self.desc) module.__file__ = self.filename.toOsSpecific() return module
def writeConfigXml(self): """ Rewrites the config.xml to the root directory. This isn't called automatically; an application may call this after adjusting some parameters (such as self.maxDiskUsage). """ from panda3d.core import TiXmlDocument, TiXmlDeclaration, TiXmlElement filename = Filename(self.rootDir, self.ConfigBasename) doc = TiXmlDocument(filename.toOsSpecific()) decl = TiXmlDeclaration("1.0", "utf-8", "") doc.InsertEndChild(decl) xconfig = TiXmlElement('config') xconfig.SetAttribute('max_disk_usage', str(self.maxDiskUsage)) doc.InsertEndChild(xconfig) # Write the file to a temporary filename, then atomically move # it to its actual filename, to avoid race conditions when # updating this file. tfile = Filename.temporary(str(self.rootDir), '.xml') if doc.SaveFile(tfile.toOsSpecific()): tfile.renameTo(filename)
def getPandaTexture(self, panda): ''' Cache the Panda3D texture object for this block layout to save time and memory. ''' from panda3d.core import Texture, Filename if not self._pandaTexture: # Write the graphics to a file because that's what panda requires. # NamedTemporaryFile sometimes returns a path with wrong casing on # Windows, so do it Panda's way instead. pandaFile = Filename.temporary('', '') f = open(pandaFile.toOsSpecific(), 'wb') try: with f: f.write(self.graphicsIO.getvalue()) self._pandaTexture = panda.loader.loadTexture( pandaFile.getFullpath()) finally: os.unlink(f.name) self._pandaTexture.setWrapU(Texture.WM_clamp) self._pandaTexture.setWrapV(Texture.WM_clamp) return self._pandaTexture
def downloadContentsFile(self, http, redownload = False, hashVal = None): """ Downloads the contents.xml file for this particular host, synchronously, and then reads it. Returns true on success, false on failure. If hashVal is not None, it should be a HashVal object, which will be filled with the hash from the new contents.xml file.""" if self.hasCurrentContentsFile(): # We've already got one. return True if self.appRunner and self.appRunner.verifyContents == self.appRunner.P3DVCNever: # Not allowed to. return False rf = None if http: if not redownload and self.appRunner and self.appRunner.superMirrorUrl: # We start with the "super mirror", if it's defined. url = self.appRunner.superMirrorUrl + 'contents.xml' request = DocumentSpec(url) self.notify.info("Downloading contents file %s" % (request)) rf = Ramfile() channel = http.makeChannel(False) channel.getDocument(request) if not channel.downloadToRam(rf): self.notify.warning("Unable to download %s" % (url)) rf = None if not rf: # Then go to the main host, if our super mirror let us # down. url = self.hostUrlPrefix + 'contents.xml' # Append a uniquifying query string to the URL to force the # download to go all the way through any caches. We use the # time in seconds; that's unique enough. url += '?' + str(int(time.time())) # We might as well explicitly request the cache to be disabled # too, since we have an interface for that via HTTPChannel. request = DocumentSpec(url) request.setCacheControl(DocumentSpec.CCNoCache) self.notify.info("Downloading contents file %s" % (request)) statusCode = None statusString = '' for attempt in range(int(ConfigVariableInt('contents-xml-dl-attempts', 3))): if attempt > 0: self.notify.info("Retrying (%s)..."%(attempt,)) rf = Ramfile() channel = http.makeChannel(False) channel.getDocument(request) if channel.downloadToRam(rf): self.notify.info("Successfully downloaded %s" % (url,)) break else: rf = None statusCode = channel.getStatusCode() statusString = channel.getStatusString() self.notify.warning("Could not contact download server at %s" % (url,)) self.notify.warning("Status code = %s %s" % (statusCode, statusString)) if not rf: self.notify.warning("Unable to download %s" % (url,)) try: # Something screwed up. if statusCode == HTTPChannel.SCDownloadOpenError or \ statusCode == HTTPChannel.SCDownloadWriteError: launcher.setPandaErrorCode(2) elif statusCode == 404: # 404 not found launcher.setPandaErrorCode(5) elif statusCode < 100: # statusCode < 100 implies the connection attempt itself # failed. This is usually due to firewall software # interfering. Apparently some firewall software might # allow the first connection and disallow subsequent # connections; how strange. launcher.setPandaErrorCode(4) else: # There are other kinds of failures, but these will # generally have been caught already by the first test; so # if we get here there may be some bigger problem. Just # give the generic "big problem" message. launcher.setPandaErrorCode(6) except NameError as e: # no launcher pass except AttributeError as e: self.notify.warning("%s" % (str(e),)) pass return False tempFilename = Filename.temporary('', 'p3d_', '.xml') if rf: f = open(tempFilename.toOsSpecific(), 'wb') f.write(rf.getData()) f.close() if hashVal: hashVal.hashString(rf.getData()) if not self.readContentsFile(tempFilename, freshDownload = True): self.notify.warning("Failure reading %s" % (url)) tempFilename.unlink() return False tempFilename.unlink() return True # Couldn't download the file. Maybe we should look for a # previously-downloaded copy already on disk? return False
def markUsed(self): """ Marks the package as having been used. This is normally called automatically by installPackage(). """ if not hasattr(core, 'TiXmlDocument'): return if self.host.appRunner and self.host.appRunner.verifyContents == self.host.appRunner.P3DVCNever: # Not allowed to write any files to the package directory. return if self.updated: # If we've just installed a new version of the package, # re-measure the actual disk space used. self.diskSpace = self.__measureDiskSpace() filename = Filename(self.getPackageDir(), self.UsageBasename) doc = TiXmlDocument(filename.toOsSpecific()) if not doc.LoadFile(): decl = TiXmlDeclaration("1.0", "utf-8", "") doc.InsertEndChild(decl) xusage = doc.FirstChildElement('usage') if not xusage: doc.InsertEndChild(TiXmlElement('usage')) xusage = doc.FirstChildElement('usage') now = int(time.time()) count = xusage.Attribute('count_app') try: count = int(count or '') except ValueError: count = 0 xusage.SetAttribute('first_use', str(now)) count += 1 xusage.SetAttribute('count_app', str(count)) xusage.SetAttribute('last_use', str(now)) if self.updated: xusage.SetAttribute('last_update', str(now)) self.updated = False else: # Since we haven't changed the disk space, we can just # read it from the previous xml file. diskSpace = xusage.Attribute('disk_space') try: diskSpace = int(diskSpace or '') except ValueError: # Unless it wasn't set already. self.diskSpace = self.__measureDiskSpace() xusage.SetAttribute('disk_space', str(self.diskSpace)) # Write the file to a temporary filename, then atomically move # it to its actual filename, to avoid race conditions when # updating this file. tfile = Filename.temporary(str(self.getPackageDir()), '.xml') if doc.SaveFile(tfile.toOsSpecific()): tfile.renameTo(filename)
launcher.setPandaErrorCode(4) else: # There are other kinds of failures, but these will # generally have been caught already by the first test; so # if we get here there may be some bigger problem. Just # give the generic "big problem" message. launcher.setPandaErrorCode(6) except NameError,e: # no launcher pass except AttributeError, e: self.notify.warning("%s" % (str(e),)) pass return False tempFilename = Filename.temporary('', 'p3d_', '.xml') if rf: f = open(tempFilename.toOsSpecific(), 'wb') f.write(rf.getData()) f.close() if hashVal: hashVal.hashString(rf.getData()) if not self.readContentsFile(tempFilename, freshDownload = True): self.notify.warning("Failure reading %s" % (url)) tempFilename.unlink() return False tempFilename.unlink() return True
launcher.setPandaErrorCode(4) else: # There are other kinds of failures, but these will # generally have been caught already by the first test; so # if we get here there may be some bigger problem. Just # give the generic "big problem" message. launcher.setPandaErrorCode(6) except NameError, e: # no launcher pass except AttributeError, e: self.notify.warning("%s" % (str(e), )) pass return False tempFilename = Filename.temporary('', 'p3d_', '.xml') if rf: f = open(tempFilename.toOsSpecific(), 'wb') f.write(rf.getData()) f.close() if hashVal: hashVal.hashString(rf.getData()) if not self.readContentsFile(tempFilename, freshDownload=True): self.notify.warning("Failure reading %s" % (url)) tempFilename.unlink() return False tempFilename.unlink() return True