def downloadIndexTask(self, task): if self.ch.run(): return task.cont if not self.ch.isValid(): self.notify.warning('Unable to download %s' % self.url) self.redownloadingNews = False return task.done self.newsFiles = [] filename = self.rf.readline() while filename: filename = filename.strip() if filename: self.newsFiles.append(filename) filename = self.rf.readline() del self.rf self.newsFiles.sort() self.newsIndexEntries = list(self.newsFiles) self.notify.info('Server lists %s news files' % len(self.newsFiles)) self.notify.debug('self.newsIndexEntries=%s' % self.newsIndexEntries) self.readNewsCache() for basename in os.listdir(self.newsDir.toOsSpecific()): if basename != self.CacheIndexFilename and basename not in self.newsCache: junk = Filename(self.newsDir, basename) self.notify.info('Removing %s' % junk) junk.unlink() self.nextNewsFile = 0 return self.downloadNextFile(task)
def __uncompressArchive(self, step): """ Turns the compressed archive into the uncompressed archive. Yields one of stepComplete, stepFailed, restartDownload, or stepContinue. """ if self.host.appRunner and self.host.appRunner.verifyContents == self.host.appRunner.P3DVCNever: # We're not allowed to! yield self.stepFailed return self.updated = True sourcePathname = Filename(self.getPackageDir(), self.compressedArchive.filename) targetPathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename) targetPathname.unlink() self.notify.info("Uncompressing %s to %s" % (sourcePathname, targetPathname)) decompressor = Decompressor() decompressor.initiate(sourcePathname, targetPathname) totalBytes = self.uncompressedArchive.size result = decompressor.run() while result == EUOk: step.bytesDone = int(totalBytes * decompressor.getProgress()) self.__updateStepProgress(step) result = decompressor.run() if taskMgr.destroyed: # If the task manager has been destroyed, we must # be shutting down. Get out of here. self.notify.warning( "Task Manager destroyed, aborting decompresss %s" % (sourcePathname)) yield self.stepFailed return yield self.stepContinue if result != EUSuccess: yield self.stepFailed return step.bytesDone = totalBytes self.__updateStepProgress(step) if not self.uncompressedArchive.quickVerify(self.getPackageDir(), notify=self.notify): self.notify.warning("after uncompressing, %s still incorrect" % (self.uncompressedArchive.filename)) yield self.stepFailed return # Now that we've verified the archive, make it read-only. os.chmod(targetPathname.toOsSpecific(), 0o444) # Now we can safely remove the compressed archive. sourcePathname.unlink() yield self.stepComplete return
def __checkArchiveStatus(self): """ Returns true if the archive and all extractable files are already correct on disk, false otherwise. """ if self.host.appRunner and self.host.appRunner.verifyContents == self.host.appRunner.P3DVCNever: # Assume that everything is just fine. return True # Get a list of all of the files in the directory, so we can # remove files that don't belong. contents = self.__scanDirectoryRecursively(self.getPackageDir()) self.__removeFileFromList(contents, self.descFileBasename) self.__removeFileFromList(contents, self.compressedArchive.filename) self.__removeFileFromList(contents, self.UsageBasename) if not self.asMirror: self.__removeFileFromList(contents, self.uncompressedArchive.filename) for file in self.extracts: self.__removeFileFromList(contents, file.filename) # Now, any files that are still in the contents list don't # belong. It's important to remove these files before we # start verifying the files that we expect to find here, in # case there is a problem with ambiguous filenames or # something (e.g. case insensitivity). for filename in contents: self.notify.info("Removing %s" % (filename)) pathname = Filename(self.getPackageDir(), filename) pathname.unlink() self.updated = True if self.asMirror: return self.compressedArchive.quickVerify(self.getPackageDir(), notify = self.notify) allExtractsOk = True if not self.uncompressedArchive.quickVerify(self.getPackageDir(), notify = self.notify): self.notify.debug("File is incorrect: %s" % (self.uncompressedArchive.filename)) allExtractsOk = False if allExtractsOk: # OK, the uncompressed archive is good; that means there # shouldn't be a compressed archive file here. pathname = Filename(self.getPackageDir(), self.compressedArchive.filename) pathname.unlink() for file in self.extracts: if not file.quickVerify(self.getPackageDir(), notify = self.notify): self.notify.debug("File is incorrect: %s" % (file.filename)) allExtractsOk = False break if allExtractsOk: self.notify.debug("All %s extracts of %s seem good." % ( len(self.extracts), self.packageName)) return allExtractsOk
def buildDmg(startDir): fstartDir = Filename.fromOsSpecific(startDir) rootFilename = Filename(fstartDir, 'bundle') output = Filename(fstartDir, 'nppanda3d.dmg') output.unlink() cmd = 'hdiutil create -fs HFS+ -srcfolder "%(dir)s" -volname "%(volname)s" "%(output)s"' % { 'dir': rootFilename.toOsSpecific(), 'volname': 'nppanda3d', 'output': output.toOsSpecific(), } os.system(cmd)
def buildDmg(startDir): fstartDir = Filename.fromOsSpecific(startDir) rootFilename = Filename(fstartDir, 'bundle') output = Filename(fstartDir, 'nppanda3d.dmg') output.unlink() cmd = 'hdiutil create -fs HFS+ -srcfolder "%(dir)s" -volname "%(volname)s" "%(output)s"' % { 'dir' : rootFilename.toOsSpecific(), 'volname' : 'nppanda3d', 'output' : output.toOsSpecific(), } os.system(cmd)
def __applyPatch(self, step, patchfile): """ Applies the indicated patching in-place to the current uncompressed archive. The patchfile is removed after the operation. Yields one of stepComplete, stepFailed, restartDownload, or stepContinue. """ self.updated = True origPathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename) patchPathname = Filename(self.getPackageDir(), patchfile.file.filename) result = Filename.temporary('', 'patch_') self.notify.info("Patching %s with %s" % (origPathname, patchPathname)) p = core.Patchfile() # The C++ class ret = p.initiate(patchPathname, origPathname, result) if ret == EUSuccess: ret = p.run() while ret == EUOk: step.bytesDone = step.bytesNeeded * p.getProgress() self.__updateStepProgress(step) if taskMgr.destroyed: # If the task manager has been destroyed, we must # be shutting down. Get out of here. self.notify.warning( "Task Manager destroyed, aborting patch %s" % (origPathname)) yield self.stepFailed return yield self.stepContinue ret = p.run() del p patchPathname.unlink() if ret < 0: self.notify.warning("Patching of %s failed." % (origPathname)) result.unlink() yield self.stepFailed return if not result.renameTo(origPathname): self.notify.warning("Couldn't rename %s to %s" % (result, origPathname)) yield self.stepFailed return yield self.stepComplete return
def __uncompressArchive(self, step): """ Turns the compressed archive into the uncompressed archive. Yields one of stepComplete, stepFailed, restartDownload, or stepContinue. """ if self.host.appRunner and self.host.appRunner.verifyContents == self.host.appRunner.P3DVCNever: # We're not allowed to! yield self.stepFailed; return self.updated = True sourcePathname = Filename(self.getPackageDir(), self.compressedArchive.filename) targetPathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename) targetPathname.unlink() self.notify.info("Uncompressing %s to %s" % (sourcePathname, targetPathname)) decompressor = Decompressor() decompressor.initiate(sourcePathname, targetPathname) totalBytes = self.uncompressedArchive.size result = decompressor.run() while result == EUOk: step.bytesDone = int(totalBytes * decompressor.getProgress()) self.__updateStepProgress(step) result = decompressor.run() if taskMgr.destroyed: # If the task manager has been destroyed, we must # be shutting down. Get out of here. self.notify.warning("Task Manager destroyed, aborting decompresss %s" % (sourcePathname)) yield self.stepFailed; return yield self.stepContinue if result != EUSuccess: yield self.stepFailed; return step.bytesDone = totalBytes self.__updateStepProgress(step) if not self.uncompressedArchive.quickVerify(self.getPackageDir(), notify= self.notify): self.notify.warning("after uncompressing, %s still incorrect" % ( self.uncompressedArchive.filename)) yield self.stepFailed; return # Now that we've verified the archive, make it read-only. os.chmod(targetPathname.toOsSpecific(), 0o444) # Now we can safely remove the compressed archive. sourcePathname.unlink() yield self.stepComplete; return
def __applyPatch(self, step, patchfile): """ Applies the indicated patching in-place to the current uncompressed archive. The patchfile is removed after the operation. Yields one of stepComplete, stepFailed, restartDownload, or stepContinue. """ self.updated = True origPathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename) patchPathname = Filename(self.getPackageDir(), patchfile.file.filename) result = Filename.temporary('', 'patch_') self.notify.info("Patching %s with %s" % (origPathname, patchPathname)) p = core.Patchfile() # The C++ class ret = p.initiate(patchPathname, origPathname, result) if ret == EUSuccess: ret = p.run() while ret == EUOk: step.bytesDone = step.bytesNeeded * p.getProgress() self.__updateStepProgress(step) if taskMgr.destroyed: # If the task manager has been destroyed, we must # be shutting down. Get out of here. self.notify.warning("Task Manager destroyed, aborting patch %s" % (origPathname)) yield self.stepFailed; return yield self.stepContinue ret = p.run() del p patchPathname.unlink() if ret < 0: self.notify.warning("Patching of %s failed." % (origPathname)) result.unlink() yield self.stepFailed; return if not result.renameTo(origPathname): self.notify.warning("Couldn't rename %s to %s" % (result, origPathname)) yield self.stepFailed; return yield self.stepComplete; return
def downloadCurrentFileTask(self, task): if self.ch.run(): return task.cont if self.ch.getStatusCode() == 304: self.notify.info('already cached: %s' % self.filename) return self.downloadNextFile(task) localFilename = Filename(self.newsDir, self.filename) if not self.ch.isValid(): self.notify.warning('Unable to download %s' % self.url) localFilename.unlink() if self.filename in self.newsCache: del self.newsCache[self.filename] self.saveNewsCache() return self.downloadNextFile(task) self.notify.info('downloaded %s' % self.filename) size = self.ch.getFileSize() doc = self.ch.getDocumentSpec() date = '' if doc.hasDate(): date = doc.getDate().getString() self.newsCache[self.filename] = (size, date) self.saveNewsCache() return self.downloadNextFile(task)
def __downloadFile(self, step, fileSpec, urlbase=None, filename=None, allowPartial=False): """ Downloads the indicated file from the host into packageDir. Yields one of stepComplete, stepFailed, restartDownload, or stepContinue. """ if self.host.appRunner and self.host.appRunner.verifyContents == self.host.appRunner.P3DVCNever: # We're not allowed to download anything. yield self.stepFailed return self.updated = True if not urlbase: urlbase = self.descFileDirname + '/' + fileSpec.filename # Build up a list of URL's to try downloading from. Unlike # the C++ implementation in P3DPackage.cxx, here we build the # URL's in forward order. tryUrls = [] if self.host.appRunner and self.host.appRunner.superMirrorUrl: # We start with the "super mirror", if it's defined. url = self.host.appRunner.superMirrorUrl + urlbase tryUrls.append((url, False)) if self.host.mirrors: # Choose two mirrors at random. mirrors = self.host.mirrors[:] for i in range(2): mirror = random.choice(mirrors) mirrors.remove(mirror) url = mirror + urlbase tryUrls.append((url, False)) if not mirrors: break # After trying two mirrors and failing (or if there are no # mirrors), go get it from the original host. url = self.host.downloadUrlPrefix + urlbase tryUrls.append((url, False)) # And finally, if the original host also fails, try again with # a cache-buster. tryUrls.append((url, True)) for url, cacheBust in tryUrls: request = DocumentSpec(url) if cacheBust: # On the last attempt to download a particular file, # we bust through the cache: append a query string to # do this. url += '?' + str(int(time.time())) request = DocumentSpec(url) request.setCacheControl(DocumentSpec.CCNoCache) self.notify.info("%s downloading %s" % (self.packageName, url)) if not filename: filename = fileSpec.filename targetPathname = Filename(self.getPackageDir(), filename) targetPathname.setBinary() channel = self.http.makeChannel(False) # If there's a previous partial download, attempt to resume it. bytesStarted = 0 if allowPartial and not cacheBust and targetPathname.exists(): bytesStarted = targetPathname.getFileSize() if bytesStarted < 1024 * 1024: # Not enough bytes downloaded to be worth the risk of # a partial download. bytesStarted = 0 elif bytesStarted >= fileSpec.size: # Couldn't possibly be our file. bytesStarted = 0 if bytesStarted: self.notify.info( "Resuming %s after %s bytes already downloaded" % (url, bytesStarted)) # Make sure the file is writable. os.chmod(targetPathname.toOsSpecific(), 0o644) channel.beginGetSubdocument(request, bytesStarted, 0) else: # No partial download possible; get the whole file. targetPathname.makeDir() targetPathname.unlink() channel.beginGetDocument(request) channel.downloadToFile(targetPathname) while channel.run(): if step: step.bytesDone = channel.getBytesDownloaded( ) + channel.getFirstByteDelivered() if step.bytesDone > step.bytesNeeded: # Oops, too much data. Might as well abort; # it's the wrong file. self.notify.warning( "Got more data than expected for download %s" % (url)) break self.__updateStepProgress(step) if taskMgr.destroyed: # If the task manager has been destroyed, we must # be shutting down. Get out of here. self.notify.warning("Task Manager destroyed, aborting %s" % (url)) yield self.stepFailed return yield self.stepContinue if step: step.bytesDone = channel.getBytesDownloaded( ) + channel.getFirstByteDelivered() self.__updateStepProgress(step) if not channel.isValid(): self.notify.warning("Failed to download %s" % (url)) elif not fileSpec.fullVerify(self.getPackageDir(), pathname=targetPathname, notify=self.notify): self.notify.warning( "After downloading, %s incorrect" % (Filename(fileSpec.filename).getBasename())) # This attempt failed. Maybe the original contents.xml # file is stale. Try re-downloading it now, just to be # sure. if self.host.redownloadContentsFile(self.http): # Yes! Go back and start over from the beginning. yield self.restartDownload return else: # Success! yield self.stepComplete return # Maybe the mirror is bad. Go back and try the next # mirror. # All attempts failed. Maybe the original contents.xml file # is stale. Try re-downloading it now, just to be sure. if self.host.redownloadContentsFile(self.http): # Yes! Go back and start over from the beginning. yield self.restartDownload return # All mirrors failed; the server (or the internet connection) # must be just fubar. yield self.stepFailed return
def __unpackArchive(self, step): """ Unpacks any files in the archive that want to be unpacked to disk. Yields one of stepComplete, stepFailed, restartDownload, or stepContinue. """ if not self.extracts: # Nothing to extract. self.hasPackage = True yield self.stepComplete return if self.host.appRunner and self.host.appRunner.verifyContents == self.host.appRunner.P3DVCNever: # We're not allowed to! yield self.stepFailed return self.updated = True mfPathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename) self.notify.info("Unpacking %s" % (mfPathname)) mf = Multifile() if not mf.openRead(mfPathname): self.notify.warning("Couldn't open %s" % (mfPathname)) yield self.stepFailed return allExtractsOk = True step.bytesDone = 0 for file in self.extracts: i = mf.findSubfile(file.filename) if i == -1: self.notify.warning("Not in Multifile: %s" % (file.filename)) allExtractsOk = False continue targetPathname = Filename(self.getPackageDir(), file.filename) targetPathname.setBinary() targetPathname.unlink() if not mf.extractSubfile(i, targetPathname): self.notify.warning("Couldn't extract: %s" % (file.filename)) allExtractsOk = False continue if not file.quickVerify(self.getPackageDir(), notify=self.notify): self.notify.warning("After extracting, still incorrect: %s" % (file.filename)) allExtractsOk = False continue # Make sure it's executable, and not writable. os.chmod(targetPathname.toOsSpecific(), 0o555) step.bytesDone += file.size self.__updateStepProgress(step) if taskMgr.destroyed: # If the task manager has been destroyed, we must # be shutting down. Get out of here. self.notify.warning( "Task Manager destroyed, aborting unpacking %s" % (mfPathname)) yield self.stepFailed return yield self.stepContinue if not allExtractsOk: yield self.stepFailed return self.hasPackage = True yield self.stepComplete return
def __unpackArchive(self, step): """ Unpacks any files in the archive that want to be unpacked to disk. Yields one of stepComplete, stepFailed, restartDownload, or stepContinue. """ if not self.extracts: # Nothing to extract. self.hasPackage = True yield self.stepComplete; return if self.host.appRunner and self.host.appRunner.verifyContents == self.host.appRunner.P3DVCNever: # We're not allowed to! yield self.stepFailed; return self.updated = True mfPathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename) self.notify.info("Unpacking %s" % (mfPathname)) mf = Multifile() if not mf.openRead(mfPathname): self.notify.warning("Couldn't open %s" % (mfPathname)) yield self.stepFailed; return allExtractsOk = True step.bytesDone = 0 for file in self.extracts: i = mf.findSubfile(file.filename) if i == -1: self.notify.warning("Not in Multifile: %s" % (file.filename)) allExtractsOk = False continue targetPathname = Filename(self.getPackageDir(), file.filename) targetPathname.setBinary() targetPathname.unlink() if not mf.extractSubfile(i, targetPathname): self.notify.warning("Couldn't extract: %s" % (file.filename)) allExtractsOk = False continue if not file.quickVerify(self.getPackageDir(), notify = self.notify): self.notify.warning("After extracting, still incorrect: %s" % (file.filename)) allExtractsOk = False continue # Make sure it's executable, and not writable. os.chmod(targetPathname.toOsSpecific(), 0o555) step.bytesDone += file.size self.__updateStepProgress(step) if taskMgr.destroyed: # If the task manager has been destroyed, we must # be shutting down. Get out of here. self.notify.warning("Task Manager destroyed, aborting unpacking %s" % (mfPathname)) yield self.stepFailed; return yield self.stepContinue if not allExtractsOk: yield self.stepFailed; return self.hasPackage = True yield self.stepComplete; return
def __downloadFile(self, step, fileSpec, urlbase = None, filename = None, allowPartial = False): """ Downloads the indicated file from the host into packageDir. Yields one of stepComplete, stepFailed, restartDownload, or stepContinue. """ if self.host.appRunner and self.host.appRunner.verifyContents == self.host.appRunner.P3DVCNever: # We're not allowed to download anything. yield self.stepFailed; return self.updated = True if not urlbase: urlbase = self.descFileDirname + '/' + fileSpec.filename # Build up a list of URL's to try downloading from. Unlike # the C++ implementation in P3DPackage.cxx, here we build the # URL's in forward order. tryUrls = [] if self.host.appRunner and self.host.appRunner.superMirrorUrl: # We start with the "super mirror", if it's defined. url = self.host.appRunner.superMirrorUrl + urlbase tryUrls.append((url, False)) if self.host.mirrors: # Choose two mirrors at random. mirrors = self.host.mirrors[:] for i in range(2): mirror = random.choice(mirrors) mirrors.remove(mirror) url = mirror + urlbase tryUrls.append((url, False)) if not mirrors: break # After trying two mirrors and failing (or if there are no # mirrors), go get it from the original host. url = self.host.downloadUrlPrefix + urlbase tryUrls.append((url, False)) # And finally, if the original host also fails, try again with # a cache-buster. tryUrls.append((url, True)) for url, cacheBust in tryUrls: request = DocumentSpec(url) if cacheBust: # On the last attempt to download a particular file, # we bust through the cache: append a query string to # do this. url += '?' + str(int(time.time())) request = DocumentSpec(url) request.setCacheControl(DocumentSpec.CCNoCache) self.notify.info("%s downloading %s" % (self.packageName, url)) if not filename: filename = fileSpec.filename targetPathname = Filename(self.getPackageDir(), filename) targetPathname.setBinary() channel = self.http.makeChannel(False) # If there's a previous partial download, attempt to resume it. bytesStarted = 0 if allowPartial and not cacheBust and targetPathname.exists(): bytesStarted = targetPathname.getFileSize() if bytesStarted < 1024*1024: # Not enough bytes downloaded to be worth the risk of # a partial download. bytesStarted = 0 elif bytesStarted >= fileSpec.size: # Couldn't possibly be our file. bytesStarted = 0 if bytesStarted: self.notify.info("Resuming %s after %s bytes already downloaded" % (url, bytesStarted)) # Make sure the file is writable. os.chmod(targetPathname.toOsSpecific(), 0o644) channel.beginGetSubdocument(request, bytesStarted, 0) else: # No partial download possible; get the whole file. targetPathname.makeDir() targetPathname.unlink() channel.beginGetDocument(request) channel.downloadToFile(targetPathname) while channel.run(): if step: step.bytesDone = channel.getBytesDownloaded() + channel.getFirstByteDelivered() if step.bytesDone > step.bytesNeeded: # Oops, too much data. Might as well abort; # it's the wrong file. self.notify.warning("Got more data than expected for download %s" % (url)) break self.__updateStepProgress(step) if taskMgr.destroyed: # If the task manager has been destroyed, we must # be shutting down. Get out of here. self.notify.warning("Task Manager destroyed, aborting %s" % (url)) yield self.stepFailed; return yield self.stepContinue if step: step.bytesDone = channel.getBytesDownloaded() + channel.getFirstByteDelivered() self.__updateStepProgress(step) if not channel.isValid(): self.notify.warning("Failed to download %s" % (url)) elif not fileSpec.fullVerify(self.getPackageDir(), pathname = targetPathname, notify = self.notify): self.notify.warning("After downloading, %s incorrect" % (Filename(fileSpec.filename).getBasename())) # This attempt failed. Maybe the original contents.xml # file is stale. Try re-downloading it now, just to be # sure. if self.host.redownloadContentsFile(self.http): # Yes! Go back and start over from the beginning. yield self.restartDownload; return else: # Success! yield self.stepComplete; return # Maybe the mirror is bad. Go back and try the next # mirror. # All attempts failed. Maybe the original contents.xml file # is stale. Try re-downloading it now, just to be sure. if self.host.redownloadContentsFile(self.http): # Yes! Go back and start over from the beginning. yield self.restartDownload; return # All mirrors failed; the server (or the internet connection) # must be just fubar. yield self.stepFailed; return