def downloadContentsFile(self, http, redownload = False, hashVal = None): """ Downloads the contents.xml file for this particular host, synchronously, and then reads it. Returns true on success, false on failure. If hashVal is not None, it should be a HashVal object, which will be filled with the hash from the new contents.xml file.""" if self.hasCurrentContentsFile(): # We've already got one. return True if self.appRunner and self.appRunner.verifyContents == self.appRunner.P3DVCNever: # Not allowed to. return False rf = None if http: if not redownload and self.appRunner and self.appRunner.superMirrorUrl: # We start with the "super mirror", if it's defined. url = self.appRunner.superMirrorUrl + 'contents.xml' request = DocumentSpec(url) self.notify.info("Downloading contents file %s" % (request)) rf = Ramfile() channel = http.makeChannel(False) channel.getDocument(request) if not channel.downloadToRam(rf): self.notify.warning("Unable to download %s" % (url)) rf = None if not rf: # Then go to the main host, if our super mirror let us # down. url = self.hostUrlPrefix + 'contents.xml' # Append a uniquifying query string to the URL to force the # download to go all the way through any caches. We use the # time in seconds; that's unique enough. url += '?' + str(int(time.time())) # We might as well explicitly request the cache to be disabled # too, since we have an interface for that via HTTPChannel. request = DocumentSpec(url) request.setCacheControl(DocumentSpec.CCNoCache) self.notify.info("Downloading contents file %s" % (request)) statusCode = None statusString = '' for attempt in range(int(ConfigVariableInt('contents-xml-dl-attempts', 3))): if attempt > 0: self.notify.info("Retrying (%s)..."%(attempt,)) rf = Ramfile() channel = http.makeChannel(False) channel.getDocument(request) if channel.downloadToRam(rf): self.notify.info("Successfully downloaded %s" % (url,)) break else: rf = None statusCode = channel.getStatusCode() statusString = channel.getStatusString() self.notify.warning("Could not contact download server at %s" % (url,)) self.notify.warning("Status code = %s %s" % (statusCode, statusString)) if not rf: self.notify.warning("Unable to download %s" % (url,)) try: # Something screwed up. if statusCode == HTTPChannel.SCDownloadOpenError or \ statusCode == HTTPChannel.SCDownloadWriteError: launcher.setPandaErrorCode(2) elif statusCode == 404: # 404 not found launcher.setPandaErrorCode(5) elif statusCode < 100: # statusCode < 100 implies the connection attempt itself # failed. This is usually due to firewall software # interfering. Apparently some firewall software might # allow the first connection and disallow subsequent # connections; how strange. launcher.setPandaErrorCode(4) else: # There are other kinds of failures, but these will # generally have been caught already by the first test; so # if we get here there may be some bigger problem. Just # give the generic "big problem" message. launcher.setPandaErrorCode(6) except NameError,e: # no launcher pass except AttributeError, e: self.notify.warning("%s" % (str(e),)) pass return False
def __downloadFile(self, step, fileSpec, urlbase=None, filename=None, allowPartial=False): """ Downloads the indicated file from the host into packageDir. Yields one of stepComplete, stepFailed, restartDownload, or stepContinue. """ if self.host.appRunner and self.host.appRunner.verifyContents == self.host.appRunner.P3DVCNever: # We're not allowed to download anything. yield self.stepFailed return self.updated = True if not urlbase: urlbase = self.descFileDirname + '/' + fileSpec.filename # Build up a list of URL's to try downloading from. Unlike # the C++ implementation in P3DPackage.cxx, here we build the # URL's in forward order. tryUrls = [] if self.host.appRunner and self.host.appRunner.superMirrorUrl: # We start with the "super mirror", if it's defined. url = self.host.appRunner.superMirrorUrl + urlbase tryUrls.append((url, False)) if self.host.mirrors: # Choose two mirrors at random. mirrors = self.host.mirrors[:] for i in range(2): mirror = random.choice(mirrors) mirrors.remove(mirror) url = mirror + urlbase tryUrls.append((url, False)) if not mirrors: break # After trying two mirrors and failing (or if there are no # mirrors), go get it from the original host. url = self.host.downloadUrlPrefix + urlbase tryUrls.append((url, False)) # And finally, if the original host also fails, try again with # a cache-buster. tryUrls.append((url, True)) for url, cacheBust in tryUrls: request = DocumentSpec(url) if cacheBust: # On the last attempt to download a particular file, # we bust through the cache: append a query string to # do this. url += '?' + str(int(time.time())) request = DocumentSpec(url) request.setCacheControl(DocumentSpec.CCNoCache) self.notify.info("%s downloading %s" % (self.packageName, url)) if not filename: filename = fileSpec.filename targetPathname = Filename(self.getPackageDir(), filename) targetPathname.setBinary() channel = self.http.makeChannel(False) # If there's a previous partial download, attempt to resume it. bytesStarted = 0 if allowPartial and not cacheBust and targetPathname.exists(): bytesStarted = targetPathname.getFileSize() if bytesStarted < 1024 * 1024: # Not enough bytes downloaded to be worth the risk of # a partial download. bytesStarted = 0 elif bytesStarted >= fileSpec.size: # Couldn't possibly be our file. bytesStarted = 0 if bytesStarted: self.notify.info( "Resuming %s after %s bytes already downloaded" % (url, bytesStarted)) # Make sure the file is writable. os.chmod(targetPathname.toOsSpecific(), 0o644) channel.beginGetSubdocument(request, bytesStarted, 0) else: # No partial download possible; get the whole file. targetPathname.makeDir() targetPathname.unlink() channel.beginGetDocument(request) channel.downloadToFile(targetPathname) while channel.run(): if step: step.bytesDone = channel.getBytesDownloaded( ) + channel.getFirstByteDelivered() if step.bytesDone > step.bytesNeeded: # Oops, too much data. Might as well abort; # it's the wrong file. self.notify.warning( "Got more data than expected for download %s" % (url)) break self.__updateStepProgress(step) if taskMgr.destroyed: # If the task manager has been destroyed, we must # be shutting down. Get out of here. self.notify.warning("Task Manager destroyed, aborting %s" % (url)) yield self.stepFailed return yield self.stepContinue if step: step.bytesDone = channel.getBytesDownloaded( ) + channel.getFirstByteDelivered() self.__updateStepProgress(step) if not channel.isValid(): self.notify.warning("Failed to download %s" % (url)) elif not fileSpec.fullVerify(self.getPackageDir(), pathname=targetPathname, notify=self.notify): self.notify.warning( "After downloading, %s incorrect" % (Filename(fileSpec.filename).getBasename())) # This attempt failed. Maybe the original contents.xml # file is stale. Try re-downloading it now, just to be # sure. if self.host.redownloadContentsFile(self.http): # Yes! Go back and start over from the beginning. yield self.restartDownload return else: # Success! yield self.stepComplete return # Maybe the mirror is bad. Go back and try the next # mirror. # All attempts failed. Maybe the original contents.xml file # is stale. Try re-downloading it now, just to be sure. if self.host.redownloadContentsFile(self.http): # Yes! Go back and start over from the beginning. yield self.restartDownload return # All mirrors failed; the server (or the internet connection) # must be just fubar. yield self.stepFailed return
def downloadContentsFile(self, http, redownload=False, hashVal=None): """ Downloads the contents.xml file for this particular host, synchronously, and then reads it. Returns true on success, false on failure. If hashVal is not None, it should be a HashVal object, which will be filled with the hash from the new contents.xml file.""" if self.hasCurrentContentsFile(): # We've already got one. return True if self.appRunner and self.appRunner.verifyContents == self.appRunner.P3DVCNever: # Not allowed to. return False rf = None if http: if not redownload and self.appRunner and self.appRunner.superMirrorUrl: # We start with the "super mirror", if it's defined. url = self.appRunner.superMirrorUrl + 'contents.xml' request = DocumentSpec(url) self.notify.info("Downloading contents file %s" % (request)) rf = Ramfile() channel = http.makeChannel(False) channel.getDocument(request) if not channel.downloadToRam(rf): self.notify.warning("Unable to download %s" % (url)) rf = None if not rf: # Then go to the main host, if our super mirror let us # down. url = self.hostUrlPrefix + 'contents.xml' # Append a uniquifying query string to the URL to force the # download to go all the way through any caches. We use the # time in seconds; that's unique enough. url += '?' + str(int(time.time())) # We might as well explicitly request the cache to be disabled # too, since we have an interface for that via HTTPChannel. request = DocumentSpec(url) request.setCacheControl(DocumentSpec.CCNoCache) self.notify.info("Downloading contents file %s" % (request)) statusCode = None statusString = '' for attempt in range( int(ConfigVariableInt('contents-xml-dl-attempts', 3))): if attempt > 0: self.notify.info("Retrying (%s)..." % (attempt, )) rf = Ramfile() channel = http.makeChannel(False) channel.getDocument(request) if channel.downloadToRam(rf): self.notify.info("Successfully downloaded %s" % (url, )) break else: rf = None statusCode = channel.getStatusCode() statusString = channel.getStatusString() self.notify.warning( "Could not contact download server at %s" % (url, )) self.notify.warning("Status code = %s %s" % (statusCode, statusString)) if not rf: self.notify.warning("Unable to download %s" % (url, )) try: # Something screwed up. if statusCode == HTTPChannel.SCDownloadOpenError or \ statusCode == HTTPChannel.SCDownloadWriteError: launcher.setPandaErrorCode(2) elif statusCode == 404: # 404 not found launcher.setPandaErrorCode(5) elif statusCode < 100: # statusCode < 100 implies the connection attempt itself # failed. This is usually due to firewall software # interfering. Apparently some firewall software might # allow the first connection and disallow subsequent # connections; how strange. launcher.setPandaErrorCode(4) else: # There are other kinds of failures, but these will # generally have been caught already by the first test; so # if we get here there may be some bigger problem. Just # give the generic "big problem" message. launcher.setPandaErrorCode(6) except NameError, e: # no launcher pass except AttributeError, e: self.notify.warning("%s" % (str(e), )) pass return False
def __downloadFile(self, step, fileSpec, urlbase = None, filename = None, allowPartial = False): """ Downloads the indicated file from the host into packageDir. Yields one of stepComplete, stepFailed, restartDownload, or stepContinue. """ if self.host.appRunner and self.host.appRunner.verifyContents == self.host.appRunner.P3DVCNever: # We're not allowed to download anything. yield self.stepFailed; return self.updated = True if not urlbase: urlbase = self.descFileDirname + '/' + fileSpec.filename # Build up a list of URL's to try downloading from. Unlike # the C++ implementation in P3DPackage.cxx, here we build the # URL's in forward order. tryUrls = [] if self.host.appRunner and self.host.appRunner.superMirrorUrl: # We start with the "super mirror", if it's defined. url = self.host.appRunner.superMirrorUrl + urlbase tryUrls.append((url, False)) if self.host.mirrors: # Choose two mirrors at random. mirrors = self.host.mirrors[:] for i in range(2): mirror = random.choice(mirrors) mirrors.remove(mirror) url = mirror + urlbase tryUrls.append((url, False)) if not mirrors: break # After trying two mirrors and failing (or if there are no # mirrors), go get it from the original host. url = self.host.downloadUrlPrefix + urlbase tryUrls.append((url, False)) # And finally, if the original host also fails, try again with # a cache-buster. tryUrls.append((url, True)) for url, cacheBust in tryUrls: request = DocumentSpec(url) if cacheBust: # On the last attempt to download a particular file, # we bust through the cache: append a query string to # do this. url += '?' + str(int(time.time())) request = DocumentSpec(url) request.setCacheControl(DocumentSpec.CCNoCache) self.notify.info("%s downloading %s" % (self.packageName, url)) if not filename: filename = fileSpec.filename targetPathname = Filename(self.getPackageDir(), filename) targetPathname.setBinary() channel = self.http.makeChannel(False) # If there's a previous partial download, attempt to resume it. bytesStarted = 0 if allowPartial and not cacheBust and targetPathname.exists(): bytesStarted = targetPathname.getFileSize() if bytesStarted < 1024*1024: # Not enough bytes downloaded to be worth the risk of # a partial download. bytesStarted = 0 elif bytesStarted >= fileSpec.size: # Couldn't possibly be our file. bytesStarted = 0 if bytesStarted: self.notify.info("Resuming %s after %s bytes already downloaded" % (url, bytesStarted)) # Make sure the file is writable. os.chmod(targetPathname.toOsSpecific(), 0o644) channel.beginGetSubdocument(request, bytesStarted, 0) else: # No partial download possible; get the whole file. targetPathname.makeDir() targetPathname.unlink() channel.beginGetDocument(request) channel.downloadToFile(targetPathname) while channel.run(): if step: step.bytesDone = channel.getBytesDownloaded() + channel.getFirstByteDelivered() if step.bytesDone > step.bytesNeeded: # Oops, too much data. Might as well abort; # it's the wrong file. self.notify.warning("Got more data than expected for download %s" % (url)) break self.__updateStepProgress(step) if taskMgr.destroyed: # If the task manager has been destroyed, we must # be shutting down. Get out of here. self.notify.warning("Task Manager destroyed, aborting %s" % (url)) yield self.stepFailed; return yield self.stepContinue if step: step.bytesDone = channel.getBytesDownloaded() + channel.getFirstByteDelivered() self.__updateStepProgress(step) if not channel.isValid(): self.notify.warning("Failed to download %s" % (url)) elif not fileSpec.fullVerify(self.getPackageDir(), pathname = targetPathname, notify = self.notify): self.notify.warning("After downloading, %s incorrect" % (Filename(fileSpec.filename).getBasename())) # This attempt failed. Maybe the original contents.xml # file is stale. Try re-downloading it now, just to be # sure. if self.host.redownloadContentsFile(self.http): # Yes! Go back and start over from the beginning. yield self.restartDownload; return else: # Success! yield self.stepComplete; return # Maybe the mirror is bad. Go back and try the next # mirror. # All attempts failed. Maybe the original contents.xml file # is stale. Try re-downloading it now, just to be sure. if self.host.redownloadContentsFile(self.http): # Yes! Go back and start over from the beginning. yield self.restartDownload; return # All mirrors failed; the server (or the internet connection) # must be just fubar. yield self.stepFailed; return