def downloadNextFile(self, task):
while self.nextNewsFile < len(self.newsFiles) and "aaver" in self.newsFiles[self.nextNewsFile]:
self.nextNewsFile += 1
if self.nextNewsFile >= len(self.newsFiles):
self.notify.info("Done downloading news.")
self.percentDownloaded = 1
del self.newsFiles
del self.nextNewsFile
del self.newsUrl
del self.newsDir
del self.ch
del self.url
if hasattr(self, "filename"):
del self.filename
self.redownloadingNews = False
if self.active:
self.parseNewsContent()
return task.done
self.percentDownloaded = float(self.nextNewsFile) / float(len(self.newsFiles))
self.filename = self.newsFiles[self.nextNewsFile]
self.nextNewsFile += 1
self.url = self.newsUrl + self.filename
localFilename = Filename(self.newsDir, self.filename)
self.notify.info("testing for %s" % localFilename.getFullpath())
doc = DocumentSpec(self.url)
if self.filename in self.newsCache:
size, date = self.newsCache[self.filename]
if date and localFilename.exists() and (size == 0 or localFilename.getFileSize() == size):
doc.setDate(date)
doc.setRequestMode(doc.RMNewer)
self.ch.beginGetDocument(doc)
self.ch.downloadToFile(localFilename)
taskMgr.remove(self.RedownloadTaskName)
taskMgr.add(self.downloadCurrentFileTask, self.RedownloadTaskName)
def run(self):
# Here is where the new Panda based HTTP code starts
# But before we hit the URL, let make sure we need to.
# Lets check to see if the VRS Collector is OFFLINE and
# host type 0 was selected. If that is the case, we can just
# return here; nothing needs to be done.
if self.CurrentHost == 0 and patcherVer() == ['OFFLINE']:
return
# Hit the URL
# The next line uses the Panda HTTP lib (blocking)
# doc = self.httpSession.getDocument(URLSpec(self.URLtoSend))
# Next line uses non-blocking
self.nonBlock = self.httpSession.makeChannel(False)
# nonBlock.setHttpTimeout(1)
# nonBlock.setConnectTimeout(.5)
# nonBlock.setBlockingConnect(False)
# doc = nonBlock.getDocument(DocumentSpec(self.URLtoSend))
self.nonBlock.beginGetDocument(DocumentSpec(self.URLtoSend))
instanceMarker = str(random.randint(1, 1000))
instanceMarker = 'FunnelLoggingRequest-%s' % instanceMarker
self.startCheckingAsyncRequest(instanceMarker)
def run(self):
if self.CurrentHost == 0 and patcherVer() == ['OFFLINE']:
return
self.nonBlock = self.httpSession.makeChannel(False)
self.nonBlock.beginGetDocument(DocumentSpec(self.URLtoSend))
instanceMarker = str(random.randint(1, 1000))
instanceMarker = 'FunnelLoggingRequest-%s' % instanceMarker
self.startCheckingAsyncRequest(instanceMarker)
def downloadNextFile(self, task):
while self.nextNewsFile < len(self.newsFiles) and 'aaver' in self.newsFiles[self.nextNewsFile]:
self.nextNewsFile += 1
if self.nextNewsFile >= len(self.newsFiles):
self.notify.info('Done downloading news.')
self.percentDownloaded = 1
del self.newsFiles
del self.nextNewsFile
del self.newsUrl
del self.newsDir
del self.ch
del self.url
if hasattr(self, 'filename'):
del self.filename
self.redownloadingNews = False
if self.active:
self.parseNewsContent()
return task.done
self.percentDownloaded = float(self.nextNewsFile) / float(len(self.newsFiles))
self.filename = self.newsFiles[self.nextNewsFile]
self.nextNewsFile += 1
self.url = self.newsUrl + self.filename
localFilename = Filename(self.newsDir, self.filename)
self.notify.info('testing for %s' % localFilename.getFullpath())
doc = DocumentSpec(self.url)
if self.filename in self.newsCache:
size, date = self.newsCache[self.filename]
if date and localFilename.exists() and (size == 0 or localFilename.getFileSize() == size):
doc.setDate(date)
doc.setRequestMode(doc.RMNewer)
self.ch.beginGetDocument(doc)
self.ch.downloadToFile(localFilename)
taskMgr.remove(self.RedownloadTaskName)
taskMgr.add(self.downloadCurrentFileTask, self.RedownloadTaskName)
def __setAdArt(self):
imageFrame = self.model.find('**/frame')
randomImageNumber = random.randint(0, len(screenShots) - 1)
imageFileName = screenShots[randomImageNumber]
self.adImage = loader.loadModel(imageFileName)
self.adImage.reparentTo(imageFrame)
self.adImage.setScale(2.15 * 5, 1, 1.2 * 5)
self.adImage.setPos(0, 0, 2.3)
self.adImage.setBin('fixed', 1)
if randomImageNumber == 0:
urlToGet = 'http://log.go.com/log?srvc=dis&guid=951C36F8-3ACD-4EB2-9F02-8E8A0A217AF5&drop=0&addata=3232:64675:408091:64675&a=0'
self.httpSession = HTTPClient()
self.nonBlockHTTP = self.httpSession.makeChannel(False)
self.nonBlockHTTP.beginGetDocument(DocumentSpec(urlToGet))
instanceMarker = 'FunnelLoggingRequest-%s' % str(random.randint(1, 1000))
self.startCheckingAsyncRequest(instanceMarker)
def downloadNextFile(self, task):
""" Starts the next news file downloading from the HTTP
server. """
if self.nextNewsFile >= len(self.newsFiles):
# Hey, we're done!
self.notify.info("Done downloading news.")
self.percentDownloaded = 1
del self.newsFiles
del self.nextNewsFile
del self.newsUrl
del self.newsDir
del self.ch
del self.url
if hasattr(self, 'filename'):
del self.filename
self.redownloadingNews = False
if self.active:
# If we're looking at the page now, go ahead and load it.
self.parseNewsContent()
return task.done
self.percentDownloaded = float(self.nextNewsFile) / float(
len(self.newsFiles))
# Get the next file on the list.
self.filename = self.newsFiles[self.nextNewsFile]
self.nextNewsFile += 1
self.url = self.newsUrl + self.filename
localFilename = Filename(self.newsDir, self.filename)
doc = DocumentSpec(self.url)
if self.filename in self.newsCache:
# We have already downloaded this file. Ask the
# server to give us another copy only if the server's
# copy is newer.
size, date = self.newsCache[self.filename]
if date and localFilename.exists() and (
size == 0 or localFilename.getFileSize() == size):
doc.setDate(date)
doc.setRequestMode(doc.RMNewer)
self.ch.beginGetDocument(doc)
self.ch.downloadToFile(localFilename)
taskMgr.remove(self.RedownloadTaskName)
taskMgr.add(self.downloadCurrentFileTask, self.RedownloadTaskName)
def downloadContentsFile(self, http, redownload = False,
hashVal = None):
""" Downloads the contents.xml file for this particular host,
synchronously, and then reads it. Returns true on success,
false on failure. If hashVal is not None, it should be a
HashVal object, which will be filled with the hash from the
new contents.xml file."""
if self.hasCurrentContentsFile():
# We've already got one.
return True
if self.appRunner and self.appRunner.verifyContents == self.appRunner.P3DVCNever:
# Not allowed to.
return False
rf = None
if http:
if not redownload and self.appRunner and self.appRunner.superMirrorUrl:
# We start with the "super mirror", if it's defined.
url = self.appRunner.superMirrorUrl + 'contents.xml'
request = DocumentSpec(url)
self.notify.info("Downloading contents file %s" % (request))
rf = Ramfile()
channel = http.makeChannel(False)
channel.getDocument(request)
if not channel.downloadToRam(rf):
self.notify.warning("Unable to download %s" % (url))
rf = None
if not rf:
# Then go to the main host, if our super mirror let us
# down.
url = self.hostUrlPrefix + 'contents.xml'
# Append a uniquifying query string to the URL to force the
# download to go all the way through any caches. We use the
# time in seconds; that's unique enough.
url += '?' + str(int(time.time()))
# We might as well explicitly request the cache to be disabled
# too, since we have an interface for that via HTTPChannel.
request = DocumentSpec(url)
request.setCacheControl(DocumentSpec.CCNoCache)
self.notify.info("Downloading contents file %s" % (request))
statusCode = None
statusString = ''
for attempt in range(ConfigVariableInt('contents-xml-dl-attempts', 3)):
if attempt > 0:
self.notify.info("Retrying (%s)..."%(attempt,))
rf = Ramfile()
channel = http.makeChannel(False)
channel.getDocument(request)
if channel.downloadToRam(rf):
self.notify.warning("Successfully downloaded %s" % (url,))
break
else:
rf = None
statusCode = channel.getStatusCode()
statusString = channel.getStatusString()
self.notify.warning("Could not contact download server at %s" % (url,))
self.notify.warning("Status code = %s %s" % (statusCode, statusString))
if not rf:
self.notify.warning("Unable to download %s" % (url,))
try:
# Something screwed up.
if statusCode == HTTPChannel.SCDownloadOpenError or \
statusCode == HTTPChannel.SCDownloadWriteError:
launcher.setPandaErrorCode(2)
elif statusCode == 404:
# 404 not found
launcher.setPandaErrorCode(5)
elif statusCode < 100:
# statusCode < 100 implies the connection attempt itself
# failed. This is usually due to firewall software
# interfering. Apparently some firewall software might
# allow the first connection and disallow subsequent
# connections; how strange.
launcher.setPandaErrorCode(4)
else:
# There are other kinds of failures, but these will
# generally have been caught already by the first test; so
# if we get here there may be some bigger problem. Just
# give the generic "big problem" message.
launcher.setPandaErrorCode(6)
except NameError,e:
# no launcher
pass
except AttributeError, e:
self.notify.warning("%s" % (str(e),))
pass
return False
def __downloadFile(self,
step,
fileSpec,
urlbase=None,
filename=None,
allowPartial=False):
""" Downloads the indicated file from the host into
packageDir. Yields one of stepComplete, stepFailed,
restartDownload, or stepContinue. """
if self.host.appRunner and self.host.appRunner.verifyContents == self.host.appRunner.P3DVCNever:
# We're not allowed to download anything.
yield self.stepFailed
return
self.updated = True
if not urlbase:
urlbase = self.descFileDirname + '/' + fileSpec.filename
# Build up a list of URL's to try downloading from. Unlike
# the C++ implementation in P3DPackage.cxx, here we build the
# URL's in forward order.
tryUrls = []
if self.host.appRunner and self.host.appRunner.superMirrorUrl:
# We start with the "super mirror", if it's defined.
url = self.host.appRunner.superMirrorUrl + urlbase
tryUrls.append((url, False))
if self.host.mirrors:
# Choose two mirrors at random.
mirrors = self.host.mirrors[:]
for i in range(2):
mirror = random.choice(mirrors)
mirrors.remove(mirror)
url = mirror + urlbase
tryUrls.append((url, False))
if not mirrors:
break
# After trying two mirrors and failing (or if there are no
# mirrors), go get it from the original host.
url = self.host.downloadUrlPrefix + urlbase
tryUrls.append((url, False))
# And finally, if the original host also fails, try again with
# a cache-buster.
tryUrls.append((url, True))
for url, cacheBust in tryUrls:
request = DocumentSpec(url)
if cacheBust:
# On the last attempt to download a particular file,
# we bust through the cache: append a query string to
# do this.
url += '?' + str(int(time.time()))
request = DocumentSpec(url)
request.setCacheControl(DocumentSpec.CCNoCache)
self.notify.info("%s downloading %s" % (self.packageName, url))
if not filename:
filename = fileSpec.filename
targetPathname = Filename(self.getPackageDir(), filename)
targetPathname.setBinary()
channel = self.http.makeChannel(False)
# If there's a previous partial download, attempt to resume it.
bytesStarted = 0
if allowPartial and not cacheBust and targetPathname.exists():
bytesStarted = targetPathname.getFileSize()
if bytesStarted < 1024 * 1024:
# Not enough bytes downloaded to be worth the risk of
# a partial download.
bytesStarted = 0
elif bytesStarted >= fileSpec.size:
# Couldn't possibly be our file.
bytesStarted = 0
if bytesStarted:
self.notify.info(
"Resuming %s after %s bytes already downloaded" %
(url, bytesStarted))
# Make sure the file is writable.
os.chmod(targetPathname.toOsSpecific(), 0644)
channel.beginGetSubdocument(request, bytesStarted, 0)
else:
# No partial download possible; get the whole file.
targetPathname.makeDir()
targetPathname.unlink()
channel.beginGetDocument(request)
channel.downloadToFile(targetPathname)
while channel.run():
if step:
step.bytesDone = channel.getBytesDownloaded(
) + channel.getFirstByteDelivered()
if step.bytesDone > step.bytesNeeded:
# Oops, too much data. Might as well abort;
# it's the wrong file.
self.notify.warning(
"Got more data than expected for download %s" %
(url))
break
self.__updateStepProgress(step)
if taskMgr.destroyed:
# If the task manager has been destroyed, we must
# be shutting down. Get out of here.
self.notify.warning("Task Manager destroyed, aborting %s" %
(url))
yield self.stepFailed
return
yield self.stepContinue
if step:
step.bytesDone = channel.getBytesDownloaded(
) + channel.getFirstByteDelivered()
self.__updateStepProgress(step)
if not channel.isValid():
self.notify.warning("Failed to download %s" % (url))
elif not fileSpec.fullVerify(self.getPackageDir(),
pathname=targetPathname,
notify=self.notify):
self.notify.warning(
"After downloading, %s incorrect" %
(Filename(fileSpec.filename).getBasename()))
# This attempt failed. Maybe the original contents.xml
# file is stale. Try re-downloading it now, just to be
# sure.
if self.host.redownloadContentsFile(self.http):
# Yes! Go back and start over from the beginning.
yield self.restartDownload
return
else:
# Success!
yield self.stepComplete
return
# Maybe the mirror is bad. Go back and try the next
# mirror.
# All attempts failed. Maybe the original contents.xml file
# is stale. Try re-downloading it now, just to be sure.
if self.host.redownloadContentsFile(self.http):
# Yes! Go back and start over from the beginning.
yield self.restartDownload
return
# All mirrors failed; the server (or the internet connection)
# must be just fubar.
yield self.stepFailed
return
def __downloadFile(self, step, fileSpec, urlbase = None, filename = None,
allowPartial = False):
""" Downloads the indicated file from the host into
packageDir. Yields one of stepComplete, stepFailed,
restartDownload, or stepContinue. """
if self.host.appRunner and self.host.appRunner.verifyContents == self.host.appRunner.P3DVCNever:
# We're not allowed to download anything.
yield self.stepFailed; return
self.updated = True
if not urlbase:
urlbase = self.descFileDirname + '/' + fileSpec.filename
# Build up a list of URL's to try downloading from. Unlike
# the C++ implementation in P3DPackage.cxx, here we build the
# URL's in forward order.
tryUrls = []
if self.host.appRunner and self.host.appRunner.superMirrorUrl:
# We start with the "super mirror", if it's defined.
url = self.host.appRunner.superMirrorUrl + urlbase
tryUrls.append((url, False))
if self.host.mirrors:
# Choose two mirrors at random.
mirrors = self.host.mirrors[:]
for i in range(2):
mirror = random.choice(mirrors)
mirrors.remove(mirror)
url = mirror + urlbase
tryUrls.append((url, False))
if not mirrors:
break
# After trying two mirrors and failing (or if there are no
# mirrors), go get it from the original host.
url = self.host.downloadUrlPrefix + urlbase
tryUrls.append((url, False))
# And finally, if the original host also fails, try again with
# a cache-buster.
tryUrls.append((url, True))
for url, cacheBust in tryUrls:
request = DocumentSpec(url)
if cacheBust:
# On the last attempt to download a particular file,
# we bust through the cache: append a query string to
# do this.
url += '?' + str(int(time.time()))
request = DocumentSpec(url)
request.setCacheControl(DocumentSpec.CCNoCache)
self.notify.info("%s downloading %s" % (self.packageName, url))
if not filename:
filename = fileSpec.filename
targetPathname = Filename(self.getPackageDir(), filename)
targetPathname.setBinary()
channel = self.http.makeChannel(False)
# If there's a previous partial download, attempt to resume it.
bytesStarted = 0
if allowPartial and not cacheBust and targetPathname.exists():
bytesStarted = targetPathname.getFileSize()
if bytesStarted < 1024*1024:
# Not enough bytes downloaded to be worth the risk of
# a partial download.
bytesStarted = 0
elif bytesStarted >= fileSpec.size:
# Couldn't possibly be our file.
bytesStarted = 0
if bytesStarted:
self.notify.info("Resuming %s after %s bytes already downloaded" % (url, bytesStarted))
# Make sure the file is writable.
os.chmod(targetPathname.toOsSpecific(), 0644)
channel.beginGetSubdocument(request, bytesStarted, 0)
else:
# No partial download possible; get the whole file.
targetPathname.makeDir()
targetPathname.unlink()
channel.beginGetDocument(request)
channel.downloadToFile(targetPathname)
while channel.run():
if step:
step.bytesDone = channel.getBytesDownloaded() + channel.getFirstByteDelivered()
if step.bytesDone > step.bytesNeeded:
# Oops, too much data. Might as well abort;
# it's the wrong file.
self.notify.warning("Got more data than expected for download %s" % (url))
break
self.__updateStepProgress(step)
if taskMgr.destroyed:
# If the task manager has been destroyed, we must
# be shutting down. Get out of here.
self.notify.warning("Task Manager destroyed, aborting %s" % (url))
yield self.stepFailed; return
yield self.stepContinue
if step:
step.bytesDone = channel.getBytesDownloaded() + channel.getFirstByteDelivered()
self.__updateStepProgress(step)
if not channel.isValid():
self.notify.warning("Failed to download %s" % (url))
elif not fileSpec.fullVerify(self.getPackageDir(), pathname = targetPathname, notify = self.notify):
self.notify.warning("After downloading, %s incorrect" % (Filename(fileSpec.filename).getBasename()))
# This attempt failed. Maybe the original contents.xml
# file is stale. Try re-downloading it now, just to be
# sure.
if self.host.redownloadContentsFile(self.http):
# Yes! Go back and start over from the beginning.
yield self.restartDownload; return
else:
# Success!
yield self.stepComplete; return
# Maybe the mirror is bad. Go back and try the next
# mirror.
# All attempts failed. Maybe the original contents.xml file
# is stale. Try re-downloading it now, just to be sure.
if self.host.redownloadContentsFile(self.http):
# Yes! Go back and start over from the beginning.
yield self.restartDownload; return
# All mirrors failed; the server (or the internet connection)
# must be just fubar.
yield self.stepFailed; return
