def download(self, step):
if self.active == MAX_DOWNLOADS_PER_SITE:
# Too busy to start a new download now. Queue this one and wait.
ticket = tasks.Blocker('queued download for ' + step.url)
self.queue.append(ticket)
yield ticket, step.dl._aborted
if step.dl._aborted.happened:
raise download.DownloadAborted()
# Start a new thread for the download
thread_blocker = _spawn_thread(step)
self.active += 1
# Wait for thread to complete download.
yield thread_blocker, step.dl._aborted
self.active -= 1
if self.active < MAX_DOWNLOADS_PER_SITE:
self.process_next() # Start next queued download, if any
if step.dl._aborted.happened:
# Don't wait for child to finish (might be stuck doing IO)
raise download.DownloadAborted()
tasks.check(thread_blocker)
if step.status == download.RESULT_REDIRECT:
assert step.redirect
return # DownloadScheduler will handle it
assert not step.redirect, step.redirect
step.dl._finish(step.status)
def download(self, step, timeout=None):
"""
Queue up this download. If it takes too long, trigger step.dl.timeout (if any), but
only count time spent actually downloading, not time spent queuing.
@type step: L{DownloadStep}"""
if self.active == MAX_DOWNLOADS_PER_SITE:
# Too busy to start a new download now. Queue this one and wait.
ticket = tasks.Blocker('queued download for ' + step.url)
self.queue.append(ticket)
yield ticket, step.dl._aborted
if step.dl._aborted.happened:
raise download.DownloadAborted()
in_progress = [True]
if timeout is not None:
def timeout_cb():
if in_progress:
step.dl.timeout.trigger()
tasks.get_loop().call_later(timeout, timeout_cb)
# Start a new thread for the download
thread_blocker = _spawn_thread(step)
self.active += 1
# Wait for thread to complete download.
yield thread_blocker, step.dl._aborted
del in_progress[0]
self.active -= 1
if self.active < MAX_DOWNLOADS_PER_SITE:
self.process_next() # Start next queued download, if any
if step.dl._aborted.happened:
# Don't wait for child to finish (might be stuck doing IO)
raise download.DownloadAborted()
tasks.check(thread_blocker)
if step.status == download.RESULT_REDIRECT:
assert step.redirect
return # DownloadScheduler will handle it
assert not step.redirect, step.redirect
step.dl._finish(step.status)
def confirm_install(self, msg):
"""We need to check something with the user before continuing with the install.
@raise download.DownloadAborted: if the user cancels"""
yield
print >>sys.stderr, msg
while True:
sys.stderr.write(_("Install [Y/N] "))
i = raw_input()
if not i: continue
if i in 'Nn':
raise download.DownloadAborted()
if i in 'Yy':
break
def download(self, step):
from ._download_child import download_in_thread
thread_blocker = tasks.Blocker("wait for thread " + step.url)
def notify_done(status, ex=None, redirect=None):
step.status = status
step.redirect = redirect
def wake_up_main():
thread_blocker.trigger(ex)
return False
gobject.idle_add(wake_up_main)
child = threading.Thread(target=lambda: download_in_thread(
step.url, step.dl.tempfile, step.dl.modification_time, notify_done)
)
child.daemon = True
child.start()
# Wait for child to complete download.
yield thread_blocker, step.dl._aborted
if step.dl._aborted.happened:
# Don't wait for child to finish (might be stuck doing IO)
raise download.DownloadAborted()
# Download is complete...
child.join()
tasks.check(thread_blocker)
if step.status == download.RESULT_REDIRECT:
assert step.redirect
return # DownloadScheduler will handle it
assert not step.redirect, step.redirect
step.dl._finish(step.status)