def download(self, step): if self.active == MAX_DOWNLOADS_PER_SITE: # Too busy to start a new download now. Queue this one and wait. ticket = tasks.Blocker('queued download for ' + step.url) self.queue.append(ticket) yield ticket, step.dl._aborted if step.dl._aborted.happened: raise download.DownloadAborted() # Start a new thread for the download thread_blocker = _spawn_thread(step) self.active += 1 # Wait for thread to complete download. yield thread_blocker, step.dl._aborted self.active -= 1 if self.active < MAX_DOWNLOADS_PER_SITE: self.process_next() # Start next queued download, if any if step.dl._aborted.happened: # Don't wait for child to finish (might be stuck doing IO) raise download.DownloadAborted() tasks.check(thread_blocker) if step.status == download.RESULT_REDIRECT: assert step.redirect return # DownloadScheduler will handle it assert not step.redirect, step.redirect step.dl._finish(step.status)
def download(self, step, timeout=None): """ Queue up this download. If it takes too long, trigger step.dl.timeout (if any), but only count time spent actually downloading, not time spent queuing. @type step: L{DownloadStep}""" if self.active == MAX_DOWNLOADS_PER_SITE: # Too busy to start a new download now. Queue this one and wait. ticket = tasks.Blocker('queued download for ' + step.url) self.queue.append(ticket) yield ticket, step.dl._aborted if step.dl._aborted.happened: raise download.DownloadAborted() in_progress = [True] if timeout is not None: def timeout_cb(): if in_progress: step.dl.timeout.trigger() tasks.get_loop().call_later(timeout, timeout_cb) # Start a new thread for the download thread_blocker = _spawn_thread(step) self.active += 1 # Wait for thread to complete download. yield thread_blocker, step.dl._aborted del in_progress[0] self.active -= 1 if self.active < MAX_DOWNLOADS_PER_SITE: self.process_next() # Start next queued download, if any if step.dl._aborted.happened: # Don't wait for child to finish (might be stuck doing IO) raise download.DownloadAborted() tasks.check(thread_blocker) if step.status == download.RESULT_REDIRECT: assert step.redirect return # DownloadScheduler will handle it assert not step.redirect, step.redirect step.dl._finish(step.status)
def confirm_install(self, msg): """We need to check something with the user before continuing with the install. @raise download.DownloadAborted: if the user cancels""" yield print >>sys.stderr, msg while True: sys.stderr.write(_("Install [Y/N] ")) i = raw_input() if not i: continue if i in 'Nn': raise download.DownloadAborted() if i in 'Yy': break
def download(self, step): from ._download_child import download_in_thread thread_blocker = tasks.Blocker("wait for thread " + step.url) def notify_done(status, ex=None, redirect=None): step.status = status step.redirect = redirect def wake_up_main(): thread_blocker.trigger(ex) return False gobject.idle_add(wake_up_main) child = threading.Thread(target=lambda: download_in_thread( step.url, step.dl.tempfile, step.dl.modification_time, notify_done) ) child.daemon = True child.start() # Wait for child to complete download. yield thread_blocker, step.dl._aborted if step.dl._aborted.happened: # Don't wait for child to finish (might be stuck doing IO) raise download.DownloadAborted() # Download is complete... child.join() tasks.check(thread_blocker) if step.status == download.RESULT_REDIRECT: assert step.redirect return # DownloadScheduler will handle it assert not step.redirect, step.redirect step.dl._finish(step.status)