def _handle_app_cast(data, up_to_date_callback): """Handle appcast data when it's correctly fetched """ try: appcast = feedparserutil.parse(data['body']) if appcast['bozo'] == '1': return up_to_date = True latest = _get_item_for_latest(appcast) if latest is None: logging.info('No updates for this platform.') # this will go through the finally clause below return serial = int(app.config.get(prefs.APP_SERIAL)) up_to_date = (serial >= _get_item_serial(latest)) if not up_to_date: logging.info('New update available.') signals.system.update_available(latest) elif up_to_date_callback: logging.info('Up to date. Notifying callback.') up_to_date_callback() else: logging.info('Up to date.') except StandardError: logging.exception("Error while handling appcast data.") finally: global check_in_progress check_in_progress = False eventloop.add_timeout(86400, check_for_updates, "Check for updates")
def _log_bug_report_progress(self, sender): current, total = sender.progress() if current < total: logging.info("Crash report progress: %0.1f", current * 100.0 / total) eventloop.add_timeout(1.0, self._log_bug_report_progress, 'log bug report progress', args=(sender,))
def check_content(data): self.check_content_data = data if len(data) == 5: # wait a bit to see if any more data comes through, which it # shouldn't eventloop.add_timeout(0.2, self.stopEventLoop, 'stop download', args=(False,)) 1/0 return True
def send_bug_report(self, report, description, send_database, quit_after=False): sender = BugReportSender(report, description, send_database) self.bug_report_senders.add(sender) sender.connect("finished", self._bug_report_sent) if quit_after: self._quit_after_bug_reports = True eventloop.add_timeout(0.5, self._start_send_bug_report_progress, "bug report progress") else: eventloop.add_timeout(0.5, self._log_bug_report_progress, "log bug report progress", args=(sender,))
def _log_bug_report_progress(self, sender): if sender.is_done: logging.info("Crash report progress: done.") return current, total = sender.progress() if current < total: logging.info("Crash report progress: %0.1f", current * 100.0 / total) eventloop.add_timeout(1.0, self._log_bug_report_progress, 'log bug report progress', args=(sender,))
def send_bug_report(self, report, description, send_database, quit_after=False): sender = BugReportSender(report, description, send_database) self.bug_report_senders.add(sender) sender.connect("finished", self._bug_report_sent) if quit_after: self._quit_after_bug_reports = True eventloop.add_timeout(0.5, self._start_send_bug_report_progress, 'bug report progress') else: eventloop.add_timeout(0.5, self._log_bug_report_progress, 'log bug report progress', args=(sender,))
def clean_up(temp_file, file_and_directory=False, attempts=0): if attempts > 5: return if os.path.exists(temp_file): try: os.remove(temp_file) except EnvironmentError, e: logging.debug("clean_up: %s kicked up while removing %s", e, temp_file) timeout = 1.0 * attempts eventloop.add_timeout( timeout, clean_up, "conversion clean_up attempt", (temp_file, file_and_directory, attempts + 1) )
def test_callbacks(self): eventloop.add_idle(self.callback, "foo") eventloop.add_timeout(0.1, self.callback, "foo", args=("chris",), kwargs={'hula': "hula"}) eventloop.add_timeout(0.2, self.callback, "foo", args=("ben",), kwargs={'hula': 'moreHula', 'stop': 1}) self.runEventLoop() self.assertEquals(self.got_args[0], ()) self.assertEquals(self.got_args[1], ("chris",)) self.assertEquals(self.got_args[2], ("ben",)) self.assertEquals(self.got_kwargs[0], {}) self.assertEquals(self.got_kwargs[1], {'hula':'hula'}) self.assertEquals(self.got_kwargs[2], {'hula': 'moreHula', 'stop': 1})
def do_update(self): try: TORRENT_SESSION.update_torrents() statuses = [] for downloader in self.to_update: statuses.append(downloader.get_status()) self.to_update = set() if statuses: command.BatchUpdateDownloadStatus(daemon.LAST_DAEMON, statuses).send() finally: eventloop.add_timeout(self.UPDATE_CLIENT_INTERVAL, self.do_update, "Download status update")
def clean_up(temp_file, file_and_directory=False, attempts=0): if attempts > 5: return if os.path.exists(temp_file): try: os.remove(temp_file) except EnvironmentError, e: logging.warning("clean_up: %s kicked up while removing %s", e, temp_file) timeout = 1.0 * attempts eventloop.add_timeout( timeout, clean_up, "conversion clean_up attempt", (temp_file, file_and_directory, attempts + 1))
def _on_thread_quit(self, thread): """Handle our thread exiting.""" # Ignore this call if it was queued from while we were in the middle # of shutdown(). if not self.is_running: return if thread is not self.thread: # If we have lost the race between the cleanup on shutdown # it should be safe to ignore. # # This can happen when the process does not immediately shut down # because the worker process is still processing pending jobs # and the quit message was not processed in time and so the # subprocess was forcibly terminated. When that happens # _cleanup_process() is called which resets the thread attribute # to None immediately, but _on_thread_quit() is only run some # time after that (when we notice the pipe to the subprocess's # close we add _on_thread_quit() to the idle loop). # # So if the self.thread attribute is None then it means we are done # and so things are all good. if self.thread is not None and thread.quit_type != thread.QUIT_NORMAL: msg = ('_on_thread_quit called by an old thread ' 'self.thread: %s thread: %s quit_type: %s' % (self.thread.name, thread.name, thread.quit_type)) app.controller.failed_soft('handling subprocess', msg) return if (self.thread.quit_type == self.thread.QUIT_NORMAL and self.sent_quit): self._cleanup_process() else: logging.warn("Subprocess quit unexpectedly (quit_type: %s, " "sent_quit: %s). Will restart subprocess", self.thread.quit_type, self.sent_quit) # NOTE: should we enforce some sort of cool-down time before # restarting the subprocess? time_since_start = clock.clock() - self.start_time delay_time = self.restart_delay - time_since_start if delay_time <= 0: logging.warn("Subprocess died after %0.1f seconds. " "Restarting", time_since_start) self.restart() else: logging.warn("Subprocess died in %0.1f seconds, waiting " "%0.1f to restart", time_since_start, delay_time) eventloop.add_timeout(delay_time, self.restart, 'restart failed subprocess')
def test_lots_of_threads(self): timeouts = [0, 0, 0.1, 0.2, 0.3] threadCount = 8 def thread(): sleep(0.5) for timeout in timeouts: eventloop.add_timeout(timeout, self.callback, "foo") for i in range(threadCount): t = threading.Thread(target=thread) t.start() eventloop.add_timeout(1.2, self.callback, "foo", kwargs={'stop':1}) self.runEventLoop() totalCalls = len(timeouts) * threadCount + 1 self.assertEquals(len(self.got_args), totalCalls)
def update_stats(self): """Update the download rate and eta based on receiving length bytes. """ if self.client is None or self.state != 'downloading': return stats = self.client.get_stats() if stats.status_code in (200, 206): self.currentSize = stats.downloaded + stats.initial_size self.rate = stats.download_rate else: self.currentSize = self.rate = 0 eventloop.add_timeout(self.CHECK_STATS_TIMEOUT, self.update_stats, 'update http downloader stats') DOWNLOAD_UPDATER.queue_update(self)
def start_download(self, resume=True): if self.retryDC: self.retryDC.cancel() self.retryDC = None if resume: resume = self._resume_sanity_check() logging.info("start_download: %s", self.url) self.client = httpclient.grab_url( self.url, self.on_download_finished, self.on_download_error, header_callback=self.on_headers, write_file=self.filename, resume=resume) self.update_client() eventloop.add_timeout(self.CHECK_STATS_TIMEOUT, self.update_stats, 'update http downloader stats')
def schedule_update(self, delay, feed, update_callback): name = "Feed update (%s)" % feed.get_title() self.timeouts[feed.id] = eventloop.add_timeout(delay, self.do_update, name, args=(feed, update_callback))
def __call__(self, database): self.database = database if self.scheduled_write: return self.scheduled_write = eventloop.add_timeout(self.SAVE_INTERVAL, self.write, 'writing device database')
def test_remove_file(self): filename = self.make_temp_path(".txt") self.httpserver.pause_after(5) def cancel_after_5_bytes(): if self.client.get_stats().downloaded == 5: self.client.cancel(remove_file=True) self.stopEventLoop(False) else: eventloop.add_timeout(0.1, cancel_after_5_bytes, 'cancel') eventloop.add_timeout(0.1, cancel_after_5_bytes, 'cancel') self.expecting_errback = True self.grab_url(self.httpserver.build_url('test.txt'), write_file=filename) self.assertEquals(self.grab_url_info, None) self.assertEquals(self.grab_url_error, None) self.wait_for_libcurl_manager() self.assert_(not os.path.exists(filename))
def startReadTimeout(self): if self.disable_read_timeout: return self.lastClock = clock() if self.readTimeout is not None: return self.readTimeout = eventloop.add_timeout(SOCKET_INITIAL_READ_TIMEOUT, self.onReadTimeout, "AsyncSocket.onReadTimeout")
def test_upload_progress(self): # upload a 100k file data = '0' * (1000 * 1024) f1 = { 'filename': 'testing.txt', 'mimetype': 'text/plain', 'handle': StringIO(data), } self.event_loop_timeout = 5 TIMEOUT = 0.001 self.last_uploaded = None self.last_total = None self.saw_total = False def check_upload_progress(): progress = self.client.get_stats() if progress.upload_total == -1: # client didn't know the total upload at this point. self.assertEquals(progress.uploaded, -1) eventloop.add_timeout(TIMEOUT, check_upload_progress, 'upload progress timeout') return self.saw_total = True if self.last_uploaded is not None: # the currently upload size should only increase self.assert_(progress.uploaded >= self.last_uploaded) self.last_uploaded = progress.uploaded if self.last_total is not None: # the total upload size shouldn't change self.assertEquals(progress.upload_total, self.last_total) self.last_total = progress.upload_total eventloop.add_timeout(TIMEOUT, check_upload_progress, 'upload progress timeout') eventloop.add_timeout(TIMEOUT, check_upload_progress, 'upload progress timeout') self.grab_url( self.httpserver.build_url('test.txt'), post_files={'file1': f1}) # make sure at least one of our sending_progress() calls worked self.assert_(self.saw_total)
def test_upload_progress(self): # upload a 100k file data = '0' * (1000 * 1024) f1 = { 'filename': 'testing.txt', 'mimetype': 'text/plain', 'handle': StringIO(data), } self.event_loop_timeout = 5 TIMEOUT = 0.001 self.last_uploaded = None self.last_total = None self.saw_total = False def check_upload_progress(): progress = self.client.get_stats() if progress.upload_total == -1: # client didn't know the total upload at this point. self.assertEquals(progress.uploaded, -1) eventloop.add_timeout(TIMEOUT, check_upload_progress, 'upload progress timeout') return self.saw_total = True if self.last_uploaded is not None: # the currently upload size should only increase self.assert_(progress.uploaded >= self.last_uploaded) self.last_uploaded = progress.uploaded if self.last_total is not None: # the total upload size shouldn't change self.assertEquals(progress.upload_total, self.last_total) self.last_total = progress.upload_total eventloop.add_timeout(TIMEOUT, check_upload_progress, 'upload progress timeout') eventloop.add_timeout(TIMEOUT, check_upload_progress, 'upload progress timeout') self.grab_url(self.httpserver.build_url('test.txt'), post_files={'file1': f1}) # make sure at least one of our sending_progress() calls worked self.assert_(self.saw_total)
def test_cancel(self): filename = self.make_temp_path(".txt") self.httpserver.pause_after(5) def cancel_after_5_bytes(): if self.client.get_stats().downloaded == 5: self.client.cancel() self.stopEventLoop(False) else: eventloop.add_timeout(0.1, cancel_after_5_bytes, 'cancel') eventloop.add_timeout(0.1, cancel_after_5_bytes, 'cancel') self.expecting_errback = True self.grab_url(self.httpserver.build_url('test.txt'), write_file=filename) # Neither the callback, nor the errback should be called self.assertEquals(self.grab_url_info, None) self.assertEquals(self.grab_url_error, None) # We shouldn't delete the file self.assert_(os.path.exists(filename))
def get_startup_activity(self): self.confirm_db_thread() activity = self.status.get("activity") if activity is None and self.status.get("retryCount", -1) > -1 and "retryTime" in self.status: activity = self._calc_retry_time() if self._update_retry_time_dc is None: self._update_retry_time_dc = eventloop.add_timeout(1, self._update_retry_time, "Updating retry time") if activity is None: return _("starting up") return activity
def test_quick_cancel(self): # Try canceling before find_http_auth returns and make sure things # work. self.setup_answer("user", "password") url = self.httpserver.build_url('protected/index.txt') self.grab_url_error = self.grab_url_info = None self.client = httpclient.grab_url(url, self.grab_url_callback, self.grab_url_errback) # at this point, our client should be finding the HTTP auth. It # shouldn't have started transfering data. self.assertEquals(len(httpclient.curl_manager.transfer_map), 0) self.client.cancel() self.expecting_errback = True eventloop.add_timeout(0.2, self.stopEventLoop, 'stopping event loop', args=(False,)) self.runEventLoop(timeout=self.event_loop_timeout) # make sure that the callback/errback weren't called and nothing is # transfering self.check_nothing_called() self.assertEquals(len(httpclient.curl_manager.transfer_map), 0)
def error_callback(self, url, error=None): self.dbItem.confirm_db_thread() if self.removed: iconCacheUpdater.update_finished() return # Don't clear the cache on an error. if self.url != url: self.url = url self.etag = None self.modified = None self.icon_changed() self.updating = False if self.needsUpdate: self.needsUpdate = False self.request_update(True) elif error is not None: eventloop.add_timeout(3600, self.request_update, "Thumbnail request for %s" % url) iconCacheUpdater.update_finished()
def _send_bug_report_progress(self): current_sent = 0 total_to_send = 0 for sender in self.bug_report_senders: sent, to_send = sender.progress() if to_send == 0: # this sender doesn't know it's total data, we can't calculate # things. current_sent = total_to_send = 0 break else: current_sent += sent total_to_send += to_send if total_to_send > 0: progress = float(current_sent) / total_to_send else: progress = -1 text = _("Sending Crash Report (%(progress)d%%)", {"progress": progress * 100}) messages.ProgressDialog(text, progress).send_to_frontend() eventloop.add_timeout(0.1, self._send_bug_report_progress, "bug report progress")
def get_startup_activity(self): self.confirm_db_thread() activity = self.activity if (activity is None and self.retry_count is not None and self.retry_time is not None): activity = self._calc_retry_time() if self._update_retry_time_dc is None: self._update_retry_time_dc = eventloop.add_timeout( 1, self._update_retry_time, 'Updating retry time') if activity is None: return _("starting up") return activity
def get_startup_activity(self): self.confirm_db_thread() activity = self.activity if (activity is None and self.retry_count is not None and self.retry_time is not None): activity = self._calc_retry_time() if self._update_retry_time_dc is None: self._update_retry_time_dc = eventloop.add_timeout(1, self._update_retry_time, 'Updating retry time') if activity is None: return _("starting up") return activity
def get_startup_activity(self): self.confirm_db_thread() activity = self.status.get('activity') if ((activity is None and self.status.get('retryCount', -1) > -1 and 'retryTime' in self.status)): activity = self._calc_retry_time() if self._update_retry_time_dc is None: self._update_retry_time_dc = eventloop.add_timeout(1, self._update_retry_time, 'Updating retry time') if activity is None: return _("starting up") return activity
def check_upload_progress(): progress = self.client.get_stats() if progress.upload_total == -1: # client didn't know the total upload at this point. self.assertEquals(progress.uploaded, -1) eventloop.add_timeout(TIMEOUT, check_upload_progress, 'upload progress timeout') return self.saw_total = True if self.last_uploaded is not None: # the currently upload size should only increase self.assert_(progress.uploaded >= self.last_uploaded) self.last_uploaded = progress.uploaded if self.last_total is not None: # the total upload size shouldn't change self.assertEquals(progress.upload_total, self.last_total) self.last_total = progress.upload_total eventloop.add_timeout(TIMEOUT, check_upload_progress, 'upload progress timeout')
def onReadTimeout(self): if self.readSomeData: timeout = SOCKET_READ_TIMEOUT else: timeout = SOCKET_INITIAL_READ_TIMEOUT if clock() < self.lastClock + timeout: self.readTimeout = eventloop.add_timeout(self.lastClock + timeout - clock(), self.onReadTimeout, "AsyncSocket.onReadTimeout") else: self.readTimeout = None self.timedOut = True self.handleEarlyClose('read')
def _send_bug_report_progress(self): current_sent = 0 total_to_send = 0 for sender in self.bug_report_senders: sent, to_send = sender.progress() if to_send == 0: # this sender doesn't know it's total data, we can't calculate # things. current_sent = total_to_send = 0 break else: current_sent += sent total_to_send += to_send if total_to_send > 0: progress = float(current_sent) / total_to_send else: progress = -1 text = _('Sending Crash Report (%(progress)d%%)', {"progress": progress * 100}) messages.ProgressDialog(text, progress).send_to_frontend() eventloop.add_timeout(0.1, self._send_bug_report_progress, 'bug report progress')
def shutdown_downloader_daemon(self, timeout=5, callback=None): """Send the downloader daemon the shutdown command. If it doesn't reply before timeout expires, kill it. (The reply is not sent until the downloader daemon has one remaining thread and that thread will immediately exit). """ self.shutdown_callback = callback c = command.ShutDownCommand(self) c.send() self.shutdown = True self._remove_config_callback() self.shutdown_timeout_dc = eventloop.add_timeout( timeout, self.shutdown_timeout_cb, "Waiting for dl_daemon shutdown")
def onReadTimeout(self): if self.readSomeData: timeout = SOCKET_READ_TIMEOUT else: timeout = SOCKET_INITIAL_READ_TIMEOUT if clock() < self.lastClock + timeout: self.readTimeout = eventloop.add_timeout( self.lastClock + timeout - clock(), self.onReadTimeout, "AsyncSocket.onReadTimeout") else: self.readTimeout = None self.timedOut = True self.handleEarlyClose('read')
def shutdown_downloader_daemon(self, timeout=5, callback=None): """Send the downloader daemon the shutdown command. If it doesn't reply before timeout expires, kill it. (The reply is not sent until the downloader daemon has one remaining thread and that thread will immediately exit). """ self._shutdown_callback = callback c = command.ShutDownCommand(self) c.send() self.shutdown = True self._remove_config_callback() self._shutdown_timeout_dc = eventloop.add_timeout( timeout, self.shutdown_timeout_cb, "Waiting for dl_daemon shutdown")
def on_download_complete(self, obj, item): try: # Re-arm count is for the next threshold, not the current one, # so add 1. rearm_count = self.donate_ask_thresholds[self.donate_nothanks + 1] except IndexError: rearm_count = self.donate_ask_thresholds[-1] self.donate_counter -= 1 # In case the donate counters are borked, then reset it if self.donate_counter < 0: self.donate_counter = 0 if self.last_donate_time < 0: self.last_donate_time = 0 # If the donate window has been shown recently, don't show it again # even if the timer is about to fire. Defuse the timer and then # continue. if self.donate_ratelimit: logging.debug('donate: rate limiting donate window popup.') return logging.debug('donate: on_download_complete %s %s %s', self.donate_nothanks, self.donate_counter, self.last_donate_time) # Show it if the donate counter has reached zero and we have asked # less than 3 times, but not if the user's already accepted in the # past 6 months HALF_YEAR = 60 * 60 * 24 * 180 show_donate = (self.donate_counter == 0 and self.donate_nothanks < 3 and time.time() - self.last_donate_time > HALF_YEAR) logging.debug('donate: show_donate = %s', show_donate) if show_donate: # re-arm the countdown self.donate_counter = rearm_count self.set_ratelimit() # 5 days self.ratelimit_dc = eventloop.add_timeout(3600 * 24 * 5, self.reset_ratelimit, 'donate ratelimiter') self.show_donate() # Set the new value of donate_counter. app.config.set(prefs.DONATE_COUNTER, self.donate_counter)
def _save_later(self): """Save the remote downloader at some point in the future. This is used to handle the fact that remote downloaders are updated often, but those updates are usually just the status dict, which is never used for SELECT statements. Continually saving those changes to disk is just a waste of time and IO. Instead, we schedule the save to happen sometime in the future. When miro quits, we call the module-level function run_delayed_saves(), which makes sure any pending objects are saved to disk. """ if self._save_later_dc is None: self._save_later_dc = eventloop.add_timeout(15, self._save_now, "Delayed RemoteDownloader save")
def handle_temporary_error(self, shortReason, reason): self.state = u"offline" self.endTime = self.startTime = 0 self.rate = 0 self.reasonFailed = reason self.shortReasonFailed = shortReason self.retryCount = self.retryCount + 1 if self.retryCount >= len(RETRY_TIMES): self.retryCount = len(RETRY_TIMES) - 1 self.retryDC = eventloop.add_timeout( RETRY_TIMES[self.retryCount], self.retry_download, "Logarithmic retry") now = datetime.datetime.now() self.retryTime = now + datetime.timedelta(seconds=RETRY_TIMES[self.retryCount]) logging.info("Temporary error: '%s' '%s'. retrying at %s %s", shortReason, reason, self.retryTime, self.retryCount) self.update_client()
def runEventLoop(self, timeout=10, timeoutNormal=False): eventloop.thread_pool_init() eventloop._eventloop.quit_flag = False eventloop._eventloop.idle_queue.quit_flag = False eventloop._eventloop.urgent_queue.quit_flag = False try: self.hadToStopEventLoop = False timeout_handle = eventloop.add_timeout(timeout, self.stopEventLoop, "Stop test event loop") eventloop._eventloop.quit_flag = False eventloop._eventloop.loop() if self.hadToStopEventLoop and not timeoutNormal: raise HadToStopEventLoop() else: timeout_handle.cancel() finally: eventloop.thread_pool_quit()
def on_frontend_started(): """Perform startup actions that should happen after the frontend is already up and running. """ logging.info("Starting auto downloader...") autodler.start_downloader() yield None feed.expire_items() yield None moviedata.movie_data_updater.start_thread() yield None commandline.startup() yield None autoupdate.check_for_updates() yield None # Delay running high CPU/IO operations for a bit eventloop.add_timeout(5, downloader.startup_downloader, "start downloader daemon") eventloop.add_timeout(30, feed.start_updates, "start feed updates") eventloop.add_timeout(60, item.update_incomplete_movie_data, "update movie data") eventloop.add_timeout(90, clear_icon_cache_orphans, "clear orphans")
def on_dialog(): self.client.cancel() eventloop.add_timeout(0.2, self.stopEventLoop, 'stopping event loop', args=(False,))
def thread(): sleep(0.5) for timeout in timeouts: eventloop.add_timeout(timeout, self.callback, "foo")
def cancel_after_5_bytes(): if self.client.get_stats().downloaded == 5: self.client.cancel(remove_file=True) self.stopEventLoop(False) else: eventloop.add_timeout(0.1, cancel_after_5_bytes, 'cancel')
def add_timeout(self, delay, function, name, args=None, kwargs=None): eventloop.add_timeout(delay, function, name, args, kwargs)
def start_updates(self): eventloop.add_timeout(self.UPDATE_INTERVAL, self.send_updates, "Send Download Command Updates")
def _handle_error(error): """Error handler""" global check_in_progress check_in_progress = False logging.warn("HTTP error while checking for updates %s", error) eventloop.add_timeout(86400, check_for_updates, "Check for updates")
def schedule_save_to_db(self): if self._save_dc is None: self._save_dc = eventloop.add_timeout(self.SAVE_INTERVAL, self.save, 'save item info cache')