def test_persist_limit(temp, size_limit, file_limit, files): temp.create_file("a", 10, -6) temp.create_file("b", 10, -5) temp.create_file("c", 10, -4) temp.create_file("d", 10, -3) temp.create_file("e", 10, -2) temp.create_file("f", 10, -1) persist_limit = PersistLimit(size_limit, file_limit) persist_limit.register_dir_content(temp.tempdir) persist_limit.remove_old_files() assert ''.join(sorted(temp.list())) == ''.join(sorted(files))
def init_worker(self, fetch_config, options): """ Create and initialize the worker. Should be subclassed to configure the worker, and should return the worker method that should start the work. """ self.options = options # global preferences global_prefs = get_prefs() self.global_prefs = global_prefs # apply the global prefs now apply_prefs(global_prefs) fetch_config.set_base_url(global_prefs["archive_base_url"]) download_dir = global_prefs["persist"] if not download_dir: download_dir = self.mainwindow.persist persist_limit = PersistLimit(abs(global_prefs["persist_size_limit"]) * 1073741824) self.download_manager = GuiBuildDownloadManager(download_dir, persist_limit) self.test_runner = GuiTestRunner() self.thread = QThread() # options for the app launcher launcher_kwargs = {} for name in ("profile", "preferences"): if name in options: value = options[name] if value: launcher_kwargs[name] = value # add add-ons paths to the app launcher launcher_kwargs["addons"] = options["addons"] self.test_runner.launcher_kwargs = launcher_kwargs launcher_kwargs["cmdargs"] = [] if options["profile_persistence"] in ("clone-first", "reuse") or options["profile"]: launcher_kwargs["cmdargs"] += ["--allow-downgrade"] # Thunderbird will fail to start if passed an URL arg if options.get("url") and fetch_config.app_name != "thunderbird": launcher_kwargs["cmdargs"] += [options["url"]] # Lang only works for firefox-l10n if options.get("lang"): if options["application"] == "firefox-l10n": fetch_config.set_lang(options["lang"]) else: raise MozRegressionError("Invalid lang argument") self.worker = self.worker_class(fetch_config, self.test_runner, self.download_manager) # Move self.bisector in the thread. This will # allow to the self.bisector slots (connected after the move) # to be automatically called in the thread. self.worker.moveToThread(self.thread) self.worker_created.emit(self.worker)
def __init__(self, destdir, persist_limit, **kwargs): QObject.__init__(self) persist_limit = PersistLimit(persist_limit) BuildDownloadManager.__init__(self, destdir, session=get_http_session(), persist_limit=persist_limit, **kwargs)
def __init__(self, cache_dir, log=None, skip_cache=False): # TODO: instead of storing N artifact packages, store M megabytes. CacheManager.__init__(self, cache_dir, 'fetch', MAX_CACHED_ARTIFACTS, cache_callback=self.delete_file, log=log, skip_cache=skip_cache) self._cache_dir = cache_dir size_limit = 1024 * 1024 * 1024 # 1Gb in bytes. file_limit = 4 # But always keep at least 4 old artifacts around. persist_limit = PersistLimit(size_limit, file_limit) self._download_manager = DownloadManager(self._cache_dir, persist_limit=persist_limit) self._last_dl_update = -1
def init_worker(self, fetch_config, options): """ Create and initialize the worker. Should be subclassed to configure the worker, and should return the worker method that should start the work. """ self.options = options # global preferences global_prefs = get_prefs() self.global_prefs = global_prefs # apply the global prefs now apply_prefs(global_prefs) fetch_config.set_base_url(global_prefs['archive_base_url']) download_dir = global_prefs['persist'] if not download_dir: download_dir = self.mainwindow.persist persist_limit = PersistLimit( abs(global_prefs['persist_size_limit']) * 1073741824) self.download_manager = GuiBuildDownloadManager( download_dir, persist_limit) self.test_runner = GuiTestRunner() self.thread = QThread() # options for the app launcher launcher_kwargs = {} for name in ('profile', 'preferences'): if name in options: value = options[name] if value: launcher_kwargs[name] = value # add add-ons paths to the app launcher launcher_kwargs['addons'] = options['addons'] self.test_runner.launcher_kwargs = launcher_kwargs if options['profile_persistence'] in ('clone-first', 'reuse') or options['profile']: launcher_kwargs['cmdargs'] = launcher_kwargs.get( 'cmdargs', []) + ['--allow-downgrade'] # Thunderbird will fail to start if passed an URL arg if 'url' in options and fetch_config.app_name != 'thunderbird': launcher_kwargs['cmdargs'] = (launcher_kwargs.get('cmdargs', []) + [options['url']]) self.worker = self.worker_class(fetch_config, self.test_runner, self.download_manager) # Move self.bisector in the thread. This will # allow to the self.bisector slots (connected after the move) # to be automatically called in the thread. self.worker.moveToThread(self.thread) self.worker_created.emit(self.worker)
def build_download_manager(self): if self._build_download_manager is None: background_dl_policy = self.options.background_dl_policy if not self.options.persist: # cancel background downloads forced background_dl_policy = "cancel" self._build_download_manager = BuildDownloadManager( self._download_dir, background_dl_policy=background_dl_policy, persist_limit=PersistLimit(self.options.persist_size_limit)) return self._build_download_manager
def __init__(self, destdir, session=requests, persist_limit=None): self.destdir = destdir self.session = session self._downloads = {} self._lock = threading.Lock() self.persist_limit = persist_limit or PersistLimit(0) self.persist_limit.register_dir_content(self.destdir) # if persist folder does not exist, create it if not os.path.isdir(destdir): os.makedirs(destdir)
def setUp(self): self.session, self.session_response = mock_session() tmpdir = tempfile.mkdtemp() tpersist_size = PersistLimit(10 * 1073741824) self.addCleanup(shutil.rmtree, tmpdir) self.dl_manager = \ build_runner.GuiBuildDownloadManager(tmpdir, tpersist_size) self.dl_manager.session = self.session self.signals = {} for sig in ('download_progress', 'download_started', 'download_finished'): self.signals[sig] = Mock() getattr(self.dl_manager, sig).connect(self.signals[sig])
def test_persist_limit(temp, size_limit, file_limit, files): temp.create_file("a", 10, -6) temp.create_file("b", 10, -5) temp.create_file("c", 10, -4) temp.create_file("d", 10, -3) temp.create_file("e", 10, -2) temp.create_file("f", 10, -1) persist_limit = PersistLimit(size_limit, file_limit) persist_limit.register_dir_content(temp.tempdir) persist_limit.remove_old_files() assert ''.join(sorted(temp.list())) == ''.join(sorted(files))