def test_persist_limit(temp, size_limit, file_limit, files):
temp.create_file("a", 10, -6)
temp.create_file("b", 10, -5)
temp.create_file("c", 10, -4)
temp.create_file("d", 10, -3)
temp.create_file("e", 10, -2)
temp.create_file("f", 10, -1)
persist_limit = PersistLimit(size_limit, file_limit)
persist_limit.register_dir_content(temp.tempdir)
persist_limit.remove_old_files()
assert ''.join(sorted(temp.list())) == ''.join(sorted(files))
def init_worker(self, fetch_config, options):
"""
Create and initialize the worker.
Should be subclassed to configure the worker, and should return the
worker method that should start the work.
"""
self.options = options
# global preferences
global_prefs = get_prefs()
self.global_prefs = global_prefs
# apply the global prefs now
apply_prefs(global_prefs)
fetch_config.set_base_url(global_prefs["archive_base_url"])
download_dir = global_prefs["persist"]
if not download_dir:
download_dir = self.mainwindow.persist
persist_limit = PersistLimit(abs(global_prefs["persist_size_limit"]) * 1073741824)
self.download_manager = GuiBuildDownloadManager(download_dir, persist_limit)
self.test_runner = GuiTestRunner()
self.thread = QThread()
# options for the app launcher
launcher_kwargs = {}
for name in ("profile", "preferences"):
if name in options:
value = options[name]
if value:
launcher_kwargs[name] = value
# add add-ons paths to the app launcher
launcher_kwargs["addons"] = options["addons"]
self.test_runner.launcher_kwargs = launcher_kwargs
launcher_kwargs["cmdargs"] = []
if options["profile_persistence"] in ("clone-first", "reuse") or options["profile"]:
launcher_kwargs["cmdargs"] += ["--allow-downgrade"]
# Thunderbird will fail to start if passed an URL arg
if options.get("url") and fetch_config.app_name != "thunderbird":
launcher_kwargs["cmdargs"] += [options["url"]]
# Lang only works for firefox-l10n
if options.get("lang"):
if options["application"] == "firefox-l10n":
fetch_config.set_lang(options["lang"])
else:
raise MozRegressionError("Invalid lang argument")
self.worker = self.worker_class(fetch_config, self.test_runner, self.download_manager)
# Move self.bisector in the thread. This will
# allow to the self.bisector slots (connected after the move)
# to be automatically called in the thread.
self.worker.moveToThread(self.thread)
self.worker_created.emit(self.worker)
def __init__(self, destdir, persist_limit, **kwargs):
QObject.__init__(self)
persist_limit = PersistLimit(persist_limit)
BuildDownloadManager.__init__(self,
destdir,
session=get_http_session(),
persist_limit=persist_limit,
**kwargs)
def __init__(self, cache_dir, log=None, skip_cache=False):
# TODO: instead of storing N artifact packages, store M megabytes.
CacheManager.__init__(self, cache_dir, 'fetch', MAX_CACHED_ARTIFACTS, cache_callback=self.delete_file, log=log, skip_cache=skip_cache)
self._cache_dir = cache_dir
size_limit = 1024 * 1024 * 1024 # 1Gb in bytes.
file_limit = 4 # But always keep at least 4 old artifacts around.
persist_limit = PersistLimit(size_limit, file_limit)
self._download_manager = DownloadManager(self._cache_dir, persist_limit=persist_limit)
self._last_dl_update = -1
def init_worker(self, fetch_config, options):
"""
Create and initialize the worker.
Should be subclassed to configure the worker, and should return the
worker method that should start the work.
"""
self.options = options
# global preferences
global_prefs = get_prefs()
self.global_prefs = global_prefs
# apply the global prefs now
apply_prefs(global_prefs)
fetch_config.set_base_url(global_prefs['archive_base_url'])
download_dir = global_prefs['persist']
if not download_dir:
download_dir = self.mainwindow.persist
persist_limit = PersistLimit(
abs(global_prefs['persist_size_limit']) * 1073741824)
self.download_manager = GuiBuildDownloadManager(
download_dir, persist_limit)
self.test_runner = GuiTestRunner()
self.thread = QThread()
# options for the app launcher
launcher_kwargs = {}
for name in ('profile', 'preferences'):
if name in options:
value = options[name]
if value:
launcher_kwargs[name] = value
# add add-ons paths to the app launcher
launcher_kwargs['addons'] = options['addons']
self.test_runner.launcher_kwargs = launcher_kwargs
if options['profile_persistence'] in ('clone-first',
'reuse') or options['profile']:
launcher_kwargs['cmdargs'] = launcher_kwargs.get(
'cmdargs', []) + ['--allow-downgrade']
# Thunderbird will fail to start if passed an URL arg
if 'url' in options and fetch_config.app_name != 'thunderbird':
launcher_kwargs['cmdargs'] = (launcher_kwargs.get('cmdargs', []) +
[options['url']])
self.worker = self.worker_class(fetch_config, self.test_runner,
self.download_manager)
# Move self.bisector in the thread. This will
# allow to the self.bisector slots (connected after the move)
# to be automatically called in the thread.
self.worker.moveToThread(self.thread)
self.worker_created.emit(self.worker)
def build_download_manager(self):
if self._build_download_manager is None:
background_dl_policy = self.options.background_dl_policy
if not self.options.persist:
# cancel background downloads forced
background_dl_policy = "cancel"
self._build_download_manager = BuildDownloadManager(
self._download_dir,
background_dl_policy=background_dl_policy,
persist_limit=PersistLimit(self.options.persist_size_limit))
return self._build_download_manager
def __init__(self, destdir, session=requests, persist_limit=None):
self.destdir = destdir
self.session = session
self._downloads = {}
self._lock = threading.Lock()
self.persist_limit = persist_limit or PersistLimit(0)
self.persist_limit.register_dir_content(self.destdir)
# if persist folder does not exist, create it
if not os.path.isdir(destdir):
os.makedirs(destdir)
def setUp(self):
self.session, self.session_response = mock_session()
tmpdir = tempfile.mkdtemp()
tpersist_size = PersistLimit(10 * 1073741824)
self.addCleanup(shutil.rmtree, tmpdir)
self.dl_manager = \
build_runner.GuiBuildDownloadManager(tmpdir, tpersist_size)
self.dl_manager.session = self.session
self.signals = {}
for sig in ('download_progress', 'download_started',
'download_finished'):
self.signals[sig] = Mock()
getattr(self.dl_manager, sig).connect(self.signals[sig])
def test_persist_limit(temp, size_limit, file_limit, files):
temp.create_file("a", 10, -6)
temp.create_file("b", 10, -5)
temp.create_file("c", 10, -4)
temp.create_file("d", 10, -3)
temp.create_file("e", 10, -2)
temp.create_file("f", 10, -1)
persist_limit = PersistLimit(size_limit, file_limit)
persist_limit.register_dir_content(temp.tempdir)
persist_limit.remove_old_files()
assert ''.join(sorted(temp.list())) == ''.join(sorted(files))