async def heavy(session, options):
metadata = {}
max = options.get("max_urls", 150)
# see Bug 1608604 - on GV we get OOM killed if we do too much...
platform = options.get("platform", "")
if "gecko" in platform:
max = 30
tabs = TabSwitcher(session, options)
await tabs.create_windows()
visited = 0
for current, url in enumerate(URL_LIST):
get_logger().visit_url(index=current + 1, total=max, url=url)
retries = 0
while retries < 3:
try:
await asyncio.wait_for(session.get(url), 5)
visited += 1
break
except asyncio.TimeoutError:
retries += 1
if max != -1 and current + 1 == max:
break
# switch to the next tab
await tabs.switch()
metadata["visited_url"] = visited
return metadata
async def heavy(session, options):
metadata = {}
max = options.get("max_urls", 150)
tabs = TabSwitcher(session, options)
await tabs.create_windows()
visited = 0
for current, url in enumerate(URL_LIST):
get_logger().visit_url(index=current + 1, total=max, url=url)
retries = 0
while retries < 3:
try:
await asyncio.wait_for(session.get(url), 5)
visited += 1
break
except asyncio.TimeoutError:
retries += 1
if max != -1 and current + 1 == max:
break
# switch to the next tab
await tabs.switch()
metadata["visited_url"] = visited
return metadata
def read_changelog(platform):
params = {"platform": platform}
changelog_url = CHANGELOG_LINK % params
get_logger().msg("Getting %s" % changelog_url)
exists, __ = check_exists(changelog_url)
if exists != 200:
raise ProfileNotFoundError(exists)
download_dir = tempfile.mkdtemp()
downloaded_changelog = os.path.join(download_dir, "changelog.json")
download_file(changelog_url, target=downloaded_changelog)
return Changelog(download_dir)
def read_changelog(platform):
params = {"platform": platform}
changelog_url = CHANGELOG_LINK % params
get_logger().msg("Getting %s" % changelog_url)
download_dir = tempfile.mkdtemp()
downloaded_changelog = os.path.join(download_dir, "changelog.json")
try:
download_file(changelog_url, target=downloaded_changelog)
except ArchiveNotFound:
shutil.rmtree(download_dir)
raise ProfileNotFoundError(changelog_url)
return Changelog(download_dir)
def get_profile(target_dir,
platform,
scenario,
customization="default",
task_id=None):
"""Extract a conditioned profile in the target directory.
If task_id is provided, will grab the profile from that task. when not
provided (default) will grab the latest profile.
"""
# XXX assert values
params = {
"platform": platform,
"scenario": scenario,
"customization": customization,
"task_id": task_id,
}
filename = ARTIFACT_NAME % params
if task_id is None:
url = TC_LINK % params + filename
else:
url = DIRECT_LINK % params + filename
download_dir = tempfile.mkdtemp()
downloaded_archive = os.path.join(download_dir, filename)
get_logger().msg("Getting %s" % url)
exists, __ = check_exists(url)
if exists != 200:
raise ProfileNotFoundError(exists)
archive = download_file(url, target=downloaded_archive)
try:
with tarfile.open(archive, "r:gz") as tar:
get_logger().msg("Extracting the tarball content in %s" %
target_dir)
size = len(list(tar))
with progress.Bar(expected_size=size) as bar:
def _extract(self, *args, **kw):
if not TASK_CLUSTER:
bar.show(bar.last_progress + 1)
return self.old(*args, **kw)
tar.old = tar.extract
tar.extract = functools.partial(_extract, tar)
tar.extractall(target_dir)
except (OSError, tarfile.ReadError) as e:
raise ProfileNotFoundError(str(e))
finally:
shutil.rmtree(download_dir)
get_logger().msg("Success, we have a profile to work with")
return target_dir