def _get_changelog():
try:
download_file(changelog_url, target=downloaded_changelog)
except ArchiveNotFound:
shutil.rmtree(download_dir)
raise ProfileNotFoundError(changelog_url)
return Changelog(download_dir)
def read_changelog(platform):
params = {"platform": platform}
changelog_url = CHANGELOG_LINK % params
get_logger().msg("Getting %s" % changelog_url)
exists, __ = check_exists(changelog_url)
if exists != 200:
raise ProfileNotFoundError(exists)
download_dir = tempfile.mkdtemp()
downloaded_changelog = os.path.join(download_dir, "changelog.json")
download_file(changelog_url, target=downloaded_changelog)
return Changelog(download_dir)
def read_changelog(platform, repo="mozilla-central"):
params = {"platform": platform, "repo": repo}
changelog_url = CHANGELOG_LINK % params
logger.info("Getting %s" % changelog_url)
download_dir = tempfile.mkdtemp()
downloaded_changelog = os.path.join(download_dir, "changelog.json")
try:
download_file(changelog_url, target=downloaded_changelog)
except ArchiveNotFound:
shutil.rmtree(download_dir)
raise ProfileNotFoundError(changelog_url)
return Changelog(download_dir)
def read_changelog(platform):
params = {"platform": platform}
changelog_url = CHANGELOG_LINK % params
get_logger().msg("Getting %s" % changelog_url)
download_dir = tempfile.mkdtemp()
downloaded_changelog = os.path.join(download_dir, "changelog.json")
try:
download_file(changelog_url, target=downloaded_changelog)
except ArchiveNotFound:
shutil.rmtree(download_dir)
raise ProfileNotFoundError(changelog_url)
return Changelog(download_dir)
def _get_profile():
logger.info("Getting %s" % url)
try:
archive = download_file(url, target=downloaded_archive)
except ArchiveNotFound:
raise ProfileNotFoundError(url)
try:
with tarfile.open(archive, "r:gz") as tar:
logger.info("Extracting the tarball content in %s" %
target_dir)
size = len(list(tar))
with progress.Bar(expected_size=size) as bar:
def _extract(self, *args, **kw):
if not TASK_CLUSTER:
bar.show(bar.last_progress + 1)
return self.old(*args, **kw)
tar.old = tar.extract
tar.extract = functools.partial(_extract, tar)
tar.extractall(target_dir)
except (OSError, tarfile.ReadError) as e:
logger.info("Failed to extract the tarball")
if download_cache and os.path.exists(archive):
logger.info("Removing cached file to attempt a new download")
os.remove(archive)
raise ProfileNotFoundError(str(e))
finally:
if not download_cache:
shutil.rmtree(download_dir)
_check_profile(target_dir)
logger.info("Success, we have a profile to work with")
return target_dir
def get_profile(args):
# getting the latest archive from the server
if TASK_CLUSTER:
url = TC_LINK % args.scenarii
basename = "today-%s.tgz" % args.scenarii
else:
basename = "%s-latest.tar.gz" % args.scenarii
url = args.archives_server + "/%s" % basename
exists, __ = check_exists(url)
if not exists:
return None
target = os.path.join(args.archives_dir, basename)
archive = download_file(url, target=target, check_file=False)
with tarfile.open(archive, "r:gz") as tar:
logger.msg("Checking the tarball content...")
size = len(list(tar))
with progress.Bar(expected_size=size) as bar:
def _extract(self, *args, **kw):
if not TASK_CLUSTER:
bar.show(bar.last_progress + 1)
try:
return self.old(*args, **kw)
finally:
pass
tar.old = tar.extract
tar.extract = functools.partial(_extract, tar)
tar.extractall(args.profile)
return args.profile
def get_profile(target_dir,
platform,
scenario,
customization="default",
task_id=None):
"""Extract a conditioned profile in the target directory.
If task_id is provided, will grab the profile from that task. when not
provided (default) will grab the latest profile.
"""
# XXX assert values
params = {
"platform": platform,
"scenario": scenario,
"customization": customization,
"task_id": task_id,
}
filename = ARTIFACT_NAME % params
if task_id is None:
url = TC_LINK % params + filename
else:
url = DIRECT_LINK % params + filename
download_dir = tempfile.mkdtemp()
downloaded_archive = os.path.join(download_dir, filename)
get_logger().msg("Getting %s" % url)
exists, __ = check_exists(url)
if exists != 200:
raise ProfileNotFoundError(exists)
archive = download_file(url, target=downloaded_archive)
try:
with tarfile.open(archive, "r:gz") as tar:
get_logger().msg("Extracting the tarball content in %s" %
target_dir)
size = len(list(tar))
with progress.Bar(expected_size=size) as bar:
def _extract(self, *args, **kw):
if not TASK_CLUSTER:
bar.show(bar.last_progress + 1)
return self.old(*args, **kw)
tar.old = tar.extract
tar.extract = functools.partial(_extract, tar)
tar.extractall(target_dir)
except (OSError, tarfile.ReadError) as e:
raise ProfileNotFoundError(str(e))
finally:
shutil.rmtree(download_dir)
get_logger().msg("Success, we have a profile to work with")
return target_dir
def get_profile(
target_dir,
platform,
scenario,
customization="default",
task_id=None,
download_cache=True,
repo="mozilla-central",
):
"""Extract a conditioned profile in the target directory.
If task_id is provided, will grab the profile from that task. when not
provided (default) will grab the latest profile.
"""
# XXX assert values
params = {
"platform": platform,
"scenario": scenario,
"customization": customization,
"task_id": task_id,
"repo": repo,
}
logger.info("Getting conditioned profile with arguments: %s" % params)
filename = ARTIFACT_NAME % params
if task_id is None:
url = TC_LINK % params + filename
else:
url = DIRECT_LINK % params + filename
logger.info("preparing download dir")
if not download_cache:
download_dir = tempfile.mkdtemp()
else:
# using a cache dir in the user home dir
download_dir = os.path.expanduser(CONDPROF_CACHE)
if not os.path.exists(download_dir):
os.makedirs(download_dir)
downloaded_archive = os.path.join(download_dir, filename)
logger.info("Downloaded archive path: %s" % downloaded_archive)
retries = 0
while retries < RETRIES:
try:
logger.info("Getting %s" % url)
try:
archive = download_file(url, target=downloaded_archive)
except ArchiveNotFound:
raise ProfileNotFoundError(url)
try:
with tarfile.open(archive, "r:gz") as tar:
logger.info("Extracting the tarball content in %s" % target_dir)
size = len(list(tar))
with progress.Bar(expected_size=size) as bar:
def _extract(self, *args, **kw):
if not TASK_CLUSTER:
bar.show(bar.last_progress + 1)
return self.old(*args, **kw)
tar.old = tar.extract
tar.extract = functools.partial(_extract, tar)
tar.extractall(target_dir)
except (OSError, tarfile.ReadError) as e:
logger.info("Failed to extract the tarball")
if download_cache and os.path.exists(archive):
logger.info("Removing cached file to attempt a new download")
os.remove(archive)
raise ProfileNotFoundError(str(e))
finally:
if not download_cache:
shutil.rmtree(download_dir)
logger.info("Success, we have a profile to work with")
return target_dir
except Exception:
logger.info("Failed to get the profile.")
retries += 1
if os.path.exists(downloaded_archive):
try:
os.remove(downloaded_archive)
except Exception:
logger.error("Could not remove the file")
time.sleep(RETRY_PAUSE)
# If we reach that point, it means all attempts failed
logger.error("All attempt failed")
raise ProfileNotFoundError(url)
