def _get_profile():
logger.info("Getting %s" % url)
try:
archive = download_file(url, target=downloaded_archive)
except ArchiveNotFound:
raise ProfileNotFoundError(url)
try:
with tarfile.open(archive, "r:gz") as tar:
logger.info("Extracting the tarball content in %s" %
target_dir)
size = len(list(tar))
with progress.Bar(expected_size=size) as bar:
def _extract(self, *args, **kw):
if not TASK_CLUSTER:
bar.show(bar.last_progress + 1)
return self.old(*args, **kw)
tar.old = tar.extract
tar.extract = functools.partial(_extract, tar)
tar.extractall(target_dir)
except (OSError, tarfile.ReadError) as e:
logger.info("Failed to extract the tarball")
if download_cache and os.path.exists(archive):
logger.info("Removing cached file to attempt a new download")
os.remove(archive)
raise ProfileNotFoundError(str(e))
finally:
if not download_cache:
shutil.rmtree(download_dir)
_check_profile(target_dir)
logger.info("Success, we have a profile to work with")
return target_dir
def create_archive(self, when, iterator=None):
if iterator is None:
def _files(tar):
files = glob.glob(os.path.join(self.profile_dir, "*"))
yield len(files)
for filename in files:
try:
tar.add(filename, os.path.basename(filename))
yield filename
except FileNotFoundError: # NOQA
# locks and such
pass
iterator = _files
if isinstance(when, str):
archive = when
else:
archive, __ = self._get_archive_path(when)
with tarfile.open(archive, "w:gz", dereference=True) as tar:
it = iterator(tar)
size = next(it)
with progress.Bar(expected_size=size) as bar:
for filename in it:
if not TASK_CLUSTER:
bar.show(bar.last_progress + 1)
return archive
def get_profile(target_dir,
platform,
scenario,
customization="default",
task_id=None):
"""Extract a conditioned profile in the target directory.
If task_id is provided, will grab the profile from that task. when not
provided (default) will grab the latest profile.
"""
# XXX assert values
params = {
"platform": platform,
"scenario": scenario,
"customization": customization,
"task_id": task_id,
}
filename = ARTIFACT_NAME % params
if task_id is None:
url = TC_LINK % params + filename
else:
url = DIRECT_LINK % params + filename
download_dir = tempfile.mkdtemp()
downloaded_archive = os.path.join(download_dir, filename)
get_logger().msg("Getting %s" % url)
exists, __ = check_exists(url)
if exists != 200:
raise ProfileNotFoundError(exists)
archive = download_file(url, target=downloaded_archive)
try:
with tarfile.open(archive, "r:gz") as tar:
get_logger().msg("Extracting the tarball content in %s" %
target_dir)
size = len(list(tar))
with progress.Bar(expected_size=size) as bar:
def _extract(self, *args, **kw):
if not TASK_CLUSTER:
bar.show(bar.last_progress + 1)
return self.old(*args, **kw)
tar.old = tar.extract
tar.extract = functools.partial(_extract, tar)
tar.extractall(target_dir)
except (OSError, tarfile.ReadError) as e:
raise ProfileNotFoundError(str(e))
finally:
shutil.rmtree(download_dir)
get_logger().msg("Success, we have a profile to work with")
return target_dir
def get_profile(
target_dir,
platform,
scenario,
customization="default",
task_id=None,
download_cache=True,
repo="mozilla-central",
):
"""Extract a conditioned profile in the target directory.
If task_id is provided, will grab the profile from that task. when not
provided (default) will grab the latest profile.
"""
# XXX assert values
params = {
"platform": platform,
"scenario": scenario,
"customization": customization,
"task_id": task_id,
"repo": repo,
}
logger.info("Getting conditioned profile with arguments: %s" % params)
filename = ARTIFACT_NAME % params
if task_id is None:
url = TC_LINK % params + filename
else:
url = DIRECT_LINK % params + filename
logger.info("preparing download dir")
if not download_cache:
download_dir = tempfile.mkdtemp()
else:
# using a cache dir in the user home dir
download_dir = os.path.expanduser(CONDPROF_CACHE)
if not os.path.exists(download_dir):
os.makedirs(download_dir)
downloaded_archive = os.path.join(download_dir, filename)
logger.info("Downloaded archive path: %s" % downloaded_archive)
retries = 0
while retries < RETRIES:
try:
logger.info("Getting %s" % url)
try:
archive = download_file(url, target=downloaded_archive)
except ArchiveNotFound:
raise ProfileNotFoundError(url)
try:
with tarfile.open(archive, "r:gz") as tar:
logger.info("Extracting the tarball content in %s" % target_dir)
size = len(list(tar))
with progress.Bar(expected_size=size) as bar:
def _extract(self, *args, **kw):
if not TASK_CLUSTER:
bar.show(bar.last_progress + 1)
return self.old(*args, **kw)
tar.old = tar.extract
tar.extract = functools.partial(_extract, tar)
tar.extractall(target_dir)
except (OSError, tarfile.ReadError) as e:
logger.info("Failed to extract the tarball")
if download_cache and os.path.exists(archive):
logger.info("Removing cached file to attempt a new download")
os.remove(archive)
raise ProfileNotFoundError(str(e))
finally:
if not download_cache:
shutil.rmtree(download_dir)
logger.info("Success, we have a profile to work with")
return target_dir
except Exception:
logger.info("Failed to get the profile.")
retries += 1
if os.path.exists(downloaded_archive):
try:
os.remove(downloaded_archive)
except Exception:
logger.error("Could not remove the file")
time.sleep(RETRY_PAUSE)
# If we reach that point, it means all attempts failed
logger.error("All attempt failed")
raise ProfileNotFoundError(url)