def unpack_rpm(package_file_name, files, tmp_dir, destdir, keeprpm, exact_files=False):
"""
Unpacks a single rpm located in tmp_dir into destdir.
Arguments:
package_file_name - name of the rpm file
files - files to extract from the rpm
tmp_dir - temporary directory where the rpm file is located
destdir - destination directory for the rpm package extraction
keeprpm - check if the user wants to delete rpms from the tmp directory
exact_files - extract only specified files
Returns:
RETURN_FAILURE in case of a serious problem
"""
package_full_path = tmp_dir + "/" + package_file_name
log1("Extracting %s to %s", package_full_path, destdir)
log2("%s", files)
print _("Extracting cpio from {0}").format(package_full_path)
unpacked_cpio_path = tmp_dir + "/unpacked.cpio"
try:
unpacked_cpio = open(unpacked_cpio_path, 'wb')
except IOError, ex:
print _("Can't write to '{0}': {1}").format(unpacked_cpio_path, ex)
return RETURN_FAILURE
def end(self, payload, status, msg):
# One may objects that this call back isn't necessary because the
# progress() callback is called when downloading finishes, but if the
# downloaded package is already in a local cache, DNF doesn't call
# progress() callback at all but yet we want to inform user about that
# progress.
if status in [STATUS_OK, STATUS_DRPM, STATUS_ALREADY_EXISTS]:
self.observer.update(str(payload), 100)
elif status == STATUS_MIRROR:
# In this case dnf (librepo) tries other mirror if available
log1("Mirror failed: %s" % (msg or "DNF did not provide more details"))
elif status == STATUS_FAILED:
log2("Downloading failed: %s" % (msg or "DNF did not provide more details"))
else:
sys.stderr.write("Unknown DNF download status: %s\n" % (msg))
def end(self, payload, status, msg):
# One may objects that this call back isn't necessary because the
# progress() callback is called when downloading finishes, but if the
# downloaded package is already in a local cache, DNF doesn't call
# progress() callback at all but yet we want to inform user about that
# progress.
if status in [STATUS_OK, STATUS_DRPM, STATUS_ALREADY_EXISTS]:
self.observer.update(str(payload), 100)
elif status == STATUS_MIRROR:
# In this case dnf (librepo) tries other mirror if available
log1("Mirror failed: %s" % (msg or "DNF did not provide more details"))
elif status == STATUS_FAILED:
log2("Downloading failed: %s" % (msg or "DNF did not provide more details"))
else:
sys.stderr.write("Unknown DNF download status: %s\n" % (msg))
def initialize_repositories(self):
# setting-up repos one-by-one, so we can skip the broken ones...
# this helps when users are using 3rd party repos like rpmfusion
# in rawhide it results in: Can't find valid base url...
for r in self.base.repos.findRepos(pattern=self.repo_pattern):
try:
rid = self.base.repos.enableRepo(r.id)
self.base.repos.doSetup(thisrepo=str(r.id))
log1("enabled repo %s", rid)
setattr(r, "skip_if_unavailable", True)
# yes, we want async download, otherwise our progressCallback
# is not called and the internal yum's one is used,
# which causes artifacts on output
try:
setattr(r, "_async", False)
except (NameError, AttributeError) as ex:
print(str(ex))
print(
_("Can't disable async download, the output might contain artifacts!"
))
except YumBaseError as ex:
print(
_("Can't setup {0}: {1}, disabling").format(r.id, str(ex)))
self.base.repos.disableRepo(r.id)
# This is somewhat "magic", it unpacks the metadata making it usable.
# Looks like this is the moment when yum talks to remote servers,
# which takes time (sometimes minutes), let user know why
# we have "paused":
try:
self.base.repos.populateSack(mdtype='metadata', cacheonly=1)
except YumBaseError as ex:
print(_("Error retrieving metadata: '{0!s}'").format(str(ex)))
#we don't want to die here, some metadata might be already retrieved
# so there is a chance we already have what we need
#return 1
try:
# Saw this exception here:
# raise Errors.NoMoreMirrorsRepoError, errstr
# NoMoreMirrorsRepoError: failure:
# repodata/7e6632b82c91a2e88a66ad848e231f14c48259cbf3a1c3e992a77b1fc0e9d2f6-filelists.sqlite.bz2
# from fedora-debuginfo: [Errno 256] No more mirrors to try.
self.base.repos.populateSack(mdtype='filelists', cacheonly=1)
except YumBaseError as ex:
print(_("Error retrieving filelists: '{0!s}'").format(str(ex)))
def initialize_repositories(self):
# setting-up repos one-by-one, so we can skip the broken ones...
# this helps when users are using 3rd party repos like rpmfusion
# in rawhide it results in: Can't find valid base url...
for r in self.base.repos.findRepos(pattern=self.repo_pattern):
try:
rid = self.base.repos.enableRepo(r.id)
self.base.repos.doSetup(thisrepo=str(r.id))
log1("enabled repo %s", rid)
setattr(r, "skip_if_unavailable", True)
# yes, we want async download, otherwise our progressCallback
# is not called and the internal yum's one is used,
# which causes artifacts on output
try:
setattr(r, "_async", False)
except (NameError, AttributeError) as ex:
print(str(ex))
print(_("Can't disable async download, the output might contain artifacts!"))
except YumBaseError as ex:
print(_("Can't setup {0}: {1}, disabling").format(r.id, str(ex)))
self.base.repos.disableRepo(r.id)
# This is somewhat "magic", it unpacks the metadata making it usable.
# Looks like this is the moment when yum talks to remote servers,
# which takes time (sometimes minutes), let user know why
# we have "paused":
try:
self.base.repos.populateSack(mdtype='metadata', cacheonly=1)
except YumBaseError as ex:
print(_("Error retrieving metadata: '{0!s}'").format(str(ex)))
#we don't want to die here, some metadata might be already retrieved
# so there is a chance we already have what we need
#return 1
try:
# Saw this exception here:
# raise Errors.NoMoreMirrorsRepoError, errstr
# NoMoreMirrorsRepoError: failure:
# repodata/7e6632b82c91a2e88a66ad848e231f14c48259cbf3a1c3e992a77b1fc0e9d2f6-filelists.sqlite.bz2
# from fedora-debuginfo: [Errno 256] No more mirrors to try.
self.base.repos.populateSack(mdtype='filelists', cacheonly=1)
except YumBaseError as ex:
print(_("Error retrieving filelists: '{0!s}'").format(str(ex)))
def unpack_rpm(package_full_path, files, tmp_dir, destdir, exact_files=False):
"""
Unpacks a single rpm located in tmp_dir into destdir.
Arguments:
package_full_path - full file system path to the rpm file
files - files to extract from the rpm
tmp_dir - temporary directory where the rpm file is located
destdir - destination directory for the rpm package extraction
exact_files - extract only specified files
Returns:
RETURN_FAILURE in case of a serious problem
"""
log1("Extracting %s to %s", package_full_path, destdir)
log2("%s", files)
print(_("Extracting cpio from {0}").format(package_full_path))
unpacked_cpio_path = tmp_dir + "/unpacked.cpio"
try:
unpacked_cpio = open(unpacked_cpio_path, 'wb')
except IOError as ex:
print(_("Can't write to '{0}': {1}").format(unpacked_cpio_path, ex))
return RETURN_FAILURE
rpm2cpio = Popen(["rpm2cpio", package_full_path],
stdout=unpacked_cpio,
bufsize=-1)
retcode = rpm2cpio.wait()
if retcode == 0:
log1("cpio written OK")
else:
unpacked_cpio.close()
print(_("Can't extract package '{0}'").format(package_full_path))
return RETURN_FAILURE
# close the file
unpacked_cpio.close()
# and open it for reading
unpacked_cpio = open(unpacked_cpio_path, 'rb')
print(
_("Caching files from {0} made from {1}").format(
"unpacked.cpio", os.path.basename(package_full_path)))
file_patterns = ""
cpio_args = ["cpio", "-idu"]
if exact_files:
for filename in files:
file_patterns += "." + filename + " "
cpio_args = ["cpio", "-idu", file_patterns.strip()]
with tempfile.NamedTemporaryFile(prefix='abrt-unpacking-',
dir='/tmp',
delete=False) as log_file:
log_file_name = log_file.name
cpio = Popen(cpio_args,
cwd=destdir,
bufsize=-1,
stdin=unpacked_cpio,
stdout=log_file,
stderr=log_file)
retcode = cpio.wait()
unpacked_cpio.close()
if retcode == 0:
log1("files extracted OK")
#print _("Removing temporary cpio file")
os.unlink(log_file_name)
os.unlink(unpacked_cpio_path)
else:
print(
_("Can't extract files from '{0}'. For more information see '{1}'"
).format(unpacked_cpio_path, log_file_name))
return RETURN_FAILURE
def download(self, files, download_exact_files=False):
"""
Downloads rpms shipping given files into a temporary directory
Arguments:
file - a list of files to download
download_exact_files - extract only specified files
Returns:
RETURN_OK if all goes well.
RETURN_FAILURE in case it cannot set up either of the directories.
"""
# nothing to download?
if not files:
return RETURN_FAILURE
# set up tmp and cache dirs so that we can check free space in both
retval = self.setup_tmp_dirs()
if retval != RETURN_OK:
return retval
if not self.find_packages_run:
self.find_packages(files)
if verbose != 0 or len(self.not_found) != 0:
print(
_("Can't find packages for {0} debuginfo files").format(
len(self.not_found)))
if verbose != 0 or len(self.package_files_dict) != 0:
print(
_("Packages to download: {0}").format(
len(self.package_files_dict)))
question = _(
"Downloading {0:.2f}Mb, installed size: {1:.2f}Mb. Continue?") \
.format(self.todownload_size / (1024 * 1024),
self.installed_size / (1024 * 1024))
if not self.noninteractive and not ask_yes_no(question):
print(_("Download cancelled by user"))
return RETURN_CANCEL_BY_USER
# check if there is enough free space in both tmp and cache
res = os.statvfs(self.tmpdir)
tmp_space = float(res.f_bsize * res.f_bavail) / (1024 * 1024)
if (self.todownload_size / (1024 * 1024)) > tmp_space:
question = _("Warning: Not enough free space in tmp dir '{0}'"
" ({1:.2f}Mb left). Continue?").format(
self.tmpdir, tmp_space)
if not self.noninteractive and not ask_yes_no(question):
print(_("Download cancelled by user"))
return RETURN_CANCEL_BY_USER
res = os.statvfs(self.cachedir)
cache_space = float(res.f_bsize * res.f_bavail) / (1024 * 1024)
if (self.installed_size / (1024 * 1024)) > cache_space:
question = _("Warning: Not enough free space in cache dir "
"'{0}' ({1:.2f}Mb left). Continue?").format(
self.cachedir, cache_space)
if not self.noninteractive and not ask_yes_no(question):
print(_("Download cancelled by user"))
return RETURN_CANCEL_BY_USER
progress_observer = DownloadProgress(len(self.package_files_dict))
self.initialize_progress(progress_observer)
for pkg, files in self.package_files_dict.items():
# Download
package_full_path, err = self.download_package(pkg)
if err:
# I observed a zero-length file left on error,
# which prevents cleanup later. Fix it:
try:
if package_full_path is not None:
os.unlink(package_full_path)
except OSError:
pass
print(_("Downloading package {0} failed").format(pkg))
else:
unpack_result = unpack_rpm(package_full_path,
files,
self.tmpdir,
self.cachedir,
exact_files=download_exact_files)
if unpack_result == RETURN_FAILURE:
# recursively delete the temp dir on failure
print(_("Unpacking failed, aborting download..."))
s = os.stat(self.cachedir)
abrt = pwd.getpwnam("abrt")
if (s.st_uid != abrt.pw_uid) or (s.st_gid != abrt.pw_gid):
print(
_("'{0}' must be owned by abrt. "
"Please run '# chown -R abrt.abrt {0}' "
"to fix the issue.").format(self.cachedir))
clean_up(self.tmpdir)
return RETURN_FAILURE
if not self.keeprpms:
log1("keeprpms = False, removing %s", package_full_path)
os.unlink(package_full_path)
progress_observer.downloaded_pkgs += 1
if not self.keeprpms and os.path.exists(self.tmpdir):
# Was: "All downloaded packages have been extracted, removing..."
# but it was appearing even if no packages were in fact extracted
# (say, when there was one package, and it has download error).
print(_("Removing {0}").format(self.tmpdir))
try:
os.rmdir(self.tmpdir)
except OSError:
error_msg(
_("Can't remove {0}, probably contains an error log").
format(self.tmpdir))
return RETURN_OK
log1("Extracting %s to %s", package_full_path, destdir)
log2("%s", files)
print _("Extracting cpio from {0}").format(package_full_path)
unpacked_cpio_path = tmp_dir + "/unpacked.cpio"
try:
unpacked_cpio = open(unpacked_cpio_path, 'wb')
except IOError, ex:
print _("Can't write to '{0}': {1}").format(unpacked_cpio_path, ex)
return RETURN_FAILURE
rpm2cpio = Popen(["rpm2cpio", package_full_path],
stdout = unpacked_cpio, bufsize = -1)
retcode = rpm2cpio.wait()
if retcode == 0:
log1("cpio written OK")
if not keeprpm:
log1("keeprpms = False, removing %s", package_full_path)
#print _("Removing temporary rpm file")
os.unlink(package_full_path)
else:
unpacked_cpio.close()
print _("Can't extract package '{0}'").format(package_full_path)
return RETURN_FAILURE
# close the file
unpacked_cpio.close()
# and open it for reading
unpacked_cpio = open(unpacked_cpio_path, 'rb')
print _("Caching files from {0} made from {1}").format("unpacked.cpio", package_file_name)
def unpack_rpm(package_full_path, files, tmp_dir, destdir, exact_files=False):
"""
Unpacks a single rpm located in tmp_dir into destdir.
Arguments:
package_full_path - full file system path to the rpm file
files - files to extract from the rpm
tmp_dir - temporary directory where the rpm file is located
destdir - destination directory for the rpm package extraction
exact_files - extract only specified files
Returns:
RETURN_FAILURE in case of a serious problem
"""
log1("Extracting %s to %s", package_full_path, destdir)
log2("%s", files)
print(_("Extracting cpio from {0}").format(package_full_path))
unpacked_cpio_path = tmp_dir + "/unpacked.cpio"
try:
unpacked_cpio = open(unpacked_cpio_path, 'wb')
except IOError as ex:
print(_("Can't write to '{0}': {1}").format(unpacked_cpio_path, ex))
return RETURN_FAILURE
rpm2cpio = Popen(["rpm2cpio", package_full_path],
stdout=unpacked_cpio, bufsize=-1)
retcode = rpm2cpio.wait()
if retcode == 0:
log1("cpio written OK")
else:
unpacked_cpio.close()
print(_("Can't extract package '{0}'").format(package_full_path))
return RETURN_FAILURE
# close the file
unpacked_cpio.close()
# and open it for reading
unpacked_cpio = open(unpacked_cpio_path, 'rb')
print(_("Caching files from {0} made from {1}").format("unpacked.cpio", os.path.basename(package_full_path)))
file_patterns = ""
cpio_args = ["cpio", "-idu"]
if exact_files:
for filename in files:
file_patterns += "." + filename + " "
cpio_args = ["cpio", "-idu", file_patterns.strip()]
with tempfile.NamedTemporaryFile(prefix='abrt-unpacking-', dir='/tmp',
delete=False) as log_file:
log_file_name = log_file.name
cpio = Popen(cpio_args, cwd=destdir, bufsize=-1,
stdin=unpacked_cpio, stdout=log_file, stderr=log_file)
retcode = cpio.wait()
unpacked_cpio.close()
if retcode == 0:
log1("files extracted OK")
#print _("Removing temporary cpio file")
os.unlink(log_file_name)
os.unlink(unpacked_cpio_path)
else:
print(_("Can't extract files from '{0}'. For more information see '{1}'")
.format(unpacked_cpio_path, log_file_name))
return RETURN_FAILURE
def download(self, files, download_exact_files=False):
"""
Downloads rpms shipping given files into a temporary directory
Arguments:
file - a list of files to download
download_exact_files - extract only specified files
Returns:
RETURN_OK if all goes well.
RETURN_FAILURE in case it cannot set up either of the directories.
"""
# nothing to download?
if not files:
return RETURN_FAILURE
# set up tmp and cache dirs so that we can check free space in both
retval = self.setup_tmp_dirs()
if retval != RETURN_OK:
return retval
print(_("Initializing package manager"))
self.prepare()
#if verbose == 0:
# # this suppress yum messages about setting up repositories
# mute_stdout()
# This takes some time, let user know what we are doing
print(_("Setting up repositories"))
self.initialize_repositories()
#if verbose == 0:
# # re-enable the output to stdout
# unmute_stdout()
print(_("Looking for needed packages in repositories"))
package_files_dict, not_found, todownload_size, installed_size = self.triage(files)
if verbose != 0 or len(not_found) != 0:
print(_("Can't find packages for {0} debuginfo files").format(len(not_found)))
if verbose != 0 or len(package_files_dict) != 0:
print(_("Packages to download: {0}").format(len(package_files_dict)))
question = _(
"Downloading {0:.2f}Mb, installed size: {1:.2f}Mb. Continue?") \
.format(todownload_size / (1024 * 1024),
installed_size / (1024 * 1024))
if not self.noninteractive and not ask_yes_no(question):
print(_("Download cancelled by user"))
return RETURN_CANCEL_BY_USER
# check if there is enough free space in both tmp and cache
res = os.statvfs(self.tmpdir)
tmp_space = float(res.f_bsize * res.f_bavail) / (1024 * 1024)
if (todownload_size / (1024 * 1024)) > tmp_space:
question = _("Warning: Not enough free space in tmp dir '{0}'"
" ({1:.2f}Mb left). Continue?").format(
self.tmpdir, tmp_space)
if not self.noninteractive and not ask_yes_no(question):
print(_("Download cancelled by user"))
return RETURN_CANCEL_BY_USER
res = os.statvfs(self.cachedir)
cache_space = float(res.f_bsize * res.f_bavail) / (1024 * 1024)
if (installed_size / (1024 * 1024)) > cache_space:
question = _("Warning: Not enough free space in cache dir "
"'{0}' ({1:.2f}Mb left). Continue?").format(
self.cachedir, cache_space)
if not self.noninteractive and not ask_yes_no(question):
print(_("Download cancelled by user"))
return RETURN_CANCEL_BY_USER
progress_observer = DownloadProgress(len(package_files_dict))
self.initialize_progress(progress_observer)
for pkg, files in package_files_dict.items():
# Download
package_full_path, err = self.download_package(pkg)
if err:
# I observed a zero-length file left on error,
# which prevents cleanup later. Fix it:
try:
os.unlink(package_full_path)
except OSError:
pass
print(_("Downloading package {0} failed").format(pkg))
else:
unpack_result = unpack_rpm(package_full_path, files, self.tmpdir,
self.cachedir, exact_files=download_exact_files)
if unpack_result == RETURN_FAILURE:
# recursively delete the temp dir on failure
print(_("Unpacking failed, aborting download..."))
s = os.stat(self.cachedir)
abrt = pwd.getpwnam("abrt")
if (s.st_uid != abrt.pw_uid) or (s.st_gid != abrt.pw_gid):
print(_("'{0}' must be owned by abrt. "
"Please run '# chown -R abrt.abrt {0}' "
"to fix the issue.").format(self.cachedir))
clean_up(self.tmpdir)
return RETURN_FAILURE
if not self.keeprpms:
log1("keeprpms = False, removing %s", package_full_path)
os.unlink(package_full_path)
progress_observer.downloaded_pkgs += 1
if not self.keeprpms and os.path.exists(self.tmpdir):
# Was: "All downloaded packages have been extracted, removing..."
# but it was appearing even if no packages were in fact extracted
# (say, when there was one package, and it has download error).
print(_("Removing {0}").format(self.tmpdir))
try:
os.rmdir(self.tmpdir)
except OSError:
error_msg(_("Can't remove {0}, probably contains an error log").format(self.tmpdir))
return RETURN_OK
def download(self, files, download_exact_files=False):
"""
Downloads rpms shipping given files into a temporary directory
Arguments:
file - a list of files to download
download_exact_files - extract only specified files
Returns:
RETURN_OK if all goes well.
RETURN_FAILURE in case it cannot set up either of the directories.
"""
# nothing to download?
if not files:
return RETURN_FAILURE
# set up tmp and cache dirs so that we can check free space in both
retval = self.setup_tmp_dirs()
if retval != RETURN_OK:
return retval
print(_("Initializing package manager"))
self.prepare()
#if verbose == 0:
# # this suppress yum messages about setting up repositories
# mute_stdout()
# This takes some time, let user know what we are doing
print(_("Setting up repositories"))
self.initialize_repositories()
#if verbose == 0:
# # re-enable the output to stdout
# unmute_stdout()
print(_("Looking for needed packages in repositories"))
package_files_dict, not_found, todownload_size, installed_size = self.triage(files)
if verbose != 0 or len(not_found) != 0:
print(_("Can't find packages for {0} debuginfo files").format(len(not_found)))
if verbose != 0 or len(package_files_dict) != 0:
print(_("Packages to download: {0}").format(len(package_files_dict)))
question = _("Downloading {0:.2f}Mb, installed size: {1:.2f}Mb. Continue?").format(
todownload_size / (1024*1024),
installed_size / (1024*1024)
)
if self.noninteractive == False and not ask_yes_no(question):
print(_("Download cancelled by user"))
return RETURN_CANCEL_BY_USER
# check if there is enough free space in both tmp and cache
res = os.statvfs(self.tmpdir)
tmp_space = float(res.f_bsize * res.f_bavail) / (1024*1024)
if (todownload_size / (1024*1024)) > tmp_space:
question = _("Warning: Not enough free space in tmp dir '{0}'"
" ({1:.2f}Mb left). Continue?").format(
self.tmpdir, tmp_space)
if not self.noninteractive and not ask_yes_no(question):
print(_("Download cancelled by user"))
return RETURN_CANCEL_BY_USER
res = os.statvfs(self.cachedir)
cache_space = float(res.f_bsize * res.f_bavail) / (1024*1024)
if (installed_size / (1024*1024)) > cache_space:
question = _("Warning: Not enough free space in cache dir "
"'{0}' ({1:.2f}Mb left). Continue?").format(
self.cachedir, cache_space)
if not self.noninteractive and not ask_yes_no(question):
print(_("Download cancelled by user"))
return RETURN_CANCEL_BY_USER
progress_observer = DownloadProgress(len(package_files_dict))
self.initialize_progress(progress_observer)
for pkg, files in package_files_dict.items():
# Download
package_full_path, err = self.download_package(pkg)
if err:
# I observed a zero-length file left on error,
# which prevents cleanup later. Fix it:
try:
os.unlink(package_full_path)
except OSError:
pass
print(_("Downloading package {0} failed").format(pkg))
else:
unpack_result = unpack_rpm(package_full_path, files, self.tmpdir,
self.cachedir, exact_files=download_exact_files)
if unpack_result == RETURN_FAILURE:
# recursively delete the temp dir on failure
print(_("Unpacking failed, aborting download..."))
clean_up(self.tmpdir)
return RETURN_FAILURE
if not self.keeprpms:
log1("keeprpms = False, removing %s", package_full_path)
os.unlink(package_full_path)
progress_observer.downloaded_pkgs += 1
if not self.keeprpms and os.path.exists(self.tmpdir):
# Was: "All downloaded packages have been extracted, removing..."
# but it was appearing even if no packages were in fact extracted
# (say, when there was one package, and it has download error).
print(_("Removing {0}").format(self.tmpdir))
try:
os.rmdir(self.tmpdir)
except OSError:
error_msg(_("Can't remove {0}, probably contains an error log").format(self.tmpdir))
return RETURN_OK