def find_packages(self, files):
self.find_packages_run = True
# nothing to download?
if not files:
return RETURN_FAILURE
print(_("Initializing package manager"))
self.prepare()
# This takes some time, let user know what we are doing
print(_("Setting up repositories"))
self.initialize_repositories()
print(_("Looking for needed packages in repositories"))
(self.package_files_dict, self.not_found, self.todownload_size,
self.installed_size) = self.triage(files)
def prepare(self):
try:
self.base.read_all_repos()
except dnf.exceptions.Error as ex:
print(
_("Error reading repository configuration: '{0!s}'").format(
str(ex)))
def find_packages(self, files):
self.find_packages_run = True;
# nothing to download?
if not files:
return RETURN_FAILURE
print(_("Initializing package manager"))
self.prepare()
# This takes some time, let user know what we are doing
print(_("Setting up repositories"))
self.initialize_repositories()
print(_("Looking for needed packages in repositories"))
(self.package_files_dict,
self.not_found,
self.todownload_size,
self.installed_size) = self.triage(files)
def update(self, name, pct):
"""
A method used to update the progress
Arguments:
name - filename
pct - percent downloaded
"""
if pct == self.last_pct:
log2("percentage is the same, not updating progress")
return
self.last_pct = pct
# if run from terminal we can have fancy output
if sys.stdout.isatty():
sys.stdout.write("\033[sDownloading (%i of %i) %s: %3u%%\033[u"
% (self.downloaded_pkgs + 1, self.total_pkgs, name, pct)
)
if pct == 100:
#print (_("Downloading (%i of %i) %s: %3u%%")
# % (self.downloaded_pkgs + 1, self.total_pkgs, name, pct)
#)
print(_("Downloading ({0} of {1}) {2}: {3:3}%").format(
self.downloaded_pkgs + 1, self.total_pkgs, name, pct
)
)
# but we want machine friendly output when spawned from abrt-server
else:
t = time.time()
if self.last_time == 0:
self.last_time = t
# update only every 5 seconds
if pct == 100 or self.last_time > t or t - self.last_time >= 5:
print(_("Downloading ({0} of {1}) {2}: {3:3}%").format(
self.downloaded_pkgs + 1, self.total_pkgs, name, pct
)
)
self.last_time = t
if pct == 100:
self.last_time = 0
sys.stdout.flush()
def clean_up(tmp_dir, silent=False):
"""
Removes the temporary directory.
"""
if tmp_dir:
try:
shutil.rmtree(tmp_dir)
except OSError as ex:
if ex.errno != errno.ENOENT and silent == False:
error_msg(_("Can't remove '{0}': {1}").format(tmp_dir, str(ex)))
def clean_up(tmp_dir):
"""
Removes the temporary directory.
"""
if tmp_dir:
try:
shutil.rmtree(tmp_dir)
except OSError as ex:
if ex.errno != errno.ENOENT:
error_msg(_("Can't remove '{0}': {1}").format(tmp_dir, str(ex)))
def initialize_repositories(self):
# enable only debug repositories
for repo in self.base.repos.all():
repo.disable()
for repo in self.base.repos.get_matching(self.repo_pattern):
repo.skip_if_unavailable = True
repo.enable()
try:
self.base.fill_sack()
except RepoError as ex:
print(_("Error setting up repositories: '{0!s}'").format(str(ex)))
def initialize_repositories(self):
# enable only debug repositories
for repo in self.base.repos.all():
repo.disable()
for repo in self.base.repos.get_matching(self.repo_pattern):
repo.skip_if_unavailable = True
repo.enable()
try:
self.base.fill_sack()
except RepoError as ex:
print(_("Error setting up repositories: '{0!s}'").format(str(ex)))
def unpack_rpm(package_full_path, files, tmp_dir, destdir, exact_files=False):
"""
Unpacks a single rpm located in tmp_dir into destdir.
Arguments:
package_full_path - full file system path to the rpm file
files - files to extract from the rpm
tmp_dir - temporary directory where the rpm file is located
destdir - destination directory for the rpm package extraction
exact_files - extract only specified files
Returns:
RETURN_FAILURE in case of a serious problem
"""
log1("Extracting %s to %s", package_full_path, destdir)
log2("%s", files)
print(_("Extracting cpio from {0}").format(package_full_path))
unpacked_cpio_path = tmp_dir + "/unpacked.cpio"
try:
unpacked_cpio = open(unpacked_cpio_path, 'wb')
except IOError as ex:
print(_("Can't write to '{0}': {1}").format(unpacked_cpio_path, ex))
return RETURN_FAILURE
rpm2cpio = Popen(["rpm2cpio", package_full_path],
stdout=unpacked_cpio,
bufsize=-1)
retcode = rpm2cpio.wait()
if retcode == 0:
log1("cpio written OK")
else:
unpacked_cpio.close()
print(_("Can't extract package '{0}'").format(package_full_path))
return RETURN_FAILURE
# close the file
unpacked_cpio.close()
# and open it for reading
unpacked_cpio = open(unpacked_cpio_path, 'rb')
print(
_("Caching files from {0} made from {1}").format(
"unpacked.cpio", os.path.basename(package_full_path)))
file_patterns = ""
cpio_args = ["cpio", "-idu"]
if exact_files:
for filename in files:
file_patterns += "." + filename + " "
cpio_args = ["cpio", "-idu", file_patterns.strip()]
with tempfile.NamedTemporaryFile(prefix='abrt-unpacking-',
dir='/tmp',
delete=False) as log_file:
log_file_name = log_file.name
cpio = Popen(cpio_args,
cwd=destdir,
bufsize=-1,
stdin=unpacked_cpio,
stdout=log_file,
stderr=log_file)
retcode = cpio.wait()
unpacked_cpio.close()
if retcode == 0:
log1("files extracted OK")
#print _("Removing temporary cpio file")
os.unlink(log_file_name)
os.unlink(unpacked_cpio_path)
else:
print(
_("Can't extract files from '{0}'. For more information see '{1}'"
).format(unpacked_cpio_path, log_file_name))
return RETURN_FAILURE
def download(self, files, download_exact_files=False):
"""
Downloads rpms shipping given files into a temporary directory
Arguments:
file - a list of files to download
download_exact_files - extract only specified files
Returns:
RETURN_OK if all goes well.
RETURN_FAILURE in case it cannot set up either of the directories.
"""
# nothing to download?
if not files:
return RETURN_FAILURE
# set up tmp and cache dirs so that we can check free space in both
retval = self.setup_tmp_dirs()
if retval != RETURN_OK:
return retval
if not self.find_packages_run:
self.find_packages(files)
if verbose != 0 or len(self.not_found) != 0:
print(
_("Can't find packages for {0} debuginfo files").format(
len(self.not_found)))
if verbose != 0 or len(self.package_files_dict) != 0:
print(
_("Packages to download: {0}").format(
len(self.package_files_dict)))
question = _(
"Downloading {0:.2f}Mb, installed size: {1:.2f}Mb. Continue?") \
.format(self.todownload_size / (1024 * 1024),
self.installed_size / (1024 * 1024))
if not self.noninteractive and not ask_yes_no(question):
print(_("Download cancelled by user"))
return RETURN_CANCEL_BY_USER
# check if there is enough free space in both tmp and cache
res = os.statvfs(self.tmpdir)
tmp_space = float(res.f_bsize * res.f_bavail) / (1024 * 1024)
if (self.todownload_size / (1024 * 1024)) > tmp_space:
question = _("Warning: Not enough free space in tmp dir '{0}'"
" ({1:.2f}Mb left). Continue?").format(
self.tmpdir, tmp_space)
if not self.noninteractive and not ask_yes_no(question):
print(_("Download cancelled by user"))
return RETURN_CANCEL_BY_USER
res = os.statvfs(self.cachedir)
cache_space = float(res.f_bsize * res.f_bavail) / (1024 * 1024)
if (self.installed_size / (1024 * 1024)) > cache_space:
question = _("Warning: Not enough free space in cache dir "
"'{0}' ({1:.2f}Mb left). Continue?").format(
self.cachedir, cache_space)
if not self.noninteractive and not ask_yes_no(question):
print(_("Download cancelled by user"))
return RETURN_CANCEL_BY_USER
progress_observer = DownloadProgress(len(self.package_files_dict))
self.initialize_progress(progress_observer)
for pkg, files in self.package_files_dict.items():
# Download
package_full_path, err = self.download_package(pkg)
if err:
# I observed a zero-length file left on error,
# which prevents cleanup later. Fix it:
try:
if package_full_path is not None:
os.unlink(package_full_path)
except OSError:
pass
print(_("Downloading package {0} failed").format(pkg))
else:
unpack_result = unpack_rpm(package_full_path,
files,
self.tmpdir,
self.cachedir,
exact_files=download_exact_files)
if unpack_result == RETURN_FAILURE:
# recursively delete the temp dir on failure
print(_("Unpacking failed, aborting download..."))
s = os.stat(self.cachedir)
abrt = pwd.getpwnam("abrt")
if (s.st_uid != abrt.pw_uid) or (s.st_gid != abrt.pw_gid):
print(
_("'{0}' must be owned by abrt. "
"Please run '# chown -R abrt.abrt {0}' "
"to fix the issue.").format(self.cachedir))
clean_up(self.tmpdir)
return RETURN_FAILURE
if not self.keeprpms:
log1("keeprpms = False, removing %s", package_full_path)
os.unlink(package_full_path)
progress_observer.downloaded_pkgs += 1
if not self.keeprpms and os.path.exists(self.tmpdir):
# Was: "All downloaded packages have been extracted, removing..."
# but it was appearing even if no packages were in fact extracted
# (say, when there was one package, and it has download error).
print(_("Removing {0}").format(self.tmpdir))
try:
os.rmdir(self.tmpdir)
except OSError:
error_msg(
_("Can't remove {0}, probably contains an error log").
format(self.tmpdir))
return RETURN_OK
def prepare(self):
try:
self.base.read_all_repos()
except dnf.exceptions.Error as ex:
print(_("Error reading repository configuration: '{0!s}'").format(str(ex)))
def unpack_rpm(package_full_path, files, tmp_dir, destdir, exact_files=False):
"""
Unpacks a single rpm located in tmp_dir into destdir.
Arguments:
package_full_path - full file system path to the rpm file
files - files to extract from the rpm
tmp_dir - temporary directory where the rpm file is located
destdir - destination directory for the rpm package extraction
exact_files - extract only specified files
Returns:
RETURN_FAILURE in case of a serious problem
"""
log1("Extracting %s to %s", package_full_path, destdir)
log2("%s", files)
print(_("Extracting cpio from {0}").format(package_full_path))
unpacked_cpio_path = tmp_dir + "/unpacked.cpio"
try:
unpacked_cpio = open(unpacked_cpio_path, 'wb')
except IOError as ex:
print(_("Can't write to '{0}': {1}").format(unpacked_cpio_path, ex))
return RETURN_FAILURE
rpm2cpio = Popen(["rpm2cpio", package_full_path],
stdout=unpacked_cpio, bufsize=-1)
retcode = rpm2cpio.wait()
if retcode == 0:
log1("cpio written OK")
else:
unpacked_cpio.close()
print(_("Can't extract package '{0}'").format(package_full_path))
return RETURN_FAILURE
# close the file
unpacked_cpio.close()
# and open it for reading
unpacked_cpio = open(unpacked_cpio_path, 'rb')
print(_("Caching files from {0} made from {1}").format("unpacked.cpio", os.path.basename(package_full_path)))
file_patterns = ""
cpio_args = ["cpio", "-idu"]
if exact_files:
for filename in files:
file_patterns += "." + filename + " "
cpio_args = ["cpio", "-idu", file_patterns.strip()]
with tempfile.NamedTemporaryFile(prefix='abrt-unpacking-', dir='/tmp',
delete=False) as log_file:
log_file_name = log_file.name
cpio = Popen(cpio_args, cwd=destdir, bufsize=-1,
stdin=unpacked_cpio, stdout=log_file, stderr=log_file)
retcode = cpio.wait()
unpacked_cpio.close()
if retcode == 0:
log1("files extracted OK")
#print _("Removing temporary cpio file")
os.unlink(log_file_name)
os.unlink(unpacked_cpio_path)
else:
print(_("Can't extract files from '{0}'. For more information see '{1}'")
.format(unpacked_cpio_path, log_file_name))
return RETURN_FAILURE
def download(self, files, download_exact_files=False):
"""
Downloads rpms shipping given files into a temporary directory
Arguments:
file - a list of files to download
download_exact_files - extract only specified files
Returns:
RETURN_OK if all goes well.
RETURN_FAILURE in case it cannot set up either of the directories.
"""
# nothing to download?
if not files:
return RETURN_FAILURE
# set up tmp and cache dirs so that we can check free space in both
retval = self.setup_tmp_dirs()
if retval != RETURN_OK:
return retval
print(_("Initializing package manager"))
self.prepare()
#if verbose == 0:
# # this suppress yum messages about setting up repositories
# mute_stdout()
# This takes some time, let user know what we are doing
print(_("Setting up repositories"))
self.initialize_repositories()
#if verbose == 0:
# # re-enable the output to stdout
# unmute_stdout()
print(_("Looking for needed packages in repositories"))
package_files_dict, not_found, todownload_size, installed_size = self.triage(files)
if verbose != 0 or len(not_found) != 0:
print(_("Can't find packages for {0} debuginfo files").format(len(not_found)))
if verbose != 0 or len(package_files_dict) != 0:
print(_("Packages to download: {0}").format(len(package_files_dict)))
question = _(
"Downloading {0:.2f}Mb, installed size: {1:.2f}Mb. Continue?") \
.format(todownload_size / (1024 * 1024),
installed_size / (1024 * 1024))
if not self.noninteractive and not ask_yes_no(question):
print(_("Download cancelled by user"))
return RETURN_CANCEL_BY_USER
# check if there is enough free space in both tmp and cache
res = os.statvfs(self.tmpdir)
tmp_space = float(res.f_bsize * res.f_bavail) / (1024 * 1024)
if (todownload_size / (1024 * 1024)) > tmp_space:
question = _("Warning: Not enough free space in tmp dir '{0}'"
" ({1:.2f}Mb left). Continue?").format(
self.tmpdir, tmp_space)
if not self.noninteractive and not ask_yes_no(question):
print(_("Download cancelled by user"))
return RETURN_CANCEL_BY_USER
res = os.statvfs(self.cachedir)
cache_space = float(res.f_bsize * res.f_bavail) / (1024 * 1024)
if (installed_size / (1024 * 1024)) > cache_space:
question = _("Warning: Not enough free space in cache dir "
"'{0}' ({1:.2f}Mb left). Continue?").format(
self.cachedir, cache_space)
if not self.noninteractive and not ask_yes_no(question):
print(_("Download cancelled by user"))
return RETURN_CANCEL_BY_USER
progress_observer = DownloadProgress(len(package_files_dict))
self.initialize_progress(progress_observer)
for pkg, files in package_files_dict.items():
# Download
package_full_path, err = self.download_package(pkg)
if err:
# I observed a zero-length file left on error,
# which prevents cleanup later. Fix it:
try:
os.unlink(package_full_path)
except OSError:
pass
print(_("Downloading package {0} failed").format(pkg))
else:
unpack_result = unpack_rpm(package_full_path, files, self.tmpdir,
self.cachedir, exact_files=download_exact_files)
if unpack_result == RETURN_FAILURE:
# recursively delete the temp dir on failure
print(_("Unpacking failed, aborting download..."))
s = os.stat(self.cachedir)
abrt = pwd.getpwnam("abrt")
if (s.st_uid != abrt.pw_uid) or (s.st_gid != abrt.pw_gid):
print(_("'{0}' must be owned by abrt. "
"Please run '# chown -R abrt.abrt {0}' "
"to fix the issue.").format(self.cachedir))
clean_up(self.tmpdir)
return RETURN_FAILURE
if not self.keeprpms:
log1("keeprpms = False, removing %s", package_full_path)
os.unlink(package_full_path)
progress_observer.downloaded_pkgs += 1
if not self.keeprpms and os.path.exists(self.tmpdir):
# Was: "All downloaded packages have been extracted, removing..."
# but it was appearing even if no packages were in fact extracted
# (say, when there was one package, and it has download error).
print(_("Removing {0}").format(self.tmpdir))
try:
os.rmdir(self.tmpdir)
except OSError:
error_msg(_("Can't remove {0}, probably contains an error log").format(self.tmpdir))
return RETURN_OK
def download(self, files, download_exact_files=False):
"""
Downloads rpms shipping given files into a temporary directory
Arguments:
file - a list of files to download
download_exact_files - extract only specified files
Returns:
RETURN_OK if all goes well.
RETURN_FAILURE in case it cannot set up either of the directories.
"""
# nothing to download?
if not files:
return RETURN_FAILURE
# set up tmp and cache dirs so that we can check free space in both
retval = self.setup_tmp_dirs()
if retval != RETURN_OK:
return retval
print(_("Initializing package manager"))
self.prepare()
#if verbose == 0:
# # this suppress yum messages about setting up repositories
# mute_stdout()
# This takes some time, let user know what we are doing
print(_("Setting up repositories"))
self.initialize_repositories()
#if verbose == 0:
# # re-enable the output to stdout
# unmute_stdout()
print(_("Looking for needed packages in repositories"))
package_files_dict, not_found, todownload_size, installed_size = self.triage(files)
if verbose != 0 or len(not_found) != 0:
print(_("Can't find packages for {0} debuginfo files").format(len(not_found)))
if verbose != 0 or len(package_files_dict) != 0:
print(_("Packages to download: {0}").format(len(package_files_dict)))
question = _("Downloading {0:.2f}Mb, installed size: {1:.2f}Mb. Continue?").format(
todownload_size / (1024*1024),
installed_size / (1024*1024)
)
if self.noninteractive == False and not ask_yes_no(question):
print(_("Download cancelled by user"))
return RETURN_CANCEL_BY_USER
# check if there is enough free space in both tmp and cache
res = os.statvfs(self.tmpdir)
tmp_space = float(res.f_bsize * res.f_bavail) / (1024*1024)
if (todownload_size / (1024*1024)) > tmp_space:
question = _("Warning: Not enough free space in tmp dir '{0}'"
" ({1:.2f}Mb left). Continue?").format(
self.tmpdir, tmp_space)
if not self.noninteractive and not ask_yes_no(question):
print(_("Download cancelled by user"))
return RETURN_CANCEL_BY_USER
res = os.statvfs(self.cachedir)
cache_space = float(res.f_bsize * res.f_bavail) / (1024*1024)
if (installed_size / (1024*1024)) > cache_space:
question = _("Warning: Not enough free space in cache dir "
"'{0}' ({1:.2f}Mb left). Continue?").format(
self.cachedir, cache_space)
if not self.noninteractive and not ask_yes_no(question):
print(_("Download cancelled by user"))
return RETURN_CANCEL_BY_USER
progress_observer = DownloadProgress(len(package_files_dict))
self.initialize_progress(progress_observer)
for pkg, files in package_files_dict.items():
# Download
package_full_path, err = self.download_package(pkg)
if err:
# I observed a zero-length file left on error,
# which prevents cleanup later. Fix it:
try:
os.unlink(package_full_path)
except OSError:
pass
print(_("Downloading package {0} failed").format(pkg))
else:
unpack_result = unpack_rpm(package_full_path, files, self.tmpdir,
self.cachedir, exact_files=download_exact_files)
if unpack_result == RETURN_FAILURE:
# recursively delete the temp dir on failure
print(_("Unpacking failed, aborting download..."))
clean_up(self.tmpdir)
return RETURN_FAILURE
if not self.keeprpms:
log1("keeprpms = False, removing %s", package_full_path)
os.unlink(package_full_path)
progress_observer.downloaded_pkgs += 1
if not self.keeprpms and os.path.exists(self.tmpdir):
# Was: "All downloaded packages have been extracted, removing..."
# but it was appearing even if no packages were in fact extracted
# (say, when there was one package, and it has download error).
print(_("Removing {0}").format(self.tmpdir))
try:
os.rmdir(self.tmpdir)
except OSError:
error_msg(_("Can't remove {0}, probably contains an error log").format(self.tmpdir))
return RETURN_OK
