def get_message(message, default_yes=True, any_input=False):
"""
Function for command line messages
:param message: prompt string
:param default_yes: If the default value is YES
:param any_input: if True, return input without checking it first
:return: True or False, based on user's input
"""
if default_yes:
choice = '[Y/n]'
else:
choice = '[y/N]'
if any_input:
msg = '{0} '.format(message)
else:
msg = '{0} {1}? '.format(message, choice)
while True:
user_input = input(msg).lower()
if not user_input or any_input:
return True if default_yes else False
try:
user_input = strtobool(user_input)
except ValueError:
logger.error('You have to type y(es) or n(o).')
continue
if any_input:
return True
else:
return bool(user_input)
def run(cls, package_name):
version = cls._get_version(package_name)
if version:
return version
logger.error("Failed to determine latest upstream version!\n"
"Check that the package exists on %s.", cls.BASE_URL)
return None
def _call_rebase_helper(self):
""" Clonning repository and call rebase-helper """
logger.info('Clonning repository %s', self.url)
try:
# git clone http://pkgs.fedoraproject.org/cgit/emacs.git/
git.Git().clone(self.url)
except git.exc.GitCommandError as gce:
logger.error(gce.message)
return
os.chdir(self.package)
pprint(self.arguments)
cli = CLI(self.arguments)
try:
app = Application(cli)
# TDO After a deep testing app.run() will be used
#app.run()
logger.info(app.kwargs)
sources = app.prepare_sources()
app.patch_sources(sources)
build = app.build_packages()
if build:
app.pkgdiff_packages()
self.patches = app.rebased_patches
self._print_patches()
logger.info(app.debug_log_file)
except RebaseHelperError as rbe:
logger.error(rbe.message)
def _get_initial_patches_list(self):
"""Method returns a list of patches from a spec file"""
patches_applied = []
patches_not_used = []
patches_list = [p for p in self.spc.sources if p[2] == 2]
patch_flags = self._get_patches_flags()
for filename, num, patch_type in patches_list:
patch_path = os.path.join(self.sources_location, filename)
if not os.path.exists(patch_path):
logger.error('Patch %s does not exist', filename)
continue
patch_num = num
if patch_flags:
if num in patch_flags:
patch_num, patch_option = patch_flags[num]
patches_applied.append(
PatchObject(patch_path, patch_num, patch_option))
else:
patches_not_used.append(
PatchObject(patch_path, patch_num, None))
else:
patches_applied.append(PatchObject(patch_path, patch_num,
None))
patches_applied = sorted(patches_applied, key=lambda x: x.get_index())
return {"applied": patches_applied, "not_applied": patches_not_used}
def _get_initial_patches_list(self):
"""
Method returns a list of patches from a spec file
"""
patches_applied = []
patches_not_used = []
patches_list = [p for p in self.spc.sources if p[2] == 2]
patch_flags = self._get_patches_flags()
for filename, num, patch_type in patches_list:
patch_path = os.path.join(self.sources_location, filename)
if not os.path.exists(patch_path):
logger.error('Patch %s does not exist', filename)
continue
patch_num = num
if patch_flags:
if num in patch_flags:
patch_num, patch_option = patch_flags[num]
patches_applied.append(PatchObject(patch_path, patch_num, patch_option))
else:
patches_not_used.append(PatchObject(patch_path, patch_num, None))
else:
patches_applied.append(PatchObject(patch_path, patch_num, None))
patches_applied = sorted(patches_applied, key=lambda x: x.get_index())
return {"applied": patches_applied, "not_applied": patches_not_used}
def get_rpm_packages(self, dirname):
"""
Function returns RPM packages stored in dirname/old and dirname/new directories
:param dirname: directory where are stored old and new RPMS
:return:
"""
found = True
for version in ['old', 'new']:
data = {}
data['name'] = self.spec_file.get_package_name()
if version == 'old':
spec_version = self.spec_file.get_version()
else:
spec_version = self.rebase_spec_file.get_version()
data['version'] = spec_version
data['rpm'] = PathHelper.find_all_files(
os.path.join(os.path.realpath(dirname), version, 'RPM'),
'*.rpm')
if not data['rpm']:
logger.error(
'Your path %s%s/RPM does not contain any RPM packages',
dirname, version)
found = False
results_store.set_build_data(version, data)
if not found:
return False
return True
def run(cls, package_name):
version = cls._get_version(package_name)
if version:
return version
logger.error("Failed to determine latest upstream version!\n"
"Check that the package exists on %s.", cls.BASE_URL)
return None
def get_message(message, default_yes=True, any_input=False):
"""
Function for command line messages
:param message: prompt string
:param default_yes: If the default value is YES
:param any_input: if True, return input without checking it first
:return: True or False, based on user's input
"""
if default_yes:
choice = '[Y/n]'
else:
choice = '[y/N]'
if any_input:
msg = '{0} '.format(message)
else:
msg = '{0} {1}? '.format(message, choice)
while True:
user_input = input(msg).lower()
if not user_input or any_input:
return True if default_yes else False
try:
user_input = strtobool(user_input)
except ValueError:
logger.error('You have to type y(es) or n(o).')
continue
if any_input:
return True
else:
return bool(user_input)
def check_output_argument(output_tool):
"""
Function checks whether output_tool argument is allowed
"""
if output_tool not in output_tools.keys():
logger.error('You have to specify one of these printing output tools %s', output_tools.keys())
sys.exit(0)
def run(self):
if self.conf.fedpkg_build_tasks:
logger.warning("Option --fedpkg-build-tasks is deprecated, use --build-tasks instead.")
if not self.conf.build_tasks:
self.conf.build_tasks = self.conf.fedpkg_build_tasks
if self.conf.build_tasks and not self.conf.builds_nowait:
if self.conf.buildtool in ['fedpkg', 'copr']:
logger.error("--builds-nowait has to be specified with --build-tasks.")
return 1
else:
logger.warning("Options are allowed only for fedpkg or copr build tools. Suppress them.")
self.conf.build_tasks = self.conf.builds_nowait = False
sources = None
if self.conf.build_tasks is None:
sources = self.prepare_sources()
if not self.conf.build_only and not self.conf.comparepkgs:
self.patch_sources(sources)
build = False
if not self.conf.patch_only:
if not self.conf.comparepkgs:
# check build dependencies for rpmbuild
if self.conf.buildtool == 'rpmbuild':
Application.check_build_requires(self.spec_file)
# Build packages
try:
build = self.build_packages()
if self.conf.builds_nowait and not self.conf.build_tasks:
if self.conf.buildtool == 'fedpkg':
self.print_koji_logs()
elif self.conf.buildtool == 'copr':
self.print_copr_logs()
return 0
except RuntimeError:
logger.error('Unknown error caused by build log analysis')
return 1
# Perform checks
else:
build = self.get_rpm_packages(self.conf.comparepkgs)
# We don't care dirname doesn't contain any RPM packages
# Therefore return 1
if build:
self.pkgdiff_packages(self.results_dir)
else:
if not self.upstream_monitoring:
logger.info('Rebase package to %s FAILED. See for more details', self.conf.sources)
return 1
self.print_summary()
if not self.conf.keep_workspace:
self._delete_workspace_dir()
if self.debug_log_file:
logger.info("Detailed debug log is located in '%s'", self.debug_log_file)
if not self.upstream_monitoring and not self.conf.patch_only:
logger.info('Rebase package to %s was SUCCESSFUL.\n', self.conf.sources)
return 0
def _parse_build_log(cls, log_path, nvr):
"""Parses a build log.
Args:
log_path (str): Path to the RPM build log.
Returns:
tuple: The first element is the type of error that was found
in the log (missing or deleted). The second element is
a list of problematic files.
"""
try:
with open(log_path, 'r') as build_log:
lines = build_log.read().splitlines()
except IOError:
logger.error('There was an error opening %s', log_path)
return None, None
error_type = None
# Use set to avoid duplicate files.
files = set()
# Two types of error can occur. The file is either in the SPEC file but missing in sources
# or the other way around. Example lines of the first case:
# File not found: /root/rpmbuild/BUILDROOT/pello-0.1.1-1.fc27.x86_64/should/not/be/here
# File not found: /root/rpmbuild/BUILDROOT/pello-0.1.1-1.fc27.x86_64/should/not/be/here2
# Example lines of the second case:
# Installed (but unpackaged) file(s) found:
# /usr/bin/pello
error_re = re.compile(
r'''
^
(BUILDSTDERR:)?
\s*
(
(?P<missing>File\s+not\s+found:\s*)|
(?P<unpackaged>Installed\s+\(but\s+unpackaged\)\s+file\(s\)\s+found:)
)?
(/.*/{}\.\w+)?
(?P<path>/.*)?
$
'''.format(re.escape(nvr)), re.VERBOSE
)
for line in lines:
match = error_re.match(line)
if match:
if match.group('missing'):
error_type = 'deleted'
files.add(match.group('path'))
elif match.group('unpackaged'):
error_type = 'missing'
elif error_type == 'missing' and match.group('path'):
files.add(match.group('path'))
elif error_type and not match.group('path'):
break
return error_type, list(files)
def run_check(cls, result_dir):
"""Compares old and new RPMs using pkgdiff"""
debug_old, rest_pkgs_old = cls._get_packages_for_abipkgdiff(OutputLogger.get_build('old'))
debug_new, rest_pkgs_new = cls._get_packages_for_abipkgdiff(OutputLogger.get_build('new'))
cmd = [cls.CMD]
if debug_old is None:
logger.warning("Package doesn't contain any debug package")
return None
try:
cmd.append('--d1')
cmd.append(debug_old[0])
except IndexError:
logger.error('Debuginfo package not found for old package.')
return None
try:
cmd.append('--d2')
cmd.append(debug_new[0])
except IndexError:
logger.error('Debuginfo package not found for new package.')
return None
reports = {}
for pkg in rest_pkgs_old:
command = list(cmd)
# Package can be <letters><numbers>-<letters>-<and_whatever>
regexp = r'^(\w*)(-\D+)?.*$'
reg = re.compile(regexp)
matched = reg.search(os.path.basename(pkg))
if matched:
file_name = matched.group(1)
command.append(pkg)
find = [x for x in rest_pkgs_new if os.path.basename(x).startswith(file_name)]
command.append(find[0])
package_name = os.path.basename(os.path.basename(pkg))
logger.debug('Package name for ABI comparision %s', package_name)
regexp_name = r'(\w-)*(\D+)*'
reg_name = re.compile(regexp_name)
matched = reg_name.search(os.path.basename(pkg))
logger.debug('Found matches %s', matched.groups())
if matched:
package_name = matched.group(0) + cls.log_name
else:
package_name = package_name + '-' + cls.log_name
output = os.path.join(cls.results_dir, result_dir, package_name)
try:
ret_code = ProcessHelper.run_subprocess(command, output=output)
except OSError:
raise CheckerNotFoundError("Checker '%s' was not found or installed." % cls.CMD)
if int(ret_code) & settings.ABIDIFF_ERROR and int(ret_code) & settings.ABIDIFF_USAGE_ERROR:
raise RebaseHelperError('Execution of %s failed.\nCommand line is: %s' % (cls.CMD, cmd))
if int(ret_code) == 0:
text = 'ABI of the compared binaries in package %s are equal.' % package_name
else:
text = 'ABI of the compared binaries in package %s are not equal.' % package_name
reports[output] = text
else:
logger.debug("Rebase-helper did not find a package name in '%s'", package_name)
return reports
def run_check(cls, result_dir):
"""Compares old and new RPMs using pkgdiff"""
debug_old, rest_pkgs_old = cls._get_packages_for_abipkgdiff(results_store.get_build('old'))
debug_new, rest_pkgs_new = cls._get_packages_for_abipkgdiff(results_store.get_build('new'))
cmd = [cls.CMD]
if debug_old is None:
logger.warning("Package doesn't contain any debug package")
return None
try:
cmd.append('--d1')
cmd.append(debug_old[0])
except IndexError:
logger.error('Debuginfo package not found for old package.')
return None
try:
cmd.append('--d2')
cmd.append(debug_new[0])
except IndexError:
logger.error('Debuginfo package not found for new package.')
return None
reports = {}
for pkg in rest_pkgs_old:
command = list(cmd)
# Package can be <letters><numbers>-<letters>-<and_whatever>
regexp = r'^(\w*)(-\D+)?.*$'
reg = re.compile(regexp)
matched = reg.search(os.path.basename(pkg))
if matched:
file_name = matched.group(1)
command.append(pkg)
find = [x for x in rest_pkgs_new if os.path.basename(x).startswith(file_name)]
command.append(find[0])
package_name = os.path.basename(os.path.basename(pkg))
logger.debug('Package name for ABI comparision %s', package_name)
regexp_name = r'(\w-)*(\D+)*'
reg_name = re.compile(regexp_name)
matched = reg_name.search(os.path.basename(pkg))
logger.debug('Found matches %s', matched.groups())
if matched:
package_name = matched.group(0) + cls.log_name
else:
package_name = package_name + '-' + cls.log_name
output = os.path.join(cls.results_dir, result_dir, package_name)
try:
ret_code = ProcessHelper.run_subprocess(command, output=output)
except OSError:
raise CheckerNotFoundError("Checker '%s' was not found or installed." % cls.CMD)
if int(ret_code) & settings.ABIDIFF_ERROR and int(ret_code) & settings.ABIDIFF_USAGE_ERROR:
raise RebaseHelperError('Execution of %s failed.\nCommand line is: %s' % (cls.CMD, cmd))
if int(ret_code) == 0:
text = 'ABI of the compared binaries in package %s are equal.' % package_name
else:
text = 'ABI of the compared binaries in package %s are not equal.' % package_name
reports[output] = text
else:
logger.debug("Rebase-helper did not find a package name in '%s'", package_name)
return reports
def _parse_build_log(cls, log_path, nvr):
"""Parses a build log.
Args:
log_path (str): Path to the RPM build log.
Returns:
tuple: The first element is the type of error that was found
in the log (missing or deleted). The second element is
a list of problematic files.
"""
try:
with open(log_path, 'r') as build_log:
lines = build_log.read().splitlines()
except IOError:
logger.error('There was an error opening %s', log_path)
return None, None
error_type = None
# Use set to avoid duplicate files.
files = set()
# Two types of error can occur. The file is either in the SPEC file but missing in sources
# or the other way around. Example lines of the first case:
# File not found: /root/rpmbuild/BUILDROOT/pello-0.1.1-1.fc27.x86_64/should/not/be/here
# File not found: /root/rpmbuild/BUILDROOT/pello-0.1.1-1.fc27.x86_64/should/not/be/here2
# Example lines of the second case:
# Installed (but unpackaged) file(s) found:
# /usr/bin/pello
error_re = re.compile(
r'''
^
(BUILDSTDERR:)?
\s*
(
(?P<missing>File\s+not\s+found:\s*)|
(?P<unpackaged>Installed\s+\(but\s+unpackaged\)\s+file\(s\)\s+found:)
)?
(/.*/{}\.\w+)?
(?P<path>/.*)?
$
'''.format(re.escape(nvr)), re.VERBOSE)
for line in lines:
match = error_re.match(line)
if match:
if match.group('missing'):
error_type = 'deleted'
files.add(match.group('path'))
elif match.group('unpackaged'):
error_type = 'missing'
elif error_type == 'missing' and match.group('path'):
files.add(match.group('path'))
elif error_type and not match.group('path'):
break
return error_type, list(files)
def build_source_packages(self):
try:
builder = plugin_manager.srpm_build_tools.get_plugin(self.conf.srpm_buildtool)
except NotImplementedError as e:
raise RebaseHelperError('{}. Supported SRPM build tools are {}'.format(
str(e), plugin_manager.srpm_build_tools.get_supported_tools()))
for version in ['old', 'new']:
koji_build_id = None
results_dir = '{}-build'.format(os.path.join(self.results_dir, version))
spec = self.spec_file if version == 'old' else self.rebase_spec_file
package_name = spec.get_package_name()
package_version = spec.get_version()
package_full_version = spec.get_full_version()
logger.info('Building source package for %s version %s', package_name, package_full_version)
if version == 'old' and self.conf.get_old_build_from_koji:
koji_build_id, package_version, package_full_version = KojiHelper.get_old_build_info(package_name,
package_version)
build_dict = dict(
name=package_name,
version=package_version,
srpm_buildtool=self.conf.srpm_buildtool,
srpm_builder_options=self.conf.srpm_builder_options)
try:
if koji_build_id:
session = KojiHelper.create_session()
build_dict['srpm'], build_dict['logs'] = KojiHelper.download_build(session,
koji_build_id,
os.path.join(
results_dir,
'SRPM'
),
arches=['src'])
else:
build_dict.update(builder.build(spec, results_dir, **build_dict))
build_dict = self._sanitize_build_dict(build_dict)
results_store.set_build_data(version, build_dict)
except RebaseHelperError: # pylint: disable=try-except-raise
raise
except SourcePackageBuildError as e:
build_dict.update(builder.get_logs())
build_dict['source_package_build_error'] = str(e)
build_dict = self._sanitize_build_dict(build_dict)
results_store.set_build_data(version, build_dict)
if e.logfile:
msg = 'Building {} SRPM packages failed; see {} for more information'.format(version, e.logfile)
else:
msg = 'Building {} SRPM packages failed; see logs in {} for more information'.format(
version, os.path.join(results_dir, 'SRPM'))
logger.error(msg)
raise RebaseHelperError(msg, logfiles=builder.get_logs().get('logs'))
except Exception:
raise RebaseHelperError('Building package failed with unknown reason. '
'Check all available log files.')
def run():
try:
cli = CLI(sys.argv[1:])
app = Application(cli)
app.run()
except KeyboardInterrupt:
logger.info('\nInterrupted by user')
except RebaseHelperError as e:
logger.error('\n%s', exc_as_decode_string(e))
sys.exit(1)
sys.exit(0)
def _call_rebase_helper(self):
""" Clonning repository and call rebase-helper """
logger.info('Clonning repository %s', self.url)
git.Git().clone(self.url)
os.chdir(self.package)
cli = CLI(self.arguments)
try:
app = Application(cli)
app.run()
self.patches = app.rebased_patches
self._print_patches()
logger.info(app.debug_log_file)
except RebaseHelperError as rbe:
logger.error(rbe.message)
def request(url, **kwargs):
"""Performs an HTTP request or an FTP RETR command.
Args:
url (str): HTTP, HTTPS or FTP URL.
**kwargs: Keyword arguments to be passed to requests.session.get().
Returns:
requests.Response: Response object.
"""
class FTPAdapter(requests.adapters.BaseAdapter):
def send(
self,
request,
stream=False,
timeout=None,
verify=True, # pylint: disable=unused-argument
cert=None,
proxies=None): # pylint: disable=unused-argument
response = requests.models.Response()
response.request = request
response.connection = self
try:
resp = urllib.request.urlopen(request.url)
except urllib.error.URLError as e:
response.status_code = 400
response.reason = e.reason
else:
response.status_code = 200
response.headers = requests.structures.CaseInsensitiveDict(
getattr(resp, 'headers', {}))
response.raw = resp
response.url = resp.url
return response
def close(self):
pass
session = requests.Session()
session.mount('ftp://', FTPAdapter())
try:
return session.get(url, **kwargs)
except requests.exceptions.RequestException as e:
logger.error('%s: %s', type(e).__name__, six.text_type(e))
return None
def run():
debug_log_file = None
try:
cli = CLI()
if hasattr(cli, 'version'):
logger.info(VERSION)
sys.exit(0)
config = Config(getattr(cli, 'config-file', None))
config.merge(cli)
for handler in [main_handler, output_tool_handler]:
handler.set_terminal_background(config.background)
ConsoleHelper.use_colors = ConsoleHelper.should_use_colors(config)
execution_dir, results_dir, debug_log_file = Application.setup(
config)
traceback_log = os.path.join(results_dir, LOGS_DIR, TRACEBACK_LOG)
if config.verbose == 0:
main_handler.setLevel(logging.INFO)
elif config.verbose == 1:
main_handler.setLevel(CustomLogger.VERBOSE)
else:
main_handler.setLevel(logging.DEBUG)
app = Application(config, execution_dir, results_dir,
debug_log_file)
app.run()
except KeyboardInterrupt:
logger.info('Interrupted by user')
except RebaseHelperError as e:
if e.msg:
logger.error('%s', e.msg)
else:
logger.error('%s', six.text_type(e))
sys.exit(1)
except SystemExit as e:
sys.exit(e.code)
except BaseException:
if debug_log_file:
logger.error(
'rebase-helper failed due to an unexpected error. Please report this problem'
'\nusing the following link: %s'
'\nand include the content of'
'\n\'%s\' and'
'\n\'%s\''
'\nin the report.'
'\nThank you!', NEW_ISSUE_LINK, debug_log_file,
traceback_log)
LoggerHelper.add_file_handler(logger_traceback, traceback_log)
logger_traceback.trace('', exc_info=1)
else:
logger.error(
'rebase-helper failed due to an unexpected error. Please report this problem'
'\nusing the following link: %s'
'\nThank you!', NEW_ISSUE_LINK)
sys.exit(1)
sys.exit(0)
def run_package_checkers(self, results_dir):
"""
Runs checkers on packages and stores results in a given directory.
:param results_dir: Path to directory in which to store the results.
:type results_dir: str
:return: None
"""
results = dict()
for checker_name in self.conf.pkgcomparetool:
try:
results[checker_name] = checkers_runner.run_checker(results_dir, checker_name)
except CheckerNotFoundError:
logger.error("Rebase-helper did not find checker '%s'." % checker_name)
for diff_name, result in six.iteritems(results):
results_store.set_checker_output(diff_name, result)
def run():
debug_log_file = None
try:
cli = CLI()
if hasattr(cli, 'version'):
logger.info(VERSION)
sys.exit(0)
config = Config(getattr(cli, 'config-file', None))
config.merge(cli)
for handler in [main_handler, output_tool_handler]:
handler.set_terminal_background(config.background)
ConsoleHelper.use_colors = ConsoleHelper.should_use_colors(config)
execution_dir, results_dir, debug_log_file = Application.setup(config)
traceback_log = os.path.join(results_dir, LOGS_DIR, TRACEBACK_LOG)
if config.verbose == 0:
main_handler.setLevel(logging.INFO)
elif config.verbose == 1:
main_handler.setLevel(CustomLogger.VERBOSE)
else:
main_handler.setLevel(logging.DEBUG)
app = Application(config, execution_dir, results_dir, debug_log_file)
app.run()
except KeyboardInterrupt:
logger.info('Interrupted by user')
except RebaseHelperError as e:
if e.msg:
logger.error('%s', e.msg)
else:
logger.error('%s', str(e))
sys.exit(1)
except SystemExit as e:
sys.exit(e.code)
except BaseException:
if debug_log_file:
logger.error('rebase-helper failed due to an unexpected error. Please report this problem'
'\nusing the following link: %s'
'\nand include the content of'
'\n\'%s\' and'
'\n\'%s\''
'\nin the report.'
'\nThank you!',
NEW_ISSUE_LINK, debug_log_file, traceback_log)
LoggerHelper.add_file_handler(logger_traceback, traceback_log)
logger_traceback.trace('', exc_info=1)
else:
logger.error('rebase-helper failed due to an unexpected error. Please report this problem'
'\nusing the following link: %s'
'\nThank you!',
NEW_ISSUE_LINK)
sys.exit(1)
sys.exit(0)
def request(url, **kwargs):
"""Performs an HTTP request or an FTP RETR command.
Args:
url (str): HTTP, HTTPS or FTP URL.
**kwargs: Keyword arguments to be passed to requests.session.get().
Returns:
requests.Response: Response object.
"""
class FTPAdapter(requests.adapters.BaseAdapter):
def send(self, request, stream=False, timeout=None, verify=True, # pylint: disable=unused-argument
cert=None, proxies=None): # pylint: disable=unused-argument
response = requests.models.Response()
response.request = request
response.connection = self
try:
resp = urllib.request.urlopen(request.url)
except urllib.error.URLError as e:
response.status_code = 400
response.reason = e.reason
else:
response.status_code = 200
response.headers = requests.structures.CaseInsensitiveDict(getattr(resp, 'headers', {}))
response.raw = resp
response.url = resp.url
return response
def close(self):
pass
session = requests.Session()
session.mount('ftp://', FTPAdapter())
try:
return session.get(url, **kwargs)
except requests.exceptions.RequestException as e:
logger.error('%s: %s', type(e).__name__, str(e))
return None
def run_package_checkers(self, results_dir):
"""
Runs checkers on packages and stores results in a given directory.
:param results_dir: Path to directory in which to store the results.
:type results_dir: str
:return: None
"""
results = dict()
for checker_name in self.conf.pkgcomparetool:
try:
results[checker_name] = checkers_runner.run_checker(
os.path.join(results_dir, 'checkers'), checker_name)
except CheckerNotFoundError:
logger.error("Rebase-helper did not find checker '%s'." %
checker_name)
for diff_name, result in six.iteritems(results):
results_store.set_checker_output(diff_name, result)
def run():
debug_log_file = None
try:
# be verbose until debug_log_file is created
cli = CLI()
if hasattr(cli, 'version'):
logger.info(VERSION)
sys.exit(0)
config = Config(getattr(cli, 'config-file', None))
config.merge(cli)
for handler in [main_handler, output_tool_handler]:
handler.set_terminal_background(config.background)
ConsoleHelper.use_colors = ConsoleHelper.should_use_colors(config)
execution_dir, results_dir, debug_log_file = Application.setup(
config)
if not config.verbose:
main_handler.setLevel(logging.INFO)
app = Application(config, execution_dir, results_dir,
debug_log_file)
app.run()
except KeyboardInterrupt:
logger.info('Interrupted by user')
except RebaseHelperError as e:
if e.msg:
logger.error('%s', e.msg)
else:
logger.error('%s', six.text_type(e))
sys.exit(1)
except SystemExit as e:
sys.exit(e.code)
except BaseException:
if debug_log_file:
logger.error(
'rebase-helper failed due to an unexpected error. Please report this problem'
'\nusing the following link: %s'
'\nand include the content of'
'\n\'%s\''
'\nfile in the report.'
'\nThank you!', NEW_ISSUE_LINK, debug_log_file)
else:
logger.error(
'rebase-helper failed due to an unexpected error. Please report this problem'
'\nusing the following link: %s'
'\nand include the traceback following this message in the report.'
'\nThank you!', NEW_ISSUE_LINK)
logger.trace('', exc_info=1)
sys.exit(1)
sys.exit(0)
def run():
debug_log_file = None
try:
# be verbose until debug_log_file is created
handler = LoggerHelper.add_stream_handler(logger, logging.DEBUG)
if "--builder-options" in sys.argv[1:]:
raise RebaseHelperError(
"Wrong format of --builder-options. It must be in the following form:"
' --builder-options="--desired-builder-option".'
)
cli = CLI()
execution_dir, results_dir, debug_log_file, report_log_file = Application.setup(cli)
if not cli.verbose:
handler.setLevel(logging.INFO)
app = Application(cli, execution_dir, results_dir, debug_log_file, report_log_file)
app.run()
except KeyboardInterrupt:
logger.info("\nInterrupted by user")
except RebaseHelperError as e:
if e.args:
logger.error("\n%s", e.args[0] % e.args[1:])
else:
logger.error("\n%s", six.text_type(e))
sys.exit(1)
except SystemExit as e:
sys.exit(e.code)
except BaseException:
if debug_log_file:
logger.error(
"\nrebase-helper failed due to an unexpected error. Please report this problem"
"\nusing the following link: %s"
"\nand include the content of"
"\n'%s'"
"\nfile in the report."
"\nThank you!",
NEW_ISSUE_LINK,
debug_log_file,
)
else:
logger.error(
"\nrebase-helper failed due to an unexpected error. Please report this problem"
"\nusing the following link: %s"
"\nand include the traceback following this message in the report."
"\nThank you!",
NEW_ISSUE_LINK,
)
logger.debug("\n", exc_info=1)
sys.exit(1)
sys.exit(0)
def run_package_checkers(self, results_dir):
"""
Runs checkers on packages and stores results in a given directory.
:param results_dir: Path to directory in which to store the results.
:type results_dir: str
:return: None
"""
results = dict()
if self.conf.pkgcomparetool:
results[self.conf.pkgcomparetool] = checkers_runner.run_checker(results_dir, self.conf.pkgcomparetool)
else:
# no specific checker was given, just run all of them
for checker_name in checkers_runner.get_supported_tools():
try:
results[checker_name] = checkers_runner.run_checker(results_dir, checker_name)
except CheckerNotFoundError:
logger.error("Rebase-helper did not find checker '%s'." % checker_name)
for diff_name, result in six.iteritems(results):
OutputLogger.set_checker_output(diff_name, result)
def get_message(message, default_yes=True, any_input=False):
"""Prompts a user with yes/no message and gets the response.
Args:
message (str): Prompt string.
default_yes (bool): If the default value should be YES.
any_input (bool): Whether to return default value regardless of input.
Returns:
bool: True or False, based on user's input.
"""
if default_yes:
choice = '[Y/n]'
else:
choice = '[y/N]'
if any_input:
msg = '{0} '.format(message)
else:
msg = '{0} {1}? '.format(message, choice)
while True:
user_input = input(msg).lower()
if not user_input or any_input:
return True if default_yes else False
try:
user_input = strtobool(user_input)
except ValueError:
logger.error('You have to type y(es) or n(o).')
continue
if any_input:
return True
else:
return bool(user_input)
def run(self):
sources = self.prepare_sources()
if not self.conf.build_only and not self.conf.comparepkgs:
self.patch_sources(sources)
if not self.conf.patch_only:
if not self.conf.comparepkgs:
# check build dependencies for rpmbuild
if self.conf.buildtool == 'rpmbuild':
Application.check_build_requires(self.spec_file)
# Build packages
try:
build = self.build_packages()
except RuntimeError:
logger.error('Not know error caused by build log analysis')
return 1
# Perform checks
else:
build = self.get_rpm_packages(self.conf.comparepkgs)
# We don't care dirname doesn't contain any RPM packages
# Therefore return 1
if build:
self.pkgdiff_packages()
else:
if not self.upstream_monitoring:
logger.info('Rebase package to %s FAILED. See for more details' % self.conf.sources)
return 1
self.print_summary()
if not self.conf.keep_workspace:
self._delete_workspace_dir()
if self.debug_log_file:
logger.info("Detailed debug log is located in '%s'", self.debug_log_file)
if not self.upstream_monitoring and not self.conf.patch_only:
logger.info('Rebase package to %s was SUCCESSFUL.\n' % self.conf.sources)
return 0
def get_message(message, default_yes=True, any_input=False):
"""Prompts a user with yes/no message and gets the response.
Args:
message (str): Prompt string.
default_yes (bool): If the default value should be YES.
any_input (bool): Whether to return default value regardless of input.
Returns:
bool: True or False, based on user's input.
"""
if default_yes:
choice = '[Y/n]'
else:
choice = '[y/N]'
if any_input:
msg = '{0} '.format(message)
else:
msg = '{0} {1}? '.format(message, choice)
while True:
user_input = input(msg).lower()
if not user_input or any_input:
return True if default_yes else False
try:
user_input = strtobool(user_input)
except ValueError:
logger.error('You have to type y(es) or n(o).')
continue
if any_input:
return True
else:
return bool(user_input)
def run():
debug_log_file = None
try:
# be verbose until debug_log_file is created
handler = LoggerHelper.add_stream_handler(logger, logging.DEBUG)
if "--builder-options" in sys.argv[1:]:
raise RebaseHelperError(
'Wrong format of --builder-options. It must be in the following form:'
' --builder-options="--desired-builder-option".')
cli = CLI()
if cli.version:
logger.info(VERSION)
sys.exit(0)
ConsoleHelper.use_colors = ConsoleHelper.should_use_colors(cli)
execution_dir, results_dir, debug_log_file = Application.setup(cli)
if not cli.verbose:
handler.setLevel(logging.INFO)
app = Application(cli, execution_dir, results_dir, debug_log_file)
app.run()
except KeyboardInterrupt:
logger.info('\nInterrupted by user')
except RebaseHelperError as e:
if e.msg:
logger.error('\n%s', e.msg)
else:
logger.error('\n%s', six.text_type(e))
sys.exit(1)
except SystemExit as e:
sys.exit(e.code)
except BaseException:
if debug_log_file:
logger.error(
'\nrebase-helper failed due to an unexpected error. Please report this problem'
'\nusing the following link: %s'
'\nand include the content of'
'\n\'%s\''
'\nfile in the report.'
'\nThank you!', NEW_ISSUE_LINK, debug_log_file)
else:
logger.error(
'\nrebase-helper failed due to an unexpected error. Please report this problem'
'\nusing the following link: %s'
'\nand include the traceback following this message in the report.'
'\nThank you!', NEW_ISSUE_LINK)
logger.debug('\n', exc_info=1)
sys.exit(1)
sys.exit(0)
def get_rpm_packages(self, dirname):
"""
Function returns RPM packages stored in dirname/old and dirname/new directories
:param dirname: directory where are stored old and new RPMS
:return:
"""
found = True
for version in ['old', 'new']:
data = {}
data['name'] = self.spec_file.get_package_name()
if version == 'old':
spec_version = self.spec_file.get_version()
else:
spec_version = self.rebase_spec_file.get_version()
data['version'] = spec_version
data['rpm'] = PathHelper.find_all_files(os.path.join(os.path.realpath(dirname), version, 'RPM'), '*.rpm')
if not data['rpm']:
logger.error('Your path %s%s/RPM does not contain any RPM packages' % (dirname, version))
found = False
OutputLogger.set_build_data(version, data)
if not found:
return False
return True
def run_package_checkers(self, results_dir, **kwargs):
"""
Runs checkers on packages and stores results in a given directory.
:param results_dir: Path to directory in which to store the results.
:type results_dir: str
:param category: checker type(SOURCE/SRPM/RPM)
:type category: str
:return: None
"""
results = dict()
for checker_name in self.conf.pkgcomparetool:
try:
data = plugin_manager.checkers.run(os.path.join(results_dir, 'checkers'),
checker_name,
**kwargs)
if data:
results[checker_name] = data
except CheckerNotFoundError:
logger.error("Rebase-helper did not find checker '%s'.", checker_name)
for diff_name, result in results.items():
results_store.set_checker_output(diff_name, result)
def build_packages(self):
"""Function calls build class for building packages"""
if self.conf.buildtool == 'fedpkg' and not koji_builder:
print ('Importing module koji failed. Switching to mockbuild.')
self.conf.buildtool = 'mock'
try:
builder = Builder(self.conf.buildtool)
except NotImplementedError as ni_e:
raise RebaseHelperError('%s. Supported build tools are %s' % six.text_type(ni_e),
Builder.get_supported_tools())
for version in ['old', 'new']:
spec_object = self.spec_file if version == 'old' else self.rebase_spec_file
build_dict = {}
task_id = None
if self.conf.build_tasks is None:
build_dict['name'] = spec_object.get_package_name()
build_dict['version'] = spec_object.get_version()
patches = [x.get_path() for x in spec_object.get_patches()]
spec = spec_object.get_path()
sources = spec_object.get_sources()
logger.info('Building packages for %s version %s',
spec_object.get_package_name(),
spec_object.get_version())
else:
if version == 'old':
task_id = self.conf.build_tasks.split(',')[0]
else:
task_id = self.conf.build_tasks.split(',')[1]
results_dir = os.path.join(self.results_dir, version)
build_dict['builds_nowait'] = self.conf.builds_nowait
build_dict['build_tasks'] = self.conf.build_tasks
files = {}
number_retries = 0
while self.conf.build_retries != number_retries:
try:
if self.conf.build_tasks is None:
build_dict.update(builder.build(spec, sources, patches, results_dir, **build_dict))
if not self.conf.builds_nowait:
if self.conf.buildtool == 'fedpkg':
while True:
kh = KojiHelper()
build_dict['rpm'], build_dict['logs'] = kh.get_koji_tasks(build_dict['koji_task_id'], results_dir)
if build_dict['rpm']:
break
else:
if self.conf.build_tasks:
if self.conf.buildtool == 'fedpkg':
kh = KojiHelper()
try:
build_dict['rpm'], build_dict['logs'] = kh.get_koji_tasks(task_id, results_dir)
OutputLogger.set_build_data(version, build_dict)
if not build_dict['rpm']:
return False
except TypeError:
logger.info('Koji tasks are not finished yet. Try again later')
return False
elif self.conf.buildtool == 'copr':
copr_helper = CoprHelper()
client = copr_helper.get_client()
build_id = int(task_id)
status = copr_helper.get_build_status(client, build_id)
if status in ['importing', 'pending', 'starting', 'running']:
logger.info('Copr build is not finished yet. Try again later')
return False
else:
build_dict['rpm'], build_dict['logs'] = copr_helper.download_build(client, build_id, results_dir)
if status not in ['succeeded', 'skipped']:
logger.info('Copr build {} did not complete successfully'.format(build_id))
return False
# Build finishes properly. Go out from while cycle
OutputLogger.set_build_data(version, build_dict)
break
except SourcePackageBuildError:
# always fail for original version
if version == 'old':
raise RebaseHelperError('Creating old SRPM package failed.')
logger.error('Building source package failed.')
# TODO: implement log analyzer for SRPMs and add the checks here!!!
raise
except BinaryPackageBuildError:
# always fail for original version
rpm_dir = os.path.join(results_dir, 'RPM')
build_dict.update(builder.get_logs())
OutputLogger.set_build_data(version, build_dict)
build_log = 'build.log'
build_log_path = os.path.join(rpm_dir, build_log)
if version == 'old':
raise RebaseHelperError('Building old RPM package failed. Check log %s', build_log_path)
logger.error('Building binary packages failed.')
msg = 'Building package failed'
try:
files = BuildLogAnalyzer.parse_log(rpm_dir, build_log)
except BuildLogAnalyzerMissingError:
raise RebaseHelperError('Build log %s does not exist', build_log_path)
except BuildLogAnalyzerMakeError:
raise RebaseHelperError('%s during build. Check log %s', msg, build_log_path)
except BuildLogAnalyzerPatchError:
raise RebaseHelperError('%s during patching. Check log %s', msg, build_log_path)
except RuntimeError:
if self.conf.build_retries == number_retries:
raise RebaseHelperError('%s with unknown reason. Check log %s', msg, build_log_path)
if 'missing' in files:
missing_files = '\n'.join(files['missing'])
logger.info('Files not packaged in the SPEC file:\n%s', missing_files)
elif 'deleted' in files:
deleted_files = '\n'.join(files['deleted'])
logger.warning('Removed files packaged in SPEC file:\n%s', deleted_files)
else:
if self.conf.build_retries == number_retries:
raise RebaseHelperError("Build failed, but no issues were found in the build log %s", build_log)
self.rebase_spec_file.modify_spec_files_section(files)
if not self.conf.non_interactive:
msg = 'Do you want rebase-helper to try build the packages one more time'
if not ConsoleHelper.get_message(msg):
raise KeyboardInterrupt
else:
logger.warning('Some patches were not successfully applied')
# build just failed, otherwise we would break out of the while loop
logger.debug('Number of retries is %s', self.conf.build_retries)
if os.path.exists(os.path.join(results_dir, 'RPM')):
shutil.rmtree(os.path.join(results_dir, 'RPM'))
if os.path.exists(os.path.join(results_dir, 'SRPM')):
shutil.rmtree(os.path.join(results_dir, 'SRPM'))
number_retries += 1
if self.conf.build_retries == number_retries:
raise RebaseHelperError('Building package failed with unknow reason. Check all available log files.')
return True
def build_packages(self):
"""
Function calls build class for building packages
"""
try:
builder = Builder(self.conf.buildtool)
except NotImplementedError as ni_e:
raise RebaseHelperError('%s. Supported build tools are %s', ni_e.message, Builder.get_supported_tools())
for version in ['old', 'new']:
spec_object = self.spec_file if version == 'old' else self.rebase_spec_file
build_dict = {}
build_dict['name'] = spec_object.get_package_name()
build_dict['version'] = spec_object.get_version()
logger.debug(build_dict)
patches = [x.get_path() for x in spec_object.get_patches()]
results_dir = os.path.join(self.results_dir, version)
spec = spec_object.get_path()
sources = spec_object.get_sources()
failed_before = False
while True:
try:
build_dict.update(builder.build(spec, sources, patches, results_dir, **build_dict))
OutputLogger.set_build_data(version, build_dict)
break
except SourcePackageBuildError:
# always fail for original version
if version == 'old':
raise RebaseHelperError('Creating old SRPM package failed.')
logger.error('Building source package failed.')
# TODO: implement log analyzer for SRPMs and add the checks here!!!
raise
except BinaryPackageBuildError:
# always fail for original version
rpm_dir = os.path.join(results_dir, 'RPM')
build_log = 'build.log'
build_log_path = os.path.join(rpm_dir, build_log)
if version == 'old':
raise RebaseHelperError('Building old RPM package failed. Check log %s', build_log_path)
logger.error('Building binary packages failed.')
try:
files = BuildLogAnalyzer.parse_log(rpm_dir, build_log)
except BuildLogAnalyzerMissingError:
raise RebaseHelperError('Build log %s does not exist', build_log_path)
except BuildLogAnalyzerMakeError:
raise RebaseHelperError('Building package failed during build. Check log %s', build_log_path)
except BuildLogAnalyzerPatchError:
raise RebaseHelperError('Building package failed during patching. Check log %s' % build_log_path)
if files['missing']:
missing_files = '\n'.join(files['added'])
logger.info('Files not packaged in the SPEC file:\n%s', missing_files)
elif files['deleted']:
deleted_files = '\n'.join(files['deleted'])
logger.warning('Removed files packaged in SPEC file:\n%s', deleted_files)
else:
raise RebaseHelperError("Build failed, but no issues were found in the build log %s", build_log)
self.rebase_spec_file.modify_spec_files_section(files)
if not self.conf.non_interactive:
if failed_before:
if not ConsoleHelper.get_message('Do you want rebase-helper to try build the packages one more time'):
raise KeyboardInterrupt
else:
logger.warning('Some patches were not successfully applied')
shutil.rmtree(os.path.join(results_dir, 'RPM'))
shutil.rmtree(os.path.join(results_dir, 'SRPM'))
return False
# build just failed, otherwise we would break out of the while loop
failed_before = True
shutil.rmtree(os.path.join(results_dir, 'RPM'))
shutil.rmtree(os.path.join(results_dir, 'SRPM'))
return True
def watch_koji_tasks(cls, session, tasklist):
"""Function is copy/paste from pyrpkg/cli.py"""
if not tasklist:
return
# Place holder for return value
rh_tasks = {}
try:
tasks = {}
for task_id in tasklist:
tasks[task_id] = TaskWatcher(task_id,
session,
logger,
quiet=False)
while True:
all_done = True
for task_id, task in tasks.items():
changed = task.update()
info = session.getTaskInfo(task_id)
state = task.info['state']
if state == koji.TASK_STATES['FAILED']:
return {info['id']: state}
else:
if info['arch'] == 'x86_64' or info['arch'] == 'noarch':
rh_tasks[info['id']] = state
if not task.is_done():
all_done = False
else:
if changed:
cls.display_task_results(tasks)
if not task.is_success():
rh_tasks = None
try:
for child in session.getTaskChildren(task_id):
child_id = child['id']
if child_id not in tasks.keys():
tasks[child_id] = TaskWatcher(child_id,
session,
logger,
task.level + 1,
quiet=False)
tasks[child_id].update()
# If we found new children, go through the list
# again, in case they have children also
info = session.getTaskInfo(child_id)
state = task.info['state']
if state == koji.TASK_STATES['FAILED']:
return {info['id']: state}
else:
if info['arch'] == 'x86_64' or info[
'arch'] == 'noarch':
rh_tasks[info['id']] = state
all_done = False
except SSL.SysCallError as e:
logger.error('Detected SSL error: %s',
six.text_type(e))
if all_done:
cls.display_task_results(tasks)
break
time.sleep(1)
except (KeyboardInterrupt):
# A ^c should return non-zero so that it doesn't continue
# on to any && commands.
rh_tasks = None
return rh_tasks
def build_packages(self):
"""Function calls build class for building packages"""
if self.conf.buildtool == KojiBuildTool.CMD and not koji_builder:
logger.info(
'Importing module koji failed. Switching to mock builder.')
self.conf.buildtool = MockBuildTool.CMD
try:
builder = Builder(self.conf.buildtool)
except NotImplementedError as ni_e:
raise RebaseHelperError(
'%s. Supported build tools are %s' % six.text_type(ni_e),
Builder.get_supported_tools())
for version in ['old', 'new']:
spec_object = self.spec_file if version == 'old' else self.rebase_spec_file
build_dict = {}
task_id = None
if self.conf.build_tasks is None:
build_dict['name'] = spec_object.get_package_name()
build_dict['version'] = spec_object.get_version()
patches = [x.get_path() for x in spec_object.get_patches()]
spec = spec_object.get_path()
sources = spec_object.get_sources()
logger.info('Building packages for %s version %s',
spec_object.get_package_name(),
spec_object.get_full_version())
else:
if version == 'old':
task_id = self.conf.build_tasks[0]
else:
task_id = self.conf.build_tasks[1]
results_dir = os.path.join(self.results_dir, version)
build_dict['builds_nowait'] = self.conf.builds_nowait
build_dict['build_tasks'] = self.conf.build_tasks
build_dict['builder_options'] = self.conf.builder_options
files = {}
number_retries = 0
while self.conf.build_retries != number_retries:
try:
if self.conf.build_tasks is None:
build_dict.update(
builder.build(spec, sources, patches, results_dir,
**build_dict))
if not self.conf.builds_nowait:
if self.conf.buildtool == KojiBuildTool.CMD:
while not build_dict['rpm']:
kh = KojiHelper()
build_dict['rpm'], build_dict[
'logs'] = kh.get_koji_tasks(
build_dict['koji_task_id'],
results_dir)
else:
if self.conf.build_tasks:
if self.conf.buildtool == KojiBuildTool.CMD:
kh = KojiHelper()
try:
build_dict['rpm'], build_dict[
'logs'] = kh.get_koji_tasks(
task_id, results_dir)
results_store.set_build_data(
version, build_dict)
if not build_dict['rpm']:
return False
except TypeError:
logger.info(
'Koji tasks are not finished yet. Try again later'
)
return False
elif self.conf.buildtool == CoprBuildTool.CMD:
copr_helper = CoprHelper()
client = copr_helper.get_client()
build_id = int(task_id)
status = copr_helper.get_build_status(
client, build_id)
if status in [
'importing', 'pending', 'starting',
'running'
]:
logger.info(
'Copr build is not finished yet. Try again later'
)
return False
else:
build_dict['rpm'], build_dict[
'logs'] = copr_helper.download_build(
client, build_id, results_dir)
if status not in ['succeeded', 'skipped']:
logger.info(
'Copr build {} did not complete successfully'
.format(build_id))
return False
# Build finishes properly. Go out from while cycle
results_store.set_build_data(version, build_dict)
break
except SourcePackageBuildError:
build_dict.update(builder.get_logs())
results_store.set_build_data(version, build_dict)
# always fail for original version
if version == 'old':
raise RebaseHelperError(
'Creating old SRPM package failed.')
logger.error('Building source package failed.')
# TODO: implement log analyzer for SRPMs and add the checks here!!!
raise
except BinaryPackageBuildError:
# always fail for original version
rpm_dir = os.path.join(results_dir, 'RPM')
build_dict.update(builder.get_logs())
results_store.set_build_data(version, build_dict)
build_log = 'build.log'
build_log_path = os.path.join(rpm_dir, build_log)
if version == 'old':
error_message = 'Building old RPM package failed. Check logs: {} '.format(
builder.get_logs().get('logs', 'N/A'))
raise RebaseHelperError(error_message)
logger.error('Building binary packages failed.')
msg = 'Building package failed'
try:
files = BuildLogAnalyzer.parse_log(rpm_dir, build_log)
except BuildLogAnalyzerMissingError:
raise RebaseHelperError('Build log %s does not exist',
build_log_path)
except BuildLogAnalyzerMakeError:
raise RebaseHelperError(
'%s during build. Check log %s', msg,
build_log_path)
except BuildLogAnalyzerPatchError:
raise RebaseHelperError(
'%s during patching. Check log %s', msg,
build_log_path)
except RuntimeError:
if self.conf.build_retries == number_retries:
raise RebaseHelperError(
'%s with unknown reason. Check log %s', msg,
build_log_path)
if 'missing' in files:
missing_files = '\n'.join(files['missing'])
logger.info('Files not packaged in the SPEC file:\n%s',
missing_files)
elif 'deleted' in files:
deleted_files = '\n'.join(files['deleted'])
logger.warning(
'Removed files packaged in SPEC file:\n%s',
deleted_files)
else:
if self.conf.build_retries == number_retries:
raise RebaseHelperError(
"Build failed, but no issues were found in the build log %s",
build_log)
self.rebase_spec_file.modify_spec_files_section(files)
if not self.conf.non_interactive:
msg = 'Do you want rebase-helper to try build the packages one more time'
if not ConsoleHelper.get_message(msg):
raise KeyboardInterrupt
else:
logger.warning(
'Some patches were not successfully applied')
# build just failed, otherwise we would break out of the while loop
logger.debug('Number of retries is %s',
self.conf.build_retries)
if os.path.exists(os.path.join(results_dir, 'RPM')):
shutil.rmtree(os.path.join(results_dir, 'RPM'))
if os.path.exists(os.path.join(results_dir, 'SRPM')):
shutil.rmtree(os.path.join(results_dir, 'SRPM'))
number_retries += 1
if self.conf.build_retries == number_retries:
raise RebaseHelperError(
'Building package failed with unknown reason. Check all available log files.'
)
return True
def run(self):
# TODO: Move this check to CliHelper OR possibly to a private method validating the configuration.
if self.conf.fedpkg_build_tasks:
logger.warning(
"Option --fedpkg-build-tasks is deprecated, use --build-tasks instead."
)
if not self.conf.build_tasks:
self.conf.build_tasks = self.conf.fedpkg_build_tasks
# Certain options can be used only with specific build tools
# here are checks for remote build tools
if self.conf.buildtool not in [KojiBuildTool.CMD, CoprBuildTool.CMD]:
options_used = []
if self.conf.build_tasks is not None:
options_used.append('--build-tasks')
if self.conf.builds_nowait is True:
options_used.append('--builds-nowait')
if options_used:
raise RebaseHelperError(
"%s can be used only with the following build tools: %s",
' and '.join(options_used),
', '.join([KojiBuildTool.CMD, CoprBuildTool.CMD]))
# here are checks for local builders
elif self.conf.buildtool not in [
RpmbuildBuildTool.CMD, MockBuildTool.CMD
]:
options_used = []
if self.conf.builder_options is not None:
options_used.append('--builder-options')
if options_used:
raise RebaseHelperError(
"%s can be used only with the following build tools: %s",
' and '.join(options_used),
', '.join([RpmbuildBuildTool.CMD, MockBuildTool.CMD]))
sources = None
if self.conf.build_tasks is None:
sources = self.prepare_sources()
if not self.conf.build_only and not self.conf.comparepkgs:
self.patch_sources(sources)
build = False
if not self.conf.patch_only:
if not self.conf.comparepkgs:
# check build dependencies for rpmbuild
if self.conf.buildtool == RpmbuildBuildTool.CMD:
Application.check_build_requires(self.spec_file)
# Build packages
try:
build = self.build_packages()
if self.conf.builds_nowait and not self.conf.build_tasks:
if self.conf.buildtool == KojiBuildTool.CMD:
self.print_koji_logs()
elif self.conf.buildtool == CoprBuildTool.CMD:
self.print_copr_logs()
return
except RuntimeError:
logger.error('Unknown error caused by build log analysis')
# TODO: exception should be raised instead of returning a value - it is never checked!
return 1
# Perform checks
else:
build = self.get_rpm_packages(self.conf.comparepkgs)
# We don't care dirname doesn't contain any RPM packages
# Therefore return 1
if build:
self.run_package_checkers(self.results_dir)
else:
if not self.upstream_monitoring:
# TODO: This should be an ERROR
logger.info(
'Rebase package to %s FAILED. See for more details',
self.conf.sources)
# TODO: exception should be raised instead of returning a value - it is never checked!
return 1
self.print_summary()
if not self.conf.keep_workspace:
self._delete_workspace_dir()
if self.debug_log_file:
logger.info("Detailed debug log is located in '%s'",
self.debug_log_file)
if not self.upstream_monitoring and not self.conf.patch_only:
logger.info('Rebase package to %s was SUCCESSFUL.\n',
self.conf.sources)
return 0
def watch_koji_tasks(cls, session, tasklist):
"""Function is copy/paste from pyrpkg/cli.py"""
if not tasklist:
return
# Place holder for return value
rh_tasks = {}
try:
tasks = {}
for task_id in tasklist:
tasks[task_id] = TaskWatcher(task_id, session, logger,
quiet=False)
while True:
all_done = True
for task_id, task in tasks.items():
changed = task.update()
info = session.getTaskInfo(task_id)
state = task.info['state']
if state == koji.TASK_STATES['FAILED']:
return {info['id']: state}
else:
if info['arch'] == 'x86_64' or info['arch'] == 'noarch':
rh_tasks[info['id']] = state
if not task.is_done():
all_done = False
else:
if changed:
cls.display_task_results(tasks)
if not task.is_success():
rh_tasks = None
try:
for child in session.getTaskChildren(task_id):
child_id = child['id']
if child_id not in tasks.keys():
tasks[child_id] = TaskWatcher(child_id,
session,
logger,
task.level + 1,
quiet=False)
tasks[child_id].update()
# If we found new children, go through the list
# again, in case they have children also
info = session.getTaskInfo(child_id)
state = task.info['state']
if state == koji.TASK_STATES['FAILED']:
return {info['id']: state}
else:
if info['arch'] == 'x86_64' or info['arch'] == 'noarch':
rh_tasks[info['id']] = state
all_done = False
except SSL.SysCallError as e:
logger.error('Detected SSL error: %s', six.text_type(e))
if all_done:
cls.display_task_results(tasks)
break
time.sleep(1)
except (KeyboardInterrupt):
# A ^c should return non-zero so that it doesn't continue
# on to any && commands.
rh_tasks = None
return rh_tasks
def run(self):
# TODO: Move this check to CliHelper OR possibly to a private method validating the configuration.
if self.conf.fedpkg_build_tasks:
logger.warning("Option --fedpkg-build-tasks is deprecated, use --build-tasks instead.")
if not self.conf.build_tasks:
self.conf.build_tasks = self.conf.fedpkg_build_tasks
# Certain options can be used only with specific build tools
# here are checks for remote build tools
if self.conf.buildtool not in [KojiBuildTool.CMD, CoprBuildTool.CMD]:
options_used = []
if self.conf.build_tasks is not None:
options_used.append('--build-tasks')
if self.conf.builds_nowait is True:
options_used.append('--builds-nowait')
if options_used:
raise RebaseHelperError("%s can be used only with the following build tools: %s",
' and '.join(options_used),
', '.join([KojiBuildTool.CMD, CoprBuildTool.CMD])
)
# here are checks for local builders
elif self.conf.buildtool not in [RpmbuildBuildTool.CMD, MockBuildTool.CMD]:
options_used = []
if self.conf.builder_options is not None:
options_used.append('--builder-options')
if options_used:
raise RebaseHelperError("%s can be used only with the following build tools: %s",
' and '.join(options_used),
', '.join([RpmbuildBuildTool.CMD, MockBuildTool.CMD])
)
sources = None
if self.conf.build_tasks is None:
sources = self.prepare_sources()
if not self.conf.build_only and not self.conf.comparepkgs:
self.patch_sources(sources)
build = False
if not self.conf.patch_only:
if not self.conf.comparepkgs:
# check build dependencies for rpmbuild
if self.conf.buildtool == RpmbuildBuildTool.CMD:
Application.check_build_requires(self.spec_file)
# Build packages
try:
build = self.build_packages()
if self.conf.builds_nowait and not self.conf.build_tasks:
if self.conf.buildtool == KojiBuildTool.CMD:
self.print_koji_logs()
elif self.conf.buildtool == CoprBuildTool.CMD:
self.print_copr_logs()
return
except RuntimeError:
logger.error('Unknown error caused by build log analysis')
# TODO: exception should be raised instead of returning a value - it is never checked!
return 1
# Perform checks
else:
build = self.get_rpm_packages(self.conf.comparepkgs)
# We don't care dirname doesn't contain any RPM packages
# Therefore return 1
if build:
self.run_package_checkers(self.results_dir)
else:
if not self.upstream_monitoring:
# TODO: This should be an ERROR
logger.info('Rebase package to %s FAILED. See for more details', self.conf.sources)
# TODO: exception should be raised instead of returning a value - it is never checked!
return 1
self.print_summary()
if not self.conf.keep_workspace:
self._delete_workspace_dir()
if self.debug_log_file:
logger.info("Detailed debug log is located in '%s'", self.debug_log_file)
if not self.upstream_monitoring and not self.conf.patch_only:
logger.info('Rebase package to %s was SUCCESSFUL.\n', self.conf.sources)
return 0
def build_binary_packages(self):
"""Function calls build class for building packages"""
try:
builder = plugin_manager.build_tools.get_plugin(self.conf.buildtool)
except NotImplementedError as e:
raise RebaseHelperError('{}. Supported build tools are {}'.format(
str(e), plugin_manager.build_tools.get_supported_tools()))
for version in ['old', 'new']:
results_dir = '{}-build'.format(os.path.join(self.results_dir, version))
spec = None
task_id = None
koji_build_id = None
build_dict = {}
if self.conf.build_tasks is None:
spec = self.spec_file if version == 'old' else self.rebase_spec_file
package_name = spec.get_package_name()
package_version = spec.get_version()
package_full_version = spec.get_full_version()
if version == 'old' and self.conf.get_old_build_from_koji:
koji_build_id, package_version, package_full_version = KojiHelper.get_old_build_info(
package_name,
package_version)
build_dict = dict(
name=package_name,
version=package_version,
builds_nowait=self.conf.builds_nowait,
build_tasks=self.conf.build_tasks,
builder_options=self.conf.builder_options,
srpm=results_store.get_build(version).get('srpm'),
srpm_logs=results_store.get_build(version).get('logs'))
# prepare for building
builder.prepare(spec, self.conf)
logger.info('Building binary packages for %s version %s', package_name, package_full_version)
else:
task_id = self.conf.build_tasks[0] if version == 'old' else self.conf.build_tasks[1]
try:
if self.conf.build_tasks is None:
if koji_build_id:
session = KojiHelper.create_session()
build_dict['rpm'], build_dict['logs'] = KojiHelper.download_build(session,
koji_build_id,
os.path.join(
results_dir,
'RPM',
),
arches=['noarch', 'x86_64'])
else:
build_dict.update(builder.build(spec, results_dir, **build_dict))
if builder.CREATES_TASKS and task_id and not koji_build_id:
if not self.conf.builds_nowait:
build_dict['rpm'], build_dict['logs'] = builder.wait_for_task(build_dict,
task_id,
results_dir)
elif self.conf.build_tasks:
build_dict['rpm'], build_dict['logs'] = builder.get_detached_task(task_id, results_dir)
build_dict = self._sanitize_build_dict(build_dict)
results_store.set_build_data(version, build_dict)
except RebaseHelperError: # pylint: disable=try-except-raise
# Proper RebaseHelperError instance was created already. Re-raise it.
raise
except BinaryPackageBuildError as e:
build_dict.update(builder.get_logs())
build_dict['binary_package_build_error'] = str(e)
build_dict = self._sanitize_build_dict(build_dict)
results_store.set_build_data(version, build_dict)
if e.logfile is None:
msg = 'Building {} RPM packages failed; see logs in {} for more information'.format(
version, os.path.join(results_dir, 'RPM')
)
else:
msg = 'Building {} RPM packages failed; see {} for more information'.format(version, e.logfile)
logger.error(msg)
raise RebaseHelperError(msg, logfiles=builder.get_logs().get('logs'))
except Exception:
raise RebaseHelperError('Building package failed with unknown reason. '
'Check all available log files.')
if self.conf.builds_nowait and not self.conf.build_tasks:
if builder.CREATES_TASKS:
self.print_task_info(builder)
def build_packages(self):
"""Function calls build class for building packages"""
try:
builder = Builder(self.conf.buildtool)
except NotImplementedError as ni_e:
raise RebaseHelperError(
'%s. Supported build tools are %s' % six.text_type(ni_e),
Builder.get_supported_tools())
for version in ['old', 'new']:
spec_object = self.spec_file if version == 'old' else self.rebase_spec_file
build_dict = {}
task_id = None
koji_build_id = None
if self.conf.build_tasks is None:
pkg_name = spec_object.get_package_name()
pkg_version = spec_object.get_version()
pkg_full_version = spec_object.get_full_version()
if version == 'old' and self.conf.get_old_build_from_koji:
if KojiHelper.functional:
koji_version, koji_build_id = KojiHelper.get_latest_build(
pkg_name)
if koji_version:
if koji_version != pkg_version:
logger.warning(
'Version of the latest Koji build (%s) with id (%s) '
'differs from version in SPEC file (%s)!',
koji_version, koji_build_id, pkg_version)
pkg_version = pkg_full_version = koji_version
else:
logger.warning(
'Unable to find the latest Koji build!')
else:
logger.warning('Unable to get the latest Koji build!')
# prepare for building
builder.prepare(spec_object, self.conf)
build_dict['name'] = pkg_name
build_dict['version'] = pkg_version
patches = [x.get_path() for x in spec_object.get_patches()]
spec = spec_object.get_path()
sources = spec_object.get_sources()
logger.info('Building packages for %s version %s', pkg_name,
pkg_full_version)
else:
if version == 'old':
task_id = self.conf.build_tasks[0]
else:
task_id = self.conf.build_tasks[1]
results_dir = os.path.join(self.results_dir, version) + '-build'
files = {}
number_retries = 0
while self.conf.build_retries != number_retries:
try:
if self.conf.build_tasks is None:
if koji_build_id:
build_dict['rpm'], build_dict[
'logs'] = KojiHelper.download_build(
koji_build_id, results_dir)
else:
build_dict.update(
builder.build(spec, sources, patches,
results_dir, **build_dict))
if builder.creates_tasks() and not koji_build_id:
if not self.conf.builds_nowait:
build_dict['rpm'], build_dict[
'logs'] = builder.wait_for_task(
build_dict, results_dir)
if build_dict['rpm'] is None:
return False
elif self.conf.build_tasks:
build_dict['rpm'], build_dict[
'logs'] = builder.get_detached_task(
task_id, results_dir)
if build_dict['rpm'] is None:
return False
# Build finishes properly. Go out from while cycle
results_store.set_build_data(version, build_dict)
break
except SourcePackageBuildError as e:
build_dict.update(builder.get_logs())
build_dict['source_package_build_error'] = six.text_type(e)
results_store.set_build_data(version, build_dict)
# always fail for original version
if version == 'old':
raise RebaseHelperError(
'Creating old SRPM package failed.')
logger.error('Building source package failed.')
# TODO: implement log analyzer for SRPMs and add the checks here!!!
raise
except BinaryPackageBuildError as e:
# always fail for original version
rpm_dir = os.path.join(results_dir, 'RPM')
build_dict.update(builder.get_logs())
build_dict['binary_package_build_error'] = six.text_type(e)
results_store.set_build_data(version, build_dict)
build_log = 'build.log'
build_log_path = os.path.join(rpm_dir, build_log)
if version == 'old':
error_message = 'Building old RPM package failed. Check logs: {} '.format(
builder.get_logs().get('logs', 'N/A'))
raise RebaseHelperError(
error_message,
logfiles=builder.get_logs().get('logs'))
logger.error('Building binary packages failed.')
msg = 'Building package failed'
try:
files = BuildLogAnalyzer.parse_log(rpm_dir, build_log)
except BuildLogAnalyzerMissingError:
raise RebaseHelperError('Build log %s does not exist',
build_log_path)
except BuildLogAnalyzerMakeError:
raise RebaseHelperError(
'%s during build. Check log %s',
msg,
build_log_path,
logfiles=[build_log_path])
except BuildLogAnalyzerPatchError:
raise RebaseHelperError(
'%s during patching. Check log %s',
msg,
build_log_path,
logfiles=[build_log_path])
except RuntimeError:
if self.conf.build_retries == number_retries:
raise RebaseHelperError(
'%s with unknown reason. Check log %s',
msg,
build_log_path,
logfiles=[build_log_path])
if 'missing' in files:
missing_files = '\n'.join(files['missing'])
logger.info('Files not packaged in the SPEC file:\n%s',
missing_files)
elif 'deleted' in files:
deleted_files = '\n'.join(files['deleted'])
logger.warning(
'Removed files packaged in SPEC file:\n%s',
deleted_files)
else:
if self.conf.build_retries == number_retries:
raise RebaseHelperError(
"Build failed, but no issues were found in the build log %s",
build_log,
logfiles=[build_log])
self.rebase_spec_file.modify_spec_files_section(files)
if not self.conf.non_interactive:
msg = 'Do you want rebase-helper to try to build the packages one more time'
if not ConsoleHelper.get_message(msg):
raise KeyboardInterrupt
else:
logger.warning(
'Some patches were not successfully applied')
# build just failed, otherwise we would break out of the while loop
logger.debug('Number of retries is %s',
self.conf.build_retries)
number_retries += 1
if self.conf.build_retries > number_retries:
# only remove builds if this retry is not the last one
if os.path.exists(os.path.join(results_dir, 'RPM')):
shutil.rmtree(os.path.join(results_dir, 'RPM'))
if os.path.exists(os.path.join(results_dir, 'SRPM')):
shutil.rmtree(os.path.join(results_dir, 'SRPM'))
if self.conf.build_retries == number_retries:
raise RebaseHelperError(
'Building package failed with unknown reason. Check all available log files.'
)
if self.conf.builds_nowait and not self.conf.build_tasks:
if builder.creates_tasks():
self.print_task_info(builder)
return True
