def test_supersede(self):
self.wf.setup_rings()
staging = self.wf.create_staging('A', freeze=True)
rq1 = self.wf.create_submit_request('devel:wine', 'wine')
ret = SelectCommand(self.wf.api, staging.name).perform(['wine'])
rq2 = self.wf.create_submit_request('devel:wine',
'wine',
text='Something new')
self.wf.api._packages_staged = None
self.osc_user('staging-bot')
Config.get(self.wf.apiurl, self.wf.project)
SupersedeCommand(self.wf.api).perform()
self.assertEqual(rq1.reviews(),
[{
'state': 'accepted',
'by_group': 'factory-staging'
}, {
'state': 'accepted',
'by_project': 'openSUSE:Factory:Staging:A'
}, {
'state': 'declined',
'by_group': 'factory-staging'
}])
self.assertEqual(rq2.reviews(),
[{
'state': 'accepted',
'by_group': 'factory-staging'
}, {
'state': 'new',
'by_project': 'openSUSE:Factory:Staging:A'
}])
def request_repository_pairs(self, request, action):
if str2bool(
Config.get(self.apiurl,
action.tgt_project).get('repo_checker-project-skip',
'False')):
# Do not change message as this should only occur in requests
# targeting multiple projects such as in maintenance workflow in
# which the message should be set by other actions.
self.logger.debug('skipping review of action targeting {}'.format(
action.tgt_project))
return True
repository = self.project_repository(action.tgt_project)
if not repository:
self.review_messages['declined'] = ERROR_REPO_SPECIFIED.format(
action.tgt_project)
return False
repository_pairs = []
# Assumes maintenance_release target project has staging disabled.
staging = Config.get(self.apiurl, action.tgt_project).get('staging')
if staging:
api = self.staging_api(staging)
stage_info = api.packages_staged.get(action.tgt_package)
if not stage_info or str(stage_info['rq_id']) != str(
request.reqid):
self.logger.info('{} not staged'.format(request.reqid))
return None
if not self.force and not self.staging_build_failure_check(
api, stage_info['prj']):
self.logger.info(
'{} not ready due to staging build failure(s)'.format(
request.reqid))
return None
# Staging setup is convoluted and thus the repository setup does not
# contain a path to the target project. Instead the ports repository
# is used to import the target prjconf. As such the staging group
# repository must be explicitly layered on top of target project.
repository_pairs.append([stage_info['prj'], repository])
repository_pairs.extend(
repository_path_expand(self.apiurl, action.tgt_project,
repository))
else:
# Find a repository which links to target project "main" repository.
repository = repository_path_search(self.apiurl,
action.src_project,
action.tgt_project, repository)
if not repository:
self.review_messages['declined'] = ERROR_REPO_SPECIFIED.format(
action.tgt_project)
return False
repository_pairs.extend(
repository_path_expand(self.apiurl, action.src_project,
repository))
return repository_pairs
def test_get_memoize_reset(self):
"""Ensure memoize_session_reset() properly forces re-fetch of config."""
self.assertEqual('remote-indeed', Config.get(obs.APIURL, obs.PROJECT)['remote-only'])
attribute_value_save(obs.APIURL, obs.PROJECT, 'Config', 'remote-only = new value\n')
memoize_session_reset()
self.assertEqual('new value', Config.get(obs.APIURL, obs.PROJECT)['remote-only'])
def test_get_memoize_reset(self):
"""Ensure memoize_session_reset() properly forces re-fetch of config."""
wf = self.setup_vcr()
self.assertEqual('remote-indeed', Config.get(wf.apiurl, wf.project)['remote-only'])
attribute_value_save(wf.apiurl, wf.project, 'Config', 'remote-only = new value\n')
memoize_session_reset()
self.assertEqual('new value', Config.get(wf.apiurl, wf.project)['remote-only'])
def test_get_memoize_reset(self):
"""Ensure memoize_session_reset() properly forces re-fetch of config."""
self.assertEqual('remote-indeed',
Config.get(APIURL, PROJECT)['remote-only'])
attribute_value_save(APIURL, PROJECT, 'Config',
'remote-only = new value\n')
memoize_session_reset()
self.assertEqual('new value',
Config.get(APIURL, PROJECT)['remote-only'])
def staging_api(self, project):
# Allow for the Staging subproject to be passed directly from config
# which should be stripped before initializing StagingAPI. This allows
# for NonFree subproject to utilize StagingAPI for main project.
if project.endswith(':Staging'):
project = project[:-8]
if project not in self.staging_apis:
Config.get(self.apiurl, project)
self.staging_apis[project] = StagingAPI(self.apiurl, project)
return self.staging_apis[project]
def test_get_memoize_reset(self):
"""Ensure memoize_session_reset() properly forces re-fetch of config."""
wf = self.setup_vcr()
self.assertEqual('remote-indeed',
Config.get(wf.apiurl, wf.project)['remote-only'])
attribute_value_save(wf.apiurl, wf.project, 'Config',
'remote-only = new value\n')
memoize_session_reset()
self.assertEqual('new value',
Config.get(wf.apiurl, wf.project)['remote-only'])
def staging_api(self, project):
# Allow for the Staging subproject to be passed directly from config
# which should be stripped before initializing StagingAPI. This allows
# for NonFree subproject to utilize StagingAPI for main project.
if project.endswith(':Staging'):
project = project[:-8]
if project not in self.staging_apis:
Config.get(self.apiurl, project)
self.staging_apis[project] = StagingAPI(self.apiurl, project)
return self.staging_apis[project]
def check_one_request(self, req):
config = Config.get(self.apiurl, req.actions[0].tgt_project)
self.needs_legal_review = False
self.needs_reviewteam = False
self.needs_release_manager = False
self.pending_factory_submission = False
self.source_in_factory = None
self.do_check_maintainer_review = not self.ibs
self.packages = {}
request_ok = ReviewBot.ReviewBot.check_one_request(self, req)
self.logger.debug("review result: %s", request_ok)
if self.pending_factory_submission:
self.logger.info("submission is waiting for a Factory request to complete")
creator = req.get_creator()
bot_name = self.bot_name.lower()
if self.automatic_submission and creator != bot_name:
self.logger.info('@{}: this request would have been automatically created by {} after the Factory submission was accepted in order to eleviate the need to manually create requests for packages sourced from Factory'.format(creator, bot_name))
elif self.source_in_factory:
self.logger.info("perfect. the submitted sources are in or accepted for Factory")
elif self.source_in_factory == False:
self.logger.warning("the submitted sources are NOT in Factory")
if request_ok == False:
self.logger.info("NOTE: if you think the automated review was wrong here, please talk to the release team before reopening the request")
if self.do_comments:
result = None
if request_ok is None:
state = 'seen'
elif request_ok:
state = 'done'
result = 'accepted'
else:
state = 'done'
result = 'declined'
self.comment_write(state, result)
add_review_groups = []
if self.needs_release_manager:
add_review_groups.append(self.release_manager_group or
config.get(self.override_group_key))
if self.needs_reviewteam:
add_review_groups.append(self.review_team_group or
config.get('review-team'))
if self.needs_legal_review:
add_review_groups.append(self.legal_review_group or
config.get('legal-review-group'))
if self.needs_check_source and self.check_source_group is not None:
add_review_groups.append(self.check_source_group)
for group in add_review_groups:
if group is None:
continue
self.logger.info("{0} needs review by [{1}](/group/show/{1})".format(req.reqid, group))
self.add_review(req, by_group=group)
return request_ok
def target_project_config(self, project):
# Load project config and allow for remote entries.
config = Config.get(self.apiurl, project)
self.single_action_require = str2bool(config.get('check-source-single-action-require', 'False'))
self.ignore_devel = not str2bool(config.get('devel-project-enforce', 'False'))
self.in_air_rename_allow = str2bool(config.get('check-source-in-air-rename-allow', 'False'))
self.add_review_team = str2bool(config.get('check-source-add-review-team', 'True'))
self.review_team = config.get('review-team')
self.staging_group = config.get('staging-group')
self.repo_checker = config.get('repo-checker')
self.devel_whitelist = config.get('devel-whitelist', '').split()
self.skip_add_reviews = False
if self.action.type == 'maintenance_incident':
# The workflow effectively enforces the names to match and the
# parent code sets target_package from source_package so this check
# becomes useless and awkward to perform.
self.in_air_rename_allow = True
# The target project will be set to product and thus inherit
# settings, but override since real target is not product.
self.single_action_require = False
# It might make sense to supersede maintbot, but for now.
self.skip_add_reviews = True
def check_action_submit(self, request, action):
repository_pairs = self.request_repository_pairs(request, action)
if not isinstance(repository_pairs, list):
return repository_pairs
# use project_only results by default as reference
whitelist = None
config = Config.get(self.apiurl, action.tgt_project)
staging = config.get('staging')
arch_whitelist = config.get('repo_checker-arch-whitelist')
cycle_packages = config.get('repo_checker-allowed-in-cycles')
if staging:
api = self.staging_api(staging)
if not api.is_adi_project(repository_pairs[0][0]):
# For "leaky" ring packages in letter stagings, where the
# repository setup does not include the target project, that are
# not intended to to have all run-time dependencies satisfied.
whitelist = set(config.get('repo_checker-binary-whitelist-ring', '').split(' '))
state_hash = self.repository_state(repository_pairs, True)
if not self.repository_check(repository_pairs, state_hash, True,
arch_whitelist=arch_whitelist,
whitelist=whitelist,
cycle_packages=cycle_packages):
return None
self.review_messages['accepted'] = 'cycle and install check passed'
return True
def check_action_submit(self, request, action):
repository_pairs = self.request_repository_pairs(request, action)
if not isinstance(repository_pairs, list):
return repository_pairs
# use project_only results by default as reference
whitelist = None
config = Config.get(self.apiurl, action.tgt_project)
staging = config.get('staging')
arch_whitelist = config.get('repo_checker-arch-whitelist')
cycle_packages = config.get('repo_checker-allowed-in-cycles')
if staging:
api = self.staging_api(staging)
if not api.is_adi_project(repository_pairs[0][0]):
# For "leaky" ring packages in letter stagings, where the
# repository setup does not include the target project, that are
# not intended to to have all run-time dependencies satisfied.
whitelist = config.get('repo_checker-binary-whitelist-ring', '').split(' ')
state_hash = self.repository_state(repository_pairs, True)
if not self.repository_check(repository_pairs, state_hash, True,
arch_whitelist=arch_whitelist,
whitelist=whitelist,
cycle_packages=cycle_packages):
return None
self.review_messages['accepted'] = 'cycle and install check passed'
return True
def request_age_wait(self,
age_min=None,
request=None,
target_project=None):
if not request:
request = self.request
if not target_project:
target_project = self.action.tgt_project
if age_min is None or isinstance(age_min, str):
key = self.request_age_min_key if age_min is None else age_min
age_min = int(
Config.get(self.apiurl,
target_project).get(key,
self.request_age_min_default))
age = request_age(request).total_seconds()
if age < age_min:
self.logger.info(
'skipping {} of age {:.2f}s since it is younger than {}s'.
format(request.reqid, age, age_min))
return True
return False
def check_one_request(self, req):
config = Config.get(self.apiurl, req.actions[0].tgt_project)
self.needs_legal_review = False
self.needs_reviewteam = False
self.needs_release_manager = False
self.pending_factory_submission = False
self.source_in_factory = None
self.do_check_maintainer_review = not self.ibs
self.packages = {}
request_ok = ReviewBot.ReviewBot.check_one_request(self, req)
self.logger.debug("review result: %s", request_ok)
if self.pending_factory_submission:
self.logger.info("submission is waiting for a Factory request to complete")
creator = req.get_creator()
bot_name = self.bot_name.lower()
if self.automatic_submission and creator != bot_name:
self.logger.info('@{}: this request would have been automatically created by {} after the Factory submission was accepted in order to eleviate the need to manually create requests for packages sourced from Factory'.format(creator, bot_name))
elif self.source_in_factory:
self.logger.info("perfect. the submitted sources are in or accepted for Factory")
elif self.source_in_factory == False:
self.logger.warn("the submitted sources are NOT in Factory")
if request_ok == False:
self.logger.info("NOTE: if you think the automated review was wrong here, please talk to the release team before reopening the request")
if self.do_comments:
result = None
if request_ok is None:
state = 'seen'
elif request_ok:
state = 'done'
result = 'accepted'
else:
state = 'done'
result = 'declined'
self.comment_write(state, result)
add_review_groups = []
if self.needs_release_manager:
add_review_groups.append(self.release_manager_group or
config.get(self.override_group_key))
if self.needs_reviewteam:
add_review_groups.append(self.review_team_group or
config.get('review-team'))
if self.needs_legal_review:
add_review_groups.append(self.legal_review_group or
config.get('legal-review-group'))
if self.needs_check_source and self.check_source_group is not None:
add_review_groups.append(self.check_source_group)
for group in add_review_groups:
if group is None:
continue
self.logger.info("{0} needs review by [{1}](/group/show/{1})".format(req.reqid, group))
self.add_review(req, by_group=group)
return request_ok
def target_project_config(self, project):
# Load project config and allow for remote entries.
config = Config.get(self.apiurl, project)
self.single_action_require = str2bool(
config.get('check-source-single-action-require', 'False'))
self.ignore_devel = not str2bool(
config.get('devel-project-enforce', 'False'))
self.in_air_rename_allow = str2bool(
config.get('check-source-in-air-rename-allow', 'False'))
self.add_review_team = str2bool(
config.get('check-source-add-review-team', 'True'))
self.review_team = config.get('review-team')
self.mail_release_list = config.get('mail-release-list')
self.staging_group = config.get('staging-group')
self.repo_checker = config.get('repo-checker')
self.devel_whitelist = config.get('devel-whitelist', '').split()
self.skip_add_reviews = False
self.security_review_team = config.get('security-review-team',
'security-team')
self.bad_rpmlint_entries = config.get('bad-rpmlint-entries',
'').split()
if self.action.type == 'maintenance_incident':
# The workflow effectively enforces the names to match and the
# parent code sets target_package from source_package so this check
# becomes useless and awkward to perform.
self.in_air_rename_allow = True
# The target project will be set to product and thus inherit
# settings, but override since real target is not product.
self.single_action_require = False
# It might make sense to supersede maintbot, but for now.
self.skip_add_reviews = True
def mail_send(apiurl,
project,
to,
subject,
body,
from_key='maintainer',
followup_to_key='release-list',
dry=False):
config = Config.get(apiurl, project)
if from_key is None:
sender = entity_email(apiurl,
conf.get_apiurl_usr(apiurl),
include_name=True)
else:
sender = config['mail-{}'.format(from_key)]
if '@' not in to:
to = config['mail-{}'.format(to)]
followup_to = config.get('mail-{}'.format(followup_to_key))
relay = config.get('mail-relay', 'relay.suse.de')
mail_send_with_details(text=body,
subject=subject,
relay=relay,
sender=sender,
followup_to=followup_to,
to=to,
dry=dry)
def __init__(self, opensuse_project, sle_project, alternative_project,
print_only, verbose):
self.upload_project = opensuse_project
self.opensuse_project = opensuse_project
if alternative_project:
self.opensuse_project = alternative_project
self.sle_project = sle_project
self.print_only = print_only
self.verbose = verbose
self.apiurl = osc.conf.config['apiurl']
self.debug = osc.conf.config['debug']
config = Config.get(self.apiurl, opensuse_project)
# binary rpms of packages from `skippkg-finder-skiplist-ignores`
# be found in the `package_binaries` thus format must to be like
# SUSE:SLE-15:Update_libcdio.12032, PROJECT-NAME_PACKAGE-NAME
self.skiplist_ignored = set(
config.get('skippkg-finder-skiplist-ignores', '').split(' '))
# supplement RPMs for skipping from the ftp-tree
self.skiplist_supplement_regex = set(
config.get('skippkg-finder-skiplist-supplement-regex',
'').split(' '))
# drops off RPM from a list of the supplement RPMs due to regex
self.skiplist_supplement_ignores = set(
config.get('skippkg-finder-skiplist-supplement-ignores',
'').split(' '))
def mail_send(apiurl, project, to, subject, body, from_key='maintainer',
followup_to_key='release-list', dry=False):
from email.mime.text import MIMEText
import email.utils
import smtplib
config = Config.get(apiurl, project)
msg = MIMEText(body)
msg['Message-ID'] = email.utils.make_msgid()
msg['Date'] = email.utils.formatdate(localtime=1)
if from_key is None:
msg['From'] = entity_email(apiurl, conf.get_apiurl_usr(apiurl), include_name=True)
else:
msg['From'] = config['mail-{}'.format(from_key)]
if '@' not in to:
to = config['mail-{}'.format(to)]
msg['To'] = to
followup_to = config.get('mail-{}'.format(followup_to_key))
if followup_to:
msg['Mail-Followup-To'] = followup_to
msg['Subject'] = subject
if dry:
print(msg.as_string())
return
s = smtplib.SMTP(config.get('mail-relay', 'relay.suse.de'))
s.sendmail(msg['From'], [msg['To']], msg.as_string())
s.quit()
def policy_get_preprocess(apiurl, origin, policy):
project = origin.rstrip('~')
config_project = Config.get(apiurl, project)
policy['pending_submission_allowed_reviews'] = filter(None, [
config_resolve_variable(v, config_project, 'config_source')
for v in policy['pending_submission_allowed_reviews']])
return policy
def policy_get_preprocess(apiurl, origin, policy):
project = origin.rstrip('~')
config_project = Config.get(apiurl, project)
policy['pending_submission_allowed_reviews'] = filter(None, [
config_resolve_variable(v, config_project, 'config_source')
for v in policy['pending_submission_allowed_reviews']])
return policy
def project_pseudometa_file_name(self, project, repository):
filename = 'repo_checker'
main_repo = Config.get(self.apiurl, project).get('main-repo')
if not main_repo:
filename += '.' + repository
return filename
def project_pseudometa_file_name(self, project, repository):
filename = 'repo_checker'
main_repo = Config.get(self.api.apiurl, project).get('main-repo')
if not main_repo:
filename += '.' + repository
return filename
def request_repository_pairs(self, request, action):
if str2bool(Config.get(self.apiurl, action.tgt_project).get('repo_checker-project-skip', 'False')):
# Do not change message as this should only occur in requests
# targeting multiple projects such as in maintenance workflow in
# which the message should be set by other actions.
self.logger.debug('skipping review of action targeting {}'.format(action.tgt_project))
return True
repository = self.project_repository(action.tgt_project)
if not repository:
self.review_messages['declined'] = ERROR_REPO_SPECIFIED.format(action.tgt_project)
return False
repository_pairs = []
# Assumes maintenance_release target project has staging disabled.
staging = Config.get(self.apiurl, action.tgt_project).get('staging')
if staging:
api = self.staging_api(staging)
stage_info = api.packages_staged.get(action.tgt_package)
if not stage_info or str(stage_info['rq_id']) != str(request.reqid):
self.logger.info('{} not staged'.format(request.reqid))
return None
if not self.force and not self.staging_build_failure_check(api, stage_info['prj']):
self.logger.info('{} not ready due to staging build failure(s)'.format(request.reqid))
return None
# Staging setup is convoluted and thus the repository setup does not
# contain a path to the target project. Instead the ports repository
# is used to import the target prjconf. As such the staging group
# repository must be explicitly layered on top of target project.
repository_pairs.append([stage_info['prj'], repository])
repository_pairs.extend(repository_path_expand(self.apiurl, action.tgt_project, repository))
else:
# Find a repository which links to target project "main" repository.
repository = repository_path_search(
self.apiurl, action.src_project, action.tgt_project, repository)
if not repository:
self.review_messages['declined'] = ERROR_REPO_SPECIFIED.format(action.tgt_project)
return False
repository_pairs.extend(repository_path_expand(self.apiurl, action.src_project, repository))
return repository_pairs
def target_archs(self, project, repository):
archs = target_archs(self.apiurl, project, repository)
# Check for arch whitelist and use intersection.
whitelist = Config.get(self.apiurl, project).get('repo_checker-arch-whitelist')
if whitelist:
archs = list(set(whitelist.split(' ')).intersection(set(archs)))
# Trick to prioritize x86_64.
return sorted(archs, reverse=True)
def origin_devel_projects(apiurl, project):
projects = set(devel_projects(apiurl, project))
for devel_project, _ in origin_devel_project_requests(apiurl, project):
projects.add(devel_project)
devel_whitelist = Config.get(apiurl, project).get('devel-whitelist',
'').split()
projects.update(devel_whitelist)
return sorted(projects)
def target_archs(self, project, repository):
archs = target_archs(self.apiurl, project, repository)
# Check for arch whitelist and use intersection.
whitelist = Config.get(self.apiurl,
project).get('repo_checker-arch-whitelist')
if whitelist:
archs = list(set(whitelist.split(' ')).intersection(set(archs)))
# Trick to prioritize x86_64.
return sorted(archs, reverse=True)
def project_only(self, project, post_comments=False):
repository = self.project_repository(project)
if not repository:
self.logger.error(ERROR_REPO_SPECIFIED.format(project))
return
config = Config.get(self.apiurl, project)
arch_whitelist = config.get('repo_checker-arch-whitelist')
repository_pairs = repository_path_expand(self.apiurl, project, repository)
state_hash = self.repository_state(repository_pairs, False)
self.repository_check(repository_pairs, state_hash, False, bool(post_comments), arch_whitelist=arch_whitelist)
def binary_whitelist(self, override_pair, overridden_pair, arch):
whitelist = self.binary_list_existing_problem(overridden_pair[0], overridden_pair[1])
staging = Config.get(self.apiurl, overridden_pair[0]).get('staging')
if staging:
additions = self.staging_api(staging).get_prj_pseudometa(
override_pair[0]).get('config', {})
prefix = 'repo_checker-binary-whitelist'
for key in [prefix, '-'.join([prefix, arch])]:
whitelist.update(additions.get(key, '').split(' '))
return set(filter(None, whitelist))
def project_only(self, project, post_comments=False):
repository = self.project_repository(project)
if not repository:
self.logger.error(ERROR_REPO_SPECIFIED.format(project))
return
config = Config.get(self.apiurl, project)
arch_whitelist = config.get('repo_checker-arch-whitelist')
repository_pairs = repository_path_expand(self.apiurl, project, repository)
state_hash = self.repository_state(repository_pairs, False)
self.repository_check(repository_pairs, state_hash, False, bool(post_comments), arch_whitelist=arch_whitelist)
def remote_config_set(self, config, replace_all=False):
if not replace_all:
config_existing = Config.get(self.apiurl, self.project)
config_existing.update(config)
config = config_existing
config_lines = []
for key, value in config.items():
config_lines.append(f'{key} = {value}')
attribute_value_save(APIURL, self.project, 'Config',
'\n'.join(config_lines))
def policy_get_preprocess(apiurl, origin, policy):
project = origin.rstrip('~')
config_project = Config.get(apiurl, project)
for suffix in ('', '_update'):
key = 'pending_submission_allowed_reviews{}'.format(suffix)
policy[key] = list(
filter(None, [
config_resolve_variable(v, config_project, 'config_source')
for v in policy[key]
]))
return policy
def binary_whitelist(self, override_pair, overridden_pair, arch):
whitelist = self.binary_list_existing_problem(overridden_pair[0], overridden_pair[1])
staging = Config.get(self.apiurl, overridden_pair[0]).get('staging')
if staging:
additions = self.staging_api(staging).get_prj_pseudometa(
override_pair[0]).get('config', {})
prefix = 'repo_checker-binary-whitelist'
for key in [prefix, '-'.join([prefix, arch])]:
whitelist.update(additions.get(key, '').split(' '))
whitelist = filter(None, whitelist)
return whitelist
def request_override_check_users(self, project):
"""Determine users allowed to override review in a comment command."""
config = Config.get(self.apiurl, project)
users = []
group = config.get('staging-group')
if group:
users += group_members(self.apiurl, group)
if self.override_group_key:
override_group = config.get(self.override_group_key)
if override_group:
users += group_members(self.apiurl, override_group)
return users
def request_override_check_users(self, project):
"""Determine users allowed to override review in a comment command."""
config = Config.get(self.apiurl, project)
users = []
group = config.get('staging-group')
if group:
users += group_members(self.apiurl, group)
if self.override_group_key:
override_group = config.get(self.override_group_key)
if override_group:
users += group_members(self.apiurl, override_group)
return users
def check(self, project, repository):
if not repository:
repository = self.project_repository(project)
if not repository:
self.logger.error('a repository must be specified via OSRT:Config main-repo for {}'.format(project))
return
config = Config.get(self.apiurl, project)
archs = target_archs(self.apiurl, project, repository)
if not len(archs):
self.logger.debug('{} has no relevant architectures'.format(project))
return None
for arch in archs:
self.check_pra(project, repository, arch)
def project_repository(self, project):
repository = Config.get(self.apiurl, project).get('main-repo')
if not repository:
self.logger.debug('no main-repo defined for {}'.format(project))
search_project = 'openSUSE:Factory'
for search_repository in ('snapshot', 'standard'):
repository = repository_path_search(
self.apiurl, project, search_project, search_repository)
if repository:
self.logger.debug('found chain to {}/{} via {}'.format(
search_project, search_repository, repository))
break
return repository
def project_repository(self, project):
repository = Config.get(self.apiurl, project).get('main-repo')
if not repository:
self.logger.debug('no main-repo defined for {}'.format(project))
search_project = 'openSUSE:Factory'
for search_repository in ('snapshot', 'standard'):
repository = repository_path_search(
self.apiurl, project, search_project, search_repository)
if repository:
self.logger.debug('found chain to {}/{} via {}'.format(
search_project, search_repository, repository))
break
return repository
def package_comments(self, project, repository):
self.logger.info('{} package comments'.format(len(
self.package_results)))
for package, sections in self.package_results.items():
if str2bool(
Config.get(self.apiurl, project).get(
'repo_checker-package-comment-devel', 'False')):
bot_name_suffix = project
comment_project, comment_package = devel_project_fallback(
self.apiurl, project, package)
if comment_project is None or comment_package is None:
self.logger.warning(
'unable to find devel project for {}'.format(package))
continue
message = 'The version of this package in [`{project}`](/package/show/{project}/{package}) ' \
'has installation issues and may not be installable:'.format(
project=project, package=package)
else:
bot_name_suffix = repository
comment_project = project
comment_package = package
message = 'This package has installation issues and may not be installable from the `{}` ' \
'repository:'.format(repository)
# Sort sections by text to group binaries together.
sections = sorted(sections, key=lambda s: s.text)
message += '\n\n<pre>\n{}\n</pre>'.format('\n'.join(
[section.text for section in sections]).strip())
# Generate a hash based on the binaries involved and the number of
# sections. This eliminates version or release changes from causing
# an update to the comment while still updating on relevant changes.
binaries = set()
for section in sections:
binaries.update(section.binaries)
info = ';'.join(['::'.join(sorted(binaries)), str(len(sections))])
reference = hashlib.sha1(info).hexdigest()[:7]
# Post comment on package in order to notifiy maintainers.
self.comment_write(state='seen',
result=reference,
bot_name_suffix=bot_name_suffix,
project=comment_project,
package=comment_package,
message=message)
def config_resolve(apiurl, project, config):
defaults = POLICY_DEFAULTS.copy()
defaults_workarounds = POLICY_DEFAULTS.copy()
origins_original = config_origin_list(config)
config_project = Config.get(apiurl, project)
config_resolve_variables(config, config_project)
origins = config['origins']
i = 0
while i < len(origins):
origin = next(iter(origins[i]))
values = origins[i][origin]
if origin == '*':
del origins[i]
defaults.update(values)
defaults_workarounds.update(values)
config_resolve_apply(config, values, until='*')
elif origin == '*~':
del origins[i]
defaults_workarounds.update(values)
config_resolve_create_workarounds(config, values, origins_original)
config_resolve_apply(config, values, workaround=True, until='*~')
elif '*' in origin:
# Does not allow for family + workaround expansion (ie. foo*~).
del origins[i]
config_resolve_create_family(apiurl, project, config, i, origin,
values)
elif origin.endswith('~'):
values_new = deepcopy(defaults_workarounds)
values_new.update(values)
values.update(values_new)
i += 1
else:
values_new = deepcopy(defaults)
values_new.update(values)
values.update(values_new)
i += 1
return config
def config_resolve(apiurl, project, config):
defaults = POLICY_DEFAULTS.copy()
defaults_workarounds = POLICY_DEFAULTS.copy()
origins_original = config_origin_list(config)
config_project = Config.get(apiurl, project)
config_resolve_variables(config, config_project)
origins = config['origins']
i = 0
while i < len(origins):
origin = origins[i].keys()[0]
values = origins[i][origin]
if origin == '*':
del origins[i]
defaults.update(values)
defaults_workarounds.update(values)
config_resolve_apply(config, values, until='*')
elif origin == '*~':
del origins[i]
defaults_workarounds.update(values)
config_resolve_create_workarounds(config, values, origins_original)
config_resolve_apply(config, values, workaround=True, until='*~')
elif '*' in origin:
# Does not allow for family + workaround expansion (ie. foo*~).
del origins[i]
config_resolve_create_family(apiurl, project, config, i, origin, values)
elif origin.endswith('~'):
values_new = deepcopy(defaults_workarounds)
values_new.update(values)
values.update(values_new)
i += 1
else:
values_new = deepcopy(defaults)
values_new.update(values)
values.update(values_new)
i += 1
return config
def check_action_maintenance_release(self, request, action):
# No reason to special case patchinfo since same source and target
# projects which is all that repo_checker cares about.
if action.tgt_project in self.checked_targets:
return True
target_config = Config.get(self.apiurl, action.tgt_project)
if str2bool(target_config.get('repo_checker-project-skip', 'False')):
# Do not change message as this should only occur in requests
# targeting multiple projects such as in maintenance workflow in
# which the message should be set by other actions.
self.logger.debug('skipping review of action targeting {}'.format(
action.tgt_project))
return True
repository = target_config.get('main-repo')
if not repository:
raise Exception('Missing main-repo in OSRT:Config')
# Find a repository which links to target project "main" repository.
repository = repository_path_search(self.apiurl, action.src_project,
action.tgt_project, repository)
if not repository:
raise Exception('Missing repositories')
repository_pairs = repository_path_expand(self.apiurl,
action.src_project,
repository)
self.checked_targets.add(action.tgt_project)
archs = set(target_archs(self.apiurl, action.src_project, repository))
arch_whitelist = target_config.get('repo_checker-arch-whitelist', None)
if arch_whitelist:
archs = set(arch_whitelist.split(' ')).intersection(archs)
if not self.repository_check(repository_pairs, archs):
return None
self.review_messages['accepted'] = 'install check passed'
return True
def package_comments(self, project, repository):
self.logger.info('{} package comments'.format(len(self.package_results)))
for package, sections in self.package_results.items():
if str2bool(Config.get(self.apiurl, project).get('repo_checker-package-comment-devel', 'False')):
bot_name_suffix = project
comment_project, comment_package = devel_project_fallback(self.apiurl, project, package)
if comment_project is None or comment_package is None:
self.logger.warning('unable to find devel project for {}'.format(package))
continue
message = 'The version of this package in [`{project}`](/package/show/{project}/{package}) ' \
'has installation issues and may not be installable:'.format(
project=project, package=package)
else:
bot_name_suffix = repository
comment_project = project
comment_package = package
message = 'This package has installation issues and may not be installable from the `{}` ' \
'repository:'.format(repository)
# Sort sections by text to group binaries together.
sections = sorted(sections, key=lambda s: s.text)
message += '\n\n<pre>\n{}\n</pre>'.format(
'\n'.join([section.text for section in sections]).strip())
# Generate a hash based on the binaries involved and the number of
# sections. This eliminates version or release changes from causing
# an update to the comment while still updating on relevant changes.
binaries = set()
for section in sections:
binaries.update(section.binaries)
info = ';'.join(['::'.join(sorted(binaries)), str(len(sections))])
reference = hashlib.sha1(info).hexdigest()[:7]
# Post comment on package in order to notifiy maintainers.
self.comment_write(state='seen', result=reference, bot_name_suffix=bot_name_suffix,
project=comment_project, package=comment_package, message=message)
def remote_config_set(self, config, replace_all=False):
"""Sets the values of the 'Config' attribute for the target project.
That attribute stores a set of values that are useful to influence the behavior of several
tools and bots in the context of the given project. For convenience, such a collection of
values is usually accessed using a :class:`osclib.Config` object. See :func:`load_config`.
:param config: values to write into the attribute
:type config: dict[str, str]
:param replace_all: whether the previous content of 'Config' should be cleared up
:type replace_all: bool
"""
if not replace_all:
config_existing = Config.get(self.apiurl, self.project)
config_existing.update(config)
config = config_existing
config_lines = []
for key, value in config.items():
config_lines.append(f'{key} = {value}')
attribute_value_save(APIURL, self.project, 'Config', '\n'.join(config_lines))
def mail_send(apiurl,
project,
to,
subject,
body,
from_key='maintainer',
followup_to_key='release-list',
dry=False):
from email.mime.text import MIMEText
import email.utils
import smtplib
config = Config.get(apiurl, project)
msg = MIMEText(body)
msg['Message-ID'] = email.utils.make_msgid()
msg['Date'] = email.utils.formatdate(localtime=1)
if from_key is None:
msg['From'] = entity_email(apiurl,
conf.get_apiurl_usr(apiurl),
include_name=True)
else:
msg['From'] = config['mail-{}'.format(from_key)]
if '@' not in to:
to = config['mail-{}'.format(to)]
msg['To'] = to
followup_to = config.get('mail-{}'.format(followup_to_key))
if followup_to:
msg['Mail-Followup-To'] = followup_to
msg['Subject'] = subject
if dry:
print(msg.as_string())
return
s = smtplib.SMTP(config.get('mail-relay', 'relay.suse.de'))
s.sendmail(msg['From'], [msg['To']], msg.as_string())
s.quit()
def project_pseudometa_package(apiurl, project):
package = Config.get(apiurl, project).get('pseudometa_package', '00Meta')
if '/' in package:
project, package = package.split('/', 2)
return project, package
default='openSUSE:Factory',
help='project to check (ex. openSUSE:Factory, openSUSE:Leap:15.1)')
parser.add_argument('-d',
'--debug',
action='store_true',
default=False,
help='enable debug information')
parser.add_argument('-A', '--apiurl', metavar='URL', help='API URL')
args = parser.parse_args()
osc.conf.get_config(override_apiurl=args.apiurl)
osc.conf.config['debug'] = args.debug
apiurl = osc.conf.config['apiurl']
config = Config.get(apiurl, args.project)
api = StagingAPI(apiurl, args.project)
staging_report = InstallChecker(api, config)
if args.debug:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
if args.staging:
if not staging_report.staging(api.prj_from_short(args.staging),
force=True):
sys.exit(1)
else:
for staging in api.get_staging_projects():
if api.is_adi_project(staging):
def check_pra(self, project, repository, arch):
config = Config.get(self.apiurl, project)
oldstate = None
self.store_filename = 'rebuildpacs.{}-{}.yaml'.format(project, repository)
if self.store_project and self.store_package:
state_yaml = source_file_load(self.apiurl, self.store_project, self.store_package,
self.store_filename)
if state_yaml:
oldstate = yaml.safe_load(state_yaml)
oldstate = oldstate or {}
oldstate.setdefault('check', {})
if not isinstance(oldstate['check'], dict):
oldstate['check'] = {}
oldstate.setdefault('leafs', {})
if not isinstance(oldstate['leafs'], dict):
oldstate['leafs'] = {}
repository_pairs = repository_path_expand(self.apiurl, project, repository)
directories = []
primaryxmls = []
for pair_project, pair_repository in repository_pairs:
mirrored = mirror(self.apiurl, pair_project, pair_repository, arch)
if os.path.isdir(mirrored):
directories.append(mirrored)
else:
primaryxmls.append(mirrored)
parsed = dict()
with tempfile.TemporaryDirectory(prefix='repochecker') as dir:
pfile = os.path.join(dir, 'packages')
SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__))
script = os.path.join(SCRIPT_PATH, 'write_repo_susetags_file.pl')
parts = ['perl', script, dir] + directories
p = subprocess.run(parts)
if p.returncode:
# technically only 126, but there is no other value atm -
# so if some other perl error happens, we don't continue
raise CorruptRepos
target_packages = []
with open(os.path.join(dir, 'catalog.yml')) as file:
catalog = yaml.safe_load(file)
if catalog is not None:
target_packages = catalog.get(directories[0], [])
parsed = parsed_installcheck([pfile] + primaryxmls, arch, target_packages, [])
for package in parsed:
parsed[package]['output'] = "\n".join(parsed[package]['output'])
# let's risk a N*N algorithm in the hope that we have a limited N
for package1 in parsed:
output = parsed[package1]['output']
for package2 in parsed:
if package1 == package2:
continue
output = output.replace(parsed[package2]['output'], 'FOLLOWUP(' + package2 + ')')
parsed[package1]['output'] = output
for package in parsed:
parsed[package]['output'] = self._split_and_filter(parsed[package]['output'])
url = makeurl(self.apiurl, ['build', project, '_result'], {
'repository': repository, 'arch': arch, 'code': 'succeeded'})
root = ET.parse(http_GET(url)).getroot()
succeeding = list(map(lambda x: x.get('package'), root.findall('.//status')))
per_source = dict()
for package, entry in parsed.items():
source = "{}/{}/{}/{}".format(project, repository, arch, entry['source'])
per_source.setdefault(source, {'output': [], 'builds': entry['source'] in succeeding})
per_source[source]['output'].extend(entry['output'])
rebuilds = set()
for source in sorted(per_source):
if not len(per_source[source]['output']):
continue
self.logger.debug("{} builds: {}".format(source, per_source[source]['builds']))
self.logger.debug(" " + "\n ".join(per_source[source]['output']))
if not per_source[source]['builds']: # nothing we can do
continue
old_output = oldstate['check'].get(source, {}).get('problem', [])
if sorted(old_output) == sorted(per_source[source]['output']):
self.logger.debug("unchanged problem")
continue
self.logger.info("rebuild %s", source)
rebuilds.add(os.path.basename(source))
for line in difflib.unified_diff(old_output, per_source[source]['output'], 'before', 'now'):
self.logger.debug(line.strip())
oldstate['check'][source] = {'problem': per_source[source]['output'],
'rebuild': str(datetime.datetime.now())}
for source in list(oldstate['check']):
if not source.startswith('{}/{}/{}/'.format(project, repository, arch)):
continue
if not os.path.basename(source) in succeeding:
continue
if source not in per_source:
self.logger.info("No known problem, erasing %s", source)
del oldstate['check'][source]
packages = config.get('rebuildpacs-leafs', '').split()
if not self.rebuild: # ignore in this case
packages = []
# first round: collect all infos from obs
infos = dict()
for package in packages:
subpacks, build_deps = self.check_leaf_package(project, repository, arch, package)
infos[package] = {'subpacks': subpacks, 'deps': build_deps}
# calculate rebuild triggers
rebuild_triggers = dict()
for package1 in packages:
for package2 in packages:
if package1 == package2:
continue
for subpack in infos[package1]['subpacks']:
if subpack in infos[package2]['deps']:
rebuild_triggers.setdefault(package1, set())
rebuild_triggers[package1].add(package2)
# ignore this depencency. we already trigger both of them
del infos[package2]['deps'][subpack]
# calculate build info hashes
for package in packages:
if not package in succeeding:
self.logger.debug("Ignore %s for the moment, not succeeding", package)
continue
m = hashlib.sha256()
for bdep in sorted(infos[package]['deps']):
m.update(bytes(bdep + '-' + infos[package]['deps'][bdep], 'utf-8'))
state_key = '{}/{}/{}/{}'.format(project, repository, arch, package)
olddigest = oldstate['leafs'].get(state_key, {}).get('buildinfo')
if olddigest == m.hexdigest():
continue
self.logger.info("rebuild leaf package %s (%s vs %s)", package, olddigest, m.hexdigest())
rebuilds.add(package)
oldstate['leafs'][state_key] = {'buildinfo': m.hexdigest(),
'rebuild': str(datetime.datetime.now())}
if self.dryrun:
if self.rebuild:
self.logger.info("To rebuild: %s", ' '.join(rebuilds))
return
if not self.rebuild or not len(rebuilds):
self.logger.debug("Nothing to rebuild")
# in case we do rebuild, wait for it to succeed before saving
self.store_yaml(oldstate, project, repository, arch)
return
query = {'cmd': 'rebuild', 'repository': repository, 'arch': arch, 'package': rebuilds}
url = makeurl(self.apiurl, ['build', project])
headers = { 'Content-Type': 'application/x-www-form-urlencoded' }
http_request('POST', url, headers, data=urlencode(query, doseq=True))
self.store_yaml(oldstate, project, repository, arch)
def repository_check(self, repository_pairs, state_hash, simulate_merge, whitelist=None, arch_whitelist=None, post_comments=False, cycle_packages=None):
comment = []
project, repository = repository_pairs[0]
self.logger.info('checking {}/{}@{}[{}]'.format(
project, repository, state_hash, len(repository_pairs)))
archs = self.target_archs(project, repository, arch_whitelist)
new_pairs = []
for pair in repository_pairs:
has_all = True
for arch in archs:
if not repository_arch_state(self.apiurl, pair[0], pair[1], arch):
has_all = False
break
# ignore repositories only inherited for config
if has_all:
new_pairs.append(pair)
repository_pairs = new_pairs
published = repositories_published(self.apiurl, repository_pairs, archs)
if not self.force:
if state_hash == self.repository_state_last(project, repository, simulate_merge):
self.logger.info('{} build unchanged'.format(project))
# TODO keep track of skipped count for cycle summary
return None
# For submit style requests, want to process if top layer is done,
# but not mark review as final until all layers are published.
if published is not True and (not simulate_merge or published[0] == project):
# Require all layers to be published except when the top layer
# is published in a simulate merge (allows quicker feedback with
# potentially incorrect resutls for staging).
self.logger.info('{}/{} not published'.format(published[0], published[1]))
return None
# Drop non-published repository information and thus reduce to boolean.
published = published is True
if not simulate_merge:
# Top of pseudometa file.
comment.append(state_hash)
if post_comments:
# Stores parsed install_check() results grouped by package.
self.package_results = {}
if not len(archs):
self.logger.debug('{} has no relevant architectures'.format(project))
return None
result = True
for arch in archs:
directories = []
for pair_project, pair_repository in repository_pairs:
directories.append(self.mirror(pair_project, pair_repository, arch))
if simulate_merge:
ignore = self.simulated_merge_ignore(repository_pairs[0], repository_pairs[1], arch)
if not whitelist:
whitelist = self.binary_whitelist(repository_pairs[0], repository_pairs[1], arch)
results = {
'cycle': self.cycle_check(repository_pairs[0][0], repository_pairs[0][1], arch, cycle_packages),
'install': self.install_check(
repository_pairs[1], arch, directories, ignore, whitelist),
}
else:
# Only products themselves will want no-filter or perhaps
# projects working on cleaning up a product.
no_filter = str2bool(Config.get(self.apiurl, project).get('repo_checker-no-filter'))
results = {
'cycle': CheckResult(True, None),
'install': self.install_check(repository_pairs[0], arch, directories,
parse=post_comments, no_filter=no_filter),
}
if not all(result.success for _, result in results.items()):
# Not all checks passed, build comment.
result = False
self.result_comment(repository, arch, results, comment)
if simulate_merge:
info_extra = {'build': state_hash}
if not result:
# Some checks in group did not pass, post comment.
# Avoid identical comments with different build hash during
# target project build phase. Once published update regardless.
self.comment_write(state='seen', result='failed', project=project,
message='\n'.join(comment).strip(), identical=True,
info_extra=info_extra, info_extra_identical=published,
bot_name_suffix=repository)
else:
# Post passed comment only if previous failed comment.
text = 'Previously reported problems have been resolved.'
self.comment_write(state='done', result='passed', project=project,
message=text, identical=True, only_replace=True,
info_extra=info_extra, bot_name_suffix=repository)
else:
text = '\n'.join(comment).strip()
if not self.dryrun:
filename = self.project_pseudometa_file_name(project, repository)
project_pseudometa_file_ensure(
self.apiurl, project, filename, text + '\n', 'repo_checker project_only run')
else:
print(text)
if post_comments:
self.package_comments(project, repository)
if result and not published:
# Wait for the complete stack to build before positive result.
self.logger.debug('demoting result from accept to ignore due to non-published layer')
result = None
return result
def repository_check(self,
repository_pairs,
state_hash,
simulate_merge,
post_comments=False):
comment = []
project, repository = repository_pairs[0]
self.logger.info('checking {}/{}@{}[{}]'.format(
project, repository, state_hash, len(repository_pairs)))
published = repositories_published(self.apiurl, repository_pairs)
if not self.force:
if state_hash == self.repository_state_last(
project, repository, not simulate_merge):
self.logger.info('{} build unchanged'.format(project))
# TODO keep track of skipped count for cycle summary
return None
# For submit style requests, want to process if top layer is done,
# but not mark review as final until all layers are published.
if published is not True and (not simulate_merge
or published[0] == project):
# Require all layers to be published except when the top layer
# is published in a simulate merge (allows quicker feedback with
# potentially incorrect resutls for staging).
self.logger.info('{}/{} not published'.format(
published[0], published[1]))
return None
# Drop non-published repository information and thus reduce to boolean.
published = published is True
if simulate_merge:
# Restrict top layer archs to the whitelisted archs from merge layer.
archs = set(target_archs(self.apiurl, project,
repository)).intersection(
set(
self.target_archs(
repository_pairs[1][0],
repository_pairs[1][1])))
else:
# Top of pseudometa file.
comment.append(state_hash)
archs = self.target_archs(project, repository)
if post_comments:
# Stores parsed install_check() results grouped by package.
self.package_results = {}
if not len(archs):
self.logger.debug(
'{} has no relevant architectures'.format(project))
return None
result = True
for arch in archs:
directories = []
for pair_project, pair_repository in repository_pairs:
directories.append(
self.mirror(pair_project, pair_repository, arch))
if simulate_merge:
ignore = self.simulated_merge_ignore(repository_pairs[0],
repository_pairs[1], arch)
whitelist = self.binary_whitelist(repository_pairs[0],
repository_pairs[1], arch)
results = {
'cycle':
self.cycle_check(repository_pairs[0], repository_pairs[1],
arch),
'install':
self.install_check(repository_pairs[1], arch, directories,
ignore, whitelist),
}
else:
# Only products themselves will want no-filter or perhaps
# projects working on cleaning up a product.
no_filter = str2bool(
Config.get(self.apiurl,
project).get('repo_checker-no-filter'))
results = {
'cycle':
CheckResult(True, None),
'install':
self.install_check(repository_pairs[0],
arch,
directories,
parse=post_comments,
no_filter=no_filter),
}
if not all(result.success for _, result in results.items()):
# Not all checks passed, build comment.
result = False
self.result_comment(repository, arch, results, comment)
if simulate_merge:
info_extra = {'build': state_hash}
if not result:
# Some checks in group did not pass, post comment.
# Avoid identical comments with different build hash during
# target project build phase. Once published update regardless.
self.comment_write(state='seen',
result='failed',
project=project,
message='\n'.join(comment).strip(),
identical=True,
info_extra=info_extra,
info_extra_identical=published,
bot_name_suffix=repository)
else:
# Post passed comment only if previous failed comment.
text = 'Previously reported problems have been resolved.'
self.comment_write(state='done',
result='passed',
project=project,
message=text,
identical=True,
only_replace=True,
info_extra=info_extra,
bot_name_suffix=repository)
else:
text = '\n'.join(comment).strip()
if not self.dryrun:
filename = self.project_pseudometa_file_name(
project, repository)
project_pseudometa_file_ensure(
self.apiurl, project, filename, text + '\n',
'repo_checker project_only run')
else:
print(text)
if post_comments:
self.package_comments(project, repository)
if result and not published:
# Wait for the complete stack to build before positive result.
self.logger.debug(
'demoting result from accept to ignore due to non-published layer'
)
result = None
return result
