def target_project_config(self, project):
# Load project config and allow for remote entries.
config = Config.get(self.apiurl, project)
self.single_action_require = str2bool(config.get('check-source-single-action-require', 'False'))
self.ignore_devel = not str2bool(config.get('devel-project-enforce', 'False'))
self.in_air_rename_allow = str2bool(config.get('check-source-in-air-rename-allow', 'False'))
self.add_review_team = str2bool(config.get('check-source-add-review-team', 'True'))
self.review_team = config.get('review-team')
self.staging_group = config.get('staging-group')
self.repo_checker = config.get('repo-checker')
self.devel_whitelist = config.get('devel-whitelist', '').split()
self.skip_add_reviews = False
if self.action.type == 'maintenance_incident':
# The workflow effectively enforces the names to match and the
# parent code sets target_package from source_package so this check
# becomes useless and awkward to perform.
self.in_air_rename_allow = True
# The target project will be set to product and thus inherit
# settings, but override since real target is not product.
self.single_action_require = False
# It might make sense to supersede maintbot, but for now.
self.skip_add_reviews = True
def target_project_config(self, project):
# Load project config and allow for remote entries.
config = Config.get(self.apiurl, project)
self.single_action_require = str2bool(
config.get('check-source-single-action-require', 'False'))
self.ignore_devel = not str2bool(
config.get('devel-project-enforce', 'False'))
self.in_air_rename_allow = str2bool(
config.get('check-source-in-air-rename-allow', 'False'))
self.add_review_team = str2bool(
config.get('check-source-add-review-team', 'True'))
self.review_team = config.get('review-team')
self.mail_release_list = config.get('mail-release-list')
self.staging_group = config.get('staging-group')
self.repo_checker = config.get('repo-checker')
self.devel_whitelist = config.get('devel-whitelist', '').split()
self.skip_add_reviews = False
self.security_review_team = config.get('security-review-team',
'security-team')
self.bad_rpmlint_entries = config.get('bad-rpmlint-entries',
'').split()
if self.action.type == 'maintenance_incident':
# The workflow effectively enforces the names to match and the
# parent code sets target_package from source_package so this check
# becomes useless and awkward to perform.
self.in_air_rename_allow = True
# The target project will be set to product and thus inherit
# settings, but override since real target is not product.
self.single_action_require = False
# It might make sense to supersede maintbot, but for now.
self.skip_add_reviews = True
def target_project_config(self, project):
# Load project config and allow for remote entries.
self.staging_api(project)
config = self.staging_config[project]
self.ignore_devel = not str2bool(
config.get('devel-project-enforce', 'False'))
self.in_air_rename_allow = str2bool(
config.get('check-source-in-air-rename-allow', 'False'))
self.add_review_team = str2bool(
config.get('check-source-add-review-team', 'True'))
self.review_team = config.get('review-team')
self.repo_checker = config.get('repo-checker')
self.devel_whitelist = config.get('devel-whitelist', '').split()
def request_repository_pairs(self, request, action):
if str2bool(
Config.get(self.apiurl,
action.tgt_project).get('repo_checker-project-skip',
'False')):
# Do not change message as this should only occur in requests
# targeting multiple projects such as in maintenance workflow in
# which the message should be set by other actions.
self.logger.debug('skipping review of action targeting {}'.format(
action.tgt_project))
return True
repository = self.project_repository(action.tgt_project)
if not repository:
self.review_messages['declined'] = ERROR_REPO_SPECIFIED.format(
action.tgt_project)
return False
repository_pairs = []
# Assumes maintenance_release target project has staging disabled.
staging = Config.get(self.apiurl, action.tgt_project).get('staging')
if staging:
api = self.staging_api(staging)
stage_info = api.packages_staged.get(action.tgt_package)
if not stage_info or str(stage_info['rq_id']) != str(
request.reqid):
self.logger.info('{} not staged'.format(request.reqid))
return None
if not self.force and not self.staging_build_failure_check(
api, stage_info['prj']):
self.logger.info(
'{} not ready due to staging build failure(s)'.format(
request.reqid))
return None
# Staging setup is convoluted and thus the repository setup does not
# contain a path to the target project. Instead the ports repository
# is used to import the target prjconf. As such the staging group
# repository must be explicitly layered on top of target project.
repository_pairs.append([stage_info['prj'], repository])
repository_pairs.extend(
repository_path_expand(self.apiurl, action.tgt_project,
repository))
else:
# Find a repository which links to target project "main" repository.
repository = repository_path_search(self.apiurl,
action.src_project,
action.tgt_project, repository)
if not repository:
self.review_messages['declined'] = ERROR_REPO_SPECIFIED.format(
action.tgt_project)
return False
repository_pairs.extend(
repository_path_expand(self.apiurl, action.src_project,
repository))
return repository_pairs
def package_comments(self, project, repository):
self.logger.info('{} package comments'.format(len(
self.package_results)))
for package, sections in self.package_results.items():
if str2bool(
Config.get(self.apiurl, project).get(
'repo_checker-package-comment-devel', 'False')):
bot_name_suffix = project
comment_project, comment_package = devel_project_fallback(
self.apiurl, project, package)
if comment_project is None or comment_package is None:
self.logger.warning(
'unable to find devel project for {}'.format(package))
continue
message = 'The version of this package in [`{project}`](/package/show/{project}/{package}) ' \
'has installation issues and may not be installable:'.format(
project=project, package=package)
else:
bot_name_suffix = repository
comment_project = project
comment_package = package
message = 'This package has installation issues and may not be installable from the `{}` ' \
'repository:'.format(repository)
# Sort sections by text to group binaries together.
sections = sorted(sections, key=lambda s: s.text)
message += '\n\n<pre>\n{}\n</pre>'.format('\n'.join(
[section.text for section in sections]).strip())
# Generate a hash based on the binaries involved and the number of
# sections. This eliminates version or release changes from causing
# an update to the comment while still updating on relevant changes.
binaries = set()
for section in sections:
binaries.update(section.binaries)
info = ';'.join(['::'.join(sorted(binaries)), str(len(sections))])
reference = hashlib.sha1(info).hexdigest()[:7]
# Post comment on package in order to notifiy maintainers.
self.comment_write(state='seen',
result=reference,
bot_name_suffix=bot_name_suffix,
project=comment_project,
package=comment_package,
message=message)
def request_repository_pairs(self, request, action):
if str2bool(Config.get(self.apiurl, action.tgt_project).get('repo_checker-project-skip', 'False')):
# Do not change message as this should only occur in requests
# targeting multiple projects such as in maintenance workflow in
# which the message should be set by other actions.
self.logger.debug('skipping review of action targeting {}'.format(action.tgt_project))
return True
repository = self.project_repository(action.tgt_project)
if not repository:
self.review_messages['declined'] = ERROR_REPO_SPECIFIED.format(action.tgt_project)
return False
repository_pairs = []
# Assumes maintenance_release target project has staging disabled.
staging = Config.get(self.apiurl, action.tgt_project).get('staging')
if staging:
api = self.staging_api(staging)
stage_info = api.packages_staged.get(action.tgt_package)
if not stage_info or str(stage_info['rq_id']) != str(request.reqid):
self.logger.info('{} not staged'.format(request.reqid))
return None
if not self.force and not self.staging_build_failure_check(api, stage_info['prj']):
self.logger.info('{} not ready due to staging build failure(s)'.format(request.reqid))
return None
# Staging setup is convoluted and thus the repository setup does not
# contain a path to the target project. Instead the ports repository
# is used to import the target prjconf. As such the staging group
# repository must be explicitly layered on top of target project.
repository_pairs.append([stage_info['prj'], repository])
repository_pairs.extend(repository_path_expand(self.apiurl, action.tgt_project, repository))
else:
# Find a repository which links to target project "main" repository.
repository = repository_path_search(
self.apiurl, action.src_project, action.tgt_project, repository)
if not repository:
self.review_messages['declined'] = ERROR_REPO_SPECIFIED.format(action.tgt_project)
return False
repository_pairs.extend(repository_path_expand(self.apiurl, action.src_project, repository))
return repository_pairs
def check_action_maintenance_release(self, request, action):
# No reason to special case patchinfo since same source and target
# projects which is all that repo_checker cares about.
if action.tgt_project in self.checked_targets:
return True
target_config = Config.get(self.apiurl, action.tgt_project)
if str2bool(target_config.get('repo_checker-project-skip', 'False')):
# Do not change message as this should only occur in requests
# targeting multiple projects such as in maintenance workflow in
# which the message should be set by other actions.
self.logger.debug('skipping review of action targeting {}'.format(
action.tgt_project))
return True
repository = target_config.get('main-repo')
if not repository:
raise Exception('Missing main-repo in OSRT:Config')
# Find a repository which links to target project "main" repository.
repository = repository_path_search(self.apiurl, action.src_project,
action.tgt_project, repository)
if not repository:
raise Exception('Missing repositories')
repository_pairs = repository_path_expand(self.apiurl,
action.src_project,
repository)
self.checked_targets.add(action.tgt_project)
archs = set(target_archs(self.apiurl, action.src_project, repository))
arch_whitelist = target_config.get('repo_checker-arch-whitelist', None)
if arch_whitelist:
archs = set(arch_whitelist.split(' ')).intersection(archs)
if not self.repository_check(repository_pairs, archs):
return None
self.review_messages['accepted'] = 'install check passed'
return True
def __init__(self, api, config):
self.api = api
self.logger = logging.getLogger('InstallChecker')
self.commentapi = CommentAPI(api.apiurl)
self.arch_whitelist = config.get('repo_checker-arch-whitelist')
if self.arch_whitelist:
self.arch_whitelist = set(self.arch_whitelist.split(' '))
self.ring_whitelist = set(
config.get('repo_checker-binary-whitelist-ring', '').split(' '))
self.cycle_packages = config.get('repo_checker-allowed-in-cycles')
self.calculate_allowed_cycles()
self.ignore_duplicated = set(
config.get('installcheck-ignore-duplicated-binaries',
'').split(' '))
self.ignore_conflicts = set(
config.get('installcheck-ignore-conflicts', '').split(' '))
self.ignore_deletes = str2bool(
config.get('installcheck-ignore-deletes', 'False'))
def package_comments(self, project, repository):
self.logger.info('{} package comments'.format(len(self.package_results)))
for package, sections in self.package_results.items():
if str2bool(Config.get(self.apiurl, project).get('repo_checker-package-comment-devel', 'False')):
bot_name_suffix = project
comment_project, comment_package = devel_project_fallback(self.apiurl, project, package)
if comment_project is None or comment_package is None:
self.logger.warning('unable to find devel project for {}'.format(package))
continue
message = 'The version of this package in [`{project}`](/package/show/{project}/{package}) ' \
'has installation issues and may not be installable:'.format(
project=project, package=package)
else:
bot_name_suffix = repository
comment_project = project
comment_package = package
message = 'This package has installation issues and may not be installable from the `{}` ' \
'repository:'.format(repository)
# Sort sections by text to group binaries together.
sections = sorted(sections, key=lambda s: s.text)
message += '\n\n<pre>\n{}\n</pre>'.format(
'\n'.join([section.text for section in sections]).strip())
# Generate a hash based on the binaries involved and the number of
# sections. This eliminates version or release changes from causing
# an update to the comment while still updating on relevant changes.
binaries = set()
for section in sections:
binaries.update(section.binaries)
info = ';'.join(['::'.join(sorted(binaries)), str(len(sections))])
reference = hashlib.sha1(info).hexdigest()[:7]
# Post comment on package in order to notifiy maintainers.
self.comment_write(state='seen', result=reference, bot_name_suffix=bot_name_suffix,
project=comment_project, package=comment_package, message=message)
def init(directory='main'):
if Cache.CACHE_DIR:
# Stick with the first initialization to allow for StagingAPI to
# ensure always enabled, but allow parent to change directory.
return
Cache.CACHE_DIR = CacheManager.directory('request', directory)
Cache.patterns = []
if str2bool(os.environ.get('OSRT_DISABLE_CACHE', '')):
if conf.config['debug']:
print('CACHE_DISABLE via $OSRT_DISABLE_CACHE', file=sys.stderr)
return
for pattern in Cache.PATTERNS:
Cache.patterns.append(re.compile(pattern))
# Replace http_request with wrapper function which needs a stored
# version of the original function to call.
if not hasattr(osc.core, '_http_request'):
osc.core._http_request = osc.core.http_request
osc.core.http_request = http_request
def update_and_solve_target(self, api, target_project, target_config, main_repo,
project, scope, force, no_checkout,
only_release_packages, stop_after_solve):
self.all_architectures = target_config.get('pkglistgen-archs').split(' ')
ignore_repos = set(target_config.get('pkglistgen-ignore_repos', '').split(' '))
self.repos = [ r for r in self.expand_repos(project, main_repo) if r[0] != project or r[1] not in ignore_repos ]
print('[{}] {}/{}: update and solve'.format(scope, project, main_repo))
group = target_config.get('pkglistgen-group', '000package-groups')
product = target_config.get('pkglistgen-product', '000product')
release = target_config.get('pkglistgen-release', '000release-packages')
oldrepos = target_config.get('pkglistgen-repos', '000update-repos')
url = api.makeurl(['source', project])
packages = ET.parse(http_GET(url)).getroot()
if packages.find('entry[@name="{}"]'.format(product)) is None:
if not self.dry_run:
undelete_package(api.apiurl, project, product, 'revive')
# TODO disable build.
print('{} undeleted, skip dvd until next cycle'.format(product))
return
elif not force:
root = ET.fromstringlist(show_results_meta(api.apiurl, project, product,
repository=[main_repo], multibuild=True))
if len(root.xpath('result[@state="building"]')) or len(root.xpath('result[@state="dirty"]')):
print('{}/{} build in progress'.format(project, product))
return
drop_list = api.item_exists(project, oldrepos)
checkout_list = [group, product, release]
if drop_list and not only_release_packages:
checkout_list.append(oldrepos)
if packages.find('entry[@name="{}"]'.format(release)) is None:
if not self.dry_run:
undelete_package(api.apiurl, project, release, 'revive')
print('{} undeleted, skip dvd until next cycle'.format(release))
return
# Cache dir specific to hostname and project.
host = urlparse(api.apiurl).hostname
cache_dir = CacheManager.directory('pkglistgen', host, project)
if not no_checkout:
if os.path.exists(cache_dir):
shutil.rmtree(cache_dir)
os.makedirs(cache_dir)
group_dir = os.path.join(cache_dir, group)
product_dir = os.path.join(cache_dir, product)
release_dir = os.path.join(cache_dir, release)
oldrepos_dir = os.path.join(cache_dir, oldrepos)
for package in checkout_list:
if no_checkout:
print('Skipping checkout of {}/{}'.format(project, package))
continue
checkout_package(api.apiurl, project, package, expand_link=True, prj_dir=cache_dir)
file_utils.unlink_all_except(release_dir, ['weakremovers.inc'])
if not only_release_packages:
file_utils.unlink_all_except(product_dir)
file_utils.copy_directory_contents(group_dir, product_dir,
['supportstatus.txt', 'groups.yml',
'reference-unsorted.yml', 'reference-summary.yml',
'package-groups.changes'])
file_utils.change_extension(product_dir, '.spec.in', '.spec')
file_utils.change_extension(product_dir, '.product.in', '.product')
self.input_dir = group_dir
self.output_dir = product_dir
print('-> do_update')
# make sure we only calculcate existant architectures
self.filter_architectures(target_archs(api.apiurl, project, main_repo))
self.update_repos(self.filtered_architectures)
if only_release_packages:
self.load_all_groups()
self.write_group_stubs()
else:
summary = self.solve_project(ignore_unresolvable=str2bool(target_config.get('pkglistgen-ignore-unresolvable')),
ignore_recommended=str2bool(target_config.get('pkglistgen-ignore-recommended')),
locale = target_config.get('pkglistgen-locale'),
locales_from = target_config.get('pkglistgen-locales-from'))
if stop_after_solve:
return
if drop_list:
weakremovers_file = os.path.join(release_dir, 'weakremovers.inc')
self.create_weakremovers(project, target_config, oldrepos_dir, output=open(weakremovers_file, 'w'))
delete_products = target_config.get('pkglistgen-delete-products', '').split(' ')
file_utils.unlink_list(product_dir, delete_products)
print('-> product service')
product_version = attribute_value_load(api.apiurl, project, 'ProductVersion')
if not product_version:
# for stagings the product version doesn't matter (I hope)
product_version = '1'
for product_file in glob.glob(os.path.join(product_dir, '*.product')):
self.replace_product_version(product_file, product_version)
print(subprocess.check_output(
[PRODUCT_SERVICE, product_file, product_dir, project], encoding='utf-8'))
for delete_kiwi in target_config.get('pkglistgen-delete-kiwis-{}'.format(scope), '').split(' '):
delete_kiwis = glob.glob(os.path.join(product_dir, delete_kiwi))
file_utils.unlink_list(product_dir, delete_kiwis)
if scope == 'staging':
self.strip_medium_from_staging(product_dir)
spec_files = glob.glob(os.path.join(product_dir, '*.spec'))
file_utils.move_list(spec_files, release_dir)
inc_files = glob.glob(os.path.join(group_dir, '*.inc'))
# filter special inc file
inc_files = filter(lambda file: file.endswith('weakremovers.inc'), inc_files)
file_utils.move_list(inc_files, release_dir)
# do not overwrite weakremovers.inc if it exists
# we will commit there afterwards if needed
if os.path.exists(os.path.join(group_dir, 'weakremovers.inc')) and \
not os.path.exists(os.path.join(release_dir, 'weakremovers.inc')):
file_utils.move_list([os.path.join(group_dir, 'weakremovers.inc')], release_dir)
file_utils.multibuild_from_glob(release_dir, '*.spec')
self.build_stub(release_dir, 'spec')
self.commit_package(release_dir)
if only_release_packages:
return
file_utils.multibuild_from_glob(product_dir, '*.kiwi')
self.build_stub(product_dir, 'kiwi')
self.commit_package(product_dir)
error_output = b''
reference_summary = os.path.join(group_dir, 'reference-summary.yml')
if os.path.isfile(reference_summary):
summary_file = os.path.join(product_dir, 'summary.yml')
with open(summary_file, 'w') as f:
f.write('# Summary of packages in groups')
for group in sorted(summary):
# the unsorted group should appear filtered by
# unneeded.yml - so we need the content of unsorted.yml
# not unsorted.group (this grew a little unnaturally)
if group == 'unsorted':
continue
f.write('\n' + group + ':\n')
for package in sorted(summary[group]):
f.write(' - ' + package + '\n')
try:
error_output += subprocess.check_output(['diff', '-u', reference_summary, summary_file])
except subprocess.CalledProcessError as e:
error_output += e.output
reference_unsorted = os.path.join(group_dir, 'reference-unsorted.yml')
unsorted_file = os.path.join(product_dir, 'unsorted.yml')
try:
error_output += subprocess.check_output(['diff', '-u', reference_unsorted, unsorted_file])
except subprocess.CalledProcessError as e:
error_output += e.output
if len(error_output) > 0:
self.logger.error('Difference in yml:\n' + error_output.decode('utf-8'))
return True
def repository_check(self,
repository_pairs,
state_hash,
simulate_merge,
post_comments=False):
comment = []
project, repository = repository_pairs[0]
self.logger.info('checking {}/{}@{}[{}]'.format(
project, repository, state_hash, len(repository_pairs)))
published = repositories_published(self.apiurl, repository_pairs)
if not self.force:
if state_hash == self.repository_state_last(
project, repository, not simulate_merge):
self.logger.info('{} build unchanged'.format(project))
# TODO keep track of skipped count for cycle summary
return None
# For submit style requests, want to process if top layer is done,
# but not mark review as final until all layers are published.
if published is not True and (not simulate_merge
or published[0] == project):
# Require all layers to be published except when the top layer
# is published in a simulate merge (allows quicker feedback with
# potentially incorrect resutls for staging).
self.logger.info('{}/{} not published'.format(
published[0], published[1]))
return None
# Drop non-published repository information and thus reduce to boolean.
published = published is True
if simulate_merge:
# Restrict top layer archs to the whitelisted archs from merge layer.
archs = set(target_archs(self.apiurl, project,
repository)).intersection(
set(
self.target_archs(
repository_pairs[1][0],
repository_pairs[1][1])))
else:
# Top of pseudometa file.
comment.append(state_hash)
archs = self.target_archs(project, repository)
if post_comments:
# Stores parsed install_check() results grouped by package.
self.package_results = {}
if not len(archs):
self.logger.debug(
'{} has no relevant architectures'.format(project))
return None
result = True
for arch in archs:
directories = []
for pair_project, pair_repository in repository_pairs:
directories.append(
self.mirror(pair_project, pair_repository, arch))
if simulate_merge:
ignore = self.simulated_merge_ignore(repository_pairs[0],
repository_pairs[1], arch)
whitelist = self.binary_whitelist(repository_pairs[0],
repository_pairs[1], arch)
results = {
'cycle':
self.cycle_check(repository_pairs[0], repository_pairs[1],
arch),
'install':
self.install_check(repository_pairs[1], arch, directories,
ignore, whitelist),
}
else:
# Only products themselves will want no-filter or perhaps
# projects working on cleaning up a product.
no_filter = str2bool(
Config.get(self.apiurl,
project).get('repo_checker-no-filter'))
results = {
'cycle':
CheckResult(True, None),
'install':
self.install_check(repository_pairs[0],
arch,
directories,
parse=post_comments,
no_filter=no_filter),
}
if not all(result.success for _, result in results.items()):
# Not all checks passed, build comment.
result = False
self.result_comment(repository, arch, results, comment)
if simulate_merge:
info_extra = {'build': state_hash}
if not result:
# Some checks in group did not pass, post comment.
# Avoid identical comments with different build hash during
# target project build phase. Once published update regardless.
self.comment_write(state='seen',
result='failed',
project=project,
message='\n'.join(comment).strip(),
identical=True,
info_extra=info_extra,
info_extra_identical=published,
bot_name_suffix=repository)
else:
# Post passed comment only if previous failed comment.
text = 'Previously reported problems have been resolved.'
self.comment_write(state='done',
result='passed',
project=project,
message=text,
identical=True,
only_replace=True,
info_extra=info_extra,
bot_name_suffix=repository)
else:
text = '\n'.join(comment).strip()
if not self.dryrun:
filename = self.project_pseudometa_file_name(
project, repository)
project_pseudometa_file_ensure(
self.apiurl, project, filename, text + '\n',
'repo_checker project_only run')
else:
print(text)
if post_comments:
self.package_comments(project, repository)
if result and not published:
# Wait for the complete stack to build before positive result.
self.logger.debug(
'demoting result from accept to ignore due to non-published layer'
)
result = None
return result
def origin_update(apiurl, target_project, package):
origin_info = origin_find(apiurl, target_project, package)
if not origin_info:
# Cases for a lack of origin:
# - package does not exist in target_project
# - initial package submission from devel (lacking devel meta on package)
# - initial package submission overriden to allow from no origin
# - origin project/package deleted
#
# Ideally, the second case should never be used and instead the first
# case should be opted for instead.
# Check for accepted source submission with devel annotation and create
# change_devel request as automatic follow-up to approval.
config = config_load(apiurl, target_project)
request_actions = request_action_list_source(apiurl,
target_project,
package,
states=['accepted'])
for request, action in sorted(request_actions,
key=lambda i: i[0].reqid,
reverse=True):
annotation = origin_annotation_load(request, action,
config['review-user'])
if not annotation:
continue
origin = annotation.get('origin')
if origin_workaround_check(origin):
continue
if origin not in config_origin_list(config, apiurl,
target_project):
message = f'Set devel project based on initial submission in request#{request.reqid}.'
return request_create_change_devel(apiurl,
origin,
package,
target_project,
message=message)
# Either the first or one of the second two cases.
exists = entity_exists(apiurl, target_project, package)
origin, version = origin_potential(apiurl, target_project, package,
not exists)
if origin is None:
if not exists:
# Package does not exist to be deleted.
return False
# Package is not found in any origin so request deletion.
message = 'Package not available from any potential origin.'
return request_create_delete(apiurl, target_project, package,
message)
if not exists:
if origin_update_initial_blacklisted(apiurl, target_project,
package):
return False
message = 'Submitting new package from highest potential origin.'
return request_create_submit(apiurl,
origin,
package,
target_project,
message=message,
ignore_if_any_request=True)
# No longer tracking previous origin (likely removed from origin) so
# submit from the highest potential origin.
message = 'Submitting package from highest potential origin.'
return request_create_submit(apiurl,
origin,
package,
target_project,
message=message)
if origin_workaround_check(origin_info.project):
# Do not attempt to update workarounds as the expected flow is to either
# to explicitely switched back to non-workaround or source to match at
# some point and implicitily switch.
return False
if origin_info.pending:
# Already accepted source ahead of origin so nothing to do.
return False
policy = policy_get(apiurl, target_project, package, origin_info.project)
if not policy['automatic_updates']:
return False
mode = origin_update_mode(apiurl, target_project, package, policy,
origin_info.project)
if mode['skip']:
return False
age = package_source_age(apiurl, origin_info.project,
package).total_seconds()
if age < int(mode['delay']):
return False
supersede = str2bool(str(mode['supersede']))
frequency = int(mode['frequency'])
if policy['pending_submission_allow']:
request_id = origin_update_pending(apiurl, origin_info.project,
package, target_project, policy,
supersede, frequency)
if request_id:
return request_id
message = 'Newer source available from package origin.'
return request_create_submit(apiurl,
origin_info.project,
package,
target_project,
message=message,
supersede=supersede,
frequency=frequency)
def update_and_solve_target(self, api, target_project, target_config,
main_repo, project, scope, force, no_checkout,
only_release_packages, stop_after_solve):
self.all_architectures = target_config.get('pkglistgen-archs').split(
' ')
self.use_newest_version = str2bool(
target_config.get('pkglistgen-use-newest-version', 'False'))
self.repos = self.expand_repos(project, main_repo)
logging.debug('[{}] {}/{}: update and solve'.format(
scope, project, main_repo))
group = target_config.get('pkglistgen-group', '000package-groups')
product = target_config.get('pkglistgen-product', '000product')
release = target_config.get('pkglistgen-release',
'000release-packages')
oldrepos = target_config.get('pkglistgen-repos', '000update-repos')
url = api.makeurl(['source', project])
packages = ET.parse(http_GET(url)).getroot()
if packages.find('entry[@name="{}"]'.format(product)) is None:
if not self.dry_run:
undelete_package(api.apiurl, project, product, 'revive')
# TODO disable build.
logging.info(
'{} undeleted, skip dvd until next cycle'.format(product))
return
elif not force:
root = ET.fromstringlist(
show_results_meta(api.apiurl,
project,
product,
repository=[main_repo],
multibuild=True))
if len(root.xpath('result[@state="building"]')) or len(
root.xpath('result[@state="dirty"]')):
logging.info('{}/{} build in progress'.format(
project, product))
return
drop_list = api.item_exists(project, oldrepos)
checkout_list = [group, product, release]
if drop_list and not only_release_packages:
checkout_list.append(oldrepos)
if packages.find('entry[@name="{}"]'.format(release)) is None:
if not self.dry_run:
undelete_package(api.apiurl, project, release, 'revive')
logging.info(
'{} undeleted, skip dvd until next cycle'.format(release))
return
# Cache dir specific to hostname and project.
host = urlparse(api.apiurl).hostname
cache_dir = CacheManager.directory('pkglistgen', host, project)
if not no_checkout:
if os.path.exists(cache_dir):
shutil.rmtree(cache_dir)
os.makedirs(cache_dir)
group_dir = os.path.join(cache_dir, group)
product_dir = os.path.join(cache_dir, product)
release_dir = os.path.join(cache_dir, release)
oldrepos_dir = os.path.join(cache_dir, oldrepos)
self.input_dir = group_dir
self.output_dir = product_dir
for package in checkout_list:
if no_checkout:
logging.debug('Skipping checkout of {}/{}'.format(
project, package))
continue
checkout_package(api.apiurl,
project,
package,
expand_link=True,
prj_dir=cache_dir,
outdir=os.path.join(cache_dir, package))
file_utils.unlink_all_except(release_dir, ['weakremovers.inc'])
if not only_release_packages:
file_utils.unlink_all_except(product_dir)
ignore_list = [
'supportstatus.txt', 'summary-staging.txt',
'package-groups.changes'
]
ignore_list += self.group_input_files()
file_utils.copy_directory_contents(group_dir, product_dir, ignore_list)
file_utils.change_extension(product_dir, '.spec.in', '.spec')
file_utils.change_extension(product_dir, '.product.in', '.product')
logging.debug('-> do_update')
# make sure we only calculcate existant architectures
self.filter_architectures(target_archs(api.apiurl, project, main_repo))
self.update_repos(self.filtered_architectures)
if only_release_packages:
self.load_all_groups()
self.write_group_stubs()
else:
summary = self.solve_project(
ignore_unresolvable=str2bool(
target_config.get('pkglistgen-ignore-unresolvable')),
ignore_recommended=str2bool(
target_config.get('pkglistgen-ignore-recommended')),
locale=target_config.get('pkglistgen-locale'),
locales_from=target_config.get('pkglistgen-locales-from'))
if stop_after_solve:
return
if drop_list and not only_release_packages:
weakremovers_file = os.path.join(release_dir, 'weakremovers.inc')
try:
self.create_weakremovers(project,
target_config,
oldrepos_dir,
output=open(weakremovers_file, 'w'))
except MismatchedRepoException:
logging.error(
"Failed to create weakremovers.inc due to mismatch in repos - project most likey started building again."
)
return
delete_products = target_config.get('pkglistgen-delete-products',
'').split(' ')
file_utils.unlink_list(product_dir, delete_products)
logging.debug('-> product service')
product_version = attribute_value_load(api.apiurl, project,
'ProductVersion')
if not product_version:
# for stagings the product version doesn't matter (I hope)
product_version = '1'
for product_file in glob.glob(os.path.join(product_dir, '*.product')):
self.replace_product_version(product_file, product_version)
logging.debug(
subprocess.check_output(
[PRODUCT_SERVICE, product_file, product_dir, project],
encoding='utf-8'))
for delete_kiwi in target_config.get(
'pkglistgen-delete-kiwis-{}'.format(scope), '').split(' '):
delete_kiwis = glob.glob(os.path.join(product_dir, delete_kiwi))
file_utils.unlink_list(product_dir, delete_kiwis)
if scope == 'staging':
self.strip_medium_from_staging(product_dir)
spec_files = glob.glob(os.path.join(product_dir, '*.spec'))
file_utils.move_list(spec_files, release_dir)
inc_files = glob.glob(os.path.join(group_dir, '*.inc'))
# filter special inc file
inc_files = filter(lambda file: file.endswith('weakremovers.inc'),
inc_files)
file_utils.move_list(inc_files, release_dir)
# do not overwrite weakremovers.inc if it exists
# we will commit there afterwards if needed
if os.path.exists(os.path.join(group_dir, 'weakremovers.inc')) and \
not os.path.exists(os.path.join(release_dir, 'weakremovers.inc')):
file_utils.move_list([os.path.join(group_dir, 'weakremovers.inc')],
release_dir)
file_utils.multibuild_from_glob(release_dir, '*.spec')
self.build_stub(release_dir, 'spec')
self.commit_package(release_dir)
if only_release_packages:
return
file_utils.multibuild_from_glob(product_dir, '*.kiwi')
self.build_stub(product_dir, 'kiwi')
reference_summary = os.path.join(group_dir, f'summary-{scope}.txt')
if os.path.isfile(reference_summary):
summary_file = os.path.join(product_dir, f'summary-{scope}.txt')
output = []
for group in summary:
for package in sorted(summary[group]):
output.append(f'{package}:{group}')
with open(summary_file, 'w') as f:
for line in sorted(output):
f.write(line + '\n')
self.commit_package(product_dir)
if os.path.isfile(reference_summary):
return self.comment.handle_package_diff(project, reference_summary,
summary_file)
def update_and_solve_target(self, api, target_project, target_config, main_repo,
project, scope, force, no_checkout,
only_release_packages, stop_after_solve):
self.all_architectures = target_config.get('pkglistgen-archs').split(' ')
self.repos = self.expand_repos(project, main_repo)
print('[{}] {}/{}: update and solve'.format(scope, project, main_repo))
group = target_config.get('pkglistgen-group', '000package-groups')
product = target_config.get('pkglistgen-product', '000product')
release = target_config.get('pkglistgen-release', '000release-packages')
oldrepos = target_config.get('pkglistgen-repos', '000update-repos')
url = api.makeurl(['source', project])
packages = ET.parse(http_GET(url)).getroot()
if packages.find('entry[@name="{}"]'.format(product)) is None:
if not self.dry_run:
undelete_package(api.apiurl, project, product, 'revive')
# TODO disable build.
print('{} undeleted, skip dvd until next cycle'.format(product))
return
elif not force:
root = ET.fromstringlist(show_results_meta(api.apiurl, project, product,
repository=[main_repo], multibuild=True))
if len(root.xpath('result[@state="building"]')) or len(root.xpath('result[@state="dirty"]')):
print('{}/{} build in progress'.format(project, product))
return
drop_list = api.item_exists(project, oldrepos)
checkout_list = [group, product, release]
if drop_list and not only_release_packages:
checkout_list.append(oldrepos)
if packages.find('entry[@name="{}"]'.format(release)) is None:
if not self.dry_run:
undelete_package(api.apiurl, project, release, 'revive')
print('{} undeleted, skip dvd until next cycle'.format(release))
return
# Cache dir specific to hostname and project.
host = urlparse(api.apiurl).hostname
cache_dir = CacheManager.directory('pkglistgen', host, project)
if not no_checkout:
if os.path.exists(cache_dir):
shutil.rmtree(cache_dir)
os.makedirs(cache_dir)
group_dir = os.path.join(cache_dir, group)
product_dir = os.path.join(cache_dir, product)
release_dir = os.path.join(cache_dir, release)
oldrepos_dir = os.path.join(cache_dir, oldrepos)
for package in checkout_list:
if no_checkout:
print('Skipping checkout of {}/{}'.format(project, package))
continue
checkout_package(api.apiurl, project, package, expand_link=True, prj_dir=cache_dir)
file_utils.unlink_all_except(release_dir, ['weakremovers.inc'])
if not only_release_packages:
file_utils.unlink_all_except(product_dir)
file_utils.copy_directory_contents(group_dir, product_dir,
['supportstatus.txt', 'groups.yml',
'reference-unsorted.yml', 'reference-summary.yml',
'package-groups.changes'])
file_utils.change_extension(product_dir, '.spec.in', '.spec')
file_utils.change_extension(product_dir, '.product.in', '.product')
self.input_dir = group_dir
self.output_dir = product_dir
print('-> do_update')
# make sure we only calculcate existant architectures
self.filter_architectures(target_archs(api.apiurl, project, main_repo))
self.update_repos(self.filtered_architectures)
if only_release_packages:
self.load_all_groups()
self.write_group_stubs()
else:
summary = self.solve_project(ignore_unresolvable=str2bool(target_config.get('pkglistgen-ignore-unresolvable')),
ignore_recommended=str2bool(target_config.get('pkglistgen-ignore-recommended')),
locale = target_config.get('pkglistgen-local'),
locales_from = target_config.get('pkglistgen-locales-from'))
if stop_after_solve:
return
if drop_list:
weakremovers_file = os.path.join(release_dir, 'weakremovers.inc')
self.create_weakremovers(project, target_config, oldrepos_dir, output=open(weakremovers_file, 'w'))
delete_products = target_config.get('pkglistgen-delete-products', '').split(' ')
file_utils.unlink_list(product_dir, delete_products)
print('-> product service')
product_version = attribute_value_load(api.apiurl, project, 'ProductVersion')
if not product_version:
# for stagings the product version doesn't matter (I hope)
product_version = '1'
for product_file in glob.glob(os.path.join(product_dir, '*.product')):
self.replace_product_version(product_file, product_version)
print(subprocess.check_output(
[PRODUCT_SERVICE, product_file, product_dir, project], encoding='utf-8'))
for delete_kiwi in target_config.get('pkglistgen-delete-kiwis-{}'.format(scope), '').split(' '):
delete_kiwis = glob.glob(os.path.join(product_dir, delete_kiwi))
file_utils.unlink_list(product_dir, delete_kiwis)
if scope == 'staging':
self.strip_medium_from_staging(product_dir)
spec_files = glob.glob(os.path.join(product_dir, '*.spec'))
file_utils.move_list(spec_files, release_dir)
inc_files = glob.glob(os.path.join(group_dir, '*.inc'))
# filter special inc file
inc_files = filter(lambda file: file.endswith('weakremovers.inc'), inc_files)
file_utils.move_list(inc_files, release_dir)
# do not overwrite weakremovers.inc if it exists
# we will commit there afterwards if needed
if not os.path.exists(os.path.join(release_dir, 'weakremovers.inc')):
file_utils.move_list([os.path.join(group_dir, 'weakremovers.inc')], release_dir)
file_utils.multibuild_from_glob(release_dir, '*.spec')
self.build_stub(release_dir, 'spec')
self.commit_package(release_dir)
if only_release_packages:
return
file_utils.multibuild_from_glob(product_dir, '*.kiwi')
self.build_stub(product_dir, 'kiwi')
self.commit_package(product_dir)
error_output = ''
reference_summary = os.path.join(group_dir, 'reference-summary.yml')
if os.path.isfile(reference_summary):
summary_file = os.path.join(product_dir, 'summary.yml')
with open(summary_file, 'w') as f:
f.write('# Summary of packages in groups')
for group in sorted(summary):
# the unsorted group should appear filtered by
# unneeded.yml - so we need the content of unsorted.yml
# not unsorted.group (this grew a little unnaturally)
if group == 'unsorted':
continue
f.write('\n' + group + ':\n')
for package in sorted(summary[group]):
f.write(' - ' + package + '\n')
try:
error_output += subprocess.check_output(['diff', '-u', reference_summary, summary_file])
except subprocess.CalledProcessError as e:
error_output += e.output
reference_unsorted = os.path.join(group_dir, 'reference-unsorted.yml')
unsorted_file = os.path.join(product_dir, 'unsorted.yml')
try:
error_output += subprocess.check_output(['diff', '-u', reference_unsorted, unsorted_file])
except subprocess.CalledProcessError as e:
error_output += e.output
if len(error_output) > 0:
self.logger.error('Difference in yml:\n' + error_output)
return True
def repository_check(self, repository_pairs, state_hash, simulate_merge, whitelist=None, arch_whitelist=None, post_comments=False, cycle_packages=None):
comment = []
project, repository = repository_pairs[0]
self.logger.info('checking {}/{}@{}[{}]'.format(
project, repository, state_hash, len(repository_pairs)))
archs = self.target_archs(project, repository, arch_whitelist)
new_pairs = []
for pair in repository_pairs:
has_all = True
for arch in archs:
if not repository_arch_state(self.apiurl, pair[0], pair[1], arch):
has_all = False
break
# ignore repositories only inherited for config
if has_all:
new_pairs.append(pair)
repository_pairs = new_pairs
published = repositories_published(self.apiurl, repository_pairs, archs)
if not self.force:
if state_hash == self.repository_state_last(project, repository, simulate_merge):
self.logger.info('{} build unchanged'.format(project))
# TODO keep track of skipped count for cycle summary
return None
# For submit style requests, want to process if top layer is done,
# but not mark review as final until all layers are published.
if published is not True and (not simulate_merge or published[0] == project):
# Require all layers to be published except when the top layer
# is published in a simulate merge (allows quicker feedback with
# potentially incorrect resutls for staging).
self.logger.info('{}/{} not published'.format(published[0], published[1]))
return None
# Drop non-published repository information and thus reduce to boolean.
published = published is True
if not simulate_merge:
# Top of pseudometa file.
comment.append(state_hash)
if post_comments:
# Stores parsed install_check() results grouped by package.
self.package_results = {}
if not len(archs):
self.logger.debug('{} has no relevant architectures'.format(project))
return None
result = True
for arch in archs:
directories = []
for pair_project, pair_repository in repository_pairs:
directories.append(self.mirror(pair_project, pair_repository, arch))
if simulate_merge:
ignore = self.simulated_merge_ignore(repository_pairs[0], repository_pairs[1], arch)
if not whitelist:
whitelist = self.binary_whitelist(repository_pairs[0], repository_pairs[1], arch)
results = {
'cycle': self.cycle_check(repository_pairs[0][0], repository_pairs[0][1], arch, cycle_packages),
'install': self.install_check(
repository_pairs[1], arch, directories, ignore, whitelist),
}
else:
# Only products themselves will want no-filter or perhaps
# projects working on cleaning up a product.
no_filter = str2bool(Config.get(self.apiurl, project).get('repo_checker-no-filter'))
results = {
'cycle': CheckResult(True, None),
'install': self.install_check(repository_pairs[0], arch, directories,
parse=post_comments, no_filter=no_filter),
}
if not all(result.success for _, result in results.items()):
# Not all checks passed, build comment.
result = False
self.result_comment(repository, arch, results, comment)
if simulate_merge:
info_extra = {'build': state_hash}
if not result:
# Some checks in group did not pass, post comment.
# Avoid identical comments with different build hash during
# target project build phase. Once published update regardless.
self.comment_write(state='seen', result='failed', project=project,
message='\n'.join(comment).strip(), identical=True,
info_extra=info_extra, info_extra_identical=published,
bot_name_suffix=repository)
else:
# Post passed comment only if previous failed comment.
text = 'Previously reported problems have been resolved.'
self.comment_write(state='done', result='passed', project=project,
message=text, identical=True, only_replace=True,
info_extra=info_extra, bot_name_suffix=repository)
else:
text = '\n'.join(comment).strip()
if not self.dryrun:
filename = self.project_pseudometa_file_name(project, repository)
project_pseudometa_file_ensure(
self.apiurl, project, filename, text + '\n', 'repo_checker project_only run')
else:
print(text)
if post_comments:
self.package_comments(project, repository)
if result and not published:
# Wait for the complete stack to build before positive result.
self.logger.debug('demoting result from accept to ignore due to non-published layer')
result = None
return result
